lsm/mdbtools
1
0
Fork 0
mirror of https://github.com/mdbtools/mdbtools.git synced 2025-06-28 15:39:02 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

Support fuzz testing (#4)

Browse Source 
Quickstart (requires Clang 6 or later):

$ export LIB_FUZZING_ENGINE=/path/to/fuzzing/library.a
$ ./configure --enable-fuzz-testing
$ make
$ cd src/fuzz
$ make fuzz_mdb
$ ./fuzz_mdb

Also add a new `mdb_open_buffer function` to facilitate in-memory
fuzz-testing. This requires fmemopen, which may not be present on all
systems. The internal API has been reworked to use file streams instead
of file descriptors. This allows reading from memory and reading from
files using a consistent API.
This commit is contained in:
Evan Miller 2020-08-31 13:03:58 -04:00 committed by GitHub
parent 8b40423f65
commit be888e0dd7
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 194 additions and 89 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
README.md
View File

@ -23,6 +23,7 @@ Intended focus areas of this fork:
- [ ] Security / stability / fuzz testing
- [x] Thread safety\*
- [x] In-memory database API
- [x] Removing GLib dependency
- [x] Improved ODBC compliance
- [x] Continuous integration with Travis and AppVeyor

10
appveyor.yml
View File

@ -39,9 +39,19 @@ test_script:
if ($env:TOOLCHAIN -eq "msys2")
{
$env:MSYSTEM="MINGW64"
C:\msys64\usr\bin\bash -l -c "cd /c/projects/mdbtools && ./src/util/mdb-ver ./test/data/ASampleDatabase.accdb"
C:\msys64\usr\bin\bash -l -c "cd /c/projects/mdbtools && ./src/util/mdb-ver ./test/data/nwind.mdb"
C:\msys64\usr\bin\bash -l -c "cd /c/projects/mdbtools && ./src/util/mdb-tables ./test/data/ASampleDatabase.accdb"
C:\msys64\usr\bin\bash -l -c "cd /c/projects/mdbtools && ./src/util/mdb-tables ./test/data/nwind.mdb"
C:\msys64\usr\bin\bash -l -c "cd /c/projects/mdbtools && ./src/util/mdb-count ./test/data/ASampleDatabase.accdb 'Asset Items'"
C:\msys64\usr\bin\bash -l -c "cd /c/projects/mdbtools && ./src/util/mdb-count ./test/data/nwind.mdb Customers"
}
else
{
C:\cygwin64\bin\sh -lc "cd /cygdrive/c/projects/mdbtools && ./src/util/mdb-ver ./test/data/ASampleDatabase.accdb"
C:\cygwin64\bin\sh -lc "cd /cygdrive/c/projects/mdbtools && ./src/util/mdb-ver ./test/data/nwind.mdb"
C:\cygwin64\bin\sh -lc "cd /cygdrive/c/projects/mdbtools && ./src/util/mdb-tables ./test/data/ASampleDatabase.accdb"
C:\cygwin64\bin\sh -lc "cd /cygdrive/c/projects/mdbtools && ./src/util/mdb-tables ./test/data/nwind.mdb"
C:\cygwin64\bin\sh -lc "cd /cygdrive/c/projects/mdbtools && ./src/util/mdb-count ./test/data/ASampleDatabase.accdb 'Asset Items'"
C:\cygwin64\bin\sh -lc "cd /cygdrive/c/projects/mdbtools && ./src/util/mdb-count ./test/data/nwind.mdb Customers"
}

32
configure.ac
View File

@ -9,6 +9,7 @@ AM_MAINTAINER_MODE([enable])
m4_ifdef([AM_SILENT_RULES], [AM_SILENT_RULES([yes])])
AC_PROG_CC(gcc)
AC_PROG_CXX
dnl Checks for programs.
AC_PROG_MAKE_SET
m4_pattern_allow([AM_PROG_AR], [AM_PROG_AR])
@ -25,6 +26,11 @@ AC_CHECK_DECLS([program_invocation_short_name], [], [], [[
#define _GNU_SOURCE
#include <errno.h>]])
dnl Checks for library functions.
VL_LIB_READLINE
AC_CHECK_FUNC(strptime,[ AC_DEFINE(HAVE_STRPTIME, 1, [strptime check]) AM_CONDITIONAL(HAVE_STRPTIME, true) ],[ AM_CONDITIONAL(HAVE_STRPTIME, false) ])
AC_CHECK_FUNC(fmemopen,[ AC_DEFINE(HAVE_FMEMOPEN, 1, [fmemopen check]) ])
dnl Checks for typedefs, structures, and compiler characteristics.
AC_C_CONST
AC_TYPE_SIZE_T
@ -76,6 +82,27 @@ AS_CASE([$host],
[*mingw*|*cygwin*], [LDFLAGS="$LDFLAGS -no-undefined"], [])
AS_CASE([$host],
[*mingw*], [LDFLAGS="$LDFLAGS -lWs2_32"], [])
dnl Fuzz testing
AC_ARG_ENABLE([fuzz-testing], AS_HELP_STRING([--enable-fuzz-testing], ["Enable fuzz testing (requires Clang 6 or later)"]), [
AC_MSG_CHECKING([whether $CC accepts -fsanitize=fuzzer])
tmp_saved_flags=$[]_AC_LANG_PREFIX[]FLAGS
_AC_LANG_PREFIX[]FLAGS="$[]_AC_LANG_PREFIX[]FLAGS -fsanitize=fuzzer"
AC_LINK_IFELSE([AC_LANG_PROGRAM()],
[
AC_MSG_RESULT(yes)
OPTDIRS="$OPTDIRS fuzz"
fuzzer=yes],
AC_MSG_RESULT(no)
AC_MSG_FAILURE([-fsanitize=fuzzer not supported (Required with --enable-fuzz-testing)]))
_AC_LANG_PREFIX[]FLAGS=$tmp_saved_flags
], [fuzzer=no])
AM_CONDITIONAL([FUZZER_ENABLED], test "x$fuzzer" = "xyes")
AC_ARG_VAR([LIB_FUZZING_ENGINE], [Location of prebuilt fuzzing engine library])
AC_SUBST([LIB_FUZZING_ENGINE])
dnl Enable -Wl,--as-needed by default to prevent overlinking
AC_ARG_ENABLE([as-needed],
@ -269,10 +296,6 @@ AM_CONDITIONAL(ENABLE_MAN, test "$enable_man" = yes)
##################################################
AM_CONDITIONAL(ENABLE_DOCBOOK, test -n "$DOCBOOK_DSL")
dnl Checks for library functions.
VL_LIB_READLINE
AC_CHECK_FUNC(strptime,[ AC_DEFINE(HAVE_STRPTIME, 1, [strptime check]) AM_CONDITIONAL(HAVE_STRPTIME, true) ],[ AM_CONDITIONAL(HAVE_STRPTIME, false) ])
localedir=${datadir}/locale
AC_SUBST(localedir)
@ -287,6 +310,7 @@ include/Makefile
src/libmdb/Makefile
src/sql/Makefile
src/odbc/Makefile
src/fuzz/Makefile
doc/Makefile
src/gmdb2/Makefile
src/gmdb2/gladefiles/Makefile

4
include/mdbtools.h
View File

@ -225,9 +225,8 @@ typedef struct {
} MdbStatistics;
typedef struct {
int fd;
FILE *stream;
gboolean writable;
char *filename;
guint32 jet_version;
guint32 db_key;
char db_passwd[14];
@ -463,6 +462,7 @@ long mdb_pg_get_int32(MdbHandle *mdb, int offset);
float mdb_pg_get_single(MdbHandle *mdb, int offset);
double mdb_pg_get_double(MdbHandle *mdb, int offset);
MdbHandle *mdb_open(const char *filename, MdbFileFlags flags);
MdbHandle *mdb_open_buffer(void *buffer, size_t len, MdbFileFlags flags);
void mdb_close(MdbHandle *mdb);
MdbHandle *mdb_clone_handle(MdbHandle *mdb);
void mdb_swap_pgbuf(MdbHandle *mdb);

9
src/fuzz/Makefile.am Normal file
View File

@ -0,0 +1,9 @@
EXTRA_PROGRAMS = fuzz_mdb
# Force C++ linking for fuzz target
nodist_EXTRA_fuzz_mdb_SOURCES = dummy.cxx
fuzz_mdb_SOURCES = fuzz_mdb.c
AM_CFLAGS = -I$(top_srcdir)/include $(GLIB_CFLAGS) -g
AM_LDFLAGS = -static
LDADD = ../libmdb/libmdb.la @LIB_FUZZING_ENGINE@

20
src/fuzz/fuzz_mdb.c Normal file
View File

@ -0,0 +1,20 @@
#include "mdbtools.h"
int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) {
MdbHandle *mdb = mdb_open_buffer((void *)Data, Size, MDB_NOFLAGS);
if (mdb) {
mdb_read_catalog(mdb, MDB_TABLE);
for (int j=0; j<mdb->num_catalog; j++) {
MdbCatalogEntry *entry = g_ptr_array_index (mdb->catalog, j);
MdbTableDef *table = mdb_read_table(entry);
if (table) {
mdb_read_columns(table);
mdb_rewind_table(table);
while (mdb_fetch_row(table));
mdb_free_tabledef(table);
}
}
mdb_close(mdb);
}
return 0;
}

4
src/libmdb/Makefile.am
View File

@ -5,4 +5,8 @@ libmdb_la_SOURCES += fakeglib.c
endif
libmdb_la_LDFLAGS = -version-info 2:1:0 -export-symbols-regex '^(mdb_|_mdb_put_int16$$|_mdb_put_int32$$)'
AM_CFLAGS = -I$(top_srcdir)/include $(GLIB_CFLAGS) -Wsign-compare
if FUZZER_ENABLED
AM_CFLAGS += -fsanitize=fuzzer-no-link -fsanitize=address
libmdb_la_LDFLAGS += -fsanitize=fuzzer -fsanitize=address
endif
LIBS = $(GLIB_LIBS) @LIBS@ @LIBICONV@

4
src/libmdb/catalog.c
View File

@ -93,6 +93,7 @@ GPtrArray *mdb_read_catalog (MdbHandle *mdb, int objtype)
table = mdb_read_table(&msysobj);
if (!table) {
fprintf(stderr, "Unable to read table %s\n", msysobj.object_name);
mdb_free_catalog(mdb);
goto cleanup;
}
@ -103,10 +104,13 @@ GPtrArray *mdb_read_catalog (MdbHandle *mdb, int objtype)
mdb_bind_column_by_name(table, "Name", obj_name, NULL) == -1 ||
mdb_bind_column_by_name(table, "Type", obj_type, NULL) == -1 ||
mdb_bind_column_by_name(table, "Flags", obj_flags, NULL) == -1) {
fprintf(stderr, "Unable to bind columns from table %s (%d columns found)\n",
msysobj.object_name, table->num_cols);
mdb_free_catalog(mdb);
goto cleanup;
}
if ((i = mdb_bind_column_by_name(table, "LvProp", obj_props, &kkd_size_ole)) == -1) {
fprintf(stderr, "Unable to bind column %s from table %s\n", "LvProp", msysobj.object_name);
mdb_free_catalog(mdb);
goto cleanup;
}

151
src/libmdb/file.c
View File

@ -156,25 +156,22 @@ static char *mdb_find_file(const char *file_name)
g_strfreev(dir);
return NULL;
}
/**
* mdb_open:
* @filename: path to MDB (database) file
* @flags: MDB_NOFLAGS for read-only, MDB_WRITABLE for read/write
* mdb_handle_from_stream:
* @stream An open file stream
* @flags MDB_NOFLAGS for read-only, MDB_WRITABLE for read/write
*
* Opens an MDB file and returns an MdbHandle to it. MDB File may be relative
* to the current directory, a full path to the file, or relative to a
* component of $MDBPATH.
* Allocates, initializes, and return an MDB handle from a file stream pointing
* to an MDB file.
*
* Return value: pointer to MdbHandle structure.
**/
MdbHandle *mdb_open(const char *filename, MdbFileFlags flags)
{
MdbHandle *mdb;
* Return value: The handle on success, NULL on failure
*/
static MdbHandle *mdb_handle_from_stream(FILE *stream, MdbFileFlags flags) {
int key[] = {0x86, 0xfb, 0xec, 0x37, 0x5d, 0x44, 0x9c, 0xfa, 0xc6, 0x5e, 0x28, 0xe6, 0x13, 0xb6};
int j, pos;
int open_flags;
mdb = (MdbHandle *) g_malloc0(sizeof(MdbHandle));
MdbHandle *mdb = (MdbHandle *) g_malloc0(sizeof(MdbHandle));
mdb_set_default_backend(mdb, "access");
mdb_set_date_fmt(mdb, "%x %X");
mdb_set_boolean_fmt_numbers(mdb);
@ -186,33 +183,13 @@ MdbHandle *mdb_open(const char *filename, MdbFileFlags flags)
mdb->fmt = &MdbJet3Constants;
mdb->f = (MdbFile *) g_malloc0(sizeof(MdbFile));
mdb->f->refs = 1;
mdb->f->fd = -1;
mdb->f->filename = mdb_find_file(filename);
if (!mdb->f->filename) {
fprintf(stderr, "File not found\n");
mdb_close(mdb);
return NULL;
}
mdb->f->stream = stream;
if (flags & MDB_WRITABLE) {
mdb->f->writable = TRUE;
open_flags = O_RDWR;
} else {
open_flags = O_RDONLY;
}
#ifdef _WIN32
open_flags |= O_BINARY;
#endif
mdb->f->fd = open(mdb->f->filename, open_flags);
if (mdb->f->fd==-1) {
fprintf(stderr,"Couldn't open file %s\n",mdb->f->filename);
mdb_close(mdb);
return NULL;
}
if (!mdb_read_pg(mdb, 0)) {
fprintf(stderr,"Couldn't read first page.\n");
// fprintf(stderr,"Couldn't read first page.\n");
mdb_close(mdb);
return NULL;
}
@ -245,23 +222,8 @@ MdbHandle *mdb_open(const char *filename, MdbFileFlags flags)
*/
mdb->f->db_key ^= 0x4ebc8afb;
/* fprintf(stderr, "Encrypted file, RC4 key seed= %d\n", mdb->f->db_key); */
if (mdb->f->db_key) {
/* write is not supported for encrypted files yet */
mdb->f->writable = FALSE;
/* that should be enough, but reopen the file read only just to be
* sure we don't write invalid data */
close(mdb->f->fd);
open_flags = O_RDONLY;
#ifdef _WIN32
open_flags |= O_BINARY;
#endif
mdb->f->fd = open(mdb->f->filename, open_flags);
if (mdb->f->fd==-1) {
fprintf(stderr, "Couldn't ropen file %s in read only\n", mdb->f->filename);
mdb_close(mdb);
return NULL;
}
}
mdb->f->writable = mdb->f->writable && !mdb->f->db_key;
/* get the db password located at 0x42 bytes into the file */
for (pos=0;pos<14;pos++) {
@ -278,6 +240,66 @@ MdbHandle *mdb_open(const char *filename, MdbFileFlags flags)
return mdb;
}
/**
* mdb_open_buffer:
* @buffer A memory buffer containing an MDB file
* @len Length of the buffer
*
* Opens an MDB file in memory and returns an MdbHandle to it.
*
* Return value: point to MdbHandle structure.
*/
MdbHandle *mdb_open_buffer(void *buffer, size_t len, MdbFileFlags flags) {
FILE *file = NULL;
#ifdef HAVE_FMEMOPEN
file = fmemopen(buffer, len, (flags & MDB_WRITABLE) ? "r+" : "r");
#else
fprintf(stderr, "mdb_open_buffer requires a platform with support for fmemopen(3)\n");
#endif
if (file == NULL) {
fprintf(stderr, "Couldn't open memory buffer\n");
return NULL;
}
return mdb_handle_from_stream(file, flags);
}
/**
* mdb_open:
* @filename: path to MDB (database) file
* @flags: MDB_NOFLAGS for read-only, MDB_WRITABLE for read/write
*
* Opens an MDB file and returns an MdbHandle to it. MDB File may be relative
* to the current directory, a full path to the file, or relative to a
* component of $MDBPATH.
*
* Return value: pointer to MdbHandle structure.
**/
MdbHandle *mdb_open(const char *filename, MdbFileFlags flags)
{
FILE *file;
char *filepath = mdb_find_file(filename);
if (!filepath) {
fprintf(stderr, "File not found\n");
return NULL;
}
#ifdef _WIN32
char *mode = (flags & MDB_WRITABLE) ? "rb+" : "rb";
#else
char *mode = (flags & MDB_WRITABLE) ? "r+" : "r";
#endif
if ((file = fopen(filepath, mode)) == NULL) {
fprintf(stderr,"Couldn't open file %s\n",filepath);
g_free(filepath);
return NULL;
}
g_free(filepath);
return mdb_handle_from_stream(file, flags);
}
/**
* mdb_close:
* @mdb: Handle to open MDB database file
@ -297,8 +319,7 @@ mdb_close(MdbHandle *mdb)
if (mdb->f->refs > 1) {
mdb->f->refs--;
} else {
if (mdb->f->fd != -1) close(mdb->f->fd);
g_free(mdb->f->filename);
if (mdb->f->stream) fclose(mdb->f->stream);
g_free(mdb->f);
}
}
@ -369,27 +390,29 @@ ssize_t mdb_read_alt_pg(MdbHandle *mdb, unsigned long pg)
static ssize_t _mdb_read_pg(MdbHandle *mdb, void *pg_buf, unsigned long pg)
{
ssize_t len;
struct stat status;
off_t offset = pg * mdb->fmt->pg_size;
fstat(mdb->f->fd, &status);
if (status.st_size < offset) {
if (fseek(mdb->f->stream, 0, SEEK_END) == -1) {
fprintf(stderr, "Unable to seek to end of file\n");
return 0;
}
if (ftello(mdb->f->stream) < offset) {
fprintf(stderr,"offset %" PRIu64 " is beyond EOF\n",(uint64_t)offset);
return 0;
}
if (mdb->stats && mdb->stats->collect)
mdb->stats->pg_reads++;
lseek(mdb->f->fd, offset, SEEK_SET);
len = read(mdb->f->fd,pg_buf,mdb->fmt->pg_size);
if (len==-1) {
if (fseek(mdb->f->stream, offset, SEEK_SET) == -1) {
fprintf(stderr, "Unable to seek to page %lu\n", pg);
return 0;
}
len = fread(pg_buf, 1, mdb->fmt->pg_size, mdb->f->stream);
if (ferror(mdb->f->stream)) {
perror("read");
return 0;
}
else if (len<mdb->fmt->pg_size) {
/* fprintf(stderr,"EOF reached %d bytes returned.\n",len, mdb->fmt->pg_size); */
return 0;
}
memset(pg_buf + len, 0, mdb->fmt->pg_size - len);
/*
* unencrypt the page if necessary.
* it might make sense to cache the unencrypted data blocks?
@ -402,7 +425,7 @@ static ssize_t _mdb_read_pg(MdbHandle *mdb, void *pg_buf, unsigned long pg)
RC4(&rc4_key, mdb->fmt->pg_size, pg_buf);
}
return len;
return mdb->fmt->pg_size;
}
void mdb_swap_pgbuf(MdbHandle *mdb)
{

14
src/libmdb/table.c
View File

@ -65,9 +65,15 @@ MdbTableDef *mdb_read_table(MdbCatalogEntry *entry)
void *buf, *pg_buf = mdb->pg_buf;
guint i;
mdb_read_pg(mdb, entry->table_pg);
if (mdb_get_byte(pg_buf, 0) != 0x02) /* not a valid table def page */
if (!mdb_read_pg(mdb, entry->table_pg)) {
fprintf(stderr, "mdb_read_table: Unable to read page %lu\n", entry->table_pg);
return NULL;
}
if (mdb_get_byte(pg_buf, 0) != 0x02) {
fprintf(stderr, "mdb_read_table: Page %lu [size=%d] is not a valid table definition page (First byte = 0x%02X, expected 0x02)\n",
entry->table_pg, (int)fmt->pg_size, mdb_get_byte(pg_buf, 0));
return NULL;
}
table = mdb_alloc_tabledef(entry);
mdb_get_int16(pg_buf, 8); /* len */
@ -81,6 +87,7 @@ MdbTableDef *mdb_read_table(MdbCatalogEntry *entry)
/* grab a copy of the usage map */
pg_row = mdb_get_int32(pg_buf, fmt->tab_usage_map_offset);
if (mdb_find_pg_row(mdb, pg_row, &buf, &row_start, &(table->map_sz))) {
fprintf(stderr, "mdb_read_table: Unable to find page row %d\n", pg_row);
mdb_free_tabledef(table);
return NULL;
}
@ -93,6 +100,7 @@ MdbTableDef *mdb_read_table(MdbCatalogEntry *entry)
/* grab a copy of the free space page map */
pg_row = mdb_get_int32(pg_buf, fmt->tab_free_map_offset);
if (mdb_find_pg_row(mdb, pg_row, &buf, &row_start, &(table->freemap_sz))) {
fprintf(stderr, "mdb_read_table: Unable to find page row %d\n", pg_row);
mdb_free_tabledef(table);
return NULL;
}
@ -300,8 +308,6 @@ GPtrArray *mdb_read_columns(MdbTableDef *table)
read_pg_if_n(mdb, tmp_buf, &cur_pos, name_sz);
mdb_unicode2ascii(mdb, tmp_buf, name_sz, pcol->name, MDB_MAX_OBJ_NAME);
g_free(tmp_buf);
}
/* Sort the columns by col_num */

11
src/libmdb/write.c
View File

@ -70,18 +70,17 @@ ssize_t
mdb_write_pg(MdbHandle *mdb, unsigned long pg)
{
ssize_t len;
struct stat status;
off_t offset = pg * mdb->fmt->pg_size;
fstat(mdb->f->fd, &status);
fseek(mdb->f->stream, 0, SEEK_END);
/* is page beyond current size + 1 ? */
if (status.st_size < offset + mdb->fmt->pg_size) {
if (ftello(mdb->f->stream) < offset + mdb->fmt->pg_size) {
fprintf(stderr,"offset %" PRIu64 " is beyond EOF\n",(uint64_t)offset);
return 0;
}
lseek(mdb->f->fd, offset, SEEK_SET);
len = write(mdb->f->fd,mdb->pg_buf,mdb->fmt->pg_size);
if (len==-1) {
fseek(mdb->f->stream, offset, SEEK_SET);
len = fwrite(mdb->pg_buf, mdb->fmt->pg_size, 1, mdb->f->stream);
if (ferror(mdb->f->stream)) {
perror("write");
return 0;
} else if (len<mdb->fmt->pg_size) {

13
src/util/mdb-count.c
View File

@ -28,12 +28,17 @@ int main(int argc, char **argv) {
if (argc < 3) {
fprintf(stderr, "Usage: %s <file> <table>\n", argv[0]);
exit(1);
return 1;
}
// open db and try to read table:
mdb = mdb_open(argv[1], MDB_NOFLAGS);
mdb_read_catalog(mdb, MDB_TABLE);
if (!mdb) {
return 1;
}
if (!mdb_read_catalog(mdb, MDB_TABLE)) {
return 1;
}
for (i = 0; i < mdb->num_catalog; i++) {
entry = g_ptr_array_index(mdb->catalog, i);
if (entry->object_type == MDB_TABLE && !g_ascii_strcasecmp(entry->object_name, argv[2])) {
@ -46,8 +51,8 @@ int main(int argc, char **argv) {
// check was found:
if (!found) {
fprintf(stderr, "No table named %s found.\n", argv[2]);
exit(1);
fprintf(stderr, "No table named %s found (among %d tables in file).\n", argv[2], mdb->num_catalog);
return 1;
}
mdb_close(mdb);