New mdb_set_bind_size function overrides MDB_BIND_SIZE

This should fix long-standing complaints about the default bind size
without causing undue memory inflation in existing applications.

Could make this adjustable on the command line later.

Supersedes:

https://github.com/mdbtools/mdbtools/pull/137

include/mdbtools.h

@@ -54,7 +54,7 @@
 #define MDB_MAX_IDX_COLS 10
 #define MDB_CATALOG_PG 18
 #define MDB_MEMO_OVERHEAD 12
-#define MDB_BIND_SIZE 16384
+#define MDB_BIND_SIZE 16384 // override with mdb_set_bind_size(MdbHandle*, size_t)
 
 // This attribute is not supported by all compilers:
 // M$VC see http://stackoverflow.com/questions/1113409/attribute-constructor-equivalent-in-vc
@@ -269,6 +269,7 @@ typedef struct {
 	unsigned char pg_buf[MDB_PGSIZE];
 	unsigned char alt_pg_buf[MDB_PGSIZE];
 	MdbFormatConstants *fmt;
+    size_t bind_size;
     char date_fmt[64];
     const char *boolean_false_value;
     const char *boolean_true_value;
@@ -512,6 +513,7 @@ int mdb_col_disp_size(MdbColumn *col);
 size_t mdb_ole_read_next(MdbHandle *mdb, MdbColumn *col, void *ole_ptr);
 size_t mdb_ole_read(MdbHandle *mdb, MdbColumn *col, void *ole_ptr, size_t chunk_size);
 void* mdb_ole_read_full(MdbHandle *mdb, MdbColumn *col, size_t *size);
+void mdb_set_bind_size(MdbHandle *mdb, size_t bind_size);
 void mdb_set_date_fmt(MdbHandle *mdb, const char *);
 void mdb_set_boolean_fmt_words(MdbHandle *mdb);
 void mdb_set_boolean_fmt_numbers(MdbHandle *mdb);

src/libmdb/backend.c

@@ -623,7 +623,7 @@ mdb_get_relationships(MdbHandle *mdb, const gchar *dbnamespace, const char* tabl
 
 		mdb_read_columns(mdb->relationships_table);
 		for (i=0;i<5;i++) {
-			bound[i] = (char *) g_malloc0(MDB_BIND_SIZE);
+			bound[i] = (char *) g_malloc0(mdb->bind_size);
 		}
 		mdb_bind_column_by_name(mdb->relationships_table, "szColumn", bound[0], NULL);
 		mdb_bind_column_by_name(mdb->relationships_table, "szObject", bound[1], NULL);

src/libmdb/data.c

@@ -22,6 +22,7 @@
 #include "mdbtools.h"
 
 #define OFFSET_MASK 0x1fff
+#define OLE_BUFFER_SIZE (MDB_BIND_SIZE*64)
 
 char *mdb_money_to_string(MdbHandle *mdb, int start);
 char *mdb_numeric_to_string(MdbHandle *mdb, int start, int prec, int scale);
@@ -46,6 +47,10 @@ static const char boolean_true_number[]  = "1";
 static const char boolean_false_word[]   = "FALSE";
 static const char boolean_true_word[]    = "TRUE";
 
+void mdb_set_bind_size(MdbHandle *mdb, size_t bind_size) {
+    mdb->bind_size = bind_size;
+}
+
 void mdb_set_date_fmt(MdbHandle *mdb, const char *fmt)
 {
     snprintf(mdb->date_fmt, sizeof(mdb->date_fmt), "%s", fmt);
@@ -255,7 +260,7 @@ int ret;
 			} else {
 				str = mdb_col_to_string(mdb, mdb->pg_buf, start, col->col_type, len);
 			}
-			snprintf(col->bind_ptr, MDB_BIND_SIZE, "%s", str);
+			snprintf(col->bind_ptr, mdb->bind_size, "%s", str);
 			g_free(str);
 		}
 		ret = strlen(col->bind_ptr);
@@ -610,18 +615,18 @@ void*
 mdb_ole_read_full(MdbHandle *mdb, MdbColumn *col, size_t *size)
 {
 	char ole_ptr[MDB_MEMO_OVERHEAD];
-	char *result = malloc(MDB_BIND_SIZE * 64);
+	char *result = malloc(OLE_BUFFER_SIZE);
-	size_t result_buffer_size = MDB_BIND_SIZE * 64;
+	size_t result_buffer_size = OLE_BUFFER_SIZE;
 	size_t len, pos;
 
 	memcpy(ole_ptr, col->bind_ptr, MDB_MEMO_OVERHEAD);
 
-	len = mdb_ole_read(mdb, col, ole_ptr, MDB_BIND_SIZE * 64);
+	len = mdb_ole_read(mdb, col, ole_ptr, OLE_BUFFER_SIZE);
 	memcpy(result, col->bind_ptr, len);
 	pos = len;
 	while ((len = mdb_ole_read_next(mdb, col, ole_ptr))) {
 		if (pos+len >= result_buffer_size) {
-			result_buffer_size += MDB_BIND_SIZE * 64;
+			result_buffer_size += OLE_BUFFER_SIZE;
 			result = realloc(result, result_buffer_size);
 		}
 		memcpy(result + pos, col->bind_ptr, len);
@@ -703,7 +708,7 @@ static char *mdb_memo_to_string(MdbHandle *mdb, int start, int size)
 	gint32 row_start, pg_row;
 	size_t len;
 	void *buf, *pg_buf = mdb->pg_buf;
-	char *text = (char *) g_malloc(MDB_BIND_SIZE);
+	char *text = (char *) g_malloc(mdb->bind_size);
 
 	if (size<MDB_MEMO_OVERHEAD) {
 		strcpy(text, "");
@@ -723,7 +728,7 @@ static char *mdb_memo_to_string(MdbHandle *mdb, int start, int size)
 	if (memo_len & 0x80000000) {
 		/* inline memo field */
 		mdb_unicode2ascii(mdb, (char*)pg_buf + start + MDB_MEMO_OVERHEAD,
-			size - MDB_MEMO_OVERHEAD, text, MDB_BIND_SIZE);
+			size - MDB_MEMO_OVERHEAD, text, mdb->bind_size);
 		return text;
 	} else if (memo_len & 0x40000000) {
 		/* single-page memo field */
@@ -740,7 +745,7 @@ static char *mdb_memo_to_string(MdbHandle *mdb, int start, int size)
 			pg_row & 0xff, row_start, len);
 		mdb_buffer_dump(buf, row_start, len);
 #endif
-		mdb_unicode2ascii(mdb, (char*)buf + row_start, len, text, MDB_BIND_SIZE);
+		mdb_unicode2ascii(mdb, (char*)buf + row_start, len, text, mdb->bind_size);
 		return text;
 	} else if ((memo_len & 0xff000000) == 0) { // assume all flags in MSB
 		/* multi-page memo field */
@@ -775,7 +780,7 @@ static char *mdb_memo_to_string(MdbHandle *mdb, int start, int size)
 		if (tmpoff < memo_len) {
 			fprintf(stderr, "Warning: incorrect memo length\n");
 		}
-		mdb_unicode2ascii(mdb, tmp, tmpoff, text, MDB_BIND_SIZE);
+		mdb_unicode2ascii(mdb, tmp, tmpoff, text, mdb->bind_size);
 		g_free(tmp);
 		return text;
 	} else {
@@ -875,12 +880,12 @@ static char *
 mdb_date_to_string(MdbHandle *mdb, void *buf, int start)
 {
 	struct tm t;
-	char *text = (char *) g_malloc(MDB_BIND_SIZE);
+	char *text = (char *) g_malloc(mdb->bind_size);
 	double td = mdb_get_double(buf, start);
 
 	mdb_date_to_tm(td, &t);
 
-	strftime(text, MDB_BIND_SIZE, mdb->date_fmt, &t);
+	strftime(text, mdb->bind_size, mdb->date_fmt, &t);
 
 	return text;
 }
@@ -974,9 +979,9 @@ char *mdb_col_to_string(MdbHandle *mdb, void *buf, int start, int datatype, int
 			if (size<0) {
 				text = g_strdup("");
 			} else {
-				text = (char *) g_malloc(MDB_BIND_SIZE);
+				text = (char *) g_malloc(mdb->bind_size);
 				mdb_unicode2ascii(mdb, (char*)buf + start,
-					size, text, MDB_BIND_SIZE);
+					size, text, mdb->bind_size);
 			}
 		break;
 		case MDB_DATETIME:

src/libmdb/file.c

@@ -174,6 +174,7 @@ static MdbHandle *mdb_handle_from_stream(FILE *stream, MdbFileFlags flags) {
 	MdbHandle *mdb = (MdbHandle *) g_malloc0(sizeof(MdbHandle));
 	mdb_set_default_backend(mdb, "access");
     mdb_set_date_fmt(mdb, "%x %X");
+    mdb_set_bind_size(mdb, MDB_BIND_SIZE);
     mdb_set_boolean_fmt_numbers(mdb);
 #ifdef HAVE_ICONV
 	mdb->iconv_in = (iconv_t)-1;

src/util/mdb-export.c

@@ -18,8 +18,7 @@
 
 #include "mdbtools.h"
 
-#undef MDB_BIND_SIZE
+#define EXPORT_BIND_SIZE 200000
-#define MDB_BIND_SIZE 200000
 
 #define is_quote_type(x) (x==MDB_TEXT || x==MDB_OLE || x==MDB_MEMO || x==MDB_DATETIME || x==MDB_BINARY || x==MDB_REPID)
 #define is_binary_type(x) (x==MDB_OLE || x==MDB_BINARY || x==MDB_REPID)
@@ -191,6 +190,8 @@ main(int argc, char **argv)
 	if (boolean_words)
 		mdb_set_boolean_fmt_words(mdb);
 
+    mdb_set_bind_size(mdb, EXPORT_BIND_SIZE);
+
 	if (insert_dialect)
 		if (!mdb_set_default_backend(mdb, insert_dialect)) {
 			fputs("Invalid backend type\n", stderr);
@@ -213,7 +214,7 @@ main(int argc, char **argv)
 	bound_lens = (int *) g_malloc(table->num_cols * sizeof(int));
 	for (i = 0; i < table->num_cols; i++) {
 		/* bind columns */
-		bound_values[i] = (char *) g_malloc0(MDB_BIND_SIZE);
+		bound_values[i] = (char *) g_malloc0(EXPORT_BIND_SIZE);
 		mdb_bind_column(table, i + 1, bound_values[i], &bound_lens[i]);
 	}
 	if (header_row) {

src/util/mdb-json.c

@@ -20,8 +20,7 @@
 
 #include "base64.h"
 
-#undef MDB_BIND_SIZE
+#define EXPORT_BIND_SIZE 200000
-#define MDB_BIND_SIZE 200000
 
 #define is_quote_type(x) (x==MDB_TEXT || x==MDB_OLE || x==MDB_MEMO || x==MDB_DATETIME || x==MDB_BINARY || x==MDB_REPID)
 #define is_binary_type(x) (x==MDB_OLE || x==MDB_BINARY || x==MDB_REPID)
@@ -144,6 +143,8 @@ main(int argc, char **argv)
 	if (date_fmt)
 		mdb_set_date_fmt(mdb, date_fmt);
 
+    mdb_set_bind_size(mdb, EXPORT_BIND_SIZE);
+
 	table = mdb_read_table_by_name(mdb, argv[2], MDB_TABLE);
 	if (!table) {
 		fprintf(stderr, "Error: Table %s does not exist in this database.\n", argv[argc-1]);
@@ -159,7 +160,7 @@ main(int argc, char **argv)
 	bound_lens = (int *) g_malloc(table->num_cols * sizeof(int));
 	for (i=0;i<table->num_cols;i++) {
 		/* bind columns */
-		bound_values[i] = (char *) g_malloc0(MDB_BIND_SIZE);
+		bound_values[i] = (char *) g_malloc0(EXPORT_BIND_SIZE);
 		mdb_bind_column(table, i+1, bound_values[i], &bound_lens[i]);
 	}
 

src/util/mdb-queries.c

@@ -33,8 +33,7 @@
 	
 #include "mdbtools.h"
 
-#undef MDB_BIND_SIZE
+#define QUERY_BIND_SIZE 200000
-#define MDB_BIND_SIZE 200000
 
 void mdb_list_queries(MdbHandle *mdb, int line_break, char *delimiter);
 char * mdb_get_query_id(MdbHandle *mdb,char *query);
@@ -50,21 +49,22 @@ int main (int argc, char **argv) {
 	int line_break=0;
 	int opt;	
 	char *query_id;
+    size_t bind_size = QUERY_BIND_SIZE;
 	
 	// variables for the msysqueries table. hopefully 256 is big enough
-	char *attribute = (char *) malloc(MDB_BIND_SIZE);
+	char *attribute = (char *) malloc(bind_size);
-	char *expression = (char *) malloc(MDB_BIND_SIZE);
+	char *expression = (char *) malloc(bind_size);
-	char *flag = (char *) malloc(MDB_BIND_SIZE);
+	char *flag = (char *) malloc(bind_size);
-	char *name1 = (char *) malloc(MDB_BIND_SIZE);
+	char *name1 = (char *) malloc(bind_size);
-	char *name2 = (char *) malloc(MDB_BIND_SIZE);
+	char *name2 = (char *) malloc(bind_size);
-	char *objectid = (char *) malloc(MDB_BIND_SIZE);
+	char *objectid = (char *) malloc(bind_size);
-	char *order = (char *) malloc(MDB_BIND_SIZE);
+	char *order = (char *) malloc(bind_size);
 	
 	//variables for the generation of sql
-	char *sql_tables = (char *) malloc(MDB_BIND_SIZE);
+	char *sql_tables = (char *) malloc(bind_size);
-	char *sql_columns = (char *) malloc(MDB_BIND_SIZE);
+	char *sql_columns = (char *) malloc(bind_size);
-	char *sql_where = (char *) malloc(MDB_BIND_SIZE);
+	char *sql_where = (char *) malloc(bind_size);
-	char *sql_sorting = (char *) malloc(MDB_BIND_SIZE);
+	char *sql_sorting = (char *) malloc(bind_size);
 	
 	/* see getopt(3) for more information on getopt and this will become clear */
 	while ((opt=getopt(argc, argv, "L1d:"))!=-1) {
@@ -102,6 +102,8 @@ int main (int argc, char **argv) {
 		exit(1);
 	}
 
+    mdb_set_bind_size(mdb, bind_size);
+
  	/* read the catalog */
  	if (!mdb_read_catalog (mdb, MDB_ANY)) {
 		fprintf(stderr,"File does not appear to be an Access database\n");