lsm/mdbtools
1
0
Fork 0
mirror of https://github.com/mdbtools/mdbtools.git synced 2025-09-18 18:22:07 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge pull request #8 from cyberemissary/bulk-insert

Browse Source 
Implemented bulk insert
This commit is contained in:
Cyber Emissary
2018-02-12 15:22:39 -05:00
committed by GitHub
parent 2ca425b3ad 278f9ba441
commit 6d9fc7d397
3 changed files with 125 additions and 63 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
include/mdbtools.h
View File

@@ -172,7 +172,8 @@ enum {
MDB_SHEXP_COMMENTS = 1<<3, /* export comments on columns & tables */ MDB_SHEXP_COMMENTS = 1<<3, /* export comments on columns & tables */
MDB_SHEXP_DEFVALUES = 1<<4, /* export default values */ MDB_SHEXP_DEFVALUES = 1<<4, /* export default values */
MDB_SHEXP_INDEXES = 1<<5, /* export indices */ MDB_SHEXP_INDEXES = 1<<5, /* export indices */
MDB_SHEXP_RELATIONS = 1<<6 /* export relation (foreign keys) */ MDB_SHEXP_RELATIONS = 1<<6, /* export relation (foreign keys) */
MDB_SHEXP_BULK_INSERT = 1 << 7 /* export data in bulk inserts */
}; };
#define MDB_SHEXP_DEFAULT (MDB_SHEXP_CST_NOTNULL | MDB_SHEXP_COMMENTS | MDB_SHEXP_INDEXES | MDB_SHEXP_RELATIONS) #define MDB_SHEXP_DEFAULT (MDB_SHEXP_CST_NOTNULL | MDB_SHEXP_COMMENTS | MDB_SHEXP_INDEXES | MDB_SHEXP_RELATIONS)

6
src/libmdb/backend.c
View File

@@ -383,7 +383,7 @@ MDB_CONSTRUCTOR(_mdb_init_backends)
"COMMENT ON TABLE %s IS %s;\n", "COMMENT ON TABLE %s IS %s;\n",
quote_schema_name_dquote); quote_schema_name_dquote);
mdb_register_backend("postgres", mdb_register_backend("postgres",
MDB_SHEXP_DROPTABLE|MDB_SHEXP_CST_NOTNULL|MDB_SHEXP_CST_NOTEMPTY|MDB_SHEXP_COMMENTS|MDB_SHEXP_INDEXES|MDB_SHEXP_RELATIONS|MDB_SHEXP_DEFVALUES, MDB_SHEXP_DROPTABLE|MDB_SHEXP_CST_NOTNULL|MDB_SHEXP_CST_NOTEMPTY|MDB_SHEXP_COMMENTS|MDB_SHEXP_INDEXES|MDB_SHEXP_RELATIONS|MDB_SHEXP_DEFVALUES|MDB_SHEXP_BULK_INSERT,
mdb_postgres_types, &mdb_postgres_shortdate_type, &mdb_postgres_serial_type, mdb_postgres_types, &mdb_postgres_shortdate_type, &mdb_postgres_serial_type,
"current_date", "now()", "current_date", "now()",
"SET client_encoding = '%s';\n", "SET client_encoding = '%s';\n",
@@ -393,7 +393,7 @@ MDB_CONSTRUCTOR(_mdb_init_backends)
"COMMENT ON TABLE %s IS %s;\n", "COMMENT ON TABLE %s IS %s;\n",
quote_schema_name_dquote); quote_schema_name_dquote);
mdb_register_backend("mysql", mdb_register_backend("mysql",
MDB_SHEXP_DROPTABLE|MDB_SHEXP_CST_NOTNULL|MDB_SHEXP_CST_NOTEMPTY|MDB_SHEXP_INDEXES|MDB_SHEXP_DEFVALUES, MDB_SHEXP_DROPTABLE|MDB_SHEXP_CST_NOTNULL|MDB_SHEXP_CST_NOTEMPTY|MDB_SHEXP_INDEXES|MDB_SHEXP_DEFVALUES|MDB_SHEXP_BULK_INSERT,
mdb_mysql_types, &mdb_mysql_shortdate_type, NULL, mdb_mysql_types, &mdb_mysql_shortdate_type, NULL,
"current_date", "now()", "current_date", "now()",
"-- That file uses encoding %s\n", "-- That file uses encoding %s\n",
@@ -403,7 +403,7 @@ MDB_CONSTRUCTOR(_mdb_init_backends)
NULL, NULL,
quote_schema_name_rquotes_merge); quote_schema_name_rquotes_merge);
mdb_register_backend("sqlite", mdb_register_backend("sqlite",
MDB_SHEXP_DROPTABLE|MDB_SHEXP_RELATIONS|MDB_SHEXP_DEFVALUES, MDB_SHEXP_DROPTABLE|MDB_SHEXP_RELATIONS|MDB_SHEXP_DEFVALUES|MDB_SHEXP_BULK_INSERT,
mdb_sqlite_types, NULL, NULL, mdb_sqlite_types, NULL, NULL,
"date('now')", "date('now')", "date('now')", "date('now')",
"-- That file uses encoding %s\n", "-- That file uses encoding %s\n",

169
src/util/mdb-export.c
View File

@@ -90,6 +90,7 @@ main(int argc, char **argv)
int header_row = 1; int header_row = 1;
int quote_text = 1; int quote_text = 1;
int boolean_words = 0; int boolean_words = 0;
int batch_size = 1000;
char *insert_dialect = NULL; char *insert_dialect = NULL;
char *date_fmt = NULL; char *date_fmt = NULL;
char *namespace = NULL; char *namespace = NULL;
@@ -100,19 +101,20 @@ main(int argc, char **argv)
size_t length; size_t length;
GOptionEntry entries[] = { GOptionEntry entries[] = {
{ "no-header", 'H', G_OPTION_FLAG_REVERSE, G_OPTION_ARG_NONE, &header_row, "Suppress header row.", NULL}, {"no-header", 'H', G_OPTION_FLAG_REVERSE, G_OPTION_ARG_NONE, &header_row, "Suppress header row.", NULL},
{ "no-quote", 'Q', G_OPTION_FLAG_REVERSE, G_OPTION_ARG_NONE, &quote_text, "Don't wrap text-like fields in quotes.", NULL}, {"no-quote", 'Q', G_OPTION_FLAG_REVERSE, G_OPTION_ARG_NONE, &quote_text, "Don't wrap text-like fields in quotes.", NULL},
{ "delimiter", 'd', 0, G_OPTION_ARG_STRING, &delimiter, "Specify an alternative column delimiter. Default is comma.", "char"}, {"delimiter", 'd', 0, G_OPTION_ARG_STRING, &delimiter, "Specify an alternative column delimiter. Default is comma.", "char"},
{ "row-delimiter", 'R', 0, G_OPTION_ARG_STRING, &row_delimiter, "Specify a row delimiter", "char"}, {"row-delimiter", 'R', 0, G_OPTION_ARG_STRING, &row_delimiter, "Specify a row delimiter", "char"},
{ "quote", 'q', 0, G_OPTION_ARG_STRING, &quote_char, "Use <char> to wrap text-like fields. Default is double quote.", "char"}, {"quote", 'q', 0, G_OPTION_ARG_STRING, &quote_char, "Use <char> to wrap text-like fields. Default is double quote.", "char"},
{ "backend", 'I', 0, G_OPTION_ARG_STRING, &insert_dialect, "INSERT statements (instead of CSV)", "backend"}, {"backend", 'I', 0, G_OPTION_ARG_STRING, &insert_dialect, "INSERT statements (instead of CSV)", "backend"},
{ "date_format", 'D', 0, G_OPTION_ARG_STRING, &date_fmt, "Set the date format (see strftime(3) for details)", "format"}, {"date_format", 'D', 0, G_OPTION_ARG_STRING, &date_fmt, "Set the date format (see strftime(3) for details)", "format"},
{ "escape", 'X', 0, G_OPTION_ARG_STRING, &escape_char, "Use <char> to escape quoted characters within a field. Default is doubling.", "format"}, {"escape", 'X', 0, G_OPTION_ARG_STRING, &escape_char, "Use <char> to escape quoted characters within a field. Default is doubling.", "format"},
{ "namespace", 'N', 0, G_OPTION_ARG_STRING, &namespace, "Prefix identifiers with namespace", "namespace"}, {"namespace", 'N', 0, G_OPTION_ARG_STRING, &namespace, "Prefix identifiers with namespace", "namespace"},
{ "null", '0', 0, G_OPTION_ARG_STRING, &null_text, "Use <char> to represent a NULL value", "char"}, {"null", '0', 0, G_OPTION_ARG_STRING, &null_text, "Use <char> to represent a NULL value", "char"},
{ "bin", 'b', 0, G_OPTION_ARG_STRING, &str_bin_mode, "Binary export mode", "strip|raw|octal"}, {"bin", 'b', 0, G_OPTION_ARG_STRING, &str_bin_mode, "Binary export mode", "strip|raw|octal"},
{ "boolean-words", 'B', 0, G_OPTION_ARG_NONE, &boolean_words, "Use TRUE/FALSE in Boolean fields (default is 0/1)", NULL}, {"boolean-words", 'B', 0, G_OPTION_ARG_NONE, &boolean_words, "Use TRUE/FALSE in Boolean fields (default is 0/1)", NULL},
{ NULL }, {"batch-size", 'S', 0, G_OPTION_ARG_INT, &batch_size, "Size of insert batches on supported platforms.", "int"},
{NULL},
}; };
GError *error = NULL; GError *error = NULL;
GOptionContext *opt_context; GOptionContext *opt_context;
@@ -205,14 +207,14 @@ main(int argc, char **argv)
bound_values = (char **) g_malloc(table->num_cols * sizeof(char *)); bound_values = (char **) g_malloc(table->num_cols * sizeof(char *));
bound_lens = (int *) g_malloc(table->num_cols * sizeof(int)); bound_lens = (int *) g_malloc(table->num_cols * sizeof(int));
for (i=0;i<table->num_cols;i++) { for (i = 0; i < table->num_cols; i++) {
/* bind columns */ /* bind columns */
bound_values[i] = (char *) g_malloc0(MDB_BIND_SIZE); bound_values[i] = (char *) g_malloc0(MDB_BIND_SIZE);
mdb_bind_column(table, i+1, bound_values[i], &bound_lens[i]); mdb_bind_column(table, i + 1, bound_values[i], &bound_lens[i]);
} }
if (header_row) { if (header_row) {
for (i=0; i<table->num_cols; i++) { for (i = 0; i < table->num_cols; i++) {
col=g_ptr_array_index(table->columns,i); col = g_ptr_array_index(table->columns, i);
if (i) if (i)
fputs(delimiter, outfile); fputs(delimiter, outfile);
fputs(col->name, outfile); fputs(col->name, outfile);
@@ -220,47 +222,106 @@ main(int argc, char **argv)
fputs(row_delimiter, outfile); fputs(row_delimiter, outfile);
} }
while(mdb_fetch_row(table)) { // TODO refactor this into functions
if (mdb->default_backend->capabilities & MDB_SHEXP_BULK_INSERT) {
if (insert_dialect) { //for efficiency do multi row insert on engines that support this
char *quoted_name; unsigned int counter = 0;
quoted_name = mdb->default_backend->quote_schema_name(namespace, argv[2]); while (mdb_fetch_row(table)) {
fprintf(outfile, "INSERT INTO %s (", quoted_name); if (counter % batch_size == 0) {
free(quoted_name); counter = 0; // reset to 0, prevent overflow on extremely large data sets.
for (i=0;i<table->num_cols;i++) { char *quoted_name;
if (i>0) fputs(", ", outfile); quoted_name = mdb->default_backend->quote_schema_name(namespace, argv[2]);
col=g_ptr_array_index(table->columns,i); fprintf(outfile, "INSERT INTO %s (", quoted_name);
quoted_name = mdb->default_backend->quote_schema_name(NULL, col->name);
fputs(quoted_name, outfile);
free(quoted_name); free(quoted_name);
} for (i = 0; i < table->num_cols; i++) {
fputs(") VALUES (", outfile); if (i > 0) fputs(", ", outfile);
} col = g_ptr_array_index(table->columns, i);
quoted_name = mdb->default_backend->quote_schema_name(NULL, col->name);
for (i=0;i<table->num_cols;i++) { fputs(quoted_name, outfile);
if (i>0) free(quoted_name);
fputs(delimiter, outfile);
col=g_ptr_array_index(table->columns,i);
if (!bound_lens[i]) {
/* Don't quote NULLs */
if (insert_dialect)
fputs("NULL", outfile);
else
fputs(null_text, outfile);
} else {
if (col->col_type == MDB_OLE) {
value = mdb_ole_read_full(mdb, col, &length);
} else {
value = bound_values[i];
length = bound_lens[i];
} }
print_col(outfile, value, quote_text, col->col_type, length, quote_char, escape_char, bin_mode); fputs(") VALUES ", outfile);
if (col->col_type == MDB_OLE) } else {
free(value); fputs(", ", outfile);
} }
fputs("(", outfile);
for (i = 0; i < table->num_cols; i++) {
if (i > 0)
fputs(delimiter, outfile);
col = g_ptr_array_index(table->columns, i);
if (!bound_lens[i]) {
/* Don't quote NULLs */
if (insert_dialect)
fputs("NULL", outfile);
else
fputs(null_text, outfile);
} else {
if (col->col_type == MDB_OLE) {
value = mdb_ole_read_full(mdb, col, &length);
} else {
value = bound_values[i];
length = bound_lens[i];
}
print_col(outfile, value, quote_text, col->col_type, length, quote_char, escape_char, bin_mode);
if (col->col_type == MDB_OLE)
free(value);
}
}
fputs(")", outfile);
if (counter % batch_size == batch_size - 1) {
fputs(";", outfile);
fputs(row_delimiter, outfile);
}
counter++;
}
if (counter % batch_size != 0) {
//if our last row did not land on closing tag, close the stement here
fputs(";", outfile);
fputs(row_delimiter, outfile);
}
} else {
while (mdb_fetch_row(table)) {
if (insert_dialect) {
char *quoted_name;
quoted_name = mdb->default_backend->quote_schema_name(namespace, argv[2]);
fprintf(outfile, "INSERT INTO %s (", quoted_name);
free(quoted_name);
for (i = 0; i < table->num_cols; i++) {
if (i > 0) fputs(", ", outfile);
col = g_ptr_array_index(table->columns, i);
quoted_name = mdb->default_backend->quote_schema_name(NULL, col->name);
fputs(quoted_name, outfile);
free(quoted_name);
}
fputs(") VALUES (", outfile);
}
for (i = 0; i < table->num_cols; i++) {
if (i > 0)
fputs(delimiter, outfile);
col = g_ptr_array_index(table->columns, i);
if (!bound_lens[i]) {
/* Don't quote NULLs */
if (insert_dialect)
fputs("NULL", outfile);
else
fputs(null_text, outfile);
} else {
if (col->col_type == MDB_OLE) {
value = mdb_ole_read_full(mdb, col, &length);
} else {
value = bound_values[i];
length = bound_lens[i];
}
print_col(outfile, value, quote_text, col->col_type, length, quote_char, escape_char, bin_mode);
if (col->col_type == MDB_OLE)
free(value);
}
}
if (insert_dialect) fputs(");", outfile);
fputs(row_delimiter, outfile);
} }
if (insert_dialect) fputs(");", outfile);
fputs(row_delimiter, outfile);
} }
/* free the memory used to bind */ /* free the memory used to bind */