/* MDB Tools - A library for reading MS Access database file * Copyright (C) 2000 Brian Bruns * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Library General Public * License as published by the Free Software Foundation; either * version 2 of the License, or (at your option) any later version. * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Library General Public License for more details. * * You should have received a copy of the GNU Library General Public * License along with this library; if not, write to the * Free Software Foundation, Inc., 59 Temple Place - Suite 330, * Boston, MA 02111-1307, USA. */ #include "mdbtools.h" #ifdef DMALLOC #include "dmalloc.h" #endif MdbIndexPage *mdb_index_read_bottom_pg(MdbHandle *mdb, MdbIndex *idx, MdbIndexChain *chain); MdbIndexPage *mdb_chain_add_page(MdbHandle *mdb, MdbIndexChain *chain, guint32 pg); char idx_to_text[] = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0-7 0x00-0x07 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 8-15 0x09-0x0f */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 16-23 0x10-0x17 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 24-31 0x19-0x1f */ ' ', 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 32-39 0x20-0x27 */ 0x00, 0x00, 0x00, 0x00, 0x00, ' ', ' ', 0x00, /* 40-47 0x29-0x2f */ 'V', 'W', 'X', 'Y', 'Z', '[', '\\', ']', /* 48-55 0x30-0x37 */ '^', '_', 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 56-63 0x39-0x3f */ 0x00, '`', 'a', 'b', 'd', 'f', 'g', 'h', /* 64-71 0x40-0x47 */ 'i', 'j', 'k', 'l', 'm', 'o', 'p', 'r', /* 72-79 0x49-0x4f H */ 's', 't', 'u', 'v', 'w', 'x', 'z', '{', /* 80-87 0x50-0x57 P */ '|', '}', '~', '5', '6', '7', '8', '9', /* 88-95 0x59-0x5f */ 0x00, '`', 'a', 'b', 'd', 'f', 'g', 'h', /* 96-103 0x60-0x67 */ 'i', 'j', 'k', 'l', 'm', 'o', 'p', 'r', /* 014-111 0x69-0x6f h */ 's', 't', 'u', 'v', 'w', 'x', 'z', '{', /* 112-119 0x70-0x77 p */ '|', '}', '~', 0x00, 0x00, 0x00, 0x00, 0x00, /* 120-127 0x78-0x7f */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 128-135 0x80-0x87 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8f */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x97 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9f */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa0-0xa7 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa8-0xaf */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb0-0xb7 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb8-0xbf */ 0x00, 0x00, 0x00, 0x00, 0x00, '`', 0x00, 0x00, /* 0xc0-0xc7 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc8-0xcf */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd0-0xd7 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd8-0xdf */ 0x00, '`', 0x00, '`', '`', '`', 0x00, 0x00, /* 0xe0-0xe7 */ 'f', 'f', 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe8-0xef */ 0x00, 0x00, 0x00, 'r', 0x00, 0x00, 'r', 0x00, /* 0xf0-0xf7 */ 0x81, 0x00, 0x00, 0x00, 'x', 0x00, 0x00, 0x00, /* 0xf8-0xff */ }; GPtrArray * mdb_read_indices(MdbTableDef *table) { MdbCatalogEntry *entry = table->entry; MdbHandle *mdb = entry->mdb; MdbFormatConstants *fmt = mdb->fmt; MdbIndex idx, *pidx; int i, j; int idx_num, key_num, col_num; int cur_pos; int name_sz, idx2_sz; gchar *tmpbuf; /* FIX ME -- doesn't handle multipage table headers */ table->indices = g_ptr_array_new(); if (IS_JET4(mdb)) { cur_pos = table->index_start + 52 * table->num_real_idxs; idx2_sz = 27; } else { cur_pos = table->index_start + 39 * table->num_real_idxs; idx2_sz = 19; } for (i=0;inum_idxs;i++) { memset(&idx, '\0', sizeof(MdbIndex)); idx.table = table; cur_pos +=4; idx.index_num = read_pg_if_16(mdb, &cur_pos); read_pg_if(mdb, &cur_pos, idx2_sz - 4); cur_pos += idx2_sz - 4; idx.index_type = mdb->pg_buf[cur_pos++]; mdb_append_index(table->indices, &idx); } for (i=0;inum_idxs;i++) { pidx = g_ptr_array_index (table->indices, i); read_pg_if(mdb, &cur_pos, 0); if (IS_JET4(mdb)) { name_sz=read_pg_if_16(mdb, &cur_pos); cur_pos += 2; tmpbuf = g_malloc((name_sz + 1)*2); read_pg_if_n(mdb, tmpbuf, &cur_pos, name_sz*2); mdb_unicode2ascii(mdb, tmpbuf, 0, name_sz, pidx->name); g_free(tmpbuf); cur_pos += name_sz; } else { name_sz=mdb->pg_buf[cur_pos++]; read_pg_if_n(mdb, pidx->name, &cur_pos, name_sz); pidx->name[name_sz]='\0'; cur_pos += name_sz; } //fprintf(stderr, "index name %s\n", pidx->name); } cur_pos = table->index_start; mdb_read_alt_pg(mdb, entry->table_pg); mdb_read_pg(mdb, entry->table_pg); idx_num=0; for (i=0;inum_real_idxs;i++) { if (IS_JET4(mdb)) cur_pos += 4; do { pidx = g_ptr_array_index (table->indices, idx_num++); } while (pidx && pidx->index_type==2); /* if there are more real indexes than index entries left after removing type 2's decrement real indexes and continue. Happens on Northwind Orders table. */ if (!pidx) { table->num_real_idxs--; continue; } pidx->num_rows = mdb_get_int32(mdb->alt_pg_buf, fmt->tab_cols_start_offset + (i*fmt->tab_ridx_entry_size)); key_num=0; for (j=0;jkey_col_num[key_num]=col_num + 1; if (mdb->pg_buf[cur_pos]) { pidx->key_col_order[key_num]=MDB_ASC; } else { pidx->key_col_order[key_num]=MDB_DESC; } key_num++; } cur_pos++; } pidx->num_keys = key_num; cur_pos += 4; pidx->first_pg = read_pg_if_32(mdb, &cur_pos); cur_pos += 4; read_pg_if(mdb, &cur_pos, 1); pidx->flags = mdb->pg_buf[cur_pos++]; if (IS_JET4(mdb)) cur_pos += 9; } return NULL; } void mdb_index_hash_text(guchar *text, guchar *hash) { int k; for (k=0;kcol_type) { case MDB_TEXT: mdb_index_hash_text(sarg->value.s, idx_sarg->value.s); break; case MDB_LONGINT: idx_sarg->value.i = mdb_index_swap_int32(sarg->value.i); //cache_int = sarg->value.i * -1; c = (unsigned char *) &(idx_sarg->value.i); c[0] |= 0x80; printf("int %08x %02x %02x %02x %02x\n", sarg->value.i, c[0], c[1], c[2], c[3]); break; case MDB_INT: break; default: break; } } #if 0 int mdb_index_test_sarg(MdbHandle *mdb, MdbColumn *col, MdbSarg *sarg, int offset, int len) { char tmpbuf[256]; int lastchar; switch (col->col_type) { case MDB_BYTE: return mdb_test_int(sarg, mdb_pg_get_byte(mdb, offset)); break; case MDB_INT: return mdb_test_int(sarg, mdb_pg_get_int16(mdb, offset)); break; case MDB_LONGINT: return mdb_test_int(sarg, mdb_pg_get_int32(mdb, offset)); break; case MDB_TEXT: strncpy(tmpbuf, &mdb->pg_buf[offset],255); lastchar = len > 255 ? 255 : len; tmpbuf[lastchar]='\0'; return mdb_test_string(sarg, tmpbuf); default: fprintf(stderr, "Calling mdb_test_sarg on unknown type. Add code to mdb_test_sarg() for type %d\n",col->col_type); break; } return 1; } #endif int mdb_index_test_sargs(MdbHandle *mdb, MdbIndex *idx, int offset, int len) { int i, j; MdbColumn *col; MdbTableDef *table = idx->table; MdbSarg *idx_sarg; MdbSarg *sarg; MdbField field; MdbSargNode node; int c_offset = 0, c_len; //fprintf(stderr,"mdb_index_test_sargs called on "); //for (i=0;ipg_buf[offset+i]); //fprintf(stderr,"\n"); for (i=0;inum_keys;i++) { c_offset++; /* the per column null indicator/flags */ col=g_ptr_array_index(table->columns,idx->key_col_num[i]-1); /* * This will go away eventually */ if (col->col_type==MDB_TEXT) { c_len = strlen(&mdb->pg_buf[offset + c_offset]); } else { c_len = col->col_size; //fprintf(stderr,"Only text types currently supported. How did we get here?\n"); } /* * If we have no cached index values for this column, * create them. */ if (col->num_sargs && !col->idx_sarg_cache) { col->idx_sarg_cache = g_ptr_array_new(); for (j=0;jnum_sargs;j++) { sarg = g_ptr_array_index (col->sargs, j); idx_sarg = g_memdup(sarg,sizeof(MdbSarg)); //printf("calling mdb_index_cache_sarg\n"); mdb_index_cache_sarg(col, sarg, idx_sarg); g_ptr_array_add(col->idx_sarg_cache, idx_sarg); } } for (j=0;jnum_sargs;j++) { sarg = g_ptr_array_index (col->idx_sarg_cache, j); /* XXX - kludge */ node.op = sarg->op; node.value = sarg->value; field.value = &mdb->pg_buf[offset + c_offset]; field.siz = c_len; field.is_null = FALSE; if (!mdb_test_sarg(mdb, col, &node, &field)) { /* sarg didn't match, no sense going on */ return 0; } } } return 1; } /* * find the next entry on a page (either index or leaf). Uses state information * stored in the MdbIndexPage across calls. */ int mdb_index_find_next_on_page(MdbHandle *mdb, MdbIndexPage *ipg) { do { //fprintf(stdout, "%d %d\n", ipg->mask_bit, ipg->mask_byte); ipg->mask_bit++; if (ipg->mask_bit==8) { ipg->mask_bit=0; ipg->mask_pos++; } ipg->mask_byte = mdb->pg_buf[ipg->mask_pos]; ipg->len++; } while (ipg->mask_pos <= 0xf8 && !((1 << ipg->mask_bit) & ipg->mask_byte)); if (ipg->mask_pos>=0xf8) return 0; return ipg->len; } void mdb_index_page_reset(MdbIndexPage *ipg) { ipg->offset = 0xf8; /* start byte of the index entries */ ipg->mask_pos = 0x16; ipg->mask_bit=0; ipg->len = 0; } void mdb_index_page_init(MdbIndexPage *ipg) { memset(ipg, 0, sizeof(MdbIndexPage)); mdb_index_page_reset(ipg); } /* * find the next leaf page if any given a chain. Assumes any exhausted leaf * pages at the end of the chain have been peeled off before the call. */ MdbIndexPage * mdb_find_next_leaf(MdbHandle *mdb, MdbIndex *idx, MdbIndexChain *chain) { MdbIndexPage *ipg, *newipg; guint32 pg; guint passed = 0; ipg = mdb_index_read_bottom_pg(mdb, idx, chain); /* * If we are at the first page deep and it's not an index page then * we are simply done. (there is no page to find */ if (mdb->pg_buf[0]==MDB_PAGE_LEAF) return ipg; /* * apply sargs here, currently we don't */ do { ipg->len = 0; //printf("finding next on pg %lu\n", ipg->pg); if (!mdb_index_find_next_on_page(mdb, ipg)) { //printf("find_next_on_page returned 0\n"); return 0; } pg = mdb_pg_get_int24_msb(mdb, ipg->offset + ipg->len - 3); //printf("Looking at pg %lu at %lu %d\n", pg, ipg->offset, ipg->len); ipg->offset += ipg->len; /* * add to the chain and call this function * recursively. */ newipg = mdb_chain_add_page(mdb, chain, pg); newipg = mdb_find_next_leaf(mdb, idx, chain); printf("returning pg %lu\n",newipg->pg); return newipg; } while (!passed); /* no more pages */ return NULL; } MdbIndexPage * mdb_chain_add_page(MdbHandle *mdb, MdbIndexChain *chain, guint32 pg) { MdbIndexPage *ipg; chain->cur_depth++; if (chain->cur_depth > MDB_MAX_INDEX_DEPTH) { fprintf(stderr,"Error! maximum index depth of %d exceeded. This is probably due to a programming bug, If you are confident that your indexes really are this deep, adjust MDB_MAX_INDEX_DEPTH in mdbtools.h and recompile.\n", MDB_MAX_INDEX_DEPTH); exit(1); } ipg = &(chain->pages[chain->cur_depth - 1]); mdb_index_page_init(ipg); ipg->pg = pg; return ipg; } /* * returns the bottom page of the IndexChain, if IndexChain is empty it * initializes it by reading idx->first_pg (the root page) */ MdbIndexPage * mdb_index_read_bottom_pg(MdbHandle *mdb, MdbIndex *idx, MdbIndexChain *chain) { MdbIndexPage *ipg; /* * if it's new use the root index page (idx->first_pg) */ if (!chain->cur_depth) { ipg = &(chain->pages[0]); mdb_index_page_init(ipg); chain->cur_depth = 1; ipg->pg = idx->first_pg; if (!(ipg = mdb_find_next_leaf(mdb, idx, chain))) return 0; } else { ipg = &(chain->pages[chain->cur_depth - 1]); ipg->len = 0; } mdb_read_pg(mdb, ipg->pg); return ipg; } /* * the main index function. * caller provides an index chain which is the current traversal of index * pages from the root page to the leaf. Initially passed as blank, * mdb_index_find_next will store it's state information here. Each invocation * then picks up where the last one left off, allowing us to scroll through * the index one by one. * * Sargs are applied here but also need to be applied on the whole row b/c * text columns may return false positives due to hashing and non-index * columns with sarg values can't be tested here. */ int mdb_index_find_next(MdbHandle *mdb, MdbIndex *idx, MdbIndexChain *chain, guint32 *pg, guint16 *row) { MdbIndexPage *ipg; int passed = 0; ipg = mdb_index_read_bottom_pg(mdb, idx, chain); /* * loop while the sargs don't match */ do { ipg->len = 0; /* * if no more rows on this leaf, try to find a new leaf */ if (!mdb_index_find_next_on_page(mdb, ipg)) { //printf("page %lu finished\n",ipg->pg); if (chain->cur_depth==1) { //printf("cur_depth == 1 we're out\n"); return 0; } /* * unwind the stack until we find something or reach * the top. */ ipg = 0; while (chain->cur_depth>1 && ipg==0) { //printf("chain depth %d\n", chain->cur_depth); chain->cur_depth--; ipg = mdb_find_next_leaf(mdb, idx, chain); if (ipg) mdb_index_find_next_on_page(mdb, ipg); } if (chain->cur_depth==1) return 0; } *row = mdb->pg_buf[ipg->offset + ipg->len - 1]; *pg = mdb_pg_get_int24_msb(mdb, ipg->offset + ipg->len - 4); //printf("row = %d pg = %lu ipg->pg = %lu offset = %lu len = %d\n", *row, *pg, ipg->pg, ipg->offset, ipg->len); passed = mdb_index_test_sargs(mdb, idx, ipg->offset, ipg->len); ipg->offset += ipg->len; } while (!passed); //fprintf(stdout,"len = %d pos %d\n", ipg->len, ipg->mask_pos); //buffer_dump(mdb->pg_buf, ipg->offset, ipg->offset+ipg->len-1); return ipg->len; } /* * XXX - FIX ME * This function is grossly inefficient. It scans the entire index building * an IndexChain to a specific row. We should be checking the index pages * for matches against the indexed fields to find the proper leaf page, but * getting it working first and then make it fast! */ int mdb_index_find_row(MdbHandle *mdb, MdbIndex *idx, MdbIndexChain *chain, guint32 pg, guint16 row) { MdbIndexPage *ipg; int passed = 0; guint32 datapg; guint16 datarow; ipg = mdb_index_read_bottom_pg(mdb, idx, chain); do { ipg->len = 0; /* * if no more rows on this leaf, try to find a new leaf */ if (!mdb_index_find_next_on_page(mdb, ipg)) { /* back to top? We're done */ if (chain->cur_depth==1) return 0; /* * unwind the stack until we find something or reach * the top. */ while (chain->cur_depth>1) { chain->cur_depth--; if (!(ipg = mdb_find_next_leaf(mdb, idx, chain))) return 0; mdb_index_find_next_on_page(mdb, ipg); } if (chain->cur_depth==1) return 0; } /* test row and pg */ datarow = mdb->pg_buf[ipg->offset + ipg->len - 1]; datapg = mdb_pg_get_int24_msb(mdb, ipg->offset + ipg->len - 4); if (datapg == pg && datarow == row) { passed = 1; } ipg->offset += ipg->len; } while (!passed); /* index chain from root to leaf should now be in "chain" */ return 1; } void mdb_index_walk(MdbTableDef *table, MdbIndex *idx) { MdbHandle *mdb = table->entry->mdb; int cur_pos = 0; unsigned char marker; MdbColumn *col; int i; if (idx->num_keys!=1) return; mdb_read_pg(mdb, idx->first_pg); cur_pos = 0xf8; for (i=0;inum_keys;i++) { marker = mdb->pg_buf[cur_pos++]; col=g_ptr_array_index(table->columns,idx->key_col_num[i]-1); //printf("column %d coltype %d col_size %d (%d)\n",i,col->col_type, mdb_col_fixed_size(col), col->col_size); } } void mdb_index_dump(MdbTableDef *table, MdbIndex *idx) { int i; MdbColumn *col; fprintf(stdout,"index number %d\n", idx->index_num); fprintf(stdout,"index name %s\n", idx->name); fprintf(stdout,"index first page %d\n", idx->first_pg); fprintf(stdout,"index rows %d\n", idx->num_rows); if (idx->index_type==1) fprintf(stdout,"index is a primary key\n"); for (i=0;inum_keys;i++) { col=g_ptr_array_index(table->columns,idx->key_col_num[i]-1); fprintf(stdout,"Column %s(%d) Sorted %s Unique: %s\n", col->name, idx->key_col_num[i], idx->key_col_order[i]==MDB_ASC ? "ascending" : "descending", idx->flags & MDB_IDX_UNIQUE ? "Yes" : "No" ); } mdb_index_walk(table, idx); } /* * compute_cost tries to assign a cost to a given index using the sargs * available in this query. * * Indexes with no matching sargs are assigned 0 * Unique indexes are preferred over non-uniques * Operator preference is equal, like, isnull, others */ int mdb_index_compute_cost(MdbTableDef *table, MdbIndex *idx) { int i; MdbColumn *col; MdbSarg *sarg; int not_all_equal = 0; if (!idx->num_keys) return 0; if (idx->num_keys > 1) { for (i=0;inum_keys;i++) { col=g_ptr_array_index(table->columns,idx->key_col_num[i]-1); if (col->sargs) sarg = g_ptr_array_index (col->sargs, 0); if (!sarg || sarg->op != MDB_EQUAL) not_all_equal++; } } col=g_ptr_array_index(table->columns,idx->key_col_num[0]-1); /* * if this is the first key column and there are no sargs, * then this index is useless. */ if (!col->num_sargs) return 0; sarg = g_ptr_array_index (col->sargs, 0); /* * a like with a wild card first is useless as a sarg */ if (sarg->op == MDB_LIKE && sarg->value.s[0]=='%') return 0; /* * this needs a lot of tweaking. */ if (idx->flags & MDB_IDX_UNIQUE) { if (idx->num_keys == 1) { //printf("op is %d\n", sarg->op); switch (sarg->op) { case MDB_EQUAL: return 1; break; case MDB_LIKE: return 4; break; case MDB_ISNULL: return 12; break; default: return 8; break; } } else { switch (sarg->op) { case MDB_EQUAL: if (not_all_equal) return 2; else return 1; break; case MDB_LIKE: return 6; break; case MDB_ISNULL: return 12; break; default: return 9; break; } } } else { if (idx->num_keys == 1) { switch (sarg->op) { case MDB_EQUAL: return 2; break; case MDB_LIKE: return 5; break; case MDB_ISNULL: return 12; break; default: return 10; break; } } else { switch (sarg->op) { case MDB_EQUAL: if (not_all_equal) return 3; else return 2; break; case MDB_LIKE: return 7; break; case MDB_ISNULL: return 12; break; default: return 11; break; } } } return 0; } /* * choose_index runs mdb_index_compute_cost for each available index and picks * the best. * * Returns strategy to use (table scan, or index scan) */ MdbStrategy mdb_choose_index(MdbTableDef *table, int *choice) { int i; MdbIndex *idx; int cost = 0; int least = 99; *choice = -1; for (i=0;inum_idxs;i++) { idx = g_ptr_array_index (table->indices, i); cost = mdb_index_compute_cost(table, idx); //printf("cost for %s is %d\n", idx->name, cost); if (cost && cost < least) { least = cost; *choice = i; } } /* and the winner is: *choice */ if (least==99) return MDB_TABLE_SCAN; return MDB_INDEX_SCAN; } void mdb_index_scan_init(MdbHandle *mdb, MdbTableDef *table) { int i; int use_index=0; char *s; if ((s=getenv("MDBOPTS"))) { if (!strcmp(s, "use_index")) use_index++; } if (use_index && mdb_choose_index(table, &i) == MDB_INDEX_SCAN) { table->strategy = MDB_INDEX_SCAN; table->scan_idx = g_ptr_array_index (table->indices, i); table->chain = g_malloc0(sizeof(MdbIndexChain)); table->mdbidx = mdb_clone_handle(mdb); mdb_read_pg(table->mdbidx, table->scan_idx->first_pg); //printf("best index is %s\n",table->scan_idx->name); } //printf("TABLE SCAN? %d\n", table->strategy); } void mdb_index_scan_free(MdbTableDef *table) { if (table->chain) { g_free(table->chain); table->chain = NULL; } if (table->mdbidx) { mdb_close(table->mdbidx); table->mdbidx = NULL; } }