patch 180-slowmap-fallback from Nirgal Vourgère

include/mdbtools.h

@@ -511,7 +511,7 @@ extern void *mdb_new_data_pg(MdbCatalogEntry *entry);
 
 /* map.c */
 extern guint32 mdb_map_find_next_freepage(MdbTableDef *table, int row_size);
-extern guint32 mdb_map_find_next(MdbHandle *mdb, unsigned char *map, unsigned int map_sz, guint32 start_pg);
+extern gint32 mdb_map_find_next(MdbHandle *mdb, unsigned char *map, unsigned int map_sz, guint32 start_pg);
 
 /* props.c */
 extern GPtrArray *mdb_read_props_list(gchar *kkd, int len);

src/libmdb/data.c

@@ -256,7 +256,10 @@ int mdb_read_row(MdbTableDef *table, unsigned int row)
 	if (table->num_rows == 0) 
 		return 0;
 
-	mdb_find_row(mdb, row, &row_start, &row_size);
+	if (mdb_find_row(mdb, row, &row_start, &row_size)) {
+		fprintf(stderr, "warning: mdb_find_row failed.");
+		return 0;
+	}
 
 	delflag = lookupflag = 0;
 	if (row_start & 0x8000) lookupflag++;
@@ -315,6 +318,8 @@ static int _mdb_attempt_bind(MdbHandle *mdb,
 	}
 	return 1;
 }
+
+/* Read next data page into mdb->pg_buf */
 int mdb_read_next_dpg(MdbTableDef *table)
 {
 	MdbCatalogEntry *entry = table->entry;
@@ -322,16 +327,28 @@ int mdb_read_next_dpg(MdbTableDef *table)
 	int next_pg;
 
 #ifndef SLOW_READ
-	next_pg = mdb_map_find_next(mdb, table->usage_map,
+	while (1) {
-		table->map_sz, table->cur_phys_pg);
+		next_pg = mdb_map_find_next(mdb, table->usage_map,
+			table->map_sz, table->cur_phys_pg);
+		if (next_pg < 0)
+			break; /* unknow map type: goto fallback */
+		if (!next_pg)
+			return 0;
 
-	if (next_pg >= 0) {
+		if (!mdb_read_pg(mdb, next_pg)) {
-		if (mdb_read_pg(mdb, next_pg)) {
+			fprintf(stderr, "error: reading page %d failed.\n", next_pg);
-			table->cur_phys_pg = next_pg;
-			return table->cur_phys_pg;
-		} else {
 			return 0;
 		}
+
+		table->cur_phys_pg = next_pg;
+		if (mdb->pg_buf[0]==MDB_PAGE_DATA && mdb_get_int32(mdb->pg_buf, 4)==entry->table_pg)
+			return table->cur_phys_pg;
+
+		/* On rare occasion, mdb_map_find_next will return a wrong page */
+		/* Found in a big file, over 4,000,000 records */
+		fprintf(stderr,
+			"warning: page %d from map doesn't match: Type=%d, buf[4..7]=%d Expected table_pg=%d\n",
+			next_pg, mdb_get_int32(mdb->pg_buf, 4), entry->table_pg);
 	}
 	fprintf(stderr, "Warning: defaulting to brute force read\n");
 #endif 
@@ -339,7 +356,7 @@ int mdb_read_next_dpg(MdbTableDef *table)
 	do {
 		if (!mdb_read_pg(mdb, table->cur_phys_pg++))
 			return 0;
-	} while (mdb->pg_buf[0]!=0x01 || mdb_get_int32(mdb->pg_buf, 4)!=entry->table_pg);
+	} while (mdb->pg_buf[0]!=MDB_PAGE_DATA || mdb_get_int32(mdb->pg_buf, 4)!=entry->table_pg);
 	/* fprintf(stderr,"returning new page %ld\n", table->cur_phys_pg); */
 	return table->cur_phys_pg;
 }
@@ -396,7 +413,7 @@ mdb_fetch_row(MdbTableDef *table)
 		} else {
 			rows = mdb_get_int16(mdb->pg_buf,fmt->row_count_offset);
 
-			/* if at end of page, find a new page */
+			/* if at end of page, find a new data page */
 			if (table->cur_row >= rows) {
 				table->cur_row=0;
 	

src/libmdb/map.c

@@ -23,7 +23,7 @@
 #include "dmalloc.h"
 #endif
 
-static guint32 
+static gint32
 mdb_map_find_next0(MdbHandle *mdb, unsigned char *map, unsigned int map_sz, guint32 start_pg)
 {
 	guint32 pgnum, i, usage_bitlen;
@@ -42,7 +42,7 @@ mdb_map_find_next0(MdbHandle *mdb, unsigned char *map, unsigned int map_sz, guin
 	/* didn't find anything */
 	return 0;
 }
-static int 
+static gint32
 mdb_map_find_next1(MdbHandle *mdb, unsigned char *map, unsigned int map_sz, guint32 start_pg)
 {
 	guint32 map_ind, max_map_pgs, offset, usage_bitlen;
@@ -83,7 +83,10 @@ mdb_map_find_next1(MdbHandle *mdb, unsigned char *map, unsigned int map_sz, guin
 	/* didn't find anything */
 	return 0;
 }
-guint32 
+
+/* returns 0 on EOF */
+/* returns -1 on error (unsupported map type) */
+gint32
 mdb_map_find_next(MdbHandle *mdb, unsigned char *map, unsigned int map_sz, guint32 start_pg)
 {
 	if (map[0] == 0) {
@@ -119,6 +122,9 @@ mdb_map_find_next_freepage(MdbTableDef *table, int row_size)
 			/* allocate new page */
 			pgnum = mdb_alloc_page(table);
 			return pgnum;
+		} else if (pgnum==-1) {
+			fprintf(stderr, "Error: mdb_map_find_next_freepage error while reading maps.\n");
+			exit(1);
 		}
 		cur_pg = pgnum;
 

src/util/prfreemap.c

@@ -48,7 +48,7 @@ unsigned char *map_buf;
 	if (map_buf[0]==0)
 		while (map_buf[map_sz]==0xff) map_sz--;
 
-	while (pgnum = mdb_map_find_next(mdb, map_buf, map_sz, pgnum)) {
+	while ((pgnum = mdb_map_find_next(mdb, map_buf, map_sz, pgnum))>0) {
 		printf("%6lu ",(long unsigned) pgnum);
 		if (coln==10) {
 			printf("\n");