# HG changeset patch
# User Timo Sirainen <tss@iki.fi>
# Date 1189839956 -10800
# Node ID 29f427039e0062ab6d407baee90f2fbb136652cd
# Parent  9b9436231ce08339425090f6251263308e720139
fsck won't fail anymore with "corrupted index", all problems are fixed.
Added mail_index_fsck_locked().

diff -r 9b9436231ce0 -r 29f427039e00 src/lib-index/mail-index-fsck.c
--- a/src/lib-index/mail-index-fsck.c	Sat Sep 15 09:56:29 2007 +0300
+++ b/src/lib-index/mail-index-fsck.c	Sat Sep 15 10:05:56 2007 +0300
@@ -24,17 +24,16 @@
 				      map->hdr.field, hdr.field); \
 	}
 
-static int
-mail_index_fsck_map(struct mail_index *index, struct mail_index_map *map,
-		    bool *lock, const char **error_r)
+static int mail_index_fsck_map(struct mail_index *index,
+			       struct mail_index_map *map, bool *lock)
 {
 	struct mail_index_header hdr;
-	const struct mail_index_record *rec;
+	struct mail_index_record *rec, *next_rec;
 	uint32_t file_seq;
 	uoff_t file_offset;
 	uint32_t i, last_uid;
-
-	*error_r = NULL;
+	bool logged_unordered_uids = FALSE, logged_zero_uids = FALSE;
+	bool records_dropped = FALSE;
 
 	if (*lock) {
 		if (mail_transaction_log_sync_lock(index->log, &file_seq,
@@ -79,10 +78,31 @@
 	hdr.first_deleted_uid_lowwater = 0;
 
 	rec = map->rec_map->records; last_uid = 0;
-	for (i = 0; i < map->rec_map->records_count; i++) {
+	for (i = 0; i < map->rec_map->records_count; ) {
+		next_rec = PTR_OFFSET(rec, hdr.record_size);
 		if (rec->uid <= last_uid) {
-			*error_r = "Record UIDs are not ordered";
-			return 0;
+			/* log an error once, and skip this record */
+			if (rec->uid == 0) {
+				if (!logged_zero_uids) {
+					mail_index_fsck_error(index,
+						"Record UIDs have zeroes");
+					logged_zero_uids = TRUE;
+				}
+			} else {
+				if (!logged_unordered_uids) {
+					mail_index_fsck_error(index,
+						"Record UIDs unordered");
+					logged_unordered_uids = TRUE;
+				}
+			}
+			/* not the fastest way when we're skipping lots of
+			   records, but this should happen rarely so don't
+			   bother optimizing. */
+			memmove(rec, next_rec, hdr.record_size *
+				(map->rec_map->records_count - i - 1));
+			map->rec_map->records_count--;
+			records_dropped = TRUE;
+			continue;
 		}
 
 		hdr.messages_count++;
@@ -99,7 +119,13 @@
 			hdr.first_deleted_uid_lowwater = rec->uid;
 
 		last_uid = rec->uid;
-		rec = CONST_PTR_OFFSET(rec, hdr.record_size);
+		rec = next_rec;
+		i++;
+	}
+
+	if (records_dropped) {
+		/* all existing views are broken now */
+		index->inconsistency_id++;
 	}
 
 	if (hdr.next_uid <= last_uid) {
@@ -134,12 +160,11 @@
 	CHECK(first_recent_uid, !=);
 
 	map->hdr = hdr;
-	return 1;
+	return 0;
 }
 
 int mail_index_fsck(struct mail_index *index)
 {
-	const char *error = NULL;
 	struct mail_index_map *map;
 	bool lock = !index->log_locked;
 	int ret;
@@ -150,22 +175,24 @@
 	mail_index_unmap(&index->map);
 	index->map = map;
 
-	ret = mail_index_fsck_map(index, map, &lock, &error);
-	if (ret > 0) {
+	ret = mail_index_fsck_map(index, map, &lock);
+	if (ret == 0) {
 		map->write_base_header = TRUE;
 		map->write_atomic = TRUE;
 
 		mail_index_write(index, FALSE);
 	}
 
-	if (error != NULL) {
-		mail_index_set_error(index, "Corrupted index file %s: %s",
-				     index->filepath, error);
-	}
-	if (ret == 0)
-		mail_index_mark_corrupted(index);
-
 	if (lock)
 		mail_transaction_log_sync_unlock(index->log);
 	return ret;
 }
+
+void mail_index_fsck_locked(struct mail_index *index)
+{
+	int ret;
+
+	i_assert(index->log_locked);
+	ret = mail_index_fsck(index);
+	i_assert(ret == 0);
+}
diff -r 9b9436231ce0 -r 29f427039e00 src/lib-index/mail-index-map.c
--- a/src/lib-index/mail-index-map.c	Sat Sep 15 09:56:29 2007 +0300
+++ b/src/lib-index/mail-index-map.c	Sat Sep 15 10:05:56 2007 +0300
@@ -685,7 +685,7 @@
 		mail_index_unlock(index, &lock_id);
 	}
 
-	for (try = 0; ret > 0 && try < 2; try++) {
+	for (try = 0; ret > 0; try++) {
 		/* make sure the header is ok before using this mapping */
 		ret = mail_index_map_check_header(new_map);
 		if (ret > 0) {
@@ -694,13 +694,13 @@
 			else if (mail_index_map_parse_keywords(new_map) < 0)
 				ret = 0;
 		}
-		if (ret != 0)
+		if (ret != 0 || try == 2)
 			break;
 
 		/* fsck and try again */
 		old_map = index->map;
 		index->map = new_map;
-		ret = mail_index_fsck(index);
+		ret = mail_index_fsck(index) < 0 ? -1 : 1;
 
 		/* fsck cloned the map, so we'll have to update it */
 		mail_index_unmap(&new_map);
diff -r 9b9436231ce0 -r 29f427039e00 src/lib-index/mail-index-private.h
--- a/src/lib-index/mail-index-private.h	Sat Sep 15 09:56:29 2007 +0300
+++ b/src/lib-index/mail-index-private.h	Sat Sep 15 10:05:56 2007 +0300
@@ -169,6 +169,8 @@
 
 	struct mail_index_map *map;
 	uint32_t indexid;
+	unsigned int inconsistency_id;
+
 	/* last_read_log_file_* contains the seq/offsets we last read from
 	   the main index file's headers. these are used to figure out when
 	   the main index file should be updated, and if we can update it
@@ -260,8 +262,7 @@
 
    If we mmap()ed the index file, the map is returned locked.
 
-   Returns 1 = ok, 0 = corrupted, -1 = error. If non-fatal problems were found,
-   1 is returned but index->fsck=TRUE is set. */
+   Returns 1 = ok, 0 = corrupted, -1 = error. */
 int mail_index_map(struct mail_index *index,
 		   enum mail_index_sync_handler_type type);
 /* Unreference given mapping and unmap it if it's dropped to zero. */
@@ -291,6 +292,8 @@
 void mail_index_view_transaction_ref(struct mail_index_view *view);
 void mail_index_view_transaction_unref(struct mail_index_view *view);
 
+void mail_index_fsck_locked(struct mail_index *index);
+
 int mail_index_set_error(struct mail_index *index, const char *fmt, ...)
 	__attr_format__(2, 3);
 /* "%s failed with index file %s: %m" */
diff -r 9b9436231ce0 -r 29f427039e00 src/lib-index/mail-index-sync.c
--- a/src/lib-index/mail-index-sync.c	Sat Sep 15 09:56:29 2007 +0300
+++ b/src/lib-index/mail-index-sync.c	Sat Sep 15 10:05:56 2007 +0300
@@ -335,10 +335,11 @@
 	   transaction log except for expunges. They're synced in
 	   mail_index_sync_commit(). */
 	if ((ret = mail_index_map(index, MAIL_INDEX_SYNC_HANDLER_HEAD)) <= 0) {
-		if (ret == 0 || mail_index_fsck(index) <= 0) {
+		if (ret == 0) {
 			mail_transaction_log_sync_unlock(index->log);
 			return -1;
 		}
+
 		/* let's try again */
 		if (mail_index_map(index, MAIL_INDEX_SYNC_HANDLER_HEAD) <= 0) {
 			mail_transaction_log_sync_unlock(index->log);
@@ -360,10 +361,7 @@
 		mail_index_set_error(index,
 			"broken sync positions in index file %s",
 			index->filepath);
-		if (mail_index_fsck(index) <= 0) {
-			mail_transaction_log_sync_unlock(index->log);
-			return -1;
-		}
+		mail_index_fsck_locked(index);
 	}
 
 	ctx = i_new(struct mail_index_sync_ctx, 1);
@@ -385,11 +383,8 @@
 					 hdr->log_file_tail_offset) < 0) {
 		/* if a log file is missing, there's nothing we can do except
 		   to skip over it. fix the problem with fsck and try again. */
+		mail_index_fsck_locked(index);
 		mail_index_sync_rollback(&ctx);
-		if (mail_index_fsck(index) <= 0) {
-			mail_transaction_log_sync_unlock(index->log);
-			return -1;
-		}
 		return mail_index_sync_begin_to(index, ctx_r, view_r, trans_r,
 						log_file_seq, log_file_offset,
 						flags);
diff -r 9b9436231ce0 -r 29f427039e00 src/lib-index/mail-index-view-private.h
--- a/src/lib-index/mail-index-view-private.h	Sat Sep 15 09:56:29 2007 +0300
+++ b/src/lib-index/mail-index-view-private.h	Sat Sep 15 10:05:56 2007 +0300
@@ -48,7 +48,9 @@
 	struct mail_index *index;
         struct mail_transaction_log_view *log_view;
 
-	unsigned int indexid;
+	uint32_t indexid;
+	unsigned int inconsistency_id;
+
 	struct mail_index_map *map;
 	/* After syncing view, map is replaced with sync_new_map. */
 	struct mail_index_map *sync_new_map;
diff -r 9b9436231ce0 -r 29f427039e00 src/lib-index/mail-index-view-sync.c
--- a/src/lib-index/mail-index-view-sync.c	Sat Sep 15 09:56:29 2007 +0300
+++ b/src/lib-index/mail-index-view-sync.c	Sat Sep 15 10:05:56 2007 +0300
@@ -267,6 +267,12 @@
 	i_assert(!view->syncing);
 	i_assert(view->transactions == 0);
 
+	if (mail_index_view_is_inconsistent(view)) {
+		mail_index_set_error(view->index, "%s view is inconsistent",
+				     view->index->filepath);
+		return -1;
+	}
+
 	sync_expunges = (flags & MAIL_INDEX_VIEW_SYNC_FLAG_NOEXPUNGES) == 0;
 	if (sync_expunges) {
 		/* get list of all expunges first */
diff -r 9b9436231ce0 -r 29f427039e00 src/lib-index/mail-index-view.c
--- a/src/lib-index/mail-index-view.c	Sat Sep 15 09:56:29 2007 +0300
+++ b/src/lib-index/mail-index-view.c	Sat Sep 15 10:05:56 2007 +0300
@@ -56,7 +56,8 @@
 
 bool mail_index_view_is_inconsistent(struct mail_index_view *view)
 {
-	if (view->index->indexid != view->indexid)
+	if (view->index->indexid != view->indexid ||
+	    view->index->inconsistency_id != view->inconsistency_id)
 		view->inconsistent = TRUE;
 	return view->inconsistent;
 }
@@ -132,6 +133,7 @@
 _view_lookup_full(struct mail_index_view *view, uint32_t seq,
 		  struct mail_index_map **map_r, bool *expunged_r)
 {
+	static struct mail_index_record broken_rec;
 	struct mail_index_map *map;
 	const struct mail_index_record *rec, *head_rec;
 
@@ -140,13 +142,18 @@
 	/* look up the record */
 	rec = MAIL_INDEX_MAP_IDX(view->map, seq-1);
 	if (rec->uid == 0) {
-		mail_index_set_error(view->index, "Corrupted Index file %s: "
-			"Record [%u].uid=0", view->index->filepath, seq);
-		mail_index_mark_corrupted(view->index);
+		if (!view->inconsistent) {
+			mail_index_set_error(view->index,
+				"Corrupted Index file %s: Record [%u].uid=0",
+				view->index->filepath, seq);
+			(void)mail_index_fsck(view->index);
+			view->inconsistent = TRUE;
+		}
 
+		/* we'll need to return something so the caller doesn't crash */
 		*map_r = view->map;
 		*expunged_r = TRUE;
-		return rec;
+		return &broken_rec;
 	}
 	if (view->map == view->index->map) {
 		/* view's mapping is latest. we can use it directly. */
@@ -610,6 +617,7 @@
 	view->log_view = mail_transaction_log_view_open(index->log);
 
 	view->indexid = index->indexid;
+	view->inconsistency_id = index->inconsistency_id;
 	view->map = map;
 	view->map->refcount++;
 
diff -r 9b9436231ce0 -r 29f427039e00 src/lib-index/mail-index.h
--- a/src/lib-index/mail-index.h	Sat Sep 15 09:56:29 2007 +0300
+++ b/src/lib-index/mail-index.h	Sat Sep 15 10:05:56 2007 +0300
@@ -278,9 +278,8 @@
 
 /* Mark index file corrupted. Invalidates all views. */
 void mail_index_mark_corrupted(struct mail_index *index);
-/* Check and fix any found problems. If index is broken beyond repair, it's
-   marked corrupted and 0 is returned. Otherwise returns -1 if there was some
-   I/O error or 1 if everything went ok. */
+/* Check and fix any found problems. Returns -1 if we couldn't lock for sync,
+   0 if everything went ok. */
 int mail_index_fsck(struct mail_index *index);
 
 /* Synchronize changes in view. You have to go through all records, or view