Mercurial > dovecot > original-hg > dovecot-1.2

changeset 3322:49071cc19102 HEAD

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
If UIDVALIDITY changes, don't invalidate the whole index. Just expunge all existing messages and update uidvalidity/nextuid fields. Now we don't have to re-login when this happens.
author Timo Sirainen <tss@iki.fi>
date Sat, 23 Apr 2005 18:18:21 +0300
parents be9bf789a1d4
children 2e153b23b830
files src/lib-index/mail-index-transaction-private.h src/lib-index/mail-index-transaction.c src/lib-index/mail-index.h src/lib-index/mail-transaction-log-append.c src/lib-storage/index/maildir/maildir-sync.c src/lib-storage/index/mbox/mbox-save.c src/lib-storage/index/mbox/mbox-sync-private.h src/lib-storage/index/mbox/mbox-sync.c
diffstat 8 files changed, 104 insertions(+), 52 deletions(-) [+]
line wrap: on
line diff
--- a/src/lib-index/mail-index-transaction-private.h	Sat Apr 23 16:28:22 2005 +0300
+++ b/src/lib-index/mail-index-transaction-private.h	Sat Apr 23 18:18:21 2005 +0300
@@ -19,8 +19,10 @@
 	array_t ARRAY_DEFINE(updates, struct mail_transaction_flag_update);
 	size_t last_update_idx;
 
-	unsigned char hdr_change[sizeof(struct mail_index_header)];
-	unsigned char hdr_mask[sizeof(struct mail_index_header)];
+	unsigned char pre_hdr_change[sizeof(struct mail_index_header)];
+	unsigned char pre_hdr_mask[sizeof(struct mail_index_header)];
+	unsigned char post_hdr_change[sizeof(struct mail_index_header)];
+	unsigned char post_hdr_mask[sizeof(struct mail_index_header)];
 
 	array_t ARRAY_DEFINE(ext_rec_updates, array_t);
 	array_t ARRAY_DEFINE(ext_resizes, struct mail_transaction_ext_intro);
@@ -35,7 +37,8 @@
 	unsigned int hide_transaction:1;
 	unsigned int no_appends:1;
 	unsigned int external:1;
-	unsigned int hdr_changed:1;
+	unsigned int pre_hdr_changed:1;
+	unsigned int post_hdr_changed:1;
 	unsigned int log_updates:1;
 };
 
--- a/src/lib-index/mail-index-transaction.c	Sat Apr 23 16:28:22 2005 +0300
+++ b/src/lib-index/mail-index-transaction.c	Sat Apr 23 18:18:21 2005 +0300
@@ -705,17 +705,25 @@
 }
 
 void mail_index_update_header(struct mail_index_transaction *t,
-			      size_t offset, const void *data, size_t size)
+			      size_t offset, const void *data, size_t size,
+			      int prepend)
 {
-	i_assert(offset < sizeof(t->hdr_change));
-	i_assert(size <= sizeof(t->hdr_change) - offset);
+	i_assert(offset < sizeof(t->pre_hdr_change));
+	i_assert(size <= sizeof(t->pre_hdr_change) - offset);
 
-	t->hdr_changed = TRUE;
 	t->log_updates = TRUE;
 
-	memcpy(t->hdr_change + offset, data, size);
-	for (; size > 0; size--)
-		t->hdr_mask[offset++] = 1;
+	if (prepend) {
+		t->pre_hdr_changed = TRUE;
+		memcpy(t->pre_hdr_change + offset, data, size);
+		for (; size > 0; size--)
+			t->pre_hdr_mask[offset++] = 1;
+	} else {
+		t->post_hdr_changed = TRUE;
+		memcpy(t->post_hdr_change + offset, data, size);
+		for (; size > 0; size--)
+			t->post_hdr_mask[offset++] = 1;
+	}
 }
 
 void mail_index_ext_resize(struct mail_index_transaction *t, uint32_t ext_id,
--- a/src/lib-index/mail-index.h	Sat Apr 23 16:28:22 2005 +0300
+++ b/src/lib-index/mail-index.h	Sat Apr 23 18:18:21 2005 +0300
@@ -326,9 +326,11 @@
 				enum modify_type modify_type,
 				struct mail_keywords *keywords);
 
-/* Update field in header. */
+/* Update field in header. If prepend is TRUE, the header change is visible
+   before message syncing begins. */
 void mail_index_update_header(struct mail_index_transaction *t,
-			      size_t offset, const void *data, size_t size);
+			      size_t offset, const void *data, size_t size,
+			      int prepend);
 
 /* Returns the last error code. */
 enum mail_index_error mail_index_get_last_error(struct mail_index *index);
--- a/src/lib-index/mail-transaction-log-append.c	Sat Apr 23 16:28:22 2005 +0300
+++ b/src/lib-index/mail-transaction-log-append.c	Sat Apr 23 18:18:21 2005 +0300
@@ -80,18 +80,22 @@
 }
 
 static const buffer_t *
-log_get_hdr_update_buffer(struct mail_index_transaction *t)
+log_get_hdr_update_buffer(struct mail_index_transaction *t, int prepend)
 {
 	buffer_t *buf;
+	const unsigned char *data, *mask;
 	struct mail_transaction_header_update u;
 	uint16_t offset;
 	int state = 0;
 
 	memset(&u, 0, sizeof(u));
 
+	data = prepend ? t->pre_hdr_change : t->post_hdr_change;
+	mask = prepend ? t->pre_hdr_mask : t->post_hdr_mask;
+
 	buf = buffer_create_dynamic(pool_datastack_create(), 256);
-	for (offset = 0; offset <= sizeof(t->hdr_change); offset++) {
-		if (offset < sizeof(t->hdr_change) && t->hdr_mask[offset]) {
+	for (offset = 0; offset <= sizeof(t->pre_hdr_change); offset++) {
+		if (offset < sizeof(t->pre_hdr_change) && mask[offset]) {
 			if (state == 0) {
 				u.offset = offset;
 				state++;
@@ -99,9 +103,8 @@
 		} else {
 			if (state > 0) {
 				u.size = offset - u.offset;
-				buffer_append(buf, &u, sizeof(uint16_t)*2);
-				buffer_append(buf, t->hdr_change + u.offset,
-					      u.size);
+				buffer_append(buf, &u, sizeof(u));
+				buffer_append(buf, data + u.offset, u.size);
 				state = 0;
 			}
 		}
@@ -411,6 +414,12 @@
 	   to avoid resize overhead as much as possible */
         ret = mail_transaction_log_append_ext_intros(file, t);
 
+	if (t->pre_hdr_changed && ret == 0) {
+		ret = log_append_buffer(file,
+					log_get_hdr_update_buffer(t, TRUE),
+					NULL, MAIL_TRANSACTION_HEADER_UPDATE,
+					t->external);
+	}
 	if (array_is_created(&t->appends) && ret == 0) {
                 visibility_changes = TRUE;
 		ret = log_append_buffer(file, t->appends.buffer, NULL,
@@ -442,16 +451,17 @@
 		ret = log_append_buffer(file, t->expunges.buffer, NULL,
 					MAIL_TRANSACTION_EXPUNGE, t->external);
 	}
-	if (t->hdr_changed && ret == 0) {
-		ret = log_append_buffer(file, log_get_hdr_update_buffer(t),
-					NULL, MAIL_TRANSACTION_HEADER_UPDATE,
-					t->external);
-	}
 
 	if (ret < 0) {
 		mail_index_file_set_syscall_error(log->index, file->filepath,
 						  "pwrite()");
 	}
+	if (t->post_hdr_changed && ret == 0) {
+		ret = log_append_buffer(file,
+					log_get_hdr_update_buffer(t, FALSE),
+					NULL, MAIL_TRANSACTION_HEADER_UPDATE,
+					t->external);
+	}
 
 	if (ret == 0 && visibility_changes && t->hide_transaction) {
 		mail_index_view_add_synced_transaction(view, file->hdr.file_seq,
--- a/src/lib-storage/index/maildir/maildir-sync.c	Sat Apr 23 16:28:22 2005 +0300
+++ b/src/lib-storage/index/maildir/maildir-sync.c	Sat Apr 23 18:18:21 2005 +0300
@@ -603,6 +603,7 @@
 	struct maildir_uidlist_iter_ctx *iter;
 	struct mail_index_transaction *trans;
 	const struct mail_index_header *hdr;
+	struct mail_index_header tmp_hdr;
 	const struct mail_index_record *rec;
 	pool_t keyword_pool;
 	uint32_t seq, uid;
@@ -613,23 +614,34 @@
 	uint32_t uid_validity, next_uid;
 	int ret;
 
+	trans = mail_index_transaction_begin(view, FALSE, TRUE);
+	sync_ctx->trans = trans;
+
 	hdr = mail_index_get_header(view);
 	uid_validity = maildir_uidlist_get_uid_validity(mbox->uidlist);
 	if (uid_validity != hdr->uid_validity &&
 	    uid_validity != 0 && hdr->uid_validity != 0) {
 		/* uidvalidity changed and mailbox isn't being initialized,
-		   index must be rebuilt */
+		   reset mailbox so we can add all messages as new */
 		mail_storage_set_critical(STORAGE(mbox->storage),
 			"Maildir %s sync: UIDVALIDITY changed (%u -> %u)",
 			mbox->path, hdr->uid_validity, uid_validity);
-		mail_index_mark_corrupted(mbox->ibox.index);
-                maildir_sync_index_abort(sync_ctx);
-		return -1;
+
+		for (seq = 1; seq < hdr->messages_count; seq++)
+			mail_index_expunge(trans, seq);
+
+		/* Reset uidvalidity and next_uid. */
+		memcpy(&tmp_hdr, hdr, sizeof(tmp_hdr));
+		tmp_hdr.uid_validity = 0;
+		tmp_hdr.next_uid = 0;
+
+		/* next_uid must be reset before message syncing begins,
+		   or we get errors about UIDs larger than next_uid. */
+		mail_index_update_header(trans,
+			offsetof(struct mail_index_header, next_uid),
+			&hdr->next_uid, sizeof(hdr->next_uid), TRUE);
 	}
 
-	trans = mail_index_transaction_begin(view, FALSE, TRUE);
-	sync_ctx->trans = trans;
-
 	keyword_pool = pool_alloconly_create("maildir keywords", 128);
 
 	seq = 0;
@@ -801,7 +813,7 @@
 
 		mail_index_update_header(trans,
 			offsetof(struct mail_index_header, sync_stamp),
-			&sync_stamp, sizeof(sync_stamp));
+			&sync_stamp, sizeof(sync_stamp), TRUE);
 	}
 
 	if (hdr->uid_validity == 0) {
@@ -822,14 +834,14 @@
 	if (uid_validity != hdr->uid_validity && uid_validity != 0) {
 		mail_index_update_header(trans,
 			offsetof(struct mail_index_header, uid_validity),
-			&uid_validity, sizeof(uid_validity));
+			&uid_validity, sizeof(uid_validity), TRUE);
 	}
 
 	next_uid = maildir_uidlist_get_next_uid(mbox->uidlist);
 	if (next_uid != 0 && hdr->next_uid != next_uid) {
 		mail_index_update_header(trans,
 			offsetof(struct mail_index_header, next_uid),
-			&next_uid, sizeof(next_uid));
+			&next_uid, sizeof(next_uid), FALSE);
 	}
 
 	if (ret < 0) {
--- a/src/lib-storage/index/mbox/mbox-save.c	Sat Apr 23 16:28:22 2005 +0300
+++ b/src/lib-storage/index/mbox/mbox-save.c	Sat Apr 23 18:18:21 2005 +0300
@@ -558,7 +558,7 @@
 	if (ctx->synced) {
 		mail_index_update_header(ctx->trans,
 			offsetof(struct mail_index_header, next_uid),
-			&ctx->next_uid, sizeof(ctx->next_uid));
+			&ctx->next_uid, sizeof(ctx->next_uid), FALSE);
 	}
 
 	if (!ctx->synced && ctx->mbox->mbox_fd != -1 &&
--- a/src/lib-storage/index/mbox/mbox-sync-private.h	Sat Apr 23 16:28:22 2005 +0300
+++ b/src/lib-storage/index/mbox/mbox-sync-private.h	Sat Apr 23 18:18:21 2005 +0300
@@ -113,7 +113,7 @@
 
 	pool_t mail_keyword_pool;
 
-	uint32_t prev_msg_uid, next_uid;
+	uint32_t prev_msg_uid, next_uid, idx_next_uid;
 	uint32_t seq, idx_seq, need_space_seq;
 	off_t expunged_space, space_diff;
 
--- a/src/lib-storage/index/mbox/mbox-sync.c	Sat Apr 23 16:28:22 2005 +0300
+++ b/src/lib-storage/index/mbox/mbox-sync.c	Sat Apr 23 18:18:21 2005 +0300
@@ -266,12 +266,12 @@
 		rec = NULL;
 	}
 
-	if (ret == 0 && uid < sync_ctx->hdr->next_uid) {
+	if (ret == 0 && uid < sync_ctx->idx_next_uid) {
 		/* this UID was already in index and it was expunged */
 		mail_storage_set_critical(STORAGE(sync_ctx->mbox->storage),
 			"mbox sync: Expunged message reappeared in mailbox %s "
 			"(UID %u < %u)", sync_ctx->mbox->path, uid,
-			sync_ctx->hdr->next_uid);
+			sync_ctx->idx_next_uid);
 		ret = 0; rec = NULL;
 	} else if (rec != NULL && rec->uid != uid) {
 		/* new UID in the middle of the mailbox - shouldn't happen */
@@ -923,7 +923,7 @@
 	const struct mail_index_record *rec;
 	uint32_t uid, messages_count;
 	uoff_t offset;
-	int ret, expunged, skipped_mails;
+	int ret, expunged, skipped_mails, uidvalidity_changed;
 
 	messages_count =
 		mail_index_view_get_messages_count(sync_ctx->sync_view);
@@ -938,10 +938,13 @@
 	while ((ret = mbox_sync_read_next_mail(sync_ctx, mail_ctx)) > 0) {
 		uid = mail_ctx->mail.uid;
 
-		if (mail_ctx->seq == 1 && sync_ctx->base_uid_validity != 0 &&
-                    sync_ctx->hdr->uid_validity != 0 &&
-		    sync_ctx->base_uid_validity !=
-		    sync_ctx->hdr->uid_validity) {
+		uidvalidity_changed = mail_ctx->seq == 1 &&
+			sync_ctx->base_uid_validity != 0 &&
+			sync_ctx->hdr->uid_validity != 0 &&
+			sync_ctx->base_uid_validity !=
+			sync_ctx->hdr->uid_validity;
+
+		if (uidvalidity_changed) {
 			mail_storage_set_critical(
 				STORAGE(sync_ctx->mbox->storage),
 				"UIDVALIDITY changed (%u -> %u) "
@@ -949,8 +952,21 @@
 				sync_ctx->hdr->uid_validity,
 				sync_ctx->base_uid_validity,
 				sync_ctx->mbox->path);
-                        mail_index_mark_corrupted(sync_ctx->mbox->ibox.index);
-			return -1;
+
+			/* we need to recreate all messages in index */
+			while (sync_ctx->idx_seq <= messages_count) {
+				mail_index_expunge(sync_ctx->t,
+						   sync_ctx->idx_seq++);
+			}
+
+			/* next_uid must be reset before message syncing
+			   begins, or we get errors about UIDs larger than
+			   next_uid. */
+			sync_ctx->idx_next_uid = 0;
+			mail_index_update_header(sync_ctx->t,
+				offsetof(struct mail_index_header, next_uid),
+				&sync_ctx->idx_next_uid,
+				sizeof(sync_ctx->idx_next_uid), TRUE);
 		}
 
 		if (mail_ctx->uid_broken && partial) {
@@ -963,7 +979,7 @@
 			uid = 0;
 
 		rec = NULL; ret = 1;
-		if (uid != 0) {
+		if (uid != 0 && !uidvalidity_changed) {
 			ret = mbox_sync_read_index_rec(sync_ctx, uid, &rec);
 			if (ret < 0)
 				return -1;
@@ -977,7 +993,8 @@
 		if (ret == 0) {
 			/* UID found but it's broken */
 			uid = 0;
-		} else if (uid == 0 && !mail_ctx->pseudo &&
+		} else if (uid == 0 && !uidvalidity_changed &&
+			   !mail_ctx->pseudo &&
 			   (sync_ctx->delay_writes ||
 			    sync_ctx->idx_seq <= messages_count)) {
 			/* If we can't use/store X-UID header, use MD5 sum.
@@ -994,7 +1011,7 @@
 				uid = mail_ctx->mail.uid = rec->uid;
 		}
 
-		if (!mail_ctx->pseudo) {
+		if (!mail_ctx->pseudo && !uidvalidity_changed) {
 			/* get all sync records related to this message */
 			if (mbox_sync_read_index_syncs(sync_ctx, uid,
 						       &expunged) < 0)
@@ -1075,7 +1092,6 @@
 
 	if (!skipped_mails)
 		sync_ctx->mbox->mbox_sync_dirty = FALSE;
-
 	return 1;
 }
 
@@ -1251,7 +1267,7 @@
 		mail_index_update_header(sync_ctx->t,
 			offsetof(struct mail_index_header, uid_validity),
 			&sync_ctx->base_uid_validity,
-			sizeof(sync_ctx->base_uid_validity));
+			sizeof(sync_ctx->base_uid_validity), TRUE);
 	}
 
 	if (istream_raw_mbox_is_eof(sync_ctx->input) &&
@@ -1259,7 +1275,7 @@
 		i_assert(sync_ctx->next_uid != 0);
 		mail_index_update_header(sync_ctx->t,
 			offsetof(struct mail_index_header, next_uid),
-			&sync_ctx->next_uid, sizeof(sync_ctx->next_uid));
+			&sync_ctx->next_uid, sizeof(sync_ctx->next_uid), FALSE);
 	}
 
 	if ((uint32_t)st->st_mtime != sync_ctx->hdr->sync_stamp &&
@@ -1268,7 +1284,7 @@
 
 		mail_index_update_header(sync_ctx->t,
 			offsetof(struct mail_index_header, sync_stamp),
-			&sync_stamp, sizeof(sync_stamp));
+			&sync_stamp, sizeof(sync_stamp), TRUE);
 	}
 
 	if ((uint64_t)st->st_size != sync_ctx->hdr->sync_size &&
@@ -1277,7 +1293,7 @@
 
 		mail_index_update_header(sync_ctx->t,
 			offsetof(struct mail_index_header, sync_size),
-			&sync_size, sizeof(sync_size));
+			&sync_size, sizeof(sync_size), TRUE);
 	}
 
 	sync_ctx->mbox->mbox_dirty_stamp = st->st_mtime;
@@ -1300,6 +1316,7 @@
 
 	sync_ctx->prev_msg_uid = 0;
 	sync_ctx->next_uid = sync_ctx->hdr->next_uid;
+	sync_ctx->idx_next_uid = sync_ctx->hdr->next_uid;
 	sync_ctx->seq = 0;
 	sync_ctx->idx_seq = 1;
 	sync_ctx->need_space_seq = 0;