Mercurial > dovecot > original-hg > dovecot-1.2

changeset 2511:710e0bf25bf8 HEAD

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Added mbox_dirty_syncs setting which delays re-reading the whole mbox when it's changed.
author Timo Sirainen <tss@iki.fi>
date Sat, 28 Aug 2004 19:39:53 +0300
parents 0f660149c7ef
children abcdec2299bd
files dovecot-example.conf src/imap/cmd-check.c src/imap/cmd-expunge.c src/imap/cmd-select.c src/lib-storage/index/index-storage.h src/lib-storage/index/mbox/mbox-file.c src/lib-storage/index/mbox/mbox-file.h src/lib-storage/index/mbox/mbox-mail.c src/lib-storage/index/mbox/mbox-save.c src/lib-storage/index/mbox/mbox-storage.c src/lib-storage/index/mbox/mbox-sync-private.h src/lib-storage/index/mbox/mbox-sync.c src/lib-storage/index/mbox/mbox-transaction.c src/lib-storage/mail-storage.h src/master/mail-process.c src/master/master-settings.c src/master/master-settings.h
diffstat 17 files changed, 262 insertions(+), 124 deletions(-) [+]
line wrap: on
line diff
--- a/dovecot-example.conf	Sat Aug 28 16:25:42 2004 +0300
+++ b/dovecot-example.conf	Sat Aug 28 19:39:53 2004 +0300
@@ -303,6 +303,17 @@
 # lock file after this many seconds.
 #mbox_dotlock_change_timeout = 30
 
+# When mbox changes unexpectedly we have to fully read it to find out what
+# changed. If the mbox is large this can take a long time. Since the change
+# is usually just a newly appended mail, it'd be faster to simply read the
+# new mails. If this setting is enabled, Dovecot does this but still safely
+# fallbacks to re-reading the whole mbox file whenever something in mbox isn't
+# how it's expected to be. The only real downside to this setting is that if
+# some other MUA changes message flags, Dovecot doesn't notice it immediately.
+# Note that a full sync is done for SELECT, EXAMINE, EXPUNGE and CHECK
+# commands.
+#mbox_dirty_syncs = yes
+
 # umask to use for mail files and directories
 #umask = 0077
 
--- a/src/imap/cmd-check.c	Sat Aug 28 16:25:42 2004 +0300
+++ b/src/imap/cmd-check.c	Sat Aug 28 19:39:53 2004 +0300
@@ -8,5 +8,5 @@
 	if (!client_verify_open_mailbox(client))
 		return TRUE;
 
-	return cmd_sync(client, 0, "OK Check completed.");
+	return cmd_sync(client, MAILBOX_SYNC_FLAG_FULL, "OK Check completed.");
 }
--- a/src/imap/cmd-expunge.c	Sat Aug 28 16:25:42 2004 +0300
+++ b/src/imap/cmd-expunge.c	Sat Aug 28 19:39:53 2004 +0300
@@ -27,9 +27,10 @@
 	if (search_arg == NULL)
 		return TRUE;
 
-	if (imap_expunge(client->mailbox, search_arg))
-		return cmd_sync(client, 0, "OK Expunge completed.");
-	else {
+	if (imap_expunge(client->mailbox, search_arg)) {
+		return cmd_sync(client, MAILBOX_SYNC_FLAG_FULL,
+				"OK Expunge completed.");
+	} else {
 		client_send_storage_error(client,
 					  mailbox_get_storage(client->mailbox));
 		return TRUE;
@@ -41,9 +42,10 @@
 	if (!client_verify_open_mailbox(client))
 		return TRUE;
 
-	if (imap_expunge(client->mailbox, NULL))
-		return cmd_sync(client, 0, "OK Expunge completed.");
-	else {
+	if (imap_expunge(client->mailbox, NULL)) {
+		return cmd_sync(client, MAILBOX_SYNC_FLAG_FULL,
+				"OK Expunge completed.");
+	} else {
 		client_send_storage_error(client,
 					  mailbox_get_storage(client->mailbox));
 		return TRUE;
--- a/src/imap/cmd-select.c	Sat Aug 28 16:25:42 2004 +0300
+++ b/src/imap/cmd-select.c	Sat Aug 28 19:39:53 2004 +0300
@@ -35,7 +35,7 @@
 		return TRUE;
 	}
 
-	if (imap_sync_nonselected(box, 0) < 0) {
+	if (imap_sync_nonselected(box, MAILBOX_SYNC_FLAG_FULL) < 0) {
 		client_send_storage_error(client, storage);
 		mailbox_close(box);
 		return TRUE;
--- a/src/lib-storage/index/index-storage.h	Sat Aug 28 16:25:42 2004 +0300
+++ b/src/lib-storage/index/index-storage.h	Sat Aug 28 19:39:53 2004 +0300
@@ -79,6 +79,8 @@
 	struct dotlock mbox_dotlock;
 	unsigned int mbox_lock_id, mbox_mail_lock_id;
 	int mbox_readonly;
+	time_t mbox_dirty_stamp;
+	off_t mbox_dirty_size;
 
 	uint32_t mbox_extra_idx, md5hdr_extra_idx;
 
@@ -99,6 +101,8 @@
 	unsigned int mail_read_mmaped:1;
 	unsigned int syncing_commit:1;
 	unsigned int unreliable_headers:1;
+	unsigned int mbox_sync_dirty:1;
+	unsigned int mbox_do_dirty_syncs:1;
 };
 
 struct index_transaction_context {
--- a/src/lib-storage/index/mbox/mbox-file.c	Sat Aug 28 16:25:42 2004 +0300
+++ b/src/lib-storage/index/mbox/mbox-file.c	Sat Aug 28 19:39:53 2004 +0300
@@ -88,3 +88,46 @@
 		ibox->mbox_stream = NULL;
 	}
 }
+
+int mbox_file_seek(struct index_mailbox *ibox, struct mail_index_view *view,
+		   uint32_t seq, int *deleted_r)
+{
+	const void *data;
+	uint64_t offset;
+	int ret;
+
+	*deleted_r = FALSE;
+
+	ret = mail_index_lookup_extra(view, seq, ibox->mbox_extra_idx, &data);
+	if (ret <= 0) {
+		if (ret < 0)
+			mail_storage_set_index_error(ibox);
+		else
+			*deleted_r = TRUE;
+		return -1;
+	}
+
+	offset = *((const uint64_t *)data);
+	if (istream_raw_mbox_seek(ibox->mbox_stream, offset) < 0) {
+		if (offset == 0) {
+			mail_storage_set_error(ibox->box.storage,
+				"Mailbox isn't a valid mbox file");
+			return -1;
+		}
+
+		if (ibox->mbox_sync_dirty)
+			return 0;
+
+		mail_storage_set_critical(ibox->box.storage,
+			"Cached message offset %s is invalid for mbox file %s",
+			dec2str(offset), ibox->path);
+		mail_index_mark_corrupted(ibox->index);
+		return -1;
+	}
+
+	if (ibox->mbox_sync_dirty) {
+		/* FIXME: we're dirty - make sure this is the correct mail */
+	}
+
+	return 1;
+}
--- a/src/lib-storage/index/mbox/mbox-file.h	Sat Aug 28 16:25:42 2004 +0300
+++ b/src/lib-storage/index/mbox/mbox-file.h	Sat Aug 28 19:39:53 2004 +0300
@@ -7,4 +7,7 @@
 int mbox_file_open_stream(struct index_mailbox *ibox);
 void mbox_file_close_stream(struct index_mailbox *ibox);
 
+int mbox_file_seek(struct index_mailbox *ibox, struct mail_index_view *view,
+		   uint32_t seq, int *deleted_r);
+
 #endif
--- a/src/lib-storage/index/mbox/mbox-mail.c	Sat Aug 28 16:25:42 2004 +0300
+++ b/src/lib-storage/index/mbox/mbox-mail.c	Sat Aug 28 19:39:53 2004 +0300
@@ -5,6 +5,7 @@
 #include "index-mail.h"
 #include "mbox-storage.h"
 #include "mbox-file.h"
+#include "mbox-lock.h"
 #include "mbox-sync-private.h"
 #include "istream-raw-mbox.h"
 #include "istream-header-filter.h"
@@ -16,42 +17,49 @@
 static int mbox_mail_seek(struct index_mail *mail)
 {
 	struct index_mailbox *ibox = mail->ibox;
-	const void *data;
-	uint64_t offset;
-	int ret;
+	enum mbox_sync_flags sync_flags = 0;
+	int ret, deleted;
 
 	if (mail->data.deleted)
 		return 0;
 
+__again:
 	if (ibox->mbox_lock_type == F_UNLCK) {
-		if (mbox_sync(ibox, FALSE, FALSE, TRUE) < 0)
+		sync_flags |= MBOX_SYNC_LOCK_READING;
+		if (mbox_sync(ibox, sync_flags) < 0)
 			return -1;
 
 		i_assert(ibox->mbox_lock_type != F_UNLCK);
-                mail->ibox->mbox_mail_lock_id = ibox->mbox_lock_id;
+		ibox->mbox_mail_lock_id = ibox->mbox_lock_id;
 	}
 
 	if (mbox_file_open_stream(ibox) < 0)
 		return -1;
 
-	ret = mail_index_lookup_extra(mail->trans->trans_view, mail->mail.seq,
-				      ibox->mbox_extra_idx, &data);
-	if (ret <= 0) {
-		if (ret < 0)
-			mail_storage_set_index_error(ibox);
-		else
+	ret = mbox_file_seek(ibox, mail->trans->trans_view, mail->mail.seq,
+			     &deleted);
+	if (ret < 0) {
+		if (deleted) {
 			mail->data.deleted = TRUE;
-		return ret;
+			return 0;
+		}
+		return -1;
 	}
 
-	offset = *((const uint64_t *)data);
-	if (istream_raw_mbox_seek(ibox->mbox_stream, offset) < 0) {
-		mail_storage_set_critical(ibox->box.storage,
-			"Cached message offset %s is invalid for mbox file %s",
-			dec2str(offset), ibox->path);
-		mail_index_mark_corrupted(ibox->index);
-		return -1;
+	if (ret == 0) {
+		/* we'll need to re-sync it completely */
+		if (ibox->mbox_lock_type == F_RDLCK) {
+			if (ibox->mbox_mail_lock_id == ibox->mbox_lock_id)
+                                ibox->mbox_mail_lock_id = 0;
+			(void)mbox_unlock(mail->ibox, ibox->mbox_lock_id);
+			ibox->mbox_lock_id = 0;
+			i_assert(ibox->mbox_lock_type == F_UNLCK);
+		}
+
+		sync_flags |= MBOX_SYNC_UNDIRTY;
+		goto __again;
 	}
+
 	return 1;
 }
 
--- a/src/lib-storage/index/mbox/mbox-save.c	Sat Aug 28 16:25:42 2004 +0300
+++ b/src/lib-storage/index/mbox/mbox-save.c	Sat Aug 28 19:39:53 2004 +0300
@@ -254,7 +254,7 @@
 
 		if (!want_mail) {
 			/* assign UIDs only if mbox doesn't require syncing */
-			ret = mbox_sync_has_changed(ibox);
+			ret = mbox_sync_has_changed(ibox, TRUE);
 			if (ret < 0)
 				return -1;
 			if (ret == 0) {
@@ -272,7 +272,7 @@
 
 	if (!ctx->synced && want_mail) {
 		/* we'll need to assign UID for the mail immediately. */
-		if (mbox_sync(ibox, FALSE, FALSE, FALSE) < 0)
+		if (mbox_sync(ibox, 0) < 0)
 			return -1;
 		if (mbox_save_init_sync(t) < 0)
 			return -1;
--- a/src/lib-storage/index/mbox/mbox-storage.c	Sat Aug 28 16:25:42 2004 +0300
+++ b/src/lib-storage/index/mbox/mbox-storage.c	Sat Aug 28 19:39:53 2004 +0300
@@ -418,6 +418,7 @@
 	ibox->is_recent = mbox_mail_is_recent;
 	ibox->mail_interface = &mbox_mail;
 	ibox->unreliable_headers = TRUE;
+        ibox->mbox_do_dirty_syncs = getenv("MBOX_DIRTY_SYNCS") != NULL;
 
 	if (access(path, R_OK|W_OK) < 0) {
 		if (errno < EACCES)
--- a/src/lib-storage/index/mbox/mbox-sync-private.h	Sat Aug 28 16:25:42 2004 +0300
+++ b/src/lib-storage/index/mbox/mbox-sync-private.h	Sat Aug 28 19:39:53 2004 +0300
@@ -4,6 +4,13 @@
 #include "md5.h"
 #include "mail-index.h"
 
+enum mbox_sync_flags {
+	MBOX_SYNC_LAST_COMMIT	= 0x01,
+	MBOX_SYNC_HEADER	= 0x02,
+	MBOX_SYNC_LOCK_READING	= 0x04,
+	MBOX_SYNC_UNDIRTY	= 0x08
+};
+
 struct mbox_flag_type {
 	char chr;
 	enum mail_flags flag;
@@ -103,9 +110,8 @@
 	unsigned int seen_first_mail:1;
 };
 
-int mbox_sync(struct index_mailbox *ibox, int last_commit,
-	      int sync_header, int lock);
-int mbox_sync_has_changed(struct index_mailbox *ibox);
+int mbox_sync(struct index_mailbox *ibox, enum mbox_sync_flags flags);
+int mbox_sync_has_changed(struct index_mailbox *ibox, int leave_dirty);
 
 void mbox_sync_parse_next_mail(struct istream *input,
 			       struct mbox_sync_mail_context *ctx);
--- a/src/lib-storage/index/mbox/mbox-sync.c	Sat Aug 28 16:25:42 2004 +0300
+++ b/src/lib-storage/index/mbox/mbox-sync.c	Sat Aug 28 19:39:53 2004 +0300
@@ -630,10 +630,59 @@
 }
 
 static int
+mbox_sync_seek_to_seq(struct mbox_sync_context *sync_ctx, uint32_t seq)
+{
+	struct index_mailbox *ibox = sync_ctx->ibox;
+	uoff_t old_offset;
+	int ret, deleted;
+
+	if (seq == 0) {
+		if (istream_raw_mbox_seek(ibox->mbox_stream, 0) < 0) {
+			mail_storage_set_error(ibox->box.storage,
+				"Mailbox isn't a valid mbox file");
+			return -1;
+		}
+		seq++;
+	} else {
+		old_offset = istream_raw_mbox_get_start_offset(sync_ctx->input);
+
+		ret = mbox_file_seek(ibox, sync_ctx->sync_view, seq, &deleted);
+		if (ret < 0) {
+			mail_storage_set_index_error(ibox);
+			return -1;
+		}
+
+		if (ret == 0) {
+			if (istream_raw_mbox_seek(ibox->mbox_stream,
+						  old_offset) < 0) {
+				mail_storage_set_critical(ibox->box.storage,
+					"Error seeking back to original "
+					"offset %s in mbox file %s",
+					dec2str(old_offset), ibox->path);
+				return -1;
+			}
+			return 0;
+		}
+	}
+
+        /* set to -1, since it's always increased later */
+	sync_ctx->seq = seq-1;
+	if (sync_ctx->seq == 0 &&
+	    istream_raw_mbox_get_start_offset(sync_ctx->input) != 0) {
+		/* this mbox has pseudo mail which contains the X-IMAP header */
+		sync_ctx->seq++;
+	}
+
+        sync_ctx->idx_seq = seq;
+	sync_ctx->dest_first_mail = sync_ctx->seq == 0;
+        (void)istream_raw_mbox_get_body_offset(sync_ctx->input);
+	return 1;
+}
+
+static int
 mbox_sync_seek_to_uid(struct mbox_sync_context *sync_ctx, uint32_t uid)
 {
 	uint32_t seq1, seq2;
-	uint64_t offset;
 
 	if (mail_index_lookup_uid_range(sync_ctx->sync_view, uid, (uint32_t)-1,
 					&seq1, &seq2) < 0) {
@@ -641,44 +690,25 @@
 		return -1;
 	}
 
-	if (seq1 == 0)
-		return 0;
-
-	if (mbox_sync_get_from_offset(sync_ctx, seq1, &offset) < 0)
-		return -1;
-
-        /* set to -1, since it's always increased later */
-	sync_ctx->seq = seq1-1;
-	if (sync_ctx->seq == 0 && offset != 0) {
-		/* this mbox has pseudo mail which contains the X-IMAP header */
-		sync_ctx->seq++;
-	}
-
-        sync_ctx->idx_seq = seq1;
-	sync_ctx->dest_first_mail = sync_ctx->seq == 0;
-	if (istream_raw_mbox_seek(sync_ctx->input, offset) < 0) {
-		mail_storage_set_critical(sync_ctx->ibox->box.storage,
-			"Cached message offset %s is invalid for mbox file %s",
-			dec2str(offset), sync_ctx->ibox->path);
-		mail_index_mark_corrupted(sync_ctx->ibox->index);
-		return -1;
-	}
-        (void)istream_raw_mbox_get_body_offset(sync_ctx->input);
-	return 1;
+	return mbox_sync_seek_to_seq(sync_ctx, seq1);
 }
 
 static int mbox_sync_loop(struct mbox_sync_context *sync_ctx,
 			  struct mbox_sync_mail_context *mail_ctx,
-			  uint32_t min_message_count)
+			  uint32_t min_message_count, int partial)
 {
 	const struct mail_index_record *rec;
 	uint32_t uid, messages_count;
 	uoff_t offset;
 	int ret, expunged;
 
-	if (min_message_count != 0)
-		ret = 0;
-	else {
+	messages_count = mail_index_view_get_message_count(sync_ctx->sync_view);
+
+	if (min_message_count != 0) {
+		ret = mbox_sync_seek_to_seq(sync_ctx,
+					    partial || messages_count == 0 ?
+					    messages_count : 1);
+	} else {
 		/* we sync only what we need to. jump to first record that
 		   needs updating */
 		const struct mail_index_sync_rec *sync_rec;
@@ -693,7 +723,7 @@
 			if (buffer_get_used_size(sync_ctx->syncs) == 0 &&
 			    sync_ctx->sync_rec.uid1 == 0) {
 				/* nothing to do */
-				return 0;
+				return 1;
 			}
 		}
 
@@ -702,21 +732,10 @@
 			sync_rec = &sync_ctx->sync_rec;
 
 		ret = mbox_sync_seek_to_uid(sync_ctx, sync_rec->uid1);
-		if (ret < 0)
-			return -1;
 	}
 
-	if (ret == 0) {
-		if (istream_raw_mbox_seek(sync_ctx->input, 0) < 0) {
-			/* doesn't begin with a From-line */
-			mail_storage_set_error(sync_ctx->ibox->box.storage,
-				"Mailbox isn't a valid mbox file");
-			return -1;
-		}
-		sync_ctx->dest_first_mail = TRUE;
-	}
-
-	messages_count = mail_index_view_get_message_count(sync_ctx->sync_view);
+	if (ret <= 0)
+		return ret;
 
 	while ((ret = mbox_sync_read_next_mail(sync_ctx, mail_ctx)) > 0) {
 		uid = mail_ctx->mail.uid;
@@ -774,10 +793,8 @@
 			mail_ctx->mail.uid = 0;
 			ret = mbox_sync_handle_expunge(mail_ctx);
 		}
-		if (ret < 0) {
-			/* -1 = error, -2 = need to restart */
-			return ret;
-		}
+		if (ret < 0)
+			return -1;
 
 		if (!mail_ctx->pseudo) {
 			if (!expunged) {
@@ -818,7 +835,9 @@
 					break;
 
 				/* we can skip forward to next record which
-				   needs updating. */
+				   needs updating. if it failes because the
+				   offset is dirty, just ignore and continue
+				   from where we are now. */
 				uid = sync_ctx->sync_rec.uid1;
 				if (mbox_sync_seek_to_uid(sync_ctx, uid) < 0)
 					return -1;
@@ -832,7 +851,10 @@
 			mail_index_expunge(sync_ctx->t, sync_ctx->idx_seq++);
 	}
 
-	return 0;
+	if (!partial)
+		sync_ctx->ibox->mbox_sync_dirty = FALSE;
+
+	return 1;
 }
 
 static int mbox_sync_handle_eof_updates(struct mbox_sync_context *sync_ctx,
@@ -946,14 +968,16 @@
 			&sync_ctx->next_uid, sizeof(sync_ctx->next_uid));
 	}
 
-	if ((uint32_t)st.st_mtime != sync_ctx->hdr->sync_stamp) {
+	if ((uint32_t)st.st_mtime != sync_ctx->hdr->sync_stamp &&
+	    !sync_ctx->ibox->mbox_sync_dirty) {
 		uint32_t sync_stamp = st.st_mtime;
 
 		mail_index_update_header(sync_ctx->t,
 			offsetof(struct mail_index_header, sync_stamp),
 			&sync_stamp, sizeof(sync_stamp));
 	}
-	if ((uint64_t)st.st_size != sync_ctx->hdr->sync_size) {
+	if ((uint64_t)st.st_size != sync_ctx->hdr->sync_size &&
+	    !sync_ctx->ibox->mbox_sync_dirty) {
 		uint64_t sync_size = st.st_size;
 
 		mail_index_update_header(sync_ctx->t,
@@ -961,6 +985,9 @@
 			&sync_size, sizeof(sync_size));
 	}
 
+	sync_ctx->ibox->mbox_dirty_stamp = st.st_mtime;
+	sync_ctx->ibox->mbox_dirty_size = st.st_size;
+
 	return 0;
 }
 
@@ -978,14 +1005,17 @@
         sync_ctx->seen_first_mail = FALSE;
 }
 
-static int mbox_sync_do(struct mbox_sync_context *sync_ctx, int sync_header)
+static int mbox_sync_do(struct mbox_sync_context *sync_ctx,
+			enum mbox_sync_flags flags)
 {
 	struct mbox_sync_mail_context mail_ctx;
 	struct stat st;
 	uint32_t min_msg_count;
-	int ret;
+	int ret, partial;
 
-	if (sync_header)
+	partial = FALSE;
+
+	if ((flags & MBOX_SYNC_HEADER) != 0)
 		min_msg_count = 1;
 	else {
 		if (fstat(sync_ctx->fd, &st) < 0) {
@@ -993,29 +1023,35 @@
 			return -1;
 		}
 
-		min_msg_count =
-			(uint32_t)st.st_mtime == sync_ctx->hdr->sync_stamp &&
-			(uint64_t)st.st_size == sync_ctx->hdr->sync_size ?
-			0 : (uint32_t)-1;
+		if ((uint32_t)st.st_mtime == sync_ctx->hdr->sync_stamp &&
+		    (uint64_t)st.st_size == sync_ctx->hdr->sync_size) {
+			/* file is fully synced */
+			sync_ctx->ibox->mbox_sync_dirty = FALSE;
+			min_msg_count = 0;
+		} else if ((flags & MBOX_SYNC_UNDIRTY) != 0) {
+			/* we want to do full syncing */
+			min_msg_count = (uint32_t)-1;
+			sync_ctx->ibox->mbox_sync_dirty = TRUE;
+		} else {
+			/* see if we can delay syncing the whole file.
+			   normally we only notice expunges and appends
+			   in partial syncing. */
+			partial = TRUE;
+			min_msg_count = (uint32_t)-1;
+			sync_ctx->ibox->mbox_sync_dirty = TRUE;
+		}
 	}
 
 	mbox_sync_restart(sync_ctx);
-	if ((ret = mbox_sync_loop(sync_ctx, &mail_ctx, min_msg_count)) == -1)
-		return -1;
+	ret = mbox_sync_loop(sync_ctx, &mail_ctx, min_msg_count, partial);
+	if (ret <= 0) {
+		if (ret < 0)
+			return -1;
 
-	if (ret == -2) {
-		/* initially we had mbox read-locked, but later we needed a
-		   write-lock. doing it required dropping the read lock.
-		   we're here because mbox was modified before we got the
-		   write-lock. so, restart the whole syncing. */
-		i_assert(sync_ctx->ibox->mbox_lock_type == F_WRLCK);
-
-		mail_index_transaction_rollback(sync_ctx->t);
-		sync_ctx->t = mail_index_transaction_begin(sync_ctx->sync_view,
-							   FALSE);
-
+		/* partial syncing didn't work, do it again */
 		mbox_sync_restart(sync_ctx);
-		if (mbox_sync_loop(sync_ctx, &mail_ctx, (uint32_t)-1) < 0)
+		if (mbox_sync_loop(sync_ctx, &mail_ctx,
+				   (uint32_t)-1, FALSE) < 0)
 			return -1;
 	}
 
@@ -1034,7 +1070,7 @@
 	return 0;
 }
 
-int mbox_sync_has_changed(struct index_mailbox *ibox)
+int mbox_sync_has_changed(struct index_mailbox *ibox, int leave_dirty)
 {
 	const struct mail_index_header *hdr;
 	struct stat st;
@@ -1049,8 +1085,18 @@
 		return -1;
 	}
 
-	return (uint32_t)st.st_mtime != hdr->sync_stamp ||
-		(uint64_t)st.st_size != hdr->sync_size;
+	if ((uint32_t)st.st_mtime == hdr->sync_stamp &&
+	    (uint64_t)st.st_size == hdr->sync_size) {
+		/* fully synced */
+		ibox->mbox_sync_dirty = FALSE;
+		return 0;
+	}
+
+	if (!ibox->mbox_sync_dirty || !leave_dirty)
+		return 1;
+
+	return st.st_mtime != ibox->mbox_dirty_stamp ||
+		st.st_size != ibox->mbox_dirty_size;
 }
 
 static int mbox_sync_update_imap_base(struct mbox_sync_context *sync_ctx)
@@ -1064,7 +1110,7 @@
 	sync_ctx->update_base_uid_last = sync_ctx->next_uid-1;
 
 	mbox_sync_restart(sync_ctx);
-	if (mbox_sync_loop(sync_ctx, &mail_ctx, 1) < 0)
+	if (mbox_sync_loop(sync_ctx, &mail_ctx, 1, 0) < 0)
 		return -1;
 
 	if (mbox_sync_handle_eof_updates(sync_ctx, &mail_ctx) < 0)
@@ -1076,8 +1122,7 @@
 	return 0;
 }
 
-int mbox_sync(struct index_mailbox *ibox, int last_commit,
-	      int sync_header, int lock)
+int mbox_sync(struct index_mailbox *ibox, enum mbox_sync_flags flags)
 {
 	struct mail_index_sync_ctx *index_sync_ctx;
 	struct mail_index_view *sync_view;
@@ -1087,22 +1132,23 @@
 	unsigned int lock_id = 0;
 	int ret, changed;
 
-	if (lock) {
+	if ((flags & MBOX_SYNC_LOCK_READING) != 0) {
 		if (mbox_lock(ibox, F_RDLCK, &lock_id) <= 0)
 			return -1;
 	}
 
-	if (sync_header)
+	if ((flags & MBOX_SYNC_HEADER) != 0)
 		changed = 1;
 	else {
-		if ((changed = mbox_sync_has_changed(ibox)) < 0) {
-			if (lock)
+		int leave_dirty = (flags & MBOX_SYNC_UNDIRTY) == 0;
+		if ((changed = mbox_sync_has_changed(ibox, leave_dirty)) < 0) {
+			if ((flags & MBOX_SYNC_LOCK_READING) != 0)
 				(void)mbox_unlock(ibox, lock_id);
 			return -1;
 		}
 	}
 
-	if (lock) {
+	if ((flags & MBOX_SYNC_LOCK_READING) != 0) {
 		/* we just want to lock it for reading. if mbox hasn't been
 		   modified don't do any syncing. */
 		if (!changed)
@@ -1124,7 +1170,7 @@
 			return -1;
 	}
 
-	if (last_commit) {
+	if ((flags & MBOX_SYNC_LAST_COMMIT) != 0) {
 		seq = ibox->commit_log_file_seq;
 		offset = ibox->commit_log_file_offset;
 	} else {
@@ -1184,7 +1230,7 @@
 	sync_ctx.input = sync_ctx.ibox->mbox_stream;
 	sync_ctx.fd = sync_ctx.ibox->mbox_fd;
 
-	if (mbox_sync_do(&sync_ctx, sync_header) < 0)
+	if (mbox_sync_do(&sync_ctx, flags) < 0)
 		ret = -1;
 
 	if (ret < 0)
@@ -1257,9 +1303,11 @@
 		}
 	}
 
-	if (lock_id != 0 && !lock) {
+	if (lock_id != 0 && (flags & MBOX_SYNC_LOCK_READING) == 0) {
 		/* FIXME: keep the lock MBOX_SYNC_SECS+1 to make sure we
-		   notice changes made by others */
+		   notice changes made by others .. and this has to be done
+		   even if lock_reading is set.. except if
+		   mbox_sync_dirty = TRUE */
 		if (mbox_unlock(ibox, lock_id) < 0)
 			ret = -1;
 	}
@@ -1276,13 +1324,18 @@
 mbox_storage_sync_init(struct mailbox *box, enum mailbox_sync_flags flags)
 {
 	struct index_mailbox *ibox = (struct index_mailbox *)box;
+	enum mbox_sync_flags mbox_sync_flags = 0;
 	int ret = 0;
 
 	if ((flags & MAILBOX_SYNC_FLAG_FAST) == 0 ||
 	    ibox->sync_last_check + MAILBOX_FULL_SYNC_INTERVAL <= ioloop_time) {
 		ibox->sync_last_check = ioloop_time;
 
-		ret = mbox_sync(ibox, FALSE, FALSE, FALSE);
+		if ((flags & MAILBOX_SYNC_FLAG_FULL) != 0 ||
+		    !ibox->mbox_do_dirty_syncs)
+			mbox_sync_flags |= MBOX_SYNC_UNDIRTY;
+
+		ret = mbox_sync(ibox, mbox_sync_flags);
 	}
 
 	return index_mailbox_sync_init(box, flags, ret < 0);
--- a/src/lib-storage/index/mbox/mbox-transaction.c	Sat Aug 28 16:25:42 2004 +0300
+++ b/src/lib-storage/index/mbox/mbox-transaction.c	Sat Aug 28 19:39:53 2004 +0300
@@ -37,7 +37,8 @@
 	t = NULL;
 
 	if (ret == 0) {
-		if (mbox_sync(ibox, TRUE, mbox_modified, FALSE) < 0)
+		if (mbox_sync(ibox, MBOX_SYNC_LAST_COMMIT |
+			      (mbox_modified ? MBOX_SYNC_HEADER : 0)) < 0)
 			ret = -1;
 	}
 
--- a/src/lib-storage/mail-storage.h	Sat Aug 28 16:25:42 2004 +0300
+++ b/src/lib-storage/mail-storage.h	Sat Aug 28 19:39:53 2004 +0300
@@ -85,9 +85,10 @@
 };
 
 enum mailbox_sync_flags {
-	MAILBOX_SYNC_FLAG_FAST		= 0x01,
-	MAILBOX_SYNC_FLAG_NO_EXPUNGES	= 0x02,
-	MAILBOX_SYNC_AUTO_STOP		= 0x04
+	MAILBOX_SYNC_FLAG_FULL		= 0x01,
+	MAILBOX_SYNC_FLAG_FAST		= 0x02,
+	MAILBOX_SYNC_FLAG_NO_EXPUNGES	= 0x04,
+	MAILBOX_SYNC_AUTO_STOP		= 0x08
 };
 
 enum mailbox_sync_type {
--- a/src/master/mail-process.c	Sat Aug 28 16:25:42 2004 +0300
+++ b/src/master/mail-process.c	Sat Aug 28 19:39:53 2004 +0300
@@ -222,6 +222,8 @@
 		env_put("FULL_FILESYSTEM_ACCESS=1");
 	if (set->pop3_mails_keep_recent)
 		env_put("POP3_MAILS_KEEP_RECENT=1");
+	if (set->mbox_dirty_syncs)
+		env_put("MBOX_DIRTY_SYNCS=1");
 	(void)umask(set->umask);
 
 	env_put(t_strconcat("MBOX_READ_LOCKS=", set->mbox_read_locks, NULL));
--- a/src/master/master-settings.c	Sat Aug 28 16:25:42 2004 +0300
+++ b/src/master/master-settings.c	Sat Aug 28 19:39:53 2004 +0300
@@ -104,6 +104,7 @@
 	DEF(SET_STR, mbox_write_locks),
 	DEF(SET_INT, mbox_lock_timeout),
 	DEF(SET_INT, mbox_dotlock_change_timeout),
+	DEF(SET_BOOL, mbox_dirty_syncs),
 	DEF(SET_INT, umask),
 	DEF(SET_BOOL, mail_drop_priv_before_exec),
 
@@ -265,6 +266,7 @@
 	MEMBER(mbox_write_locks) "dotlock fcntl",
 	MEMBER(mbox_lock_timeout) 300,
 	MEMBER(mbox_dotlock_change_timeout) 30,
+	MEMBER(mbox_dirty_syncs) TRUE,
 	MEMBER(umask) 0077,
 	MEMBER(mail_drop_priv_before_exec) FALSE,
 
--- a/src/master/master-settings.h	Sat Aug 28 16:25:42 2004 +0300
+++ b/src/master/master-settings.h	Sat Aug 28 19:39:53 2004 +0300
@@ -75,6 +75,7 @@
 	const char *mbox_write_locks;
 	unsigned int mbox_lock_timeout;
 	unsigned int mbox_dotlock_change_timeout;
+	int mbox_dirty_syncs;
 	unsigned int umask;
 	int mail_drop_priv_before_exec;