Mercurial > dovecot > original-hg > dovecot-1.2

changeset 2662:add94f9aa9e1 HEAD

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Added mbox_lazy_writes setting.
author Timo Sirainen <tss@iki.fi>
date Fri, 24 Sep 2004 01:29:13 +0300
parents 0d585c330c18
children cecf6f5a0d80
files dovecot-example.conf src/lib-storage/index/mbox/mbox-sync-private.h src/lib-storage/index/mbox/mbox-sync-rewrite.c src/lib-storage/index/mbox/mbox-sync-update.c src/lib-storage/index/mbox/mbox-sync.c src/master/mail-process.c src/master/master-settings.c src/master/master-settings.h
diffstat 8 files changed, 117 insertions(+), 30 deletions(-) [+]
line wrap: on
line diff
--- a/dovecot-example.conf	Fri Sep 24 01:27:16 2004 +0300
+++ b/dovecot-example.conf	Fri Sep 24 01:29:13 2004 +0300
@@ -306,10 +306,14 @@
 # fallbacks to re-reading the whole mbox file whenever something in mbox isn't
 # how it's expected to be. The only real downside to this setting is that if
 # some other MUA changes message flags, Dovecot doesn't notice it immediately.
-# Note that a full sync is done for SELECT, EXAMINE, EXPUNGE and CHECK
-# commands.
+# Note that a full sync is done with EXPUNGE, CHECK, CLOSE and LOGOUT commands.
 #mbox_dirty_syncs = yes
 
+# Delay writing mbox headers until doing a full sync (see above). This is
+# especially useful for POP3 where clients often delete all mails. The downside
+# is that our changes aren't immediately visible to other MUAs.
+#mbox_lazy_writes = yes
+
 # umask to use for mail files and directories
 #umask = 0077
 
--- a/src/lib-storage/index/mbox/mbox-sync-private.h	Fri Sep 24 01:27:16 2004 +0300
+++ b/src/lib-storage/index/mbox/mbox-sync-private.h	Fri Sep 24 01:29:13 2004 +0300
@@ -81,12 +81,14 @@
 	unsigned int pseudo:1;
 	unsigned int updated:1;
 	unsigned int recent:1;
+	unsigned int dirty:1;
 	unsigned int seen_received_hdr:1;
 	unsigned int uid_broken:1;
 };
 
 struct mbox_sync_context {
 	struct index_mailbox *ibox;
+        enum mbox_sync_flags flags;
 	struct istream *input, *file_input;
 	int fd;
 
@@ -111,6 +113,7 @@
 
 	unsigned int dest_first_mail:1;
 	unsigned int seen_first_mail:1;
+	unsigned int delay_writes:1;
 };
 
 int mbox_sync(struct index_mailbox *ibox, enum mbox_sync_flags flags);
--- a/src/lib-storage/index/mbox/mbox-sync-rewrite.c	Fri Sep 24 01:27:16 2004 +0300
+++ b/src/lib-storage/index/mbox/mbox-sync-rewrite.c	Fri Sep 24 01:29:13 2004 +0300
@@ -370,6 +370,7 @@
 	offset = sync_ctx->input->v_offset;
 	dest_offset = offset + space_diff;
 	if (mbox_move(sync_ctx, dest_offset, offset,
+		      end_offset == (uoff_t)-1 ? mail_ctx.mail.body_size :
 		      end_offset - dest_offset) < 0)
 		return -1;
 
@@ -412,12 +413,29 @@
 	size_t size;
 	int ret = 0;
 
-	i_assert(first_seq != last_seq);
 	i_assert(sync_ctx->ibox->mbox_lock_type == F_WRLCK);
 
 	mails = buffer_get_modifyable_data(sync_ctx->mails, &size);
 	i_assert(size / sizeof(*mails) == last_seq - first_seq + 1);
 
+	if (first_seq == last_seq) {
+		/* just move this mail forward */
+		if (mbox_sync_read_and_move(sync_ctx, mails, first_seq, 0,
+					    -mails[0].space + extra_space,
+					    (uoff_t)-1) < 0)
+			return -1;
+
+		if ((mails[0].flags & MBOX_DIRTY_SPACE) != 0) {
+                        mails[0].flags &= ~MBOX_DIRTY_SPACE;
+			if (mbox_fill_space(sync_ctx, mails[0].offset,
+					    mails[0].space) < 0)
+				return -1;
+		}
+
+		istream_raw_mbox_flush(sync_ctx->input);
+		return 0;
+	}
+
 	/* if there's expunges in mails[], we would get more correct balancing
 	   by counting only them here. however, that might make us overwrite
 	   data which hasn't yet been copied backwards. to avoid too much
--- a/src/lib-storage/index/mbox/mbox-sync-update.c	Fri Sep 24 01:27:16 2004 +0300
+++ b/src/lib-storage/index/mbox/mbox-sync-update.c	Fri Sep 24 01:29:13 2004 +0300
@@ -279,6 +279,8 @@
 	uint8_t old_flags;
 	keywords_mask_t old_keywords;
 
+	i_assert(ctx->mail.uid != 0 || ctx->pseudo);
+
 	sync = buffer_get_data(syncs_buf, &size);
 	size /= sizeof(*sync);
 
--- a/src/lib-storage/index/mbox/mbox-sync.c	Fri Sep 24 01:27:16 2004 +0300
+++ b/src/lib-storage/index/mbox/mbox-sync.c	Fri Sep 24 01:29:13 2004 +0300
@@ -45,6 +45,7 @@
 #include "mbox-sync-private.h"
 
 #include <stddef.h>
+#include <stdlib.h>
 #include <sys/stat.h>
 
 #define MBOX_SYNC_SECS 1
@@ -95,7 +96,7 @@
 	size /= sizeof(*sync);
 
 	for (src = dest = 0; src < size; src++) {
-		if (sync[src].uid2 >= uid) {
+		if (sync[src].uid2 > uid) {
 			if (src != dest)
 				sync[dest] = sync[src];
 			dest++;
@@ -176,7 +177,7 @@
 
 	*sync_expunge_r = FALSE;
 
-	if (sync_ctx->ibox->mbox_readonly || sync_ctx->index_sync_ctx == NULL)
+	if (sync_ctx->index_sync_ctx == NULL)
 		return 0;
 
 	if (uid == 0) {
@@ -187,7 +188,9 @@
 	mbox_sync_buffer_delete_old(sync_ctx->syncs, uid);
 	while (uid >= sync_rec->uid1) {
 		if (uid <= sync_rec->uid2 &&
-		    sync_rec->type != MAIL_INDEX_SYNC_TYPE_APPEND) {
+		    sync_rec->type != MAIL_INDEX_SYNC_TYPE_APPEND &&
+		    (sync_rec->type != MAIL_INDEX_SYNC_TYPE_EXPUNGE ||
+		     !sync_ctx->ibox->mbox_readonly)) {
 			buffer_append(sync_ctx->syncs, sync_rec,
 				      sizeof(*sync_rec));
 
@@ -364,6 +367,8 @@
 		/* new message */
 		mail_index_append(sync_ctx->t, mail->uid, &sync_ctx->idx_seq);
 		mbox_flags = mail->flags & (MAIL_FLAGS_MASK^MAIL_RECENT);
+		if (mail_ctx->dirty)
+			mbox_flags |= MAIL_INDEX_MAIL_FLAG_DIRTY;
 		if (sync_ctx->ibox->keep_recent &&
 		    (mail->flags & MBOX_NONRECENT) == 0)
 			mbox_flags |= MAIL_RECENT;
@@ -391,13 +396,40 @@
 		mbox_sync_apply_index_syncs(sync_ctx->syncs,
 					    &idx_flags, idx_keywords);
 
-		mbox_flags = (rec->flags & ~MAIL_FLAGS_MASK) |
-			(mail->flags & MAIL_FLAGS_MASK);
-		mbox_flags ^= MAIL_RECENT;
+		if ((idx_flags & MAIL_INDEX_MAIL_FLAG_DIRTY) != 0) {
+			/* flags are dirty, ignore whatever was in the file */
+			mbox_flags = idx_flags;
+		} else {
+			mbox_flags = (rec->flags & ~MAIL_FLAGS_MASK) |
+				(mail->flags & MAIL_FLAGS_MASK);
+			mbox_flags ^= MAIL_RECENT;
+		}
+
+		if (mail_ctx->dirty)
+			mbox_flags |= MAIL_INDEX_MAIL_FLAG_DIRTY;
+		else if ((sync_ctx->flags & MBOX_SYNC_UNDIRTY) != 0)
+			mbox_flags &= ~MAIL_INDEX_MAIL_FLAG_DIRTY;
 
-		if ((idx_flags & ~MAIL_RECENT) != (mbox_flags & ~MAIL_RECENT) ||
+		if ((idx_flags & ~MAIL_INDEX_MAIL_FLAG_DIRTY) ==
+		    (mbox_flags & ~MAIL_INDEX_MAIL_FLAG_DIRTY) &&
 		    memcmp(idx_keywords, mail->keywords,
-			   INDEX_KEYWORDS_BYTE_COUNT) != 0) {
+			   INDEX_KEYWORDS_BYTE_COUNT) == 0) {
+			if (idx_flags != mbox_flags) {
+				/* dirty flag state changed */
+				int dirty = (mbox_flags &
+					     MAIL_INDEX_MAIL_FLAG_DIRTY) != 0;
+				memset(idx_keywords, 0,
+				       INDEX_KEYWORDS_BYTE_COUNT);
+				mail_index_update_flags(sync_ctx->t,
+					sync_ctx->idx_seq,
+					dirty ? MODIFY_ADD : MODIFY_REMOVE,
+					MAIL_INDEX_MAIL_FLAG_DIRTY,
+					idx_keywords);
+			}
+		} else if ((idx_flags & ~MAIL_RECENT) !=
+			   (mbox_flags & ~MAIL_RECENT) ||
+			   memcmp(idx_keywords, mail->keywords,
+				  INDEX_KEYWORDS_BYTE_COUNT) != 0) {
 			mail_index_update_flags(sync_ctx->t, sync_ctx->idx_seq,
 						MODIFY_REPLACE, mbox_flags,
 						mail->keywords);
@@ -478,10 +510,10 @@
 
 	mails = buffer_get_modifyable_data(sync_ctx->mails, &size);
 	size /= sizeof(*mails);
-	i_assert(sync_ctx->seq - sync_ctx->need_space_seq + 1 == size);
 
 	for (idx = 0; idx < size; idx++) {
-		if (mails[idx].idx_seq == 0)
+		if (mails[idx].idx_seq == 0 ||
+		    (mails[idx].flags & MBOX_EXPUNGED) != 0)
 			continue;
 
 		offset = mails[idx].from_offset;
@@ -492,6 +524,7 @@
 
 static void mbox_sync_handle_expunge(struct mbox_sync_mail_context *mail_ctx)
 {
+	mail_ctx->mail.flags = MBOX_EXPUNGED;
 	mail_ctx->mail.offset = mail_ctx->mail.from_offset;
 	mail_ctx->mail.space =
 		mail_ctx->body_offset - mail_ctx->mail.from_offset +
@@ -513,9 +546,6 @@
 	off_t move_diff;
 	int ret;
 
-	if (sync_ctx->ibox->mbox_readonly)
-		return 0;
-
 	if (sync_ctx->expunged_space > 0 && sync_ctx->need_space_seq == 0) {
 		/* move the header backwards to fill expunged space */
 		move_diff = -sync_ctx->expunged_space;
@@ -542,6 +572,12 @@
 		   (mail_ctx->seq == 1 &&
 		    sync_ctx->update_base_uid_last != 0)) {
 		mbox_sync_update_header(mail_ctx, sync_ctx->syncs);
+		if (sync_ctx->delay_writes) {
+			/* mark it dirty and do it later */
+			mail_ctx->dirty = TRUE;
+			return 0;
+		}
+
 		if ((ret = mbox_sync_try_rewrite(mail_ctx, 0, FALSE)) < 0)
 			return -1;
 	} else {
@@ -576,6 +612,7 @@
 {
 	struct mbox_sync_context *sync_ctx = mail_ctx->sync_ctx;
 	uoff_t padding;
+	uint32_t last_seq;
 
 	buffer_append(sync_ctx->mails, &mail_ctx->mail, sizeof(mail_ctx->mail));
 
@@ -592,12 +629,16 @@
 		/* don't waste too much on padding */
 		sync_ctx->expunged_space = sync_ctx->space_diff - padding;
 		sync_ctx->space_diff = padding;
+		last_seq = sync_ctx->seq - 1;
+		buffer_set_used_size(sync_ctx->mails, sync_ctx->mails->used -
+				     sizeof(mail_ctx->mail));
 	} else {
 		sync_ctx->expunged_space = 0;
+		last_seq = sync_ctx->seq;
 	}
 
 	if (mbox_sync_rewrite(sync_ctx, sync_ctx->space_diff,
-			      sync_ctx->need_space_seq, sync_ctx->seq) < 0)
+			      sync_ctx->need_space_seq, last_seq) < 0)
 		return -1;
 
 	update_from_offsets(sync_ctx);
@@ -755,12 +796,8 @@
 		if (mail_ctx->pseudo)
 			uid = 0;
 
-		/* get all sync records related to this message */
-		if (mbox_sync_read_index_syncs(sync_ctx, uid, &expunged) < 0)
-			return -1;
-
 		rec = NULL;
-		if (uid != 0 && sync_ctx->ibox->md5hdr_extra_idx == 0) {
+		if (uid != 0) {
 			ret = mbox_sync_read_index_rec(sync_ctx, uid, &rec);
 			if (ret < 0)
 				return -1;
@@ -768,8 +805,12 @@
 				uid = 0;
 		}
 
-		if (sync_ctx->ibox->mbox_readonly && !mail_ctx->pseudo) {
-			/* Read-only mboxes use MD5 sums. */
+		if (uid == 0 && !mail_ctx->pseudo &&
+		    (sync_ctx->delay_writes ||
+		     sync_ctx->idx_seq <= messages_count)) {
+			/* If we can't use/store X-UID header, use MD5 sum.
+			   Also check for existing MD5 sums when we're actually
+			   able to write X-UIDs. */
 			if (sync_ctx->ibox->md5hdr_extra_idx == 0) {
 				sync_ctx->ibox->md5hdr_extra_idx =
 					mail_index_register_record_extra(
@@ -783,7 +824,16 @@
 				return -1;
 
 			if (rec != NULL)
-				uid = rec->uid;
+				uid = mail_ctx->mail.uid = rec->uid;
+		}
+
+		if (!mail_ctx->pseudo) {
+			/* get all sync records related to this message */
+			if (mbox_sync_read_index_syncs(sync_ctx, uid,
+						       &expunged) < 0)
+				return -1;
+		} else {
+			expunged = FALSE;
 		}
 
 		if (uid == 0 && !mail_ctx->pseudo) {
@@ -840,7 +890,7 @@
 					return -1;
 			}
 		} else if (sync_ctx->seq >= min_message_count) {
-			mbox_sync_buffer_delete_old(sync_ctx->syncs, uid+1);
+			mbox_sync_buffer_delete_old(sync_ctx->syncs, uid);
 			if (buffer_get_used_size(sync_ctx->syncs) == 0) {
 				/* if there's no sync records left,
 				   we can stop */
@@ -962,9 +1012,9 @@
 	     sync_ctx->base_uid_validity != sync_ctx->hdr->uid_validity) ||
 	    (sync_ctx->hdr->uid_validity == 0 && sync_ctx->seen_first_mail)) {
 		if (sync_ctx->base_uid_validity == 0) {
-			/* we couldn't rewrite X-IMAPbase because it's
-			   a read-only mbox */
-			i_assert(sync_ctx->ibox->mbox_readonly);
+			/* we didn't rewrite X-IMAPbase header because
+			   a) mbox is read-only, b) we're lazy-writing */
+			i_assert(sync_ctx->delay_writes);
                         sync_ctx->base_uid_validity = time(NULL);
 		}
 		mail_index_update_header(sync_ctx->t,
@@ -1260,6 +1310,11 @@
 	sync_ctx.file_input = sync_ctx.ibox->mbox_file_stream;
 	sync_ctx.input = sync_ctx.ibox->mbox_stream;
 	sync_ctx.fd = sync_ctx.ibox->mbox_fd;
+	sync_ctx.flags = flags;
+	sync_ctx.delay_writes = sync_ctx.ibox->mbox_readonly ||
+		((flags & MBOX_SYNC_UNDIRTY) == 0 &&
+		 getenv("MBOX_LAZY_WRITES") != NULL);
+
 
 	if (mbox_sync_do(&sync_ctx, flags) < 0)
 		ret = -1;
@@ -1284,7 +1339,7 @@
 
 	if (sync_ctx.seen_first_mail &&
 	    sync_ctx.base_uid_last != sync_ctx.next_uid-1 &&
-	    ret == 0 && !ibox->mbox_readonly) {
+	    ret == 0 && !sync_ctx.delay_writes) {
 		/* rewrite X-IMAPbase header. do it after mail_index_sync_end()
 		   so previous transactions have been committed. */
 		/* FIXME: ugly .. */
--- a/src/master/mail-process.c	Fri Sep 24 01:27:16 2004 +0300
+++ b/src/master/mail-process.c	Fri Sep 24 01:29:13 2004 +0300
@@ -222,6 +222,8 @@
 		env_put("POP3_MAILS_KEEP_RECENT=1");
 	if (set->mbox_dirty_syncs)
 		env_put("MBOX_DIRTY_SYNCS=1");
+	if (set->mbox_lazy_writes)
+		env_put("MBOX_LAZY_WRITES=1");
 	(void)umask(set->umask);
 
 	env_put(t_strconcat("MBOX_READ_LOCKS=", set->mbox_read_locks, NULL));
--- a/src/master/master-settings.c	Fri Sep 24 01:27:16 2004 +0300
+++ b/src/master/master-settings.c	Fri Sep 24 01:29:13 2004 +0300
@@ -104,6 +104,7 @@
 	DEF(SET_INT, mbox_lock_timeout),
 	DEF(SET_INT, mbox_dotlock_change_timeout),
 	DEF(SET_BOOL, mbox_dirty_syncs),
+	DEF(SET_BOOL, mbox_lazy_writes),
 	DEF(SET_INT, umask),
 	DEF(SET_BOOL, mail_drop_priv_before_exec),
 
@@ -270,6 +271,7 @@
 	MEMBER(mbox_lock_timeout) 300,
 	MEMBER(mbox_dotlock_change_timeout) 30,
 	MEMBER(mbox_dirty_syncs) TRUE,
+	MEMBER(mbox_lazy_writes) TRUE,
 	MEMBER(umask) 0077,
 	MEMBER(mail_drop_priv_before_exec) FALSE,
 
--- a/src/master/master-settings.h	Fri Sep 24 01:27:16 2004 +0300
+++ b/src/master/master-settings.h	Fri Sep 24 01:29:13 2004 +0300
@@ -75,6 +75,7 @@
 	unsigned int mbox_lock_timeout;
 	unsigned int mbox_dotlock_change_timeout;
 	int mbox_dirty_syncs;
+	int mbox_lazy_writes;
 	unsigned int umask;
 	int mail_drop_priv_before_exec;