Mercurial > dovecot > core-2.2

changeset 559:c834e77b624c HEAD

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Mostly mbox locking/syncing fixes. Still some problems though.
author Timo Sirainen <tss@iki.fi>
date Sat, 02 Nov 2002 22:10:20 +0200
parents 4bf3e0cb8159
children 08d5e82b9068
files src/imap/cmd-close.c src/lib-index/mail-index-open.c src/lib-index/mail-index-update-cache.c src/lib-index/mail-index.c src/lib-index/mail-index.h src/lib-index/mail-tree.c src/lib-index/maildir/maildir-index.h src/lib-index/maildir/maildir-sync.c src/lib-index/mbox/mbox-index.c src/lib-index/mbox/mbox-index.h src/lib-index/mbox/mbox-lock.c src/lib-index/mbox/mbox-rewrite.c src/lib-index/mbox/mbox-sync-full.c src/lib-index/mbox/mbox-sync.c src/lib-storage/index/index-copy.c src/lib-storage/index/index-expunge.c src/lib-storage/index/index-fetch.c src/lib-storage/index/index-search.c src/lib-storage/index/index-status.c src/lib-storage/index/index-storage.c src/lib-storage/index/index-storage.h src/lib-storage/index/index-sync.c src/lib-storage/index/index-update-flags.c src/lib-storage/index/maildir/maildir-copy.c src/lib-storage/index/mbox/mbox-expunge.c src/lib-storage/index/mbox/mbox-storage.c src/lib-storage/index/mbox/mbox-storage.h src/lib-storage/mail-storage.h
diffstat 28 files changed, 245 insertions(+), 200 deletions(-) [+]
line wrap: on
line diff
--- a/src/imap/cmd-close.c	Fri Nov 01 18:02:23 2002 +0200
+++ b/src/imap/cmd-close.c	Sat Nov 02 22:10:20 2002 +0200
@@ -3,19 +3,23 @@
 #include "common.h"
 #include "commands.h"
 
+static void client_send_untagged_storage_error(Client *client)
+{
+	client_send_line(client, t_strconcat("* NO ",
+		client->storage->get_last_error(client->storage), NULL));
+}
+
 int cmd_close(Client *client)
 {
 	if (!client_verify_open_mailbox(client))
 		return TRUE;
 
-	if (!client->mailbox->expunge(client->mailbox, FALSE)) {
-		/* just warn about the error */
-		client_send_tagline(client, t_strconcat("* NO ",
-			client->storage->get_last_error(client->storage),
-			NULL));
-	}
+	if (!client->mailbox->expunge(client->mailbox, FALSE))
+                client_send_untagged_storage_error(client);
 
-	client->mailbox->close(client->mailbox);
+	if (!client->mailbox->close(client->mailbox))
+                client_send_untagged_storage_error(client);
+
 	client->mailbox = NULL;
 
 	client_send_tagline(client, "OK Close completed.");
--- a/src/lib-index/mail-index-open.c	Fri Nov 01 18:02:23 2002 +0200
+++ b/src/lib-index/mail-index-open.c	Sat Nov 02 22:10:20 2002 +0200
@@ -212,9 +212,9 @@
 			return FALSE;
 	}
 
-	/* sync before updating cached fields so it won't print
-	   warnings if mails were deleted */
-	if (!index->sync(index))
+	/* sync ourself - before updating cache and compression which
+	   may happen because of this. */
+	if (!index->sync_and_lock(index, MAIL_LOCK_EXCLUSIVE, NULL))
 		return FALSE;
 
 	if (!fast && (index->header->flags & MAIL_INDEX_FLAG_CACHE_FIELDS)) {
--- a/src/lib-index/mail-index-update-cache.c	Fri Nov 01 18:02:23 2002 +0200
+++ b/src/lib-index/mail-index-update-cache.c	Sat Nov 02 22:10:20 2002 +0200
@@ -38,10 +38,6 @@
 	if (!index->set_lock(index, MAIL_LOCK_EXCLUSIVE))
 		return FALSE;
 
-	/* make sure the index is ok before doing this */
-	if (!index->fsck(index))
-		return FALSE;
-
 	cache_fields = index->header->cache_fields;
 
 	rec = index->lookup(index, 1);
--- a/src/lib-index/mail-index.c	Fri Nov 01 18:02:23 2002 +0200
+++ b/src/lib-index/mail-index.c	Sat Nov 02 22:10:20 2002 +0200
@@ -248,23 +248,6 @@
 	}
 }
 
-int mail_index_try_lock(MailIndex *index, MailLockType lock_type)
-{
-	int ret;
-
-	if (index->lock_type == lock_type)
-		return TRUE;
-
-	if (index->anon_mmap)
-		return TRUE;
-
-	ret = file_try_lock(index->fd, MAIL_LOCK_TO_FLOCK(lock_type));
-	if (ret < 0)
-		index_set_syscall_error(index, "file_try_lock()");
-
-	return ret > 0;
-}
-
 static int mail_index_write_header_changes(MailIndex *index)
 {
 	int failed;
@@ -274,12 +257,21 @@
 	if (file_wait_lock(index->fd, F_WRLCK, DEFAULT_LOCK_TIMEOUT) <= 0)
 		return index_set_syscall_error(index, "file_wait_lock()");
 
+#ifdef DEBUG
+	mprotect(index->mmap_base, index->mmap_used_length,
+		 PROT_READ|PROT_WRITE);
+#endif
+
 	mail_index_update_header_changes(index);
 
 	failed = msync(index->mmap_base, sizeof(MailIndexHeader), MS_SYNC) < 0;
 	if (failed)
 		index_set_syscall_error(index, "msync()");
 
+#ifdef DEBUG
+	mprotect(index->mmap_base, index->mmap_used_length, PROT_NONE);
+#endif
+
 	if (file_wait_lock(index->fd, F_UNLCK, 0) <= 0)
 		return index_set_syscall_error(index, "file_wait_lock()");
 
@@ -313,10 +305,13 @@
 	return TRUE;
 }
 
-static int mail_index_lock_change(MailIndex *index, MailLockType lock_type)
+static int mail_index_lock_change(MailIndex *index, MailLockType lock_type,
+				  int try_lock)
 {
-	/* shared -> exclusive isn't allowed */
-	i_assert(lock_type != MAIL_LOCK_EXCLUSIVE ||
+	int ret, fd_lock_type;
+
+	/* shared -> exclusive isn't allowed without try_lock */
+	i_assert(try_lock || lock_type != MAIL_LOCK_EXCLUSIVE ||
 		 index->lock_type != MAIL_LOCK_SHARED);
 
 	if (index->inconsistent) {
@@ -325,9 +320,20 @@
 		return FALSE;
 	}
 
-	if (file_wait_lock(index->fd, MAIL_LOCK_TO_FLOCK(lock_type),
-			   DEFAULT_LOCK_TIMEOUT) <= 0)
-		return index_set_syscall_error(index, "file_wait_lock()");
+	fd_lock_type = MAIL_LOCK_TO_FLOCK(lock_type);
+	if (try_lock) {
+		ret = file_try_lock(index->fd, fd_lock_type);
+		if (ret < 0)
+			index_set_syscall_error(index, "file_try_lock()");
+		if (ret <= 0)
+			return FALSE;
+	} else {
+		if (file_wait_lock(index->fd, fd_lock_type,
+				   DEFAULT_LOCK_TIMEOUT) <= 0) {
+			return index_set_syscall_error(index,
+						       "file_wait_lock()");
+		}
+	}
 
 	index->lock_type = lock_type;
 	debug_mprotect(index->mmap_base, index->mmap_full_length, index);
@@ -389,7 +395,8 @@
 	return TRUE;
 }
 
-int mail_index_set_lock(MailIndex *index, MailLockType lock_type)
+int mail_index_lock_full(MailIndex *index, MailLockType lock_type,
+			 int try_lock)
 {
 	int keep_fsck;
 
@@ -425,7 +432,17 @@
 	if (lock_type == MAIL_LOCK_UNLOCK)
 		return mail_index_lock_remove(index);
 	else
-		return mail_index_lock_change(index, lock_type);
+		return mail_index_lock_change(index, lock_type, try_lock);
+}
+
+int mail_index_set_lock(MailIndex *index, MailLockType lock_type)
+{
+	return mail_index_lock_full(index, lock_type, FALSE);
+}
+
+int mail_index_try_lock(MailIndex *index, MailLockType lock_type)
+{
+	return mail_index_lock_full(index, lock_type, TRUE);
 }
 
 int mail_index_verify_hole_range(MailIndex *index)
--- a/src/lib-index/mail-index.h	Fri Nov 01 18:02:23 2002 +0200
+++ b/src/lib-index/mail-index.h	Sat Nov 02 22:10:20 2002 +0200
@@ -199,9 +199,13 @@
 	   Same locking issues as with rebuild(). */
 	int (*fsck)(MailIndex *index);
 
-	/* Synchronize the index with the mailbox. Same locking issues as
-	   with rebuild(). */
-	int (*sync)(MailIndex *index);
+	/* Synchronize the index with the mailbox. Index must not have shared
+	   lock when calling this function. The lock_type specifies what
+	   locking state the index will be left, also locking mailbox file
+	   if needed. If changes is non-NULL, it's set to TRUE if any changes
+	   were noticed. */
+	int (*sync_and_lock)(MailIndex *index, MailLockType lock_type,
+			     int *changes);
 
 	/* Returns the index header (never fails). The index needs to be
 	   locked before calling this function, and must be kept locked as
@@ -329,7 +333,6 @@
 	int mbox_fd;
 	IBuffer *mbox_inbuf;
 	MailLockType mbox_lock_type;
-	MailLockType mbox_lock_next_sync;
 
 	/* these counters can be used to check that we've synced the mailbox
 	   after locking it */
@@ -371,7 +374,7 @@
 	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, \
 	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, \
 	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, \
-	0, 0, 0, 0, 0
+	0, 0, 0, 0
 
 /* defaults - same as above but prefixed with mail_index_. */
 int mail_index_open(MailIndex *index, int update_recent, int fast);
--- a/src/lib-index/mail-tree.c	Fri Nov 01 18:02:23 2002 +0200
+++ b/src/lib-index/mail-tree.c	Sat Nov 02 22:10:20 2002 +0200
@@ -125,6 +125,9 @@
 
 int _mail_tree_mmap_update(MailTree *tree, int forced)
 {
+	debug_mprotect(tree->mmap_base, tree->mmap_full_length,
+		       tree->index);
+
 	if (!forced && tree->header != NULL &&
 	    tree->sync_id == tree->header->sync_id) {
 		/* make sure file size hasn't changed */
--- a/src/lib-index/maildir/maildir-index.h	Fri Nov 01 18:02:23 2002 +0200
+++ b/src/lib-index/maildir/maildir-index.h	Sat Nov 02 22:10:20 2002 +0200
@@ -13,7 +13,7 @@
 const char *maildir_filename_set_flags(const char *fname, MailFlags flags);
 
 int maildir_index_rebuild(MailIndex *index);
-int maildir_index_sync(MailIndex *index);
+int maildir_index_sync(MailIndex *index, MailLockType lock_type, int *changes);
 
 int maildir_index_append_file(MailIndex *index, const char *dir,
 			      const char *fname);
--- a/src/lib-index/maildir/maildir-sync.c	Fri Nov 01 18:02:23 2002 +0200
+++ b/src/lib-index/maildir/maildir-sync.c	Sat Nov 02 22:10:20 2002 +0200
@@ -249,7 +249,7 @@
 	return !failed;
 }
 
-int maildir_index_sync(MailIndex *index)
+int maildir_index_sync(MailIndex *index, MailLockType lock_type, int *changes)
 {
 	struct stat sti, std;
 	struct utimbuf ut;
@@ -258,6 +258,9 @@
 
 	i_assert(index->lock_type != MAIL_LOCK_SHARED);
 
+	if (changes != NULL)
+		*changes = FALSE;
+
 	if (index->fd == -1) {
 		/* anon-mmaped */
 		index_mtime = index->file_sync_stamp;
@@ -275,6 +278,7 @@
 		return index_file_set_syscall_error(index, cur_dir, "stat()");
 
 	if (std.st_mtime != index_mtime) {
+		if (changes != NULL) *changes = TRUE;
 		if (!maildir_index_sync_dir(index, cur_dir))
 			return FALSE;
 	}
@@ -285,6 +289,8 @@
 		return index_file_set_syscall_error(index, new_dir, "stat()");
 
 	if (std.st_mtime != index_mtime) {
+		if (changes != NULL) *changes = TRUE;
+
 		if (!maildir_index_build_dir(index, new_dir, cur_dir))
 			return FALSE;
 
@@ -324,5 +330,5 @@
 			return index_set_syscall_error(index, "utime()");
 	}
 
-	return TRUE;
+	return index->set_lock(index, lock_type);
 }
--- a/src/lib-index/mbox/mbox-index.c	Fri Nov 01 18:02:23 2002 +0200
+++ b/src/lib-index/mbox/mbox-index.c	Sat Nov 02 22:10:20 2002 +0200
@@ -701,6 +701,20 @@
 	i_free(index);
 }
 
+static int mbox_index_set_lock(MailIndex *index, MailLockType lock_type)
+{
+	if (lock_type == MAIL_LOCK_UNLOCK)
+		(void)mbox_unlock(index);
+	return mail_index_set_lock(index, lock_type);
+}
+
+static int mbox_index_try_lock(MailIndex *index, MailLockType lock_type)
+{
+	if (lock_type == MAIL_LOCK_UNLOCK)
+		(void)mbox_unlock(index);
+	return mail_index_try_lock(index, lock_type);
+}
+
 static int mbox_index_update_flags(MailIndex *index, MailIndexRecord *rec,
 				   unsigned int seq, MailFlags flags,
 				   int external_change)
@@ -719,8 +733,8 @@
 	mail_index_open,
 	mail_index_open_or_create,
 	mbox_index_free,
-	mail_index_set_lock,
-	mail_index_try_lock,
+	mbox_index_set_lock,
+	mbox_index_try_lock,
 	mbox_index_rebuild,
 	mail_index_fsck,
 	mbox_index_sync,
--- a/src/lib-index/mbox/mbox-index.h	Fri Nov 01 18:02:23 2002 +0200
+++ b/src/lib-index/mbox/mbox-index.h	Sat Nov 02 22:10:20 2002 +0200
@@ -48,7 +48,7 @@
 
 MailIndex *mbox_index_alloc(const char *dir, const char *mbox_path);
 int mbox_index_rebuild(MailIndex *index);
-int mbox_index_sync(MailIndex *index);
+int mbox_index_sync(MailIndex *index, MailLockType lock_type, int *changes);
 int mbox_sync_full(MailIndex *index);
 IBuffer *mbox_open_mail(MailIndex *index, MailIndexRecord *rec,
 			time_t *internal_date, int *deleted);
--- a/src/lib-index/mbox/mbox-lock.c	Fri Nov 01 18:02:23 2002 +0200
+++ b/src/lib-index/mbox/mbox-lock.c	Sat Nov 02 22:10:20 2002 +0200
@@ -151,6 +151,10 @@
 {
 	struct stat st;
 
+	/* index must be locked before mbox file, to avoid deadlocks */
+	i_assert(index->lock_type != MAIL_LOCK_UNLOCK);
+
+	/* allow only unlock -> shared/exclusive or exclusive -> shared */
 	i_assert(lock_type == MAIL_LOCK_SHARED ||
 		 lock_type == MAIL_LOCK_EXCLUSIVE);
 	i_assert(lock_type != MAIL_LOCK_EXCLUSIVE ||
@@ -200,7 +204,6 @@
 	int failed;
 
 	index->mbox_lock_counter++;
-	index->mbox_lock_next_sync = MAIL_LOCK_UNLOCK;
 
 	if (index->mbox_lock_type == MAIL_LOCK_UNLOCK)
 		return TRUE;
--- a/src/lib-index/mbox/mbox-rewrite.c	Fri Nov 01 18:02:23 2002 +0200
+++ b/src/lib-index/mbox/mbox-rewrite.c	Sat Nov 02 22:10:20 2002 +0200
@@ -406,9 +406,17 @@
 	unsigned int seq;
 	int tmp_fd, failed, dirty_found, rewrite;
 
-	i_assert(index->lock_type == MAIL_LOCK_EXCLUSIVE);
+	i_assert(index->lock_type == MAIL_LOCK_UNLOCK);
+
+	if (!index->set_lock(index, MAIL_LOCK_SHARED))
+		return FALSE;
 
-	if ((index->header->flags & MAIL_INDEX_FLAG_DIRTY_MESSAGES) == 0) {
+        rewrite = (index->header->flags & MAIL_INDEX_FLAG_DIRTY_MESSAGES);
+
+	if (!index->set_lock(index, MAIL_LOCK_UNLOCK))
+		return FALSE;
+
+	if (!rewrite) {
 		/* no need to rewrite */
 		return TRUE;
 	}
@@ -417,8 +425,7 @@
 	failed = TRUE; rewrite = FALSE;
 	do {
 		/* make sync() lock the file to prevent race conditions */
-                index->mbox_lock_next_sync = MAIL_LOCK_EXCLUSIVE;
-		if (!index->sync(index))
+		if (!index->sync_and_lock(index, MAIL_LOCK_EXCLUSIVE, NULL))
 			break;
 
 		inbuf = mbox_get_inbuf(index, 0, MAIL_LOCK_EXCLUSIVE);
@@ -442,6 +449,8 @@
 
 	if (!rewrite) {
 		(void)mbox_unlock(index);
+		if (!index->set_lock(index, MAIL_LOCK_EXCLUSIVE))
+			failed = TRUE;
 		if (inbuf != NULL)
 			i_buffer_unref(inbuf);
 		return !failed;
@@ -455,10 +464,9 @@
 	}
 	dirty_offset = 0;
 
-	//offset = hdr_size = body_size = 0; /* just to keep compiler happy */
-
-	t_push();
-	outbuf = o_buffer_create_file(tmp_fd, data_stack_pool, 8192, 0, FALSE);
+	/* note: we can't use data_stack_pool with outbuf because it's
+	   being written to inside t_push() .. t_pop() calls */
+	outbuf = o_buffer_create_file(tmp_fd, system_pool, 8192, 0, FALSE);
 
 	failed = FALSE; seq = 1;
 	rec = index->lookup(index, 1);
@@ -535,7 +543,6 @@
 
 	i_buffer_unref(inbuf);
 	o_buffer_unref(outbuf);
-	t_pop();
 
 	if (!failed) {
 		/* POSSIBLE DATA LOSS HERE. We're writing to the mbox file,
@@ -565,10 +572,13 @@
 		}
 	}
 
+	if (!index->set_lock(index, MAIL_LOCK_UNLOCK))
+		failed = TRUE;
+
 	(void)mbox_unlock(index);
 	(void)unlink(path);
 
 	if (close(tmp_fd) < 0)
 		index_file_set_syscall_error(index, path, "close()");
-	return failed;
+	return !failed;
 }
--- a/src/lib-index/mbox/mbox-sync-full.c	Fri Nov 01 18:02:23 2002 +0200
+++ b/src/lib-index/mbox/mbox-sync-full.c	Sat Nov 02 22:10:20 2002 +0200
@@ -47,6 +47,7 @@
 {
 	const void *part_data;
 	void *part_data_copy;
+	uoff_t virtual_size;
 	size_t size;
 
 	/* update FIELD_HDR_HEADER_SIZE */
@@ -54,10 +55,15 @@
 				&hdr_size->physical_size,
 				sizeof(hdr_size->physical_size));
 
+	/* reset FIELD_HDR_VIRTUAL_SIZE - we don't know it anymore */
+        virtual_size = (uoff_t)-1;
+	index->update_field_raw(update, DATA_HDR_VIRTUAL_SIZE,
+				&virtual_size, sizeof(virtual_size));
+
+	/* update DATA_FIELD_MESSAGEPART */
 	if ((rec->data_fields & DATA_FIELD_MESSAGEPART) == 0)
 		return TRUE;
 
-	/* update DATA_FIELD_MESSAGEPART */
 	part_data = index->lookup_field_raw(index, rec, DATA_FIELD_MESSAGEPART,
 					    &size);
 	if (part_data == NULL) {
@@ -77,10 +83,9 @@
 		return FALSE;
 	}
 
-	t_pop();
-
 	index->update_field_raw(update, DATA_FIELD_MESSAGEPART,
 				part_data_copy, size);
+	t_pop();
 	return TRUE;
 }
 
@@ -183,9 +188,6 @@
 	unsigned int seq;
 	int dirty;
 
-	if (!index->set_lock(index, MAIL_LOCK_EXCLUSIVE))
-		return FALSE;
-
 	mbox_skip_empty_lines(inbuf);
 
 	/* first make sure we start with a "From " line. If file is too
@@ -217,6 +219,10 @@
 			if (!mbox_skip_crlf(inbuf)) {
 				/* they just went and broke it, even while
 				   we had it locked. */
+				index_set_error(index,
+						"Error syncing mbox file %s: "
+						"LF not found where expected",
+						index->mbox_path);
 				return FALSE;
 			}
 		}
@@ -258,9 +264,14 @@
 int mbox_sync_full(MailIndex *index)
 {
 	IBuffer *inbuf;
-	int failed, unlock;
+	int failed;
 
-	unlock = index->mbox_lock_type == MAIL_LOCK_UNLOCK;
+	if (index->lock_type == MAIL_LOCK_SHARED)
+		(void)mail_index_set_lock(index, MAIL_LOCK_UNLOCK);
+
+	if (!index->set_lock(index, MAIL_LOCK_EXCLUSIVE))
+		return FALSE;
+
 	inbuf = mbox_get_inbuf(index, 0, MAIL_LOCK_SHARED);
 	if (inbuf == NULL)
 		return FALSE;
@@ -268,8 +279,5 @@
 	failed = !mbox_sync_buf(index, inbuf);
 	i_buffer_unref(inbuf);
 
-	if (unlock)
-		(void)mbox_unlock(index);
-
 	return !failed;
 }
--- a/src/lib-index/mbox/mbox-sync.c	Fri Nov 01 18:02:23 2002 +0200
+++ b/src/lib-index/mbox/mbox-sync.c	Sat Nov 02 22:10:20 2002 +0200
@@ -33,9 +33,6 @@
 			offset += hdr_size + body_size;
 	}
 
-	if (index->lock_type == MAIL_LOCK_SHARED)
-		(void)mail_index_set_lock(index, MAIL_LOCK_UNLOCK);
-
 	if (offset > OFF_T_MAX) {
 		/* too large to fit in off_t */
 		return 0;
@@ -44,17 +41,16 @@
 	return offset;
 }
 
-int mbox_index_sync(MailIndex *index)
+int mbox_index_sync(MailIndex *index, MailLockType lock_type, int *changes)
 {
 	struct stat st;
-	MailLockType lock_type;
 	time_t index_mtime;
 	uoff_t filesize;
 
 	i_assert(index->lock_type != MAIL_LOCK_SHARED);
 
-	lock_type = index->mbox_lock_next_sync;
-	index->mbox_lock_next_sync = MAIL_LOCK_UNLOCK;
+	if (changes != NULL)
+		*changes = FALSE;
 	index->mbox_sync_counter = index->mbox_lock_counter;
 
 	if (index->fd == -1) {
@@ -79,34 +75,60 @@
                 mbox_file_close_fd(index);
 	}
 
-	if (lock_type != MAIL_LOCK_UNLOCK) {
-		if (!mbox_lock(index, lock_type))
+	if (lock_type == MAIL_LOCK_EXCLUSIVE) {
+		/* if we know that we want exclusive lock, we might get
+		   it immediately to save extra lock changes */
+		if (!index->set_lock(index, MAIL_LOCK_EXCLUSIVE))
 			return FALSE;
 	}
 
-	if (index_mtime == st.st_mtime && index->mbox_size == filesize)
-		return TRUE;
+	if (index_mtime != st.st_mtime || index->mbox_size != filesize) {
+		mbox_file_close_inbuf(index);
 
-	mbox_file_close_inbuf(index);
+		/* problem .. index->mbox_size points to data after the last
+		   message. that should be \n or end of file. modify filesize
+		   accordingly to allow the extra byte. Don't actually bother
+		   to open the file and verify it, it'd just slow things.. */
+		index->mbox_size = get_indexed_mbox_size(index);
+		if (filesize == index->mbox_size+1)
+			index->mbox_size = filesize;
 
-	/* problem .. index->mbox_size points to data after the last message.
-	   that should be \n, \r\n, or end of file. modify filesize
-	   accordingly to allow any of the extra 0-2 bytes. Don't actually
-	   bother to open the file and verify it, it'd just slow things.. */
-	index->mbox_size = get_indexed_mbox_size(index);
-	if (filesize == index->mbox_size+1 ||
-	    filesize == index->mbox_size+2)
-		index->mbox_size = filesize;
+		if (index->file_sync_stamp == 0 &&
+		    index->mbox_size == filesize) {
+			/* just opened the mailbox, and the file size is same
+			   as we expected. don't bother checking it any
+			   further. */
+		} else {
+			if (changes != NULL)
+				*changes = TRUE;
 
-	if (index->file_sync_stamp == 0 && index->mbox_size == filesize) {
-		/* just opened the mailbox, and the file size is same as
-		   we expected. don't bother checking it any further. */
+			/* file has changed, scan through the whole mbox */
+			if (!mbox_sync_full(index)) {
+				(void)index->set_lock(index, MAIL_LOCK_UNLOCK);
+				return FALSE;
+			}
+
+			if (lock_type == MAIL_LOCK_EXCLUSIVE &&
+			    index->mbox_lock_type == MAIL_LOCK_SHARED) {
+				/* mbox_sync_full() left it */
+				if (!mbox_unlock(index))
+					return FALSE;
+			}
+		}
+
 		index->file_sync_stamp = st.st_mtime;
-		return TRUE;
 	}
 
-	index->file_sync_stamp = st.st_mtime;
+	if (!index->set_lock(index, lock_type))
+		return FALSE;
 
-	/* file has changed, scan through the whole mbox */
-	return mbox_sync_full(index);
+	if (lock_type != MAIL_LOCK_UNLOCK) {
+		if (!mbox_lock(index, lock_type))
+			return FALSE;
+	} else {
+		if (!mbox_unlock(index))
+			return FALSE;
+	}
+
+	return TRUE;
 }
--- a/src/lib-storage/index/index-copy.c	Fri Nov 01 18:02:23 2002 +0200
+++ b/src/lib-storage/index/index-copy.c	Sat Nov 02 22:10:20 2002 +0200
@@ -40,6 +40,7 @@
 {
 	IndexMailbox *ibox = (IndexMailbox *) box;
         CopyContext ctx;
+	MailLockType lock_type;
 	int failed;
 
 	if (destbox->readonly) {
@@ -48,10 +49,11 @@
 		return FALSE;
 	}
 
-	if (!ibox->index->sync(ibox->index))
-		return mail_storage_set_index_error(ibox);
+	lock_type = destbox->storage == box->storage &&
+		strcmp(destbox->name, box->name) == 0 ?
+		MAIL_LOCK_EXCLUSIVE : MAIL_LOCK_SHARED;
 
-	if (!ibox->index->set_lock(ibox->index, MAIL_LOCK_SHARED))
+	if (!ibox->index->sync_and_lock(ibox->index, lock_type, NULL))
 		return mail_storage_set_index_error(ibox);
 
 	ctx.custom_flags =
--- a/src/lib-storage/index/index-expunge.c	Fri Nov 01 18:02:23 2002 +0200
+++ b/src/lib-storage/index/index-expunge.c	Sat Nov 02 22:10:20 2002 +0200
@@ -70,12 +70,9 @@
 		return FALSE;
 	}
 
-	if (!index_storage_sync_index_if_possible(ibox, FALSE))
+	if (!index_storage_sync_and_lock(ibox, FALSE, MAIL_LOCK_EXCLUSIVE))
 		return FALSE;
 
-	if (!ibox->index->set_lock(ibox->index, MAIL_LOCK_EXCLUSIVE))
-		return mail_storage_set_index_error(ibox);
-
 	/* modifylog must be marked synced before expunging anything new */
 	failed = !index_storage_sync_modifylog(ibox, TRUE);
 
--- a/src/lib-storage/index/index-fetch.c	Fri Nov 01 18:02:23 2002 +0200
+++ b/src/lib-storage/index/index-fetch.c	Sat Nov 02 22:10:20 2002 +0200
@@ -347,9 +347,6 @@
 	MailFetchBodyData *sect;
 	int ret;
 
-	if (!index_storage_sync_index_if_possible(ibox, TRUE))
-		return FALSE;
-
 	memset(&ctx, 0, sizeof(ctx));
 
 	if (!box->readonly) {
@@ -364,21 +361,22 @@
 		}
 	}
 
+	/* need exclusive lock to update the \Seen flags */
+	if (!index_storage_sync_and_lock(ibox, TRUE, ctx.update_seen ?
+					 MAIL_LOCK_EXCLUSIVE :
+					 MAIL_LOCK_SHARED))
+		return FALSE;
+
 	if (ctx.update_seen) {
-		/* need exclusive lock to update the \Seen flags */
-		if (!ibox->index->set_lock(ibox->index, MAIL_LOCK_EXCLUSIVE))
-			return mail_storage_set_index_error(ibox);
-
 		/* if all messages are already seen, there's no point in
 		   keeping exclusive lock */
 		if (ibox->index->header->messages_count ==
-		    ibox->index->header->seen_messages_count)
+		    ibox->index->header->seen_messages_count) {
 			ctx.update_seen = FALSE;
-	}
 
-	if (!ctx.update_seen) {
-		if (!ibox->index->set_lock(ibox->index, MAIL_LOCK_SHARED))
-			return mail_storage_set_index_error(ibox);
+			(void)ibox->index->set_lock(ibox->index,
+						    MAIL_LOCK_SHARED);
+		}
 	}
 
 	ctx.box = box;
--- a/src/lib-storage/index/index-search.c	Fri Nov 01 18:02:23 2002 +0200
+++ b/src/lib-storage/index/index-search.c	Sat Nov 02 22:10:20 2002 +0200
@@ -903,12 +903,9 @@
 	IndexMailbox *ibox = (IndexMailbox *) box;
 	int failed;
 
-	if (!index_storage_sync_index_if_possible(ibox, TRUE))
+	if (!index_storage_sync_and_lock(ibox, TRUE, MAIL_LOCK_SHARED))
 		return FALSE;
 
-	if (!ibox->index->set_lock(ibox->index, MAIL_LOCK_SHARED))
-		return mail_storage_set_index_error(ibox);
-
 	o_buffer_send(outbuf, "* SEARCH", 8);
 	failed = !search_messages(ibox, args, outbuf, uid_result);
 	o_buffer_send(outbuf, "\r\n", 2);
--- a/src/lib-storage/index/index-status.c	Fri Nov 01 18:02:23 2002 +0200
+++ b/src/lib-storage/index/index-status.c	Sat Nov 02 22:10:20 2002 +0200
@@ -75,12 +75,9 @@
 
 	/* if we're doing STATUS for selected mailbox, we have to sync it
 	   first or STATUS reply may give different data */
-	if (!index_storage_sync_index_if_possible(ibox, TRUE))
+	if (!index_storage_sync_and_lock(ibox, TRUE, MAIL_LOCK_SHARED))
 		return FALSE;
 
-	if (!ibox->index->set_lock(ibox->index, MAIL_LOCK_SHARED))
-		return mail_storage_set_index_error(ibox);
-
 	if (!index_storage_sync_modifylog(ibox, FALSE)) {
 		if (!ibox->index->set_lock(ibox->index, MAIL_LOCK_UNLOCK))
 			return mail_storage_set_index_error(ibox);
--- a/src/lib-storage/index/index-storage.c	Fri Nov 01 18:02:23 2002 +0200
+++ b/src/lib-storage/index/index-storage.c	Sat Nov 02 22:10:20 2002 +0200
@@ -191,7 +191,7 @@
 	return NULL;
 }
 
-void index_storage_close(Mailbox *box)
+int index_storage_close(Mailbox *box)
 {
 	IndexMailbox *ibox = (IndexMailbox *) box;
 
@@ -199,6 +199,8 @@
 	index_storage_unref(ibox->index);
 	i_free(box->name);
 	i_free(box);
+
+	return TRUE;
 }
 
 void index_storage_set_sync_callbacks(Mailbox *box,
--- a/src/lib-storage/index/index-storage.h	Fri Nov 01 18:02:23 2002 +0200
+++ b/src/lib-storage/index/index-storage.h	Sat Nov 02 22:10:20 2002 +0200
@@ -35,9 +35,10 @@
 IndexMailbox *index_storage_init(MailStorage *storage, Mailbox *box,
 				 MailIndex *index, const char *name,
 				 int readonly, int fast);
-void index_storage_close(Mailbox *box);
+int index_storage_close(Mailbox *box);
 
-int index_storage_sync_index_if_possible(IndexMailbox *ibox, int sync_size);
+int index_storage_sync_and_lock(IndexMailbox *ibox, int sync_size,
+				MailLockType lock_type);
 int index_storage_sync_modifylog(IndexMailbox *ibox, int hide_deleted);
 
 int index_mailbox_fix_custom_flags(IndexMailbox *ibox, MailFlags *flags,
--- a/src/lib-storage/index/index-sync.c	Fri Nov 01 18:02:23 2002 +0200
+++ b/src/lib-storage/index/index-sync.c	Sat Nov 02 22:10:20 2002 +0200
@@ -27,12 +27,20 @@
 	}
 }
 
-/* may leave the index locked */
-int index_storage_sync_index_if_possible(IndexMailbox *ibox, int sync_size)
+int index_storage_sync_and_lock(IndexMailbox *ibox, int sync_size,
+				MailLockType lock_type)
 {
 	MailIndex *index = ibox->index;
+	int unlock, changes;
 
-	if (index->sync(index)) {
+	if (lock_type != MAIL_LOCK_UNLOCK)
+		unlock = FALSE;
+	else {
+		unlock = TRUE;
+		lock_type = MAIL_LOCK_SHARED;
+	}
+
+	if (index->sync_and_lock(index, lock_type, &changes)) {
 		/* reset every time it has worked */
 		ibox->sent_diskspace_warning = FALSE;
 	} else {
@@ -53,8 +61,11 @@
 	}
 
 	/* notify about changes in mailbox size. */
-	if (index->lock_type == MAIL_LOCK_UNLOCK)
+	if (!changes) {
+		if (unlock)
+			(void)index->set_lock(index, MAIL_LOCK_UNLOCK);
 		return TRUE; /* no changes - must be no new mail either */
+	}
 
 	if (sync_size)
 		index_storage_sync_size(ibox);
@@ -67,6 +78,9 @@
 			MAIL_CUSTOM_FLAGS_COUNT, ibox->sync_context);
 	}
 
+	if (unlock)
+		(void)index->set_lock(index, MAIL_LOCK_UNLOCK);
+
 	return TRUE;
 }
 
@@ -194,16 +208,13 @@
 	IndexMailbox *ibox = (IndexMailbox *) box;
 	int failed;
 
-	if (!index_storage_sync_index_if_possible(ibox, FALSE))
+	if (!index_storage_sync_and_lock(ibox, FALSE, MAIL_LOCK_SHARED))
 		return FALSE;
 
 	if (!sync_expunges) {
 		/* FIXME: we could still send flag changes */
 		failed = FALSE;
 	} else {
-		if (!ibox->index->set_lock(ibox->index, MAIL_LOCK_SHARED))
-			return mail_storage_set_index_error(ibox);
-
 		failed = !index_storage_sync_modifylog(ibox, FALSE);
 	}
 
--- a/src/lib-storage/index/index-update-flags.c	Fri Nov 01 18:02:23 2002 +0200
+++ b/src/lib-storage/index/index-update-flags.c	Sat Nov 02 22:10:20 2002 +0200
@@ -72,14 +72,11 @@
 		return FALSE;
 	}
 
-	if (!index_storage_sync_index_if_possible(ibox, TRUE))
-		return FALSE;
-
 	if (!index_mailbox_fix_custom_flags(ibox, &flags, custom_flags))
 		return FALSE;
 
-	if (!ibox->index->set_lock(ibox->index, MAIL_LOCK_EXCLUSIVE))
-		return mail_storage_set_index_error(ibox);
+	if (!index_storage_sync_and_lock(ibox, TRUE, MAIL_LOCK_EXCLUSIVE))
+		return FALSE;
 
 	ctx.ibox = ibox;
 	ctx.flags = flags & ~MAIL_RECENT; /* \Recent can't be changed */
--- a/src/lib-storage/index/maildir/maildir-copy.c	Fri Nov 01 18:02:23 2002 +0200
+++ b/src/lib-storage/index/maildir/maildir-copy.c	Sat Nov 02 22:10:20 2002 +0200
@@ -58,10 +58,7 @@
 	CopyHardContext ctx;
 	int ret;
 
-	if (!src->index->sync(src->index))
-		return mail_storage_set_index_error(src);
-
-	if (!src->index->set_lock(src->index, MAIL_LOCK_SHARED))
+	if (!src->index->sync_and_lock(src->index, MAIL_LOCK_SHARED, NULL))
 		return mail_storage_set_index_error(src);
 
 	ctx.storage = src->box.storage;
--- a/src/lib-storage/index/mbox/mbox-expunge.c	Fri Nov 01 18:02:23 2002 +0200
+++ b/src/lib-storage/index/mbox/mbox-expunge.c	Sat Nov 02 22:10:20 2002 +0200
@@ -150,15 +150,3 @@
 
 	return !failed;
 }
-
-int mbox_storage_expunge(Mailbox *box, int notify)
-{
-	IndexMailbox *ibox = (IndexMailbox *) box;
-	int ret;
-
-	ibox->index->mbox_lock_next_sync = MAIL_LOCK_EXCLUSIVE;
-	ret = index_storage_expunge(box, notify);
-	(void)mbox_unlock(ibox->index);
-
-	return ret;
-}
--- a/src/lib-storage/index/mbox/mbox-storage.c	Fri Nov 01 18:02:23 2002 +0200
+++ b/src/lib-storage/index/mbox/mbox-storage.c	Sat Nov 02 22:10:20 2002 +0200
@@ -371,46 +371,18 @@
 	}
 }
 
-static void mbox_storage_close(Mailbox *box)
-{
-	IndexMailbox *ibox = (IndexMailbox *) box;
-
-	if (!ibox->index->set_lock(ibox->index, MAIL_LOCK_EXCLUSIVE))
-		mail_storage_set_index_error(ibox);
-	else {
-		/* update flags by rewrite mbox file */
-		mbox_index_rewrite(ibox->index);
-
-		(void)ibox->index->set_lock(ibox->index, MAIL_LOCK_UNLOCK);
-	}
-
-	index_storage_close(box);
-}
-
-static int mbox_storage_fetch(Mailbox *box, MailFetchData *fetch_data,
-			      OBuffer *outbuf, int *all_found)
+static int mbox_storage_close(Mailbox *box)
 {
 	IndexMailbox *ibox = (IndexMailbox *) box;
-	int ret;
-
-	ibox->index->mbox_lock_next_sync = MAIL_LOCK_SHARED;
-	ret = index_storage_fetch(box, fetch_data, outbuf, all_found);
-	(void)mbox_unlock(ibox->index);
-
- 	return ret;
-}
+	int failed = FALSE;
 
-static int mbox_storage_search(Mailbox *box, MailSearchArg *args,
-			       OBuffer *outbuf, int uid_result)
-{
-	IndexMailbox *ibox = (IndexMailbox *) box;
-	int ret;
+	/* update flags by rewrite mbox file */
+	if (!mbox_index_rewrite(ibox->index)) {
+		mail_storage_set_index_error(ibox);
+		failed = TRUE;
+	}
 
-	ibox->index->mbox_lock_next_sync = MAIL_LOCK_SHARED;
-	ret = index_storage_search(box, args, outbuf, uid_result);
-	(void)mbox_unlock(ibox->index);
-
- 	return ret;
+	return index_storage_close(box) && !failed;
 }
 
 MailStorage mbox_storage = {
@@ -444,11 +416,11 @@
 	index_storage_set_sync_callbacks,
 	index_storage_get_status,
 	index_storage_sync,
-	mbox_storage_expunge,
+	index_storage_expunge,
 	index_storage_update_flags,
 	index_storage_copy,
-	mbox_storage_fetch,
-	mbox_storage_search,
+	index_storage_fetch,
+	index_storage_search,
 	mbox_storage_save,
 	mail_storage_is_inconsistency_error,
 
--- a/src/lib-storage/index/mbox/mbox-storage.h	Fri Nov 01 18:02:23 2002 +0200
+++ b/src/lib-storage/index/mbox/mbox-storage.h	Sat Nov 02 22:10:20 2002 +0200
@@ -14,7 +14,6 @@
 int mbox_find_subscribed(MailStorage *storage, const char *mask,
 			 MailboxFunc func, void *context);
 
-int mbox_storage_expunge(Mailbox *box, int notify);
 int mbox_expunge_locked(IndexMailbox *ibox, int notify);
 
 #endif
--- a/src/lib-storage/mail-storage.h	Fri Nov 01 18:02:23 2002 +0200
+++ b/src/lib-storage/mail-storage.h	Sat Nov 02 22:10:20 2002 +0200
@@ -119,8 +119,9 @@
 
 	MailStorage *storage;
 
-	/* Close the box */
-	void (*close)(Mailbox *box);
+	/* Close the box. Returns FALSE if some cleanup errors occured, but
+	   the mailbox was closed anyway. */
+	int (*close)(Mailbox *box);
 
 	/* Set synchronization callback functions to use. */
 	void (*set_sync_callbacks)(Mailbox *box,