Mercurial > dovecot > original-hg > dovecot-1.2

changeset 450:925d6eb5f8be HEAD

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
MailStorage interface change. We now let storage call several sync-functions whenever it wants instead of forcing it through sync() function. This now allows us to send client FLAGS and PERMANENTFLAGS messages whenever new custom flags are added. Also EXPUNGE command now goes nicely through expunge() function (which CLOSE used) instead of through flag in sync(). "Out of disk space" alert is also sent now whenever it happens, before it was sent only when opening mailbox. Message set parser now complains when referencing mail sequences outside the possible range. Modify log before sent flags/expunges changes to client even if it didn't know about those messages yet. Also some other smaller cleanups. Beginnings of fixing APPEND and COPY when trying to store mails into selected mailbox. Before this didn't work too well or simply broke things. Still broken, needs a bit more changes..
author Timo Sirainen <tss@iki.fi>
date Sat, 19 Oct 2002 17:51:59 +0300
parents 98588f02693b
children af7b05f88dae
files src/imap/Makefile.am src/imap/client.h src/imap/cmd-append.c src/imap/cmd-capability.c src/imap/cmd-check.c src/imap/cmd-close.c src/imap/cmd-copy.c src/imap/cmd-expunge.c src/imap/cmd-fetch.c src/imap/cmd-noop.c src/imap/cmd-search.c src/imap/cmd-select.c src/imap/cmd-store.c src/imap/commands-util.c src/imap/commands-util.h src/lib-imap/imap-util.c src/lib-imap/imap-util.h src/lib-index/mail-custom-flags.c src/lib-index/mail-custom-flags.h src/lib-index/mail-index.c src/lib-index/mail-index.h src/lib-index/mail-modifylog.c src/lib-storage/index/index-expunge.c src/lib-storage/index/index-fetch.c src/lib-storage/index/index-fetch.h src/lib-storage/index/index-messageset.c src/lib-storage/index/index-search.c src/lib-storage/index/index-status.c src/lib-storage/index/index-storage.c src/lib-storage/index/index-storage.h src/lib-storage/index/index-sync.c src/lib-storage/index/index-update-flags.c src/lib-storage/index/maildir/maildir-expunge.c src/lib-storage/index/maildir/maildir-storage.c src/lib-storage/index/maildir/maildir-storage.h src/lib-storage/index/mbox/mbox-expunge.c src/lib-storage/index/mbox/mbox-storage.c src/lib-storage/index/mbox/mbox-storage.h src/lib-storage/mail-storage.h
diffstat 39 files changed, 576 insertions(+), 411 deletions(-) [+]
line wrap: on
line diff
--- a/src/imap/Makefile.am	Fri Oct 18 02:57:44 2002 +0300
+++ b/src/imap/Makefile.am	Sat Oct 19 17:51:59 2002 +0300
@@ -50,6 +50,7 @@
 	client.c \
 	commands.c \
 	commands-util.c \
+	mailbox-sync.c \
 	main.c
 
 noinst_HEADERS = \
--- a/src/imap/client.h	Fri Oct 18 02:57:44 2002 +0300
+++ b/src/imap/client.h	Sat Oct 19 17:51:59 2002 +0300
@@ -27,6 +27,7 @@
 
 	unsigned int cmd_error:1;
 	unsigned int cmd_uid:1; /* used UID command */
+	unsigned int sync_flags_send_uid:1;
 	unsigned int inbuf_skip_line:1; /* skip all the data until we've
 					   found a new line */
 };
--- a/src/imap/cmd-append.c	Fri Oct 18 02:57:44 2002 +0300
+++ b/src/imap/cmd-append.c	Sat Oct 19 17:51:59 2002 +0300
@@ -92,9 +92,16 @@
 		return FALSE;
 	}
 
-	if (!client_parse_mail_flags(client, flags_list->args, flags_list->size,
-				     &flags, custom_flags))
-		return TRUE;
+	if (flags_list != NULL) {
+		if (!client_parse_mail_flags(client, flags_list->args,
+					     flags_list->size,
+					     &flags, custom_flags))
+			return TRUE;
+	} else {
+		if (!client_parse_mail_flags(client, NULL, 0,
+					     &flags, custom_flags))
+			return TRUE;
+	}
 
 	if (internal_date_str == NULL) {
 		/* no time given, default to now. */
@@ -104,21 +111,15 @@
 		return TRUE;
 	}
 
-	if (client->mailbox != NULL &&
-	    strcmp(client->mailbox->name, mailbox) == 0) {
-		/* this mailbox is selected */
-		box = client->mailbox;
-	} else {
-		/* open the mailbox */
-		if (!client_verify_mailbox_name(client, mailbox, TRUE, FALSE))
-			return TRUE;
+	/* open the mailbox */
+	if (!client_verify_mailbox_name(client, mailbox, TRUE, FALSE))
+		return TRUE;
 
-		box = client->storage->open_mailbox(client->storage,
-						    mailbox, FALSE, TRUE);
-		if (box == NULL) {
-			client_send_storage_error(client);
-			return TRUE;
-		}
+	box = client->storage->open_mailbox(client->storage,
+					    mailbox, FALSE, TRUE);
+	if (box == NULL) {
+		client_send_storage_error(client);
+		return TRUE;
 	}
 
 	o_buffer_send(client->outbuf, "+ OK\r\n", 6);
@@ -127,13 +128,12 @@
 	/* save the mail */
 	failed = !box->save(box, flags, custom_flags, internal_date,
 			    client->inbuf, msg_size);
-	if (box != client->mailbox)
-		box->close(box);
+	box->close(box);
 
 	if (failed) {
 		client_send_storage_error(client);
 	} else {
-		client_sync_mailbox(client);
+		client_sync_full(client);
 		client_send_tagline(client, "OK Append completed.");
 	}
 	return TRUE;
--- a/src/imap/cmd-capability.c	Fri Oct 18 02:57:44 2002 +0300
+++ b/src/imap/cmd-capability.c	Sat Oct 19 17:51:59 2002 +0300
@@ -7,7 +7,7 @@
 {
 	client_send_line(client, "* CAPABILITY " CAPABILITY_STRING);
 
-	client_sync_mailbox(client);
+	client_sync_full(client);
 	client_send_tagline(client, "OK Capability completed.");
 	return TRUE;
 }
--- a/src/imap/cmd-check.c	Fri Oct 18 02:57:44 2002 +0300
+++ b/src/imap/cmd-check.c	Sat Oct 19 17:51:59 2002 +0300
@@ -9,7 +9,7 @@
 		return TRUE;
 
 	/* we don't need this command, but sync the mailbox anyway. */
-	client_sync_mailbox(client);
+	client_sync_full(client);
 	client_send_tagline(client, "OK Check completed.");
 	return TRUE;
 }
--- a/src/imap/cmd-close.c	Fri Oct 18 02:57:44 2002 +0300
+++ b/src/imap/cmd-close.c	Sat Oct 19 17:51:59 2002 +0300
@@ -8,8 +8,12 @@
 	if (!client_verify_open_mailbox(client))
 		return TRUE;
 
-	/* Ignore expunge errors - we can't really do anything about it */
-	(void)client->mailbox->expunge(client->mailbox);
+	if (!client->mailbox->expunge(client->mailbox, FALSE)) {
+		/* just warn about the error */
+		client_send_tagline(client, t_strconcat("* NO ",
+			client->storage->get_last_error(client->storage),
+			NULL));
+	}
 
 	client->mailbox->close(client->mailbox);
 	client->mailbox = NULL;
--- a/src/imap/cmd-copy.c	Fri Oct 18 02:57:44 2002 +0300
+++ b/src/imap/cmd-copy.c	Sat Oct 19 17:51:59 2002 +0300
@@ -29,7 +29,7 @@
 	/* copy the mail */
 	if (client->mailbox->copy(client->mailbox, destbox,
 				  messageset, client->cmd_uid)) {
-                client_sync_mailbox(client);
+                client_sync_full(client);
 		client_send_tagline(client, "OK Copy completed.");
 	} else
 		client_send_storage_error(client);
--- a/src/imap/cmd-expunge.c	Fri Oct 18 02:57:44 2002 +0300
+++ b/src/imap/cmd-expunge.c	Sat Oct 19 17:51:59 2002 +0300
@@ -8,7 +8,7 @@
 	if (!client_verify_open_mailbox(client))
 		return TRUE;
 
-	if (client_sync_and_expunge_mailbox(client))
+	if (client->mailbox->expunge(client->mailbox, TRUE))
 		client_send_tagline(client, "OK Expunge completed.");
 	else
 		client_send_storage_error(client);
--- a/src/imap/cmd-fetch.c	Fri Oct 18 02:57:44 2002 +0300
+++ b/src/imap/cmd-fetch.c	Sat Oct 19 17:51:59 2002 +0300
@@ -266,7 +266,7 @@
 	if (client->mailbox->fetch(client->mailbox, &data,
 				   client->outbuf, &all_found)) {
 		/* NOTE: syncing isn't allowed here */
-                client_check_new_mail(client);
+                client_sync_without_expunges(client);
 		client_send_tagline(client, all_found ? "OK Fetch completed." :
 				    "NO Some of the requested messages "
 				    "no longer exist.");
--- a/src/imap/cmd-noop.c	Fri Oct 18 02:57:44 2002 +0300
+++ b/src/imap/cmd-noop.c	Sat Oct 19 17:51:59 2002 +0300
@@ -5,7 +5,7 @@
 
 int cmd_noop(Client *client)
 {
-	client_sync_mailbox(client);
+	client_sync_full(client);
 	client_send_tagline(client, "OK NOOP completed.");
 	return TRUE;
 }
--- a/src/imap/cmd-search.c	Fri Oct 18 02:57:44 2002 +0300
+++ b/src/imap/cmd-search.c	Sat Oct 19 17:51:59 2002 +0300
@@ -35,7 +35,7 @@
 		if (client->mailbox->search(client->mailbox, sargs,
 					    client->outbuf, client->cmd_uid)) {
 			/* NOTE: syncing isn't allowed here */
-			client_check_new_mail(client);
+			client_sync_without_expunges(client);
 			client_send_tagline(client, "OK Search completed.");
 		} else {
 			client_send_storage_error(client);
--- a/src/imap/cmd-select.c	Fri Oct 18 02:57:44 2002 +0300
+++ b/src/imap/cmd-select.c	Sat Oct 19 17:51:59 2002 +0300
@@ -4,40 +4,13 @@
 #include "temp-string.h"
 #include "commands.h"
 
-#define SYSTEM_PERMANENT_FLAGS \
-	"\\* \\Answered \\Flagged \\Deleted \\Seen \\Draft"
-#define SYSTEM_FLAGS SYSTEM_PERMANENT_FLAGS " \\Recent"
-
-static const char *
-get_custom_flags_string(const char *custom_flags[MAIL_CUSTOM_FLAGS_COUNT])
-{
-	TempString *str;
-	int i;
-
-	/* first see if there even is custom flags */
-	for (i = 0; i < MAIL_CUSTOM_FLAGS_COUNT; i++) {
-		if (custom_flags[i] != NULL)
-			break;
-	}
-
-	if (i == MAIL_CUSTOM_FLAGS_COUNT)
-		return "";
-
-	str = t_string_new(256);
-	for (; i < MAIL_CUSTOM_FLAGS_COUNT; i++) {
-		if (custom_flags[i] != NULL) {
-			t_string_append_c(str, ' ');
-			t_string_append(str, custom_flags[i]);
-		}
-	}
-	return str->str;
-}
+extern MailboxSyncCallbacks sync_callbacks;
 
 int cmd_select_full(Client *client, int readonly)
 {
 	Mailbox *box;
 	MailboxStatus status;
-	const char *mailbox, *custom_flags;
+	const char *mailbox;
 
 	/* <mailbox> */
 	if (!client_read_string_args(client, 1, &mailbox))
@@ -62,19 +35,10 @@
 		return TRUE;
 	}
 
-	custom_flags = get_custom_flags_string(status.custom_flags);
+	box->set_sync_callbacks(box, &sync_callbacks, client);
 
-	client_send_line(client, t_strconcat("* FLAGS ("SYSTEM_FLAGS,
-					     custom_flags, ")", NULL));
-	if (box->readonly) {
-		client_send_line(client, "* OK [PERMANENTFLAGS ()] "
-				 "Read-only mailbox.");
-	} else {
-		client_send_line(client, t_strconcat("* OK [PERMANENTFLAGS ("
-						     SYSTEM_PERMANENT_FLAGS,
-						     custom_flags, ")] "
-						     "Flags permitted.", NULL));
-	}
+	client_send_mailbox_flags(client, box, status.custom_flags,
+				  status.custom_flags_count);
 
 	client_send_line(client,
 		t_strdup_printf("* %u EXISTS", status.messages));
@@ -92,8 +56,8 @@
 				status.uidvalidity));
 
 	if (status.diskspace_full) {
-		client_send_line(client, "* OK [ALERT] Disk space is full, "
-				 "delete some messages.");
+		client_send_line(client, "* OK [ALERT] "
+				 "Disk space is full, delete some messages.");
 	}
 
 	client_send_tagline(client, box->readonly ?
--- a/src/imap/cmd-store.c	Fri Oct 18 02:57:44 2002 +0300
+++ b/src/imap/cmd-store.c	Sat Oct 19 17:51:59 2002 +0300
@@ -2,35 +2,6 @@
 
 #include "common.h"
 #include "commands.h"
-#include "imap-util.h"
-
-static void update_func(Mailbox *mailbox __attr_unused__, unsigned int seq,
-			unsigned int uid __attr_unused__, MailFlags flags,
-			const char *custom_flags[], void *context)
-{
-	Client *client = context;
-	const char *str;
-
-	t_push();
-	str = imap_write_flags(flags, custom_flags);
-	client_send_line(client,
-			 t_strdup_printf("* %u FETCH (FLAGS (%s))", seq, str));
-	t_pop();
-}
-
-static void update_func_uid(Mailbox *mailbox __attr_unused__, unsigned int seq,
-			    unsigned int uid, MailFlags flags,
-			    const char *custom_flags[], void *context)
-{
-	Client *client = context;
-	const char *str;
-
-	t_push();
-	str = imap_write_flags(flags, custom_flags);
-	client_send_line(client, t_strdup_printf(
-		"* %u FETCH (FLAGS (%s) UID %u)", seq, str, uid));
-	t_pop();
-}
 
 static int get_modify_type(Client *client, const char *item,
 			   ModifyType *modify_type, int *silent)
@@ -66,7 +37,6 @@
 	ImapArg *args;
 	MailFlags flags;
 	ModifyType modify_type;
-	MailFlagUpdateFunc func;
 	const char *custflags[MAIL_CUSTOM_FLAGS_COUNT];
 	const char *messageset, *item;
 	int silent, all_found;
@@ -101,18 +71,18 @@
 	}
 
 	/* and update the flags */
-	func = silent ? NULL :
-		client->cmd_uid ? update_func_uid : update_func;
+	client->sync_flags_send_uid = client->cmd_uid;
 	if (client->mailbox->update_flags(client->mailbox, messageset,
 					  client->cmd_uid, flags, custflags,
-					  modify_type, func, client,
-					  &all_found)) {
+					  modify_type, !silent, &all_found)) {
 		/* NOTE: syncing isn't allowed here */
-                client_check_new_mail(client);
+		client_sync_without_expunges(client);
 		client_send_tagline(client, all_found ? "OK Store completed." :
 				    "NO Some of the messages no longer exist.");
-	} else
+	} else {
 		client_send_storage_error(client);
+	}
 
+	client->sync_flags_send_uid = FALSE;
 	return TRUE;
 }
--- a/src/imap/commands-util.c	Fri Oct 18 02:57:44 2002 +0300
+++ b/src/imap/commands-util.c	Sat Oct 19 17:51:59 2002 +0300
@@ -1,6 +1,7 @@
 /* Copyright (C) 2002 Timo Sirainen */
 
 #include "common.h"
+#include "temp-string.h"
 #include "commands-util.h"
 #include "imap-util.h"
 
@@ -72,68 +73,16 @@
 	}
 }
 
-static void sync_expunge_func(Mailbox *mailbox __attr_unused__,
-			      unsigned int seq,
-			      unsigned int uid __attr_unused__, void *context)
+void client_sync_full(Client *client)
 {
-	Client *client = context;
-	char str[MAX_LARGEST_T_STRLEN+20];
-
-	i_snprintf(str, sizeof(str), "* %u EXPUNGE", seq);
-	client_send_line(client, str);
-}
-
-static void sync_flags_func(Mailbox *mailbox __attr_unused__, unsigned int seq,
-			    unsigned int uid __attr_unused__, MailFlags flags,
-			    const char *custom_flags[], void *context)
-{
-	Client *client = context;
-	const char *str;
-
-	t_push();
-	str = imap_write_flags(flags, custom_flags);
-	client_send_line(client,
-			 t_strdup_printf("* %u FETCH (FLAGS (%s))", seq, str));
-	t_pop();
+	if (client->mailbox != NULL)
+		(void)client->mailbox->sync(client->mailbox, TRUE);
 }
 
-static int client_sync_full(Client *client, int sync_log, int expunge)
+void client_sync_without_expunges(Client *client)
 {
-	unsigned int messages, recent;
-	char str[MAX_LARGEST_T_STRLEN+20];
-
-	if (client->mailbox == NULL)
-		return TRUE;
-
-	if (!client->mailbox->sync(client->mailbox, expunge, &messages, &recent,
-				   sync_log ? sync_expunge_func : NULL,
-				   sync_log ? sync_flags_func : NULL, client))
-		return FALSE;
-
-	if (messages != 0) {
-		i_snprintf(str, sizeof(str), "* %u EXISTS", messages);
-		client_send_line(client, str);
-
-		i_snprintf(str, sizeof(str), "* %u RECENT", recent);
-		client_send_line(client, str);
-	}
-
-	return TRUE;
-}
-
-void client_check_new_mail(Client *client)
-{
-	(void)client_sync_full(client, FALSE, FALSE);
-}
-
-void client_sync_mailbox(Client *client)
-{
-	(void)client_sync_full(client, TRUE, FALSE);
-}
-
-int client_sync_and_expunge_mailbox(Client *client)
-{
-	return client_sync_full(client, TRUE, TRUE);
+	if (client->mailbox != NULL)
+		(void)client->mailbox->sync(client->mailbox, FALSE);
 }
 
 void client_send_storage_error(Client *client)
@@ -212,3 +161,51 @@
 
 	return TRUE;
 }
+
+static const char *get_custom_flags_string(const char *custom_flags[],
+					   unsigned int custom_flags_count)
+{
+	TempString *str;
+	unsigned int i;
+
+	/* first see if there even is custom flags */
+	for (i = 0; i < custom_flags_count; i++) {
+		if (custom_flags[i] != NULL)
+			break;
+	}
+
+	if (i == custom_flags_count)
+		return "";
+
+	str = t_string_new(256);
+	for (; i < custom_flags_count; i++) {
+		if (custom_flags[i] != NULL) {
+			t_string_append_c(str, ' ');
+			t_string_append(str, custom_flags[i]);
+		}
+	}
+	return str->str;
+}
+
+#define SYSTEM_FLAGS "\\Answered \\Flagged \\Deleted \\Seen \\Draft"
+
+void client_send_mailbox_flags(Client *client, Mailbox *box,
+			       const char *custom_flags[],
+			       unsigned int custom_flags_count)
+{
+	const char *str;
+
+	str = get_custom_flags_string(custom_flags, custom_flags_count);
+	client_send_line(client,
+		t_strconcat("* FLAGS ("SYSTEM_FLAGS, str, ")", NULL));
+
+	if (box->readonly) {
+		client_send_line(client, "* OK [PERMANENTFLAGS ()] "
+				 "Read-only mailbox.");
+	} else {
+		client_send_line(client,
+			t_strconcat("* OK [PERMANENTFLAGS ("SYSTEM_FLAGS, str,
+				    box->allow_custom_flags ? " \\*" : "",
+				    ")] Flags permitted.", NULL));
+	}
+}
--- a/src/imap/commands-util.h	Fri Oct 18 02:57:44 2002 +0300
+++ b/src/imap/commands-util.h	Sat Oct 19 17:51:59 2002 +0300
@@ -14,16 +14,12 @@
    error message to client. */
 int client_verify_open_mailbox(Client *client);
 
-/* Check if there's new mail in mailbox. If yes, notify client by sending
-   EXISTS and RECENT. */
-void client_check_new_mail(Client *client);
+/* Synchronize selected mailbox with client by sending EXPUNGE,
+   FETCH FLAGS, EXISTS and RECENT responses. */
+void client_sync_full(Client *client);
 
-/* Synchronize selected mailbox with client by sending EXPUNGE and
-   FETCH FLAGS responses. Also does new mail checking. */
-void client_sync_mailbox(Client *client);
-
-/* Synchronize selected mailbox and expunge messages with \Deleted flag. */
-int client_sync_and_expunge_mailbox(Client *client);
+/* Synchronize all but expunges with client. */
+void client_sync_without_expunges(Client *client);
 
 /* Send last mail storage error message to client. */
 void client_send_storage_error(Client *client);
@@ -35,4 +31,9 @@
 			    MailFlags *flags,
 			    const char *custflags[MAIL_CUSTOM_FLAGS_COUNT]);
 
+/* Send FLAGS + PERMANENTFLAGS to client. */
+void client_send_mailbox_flags(Client *client, Mailbox *box,
+			       const char *custom_flags[],
+			       unsigned int custom_flags_count);
+
 #endif
--- a/src/lib-imap/imap-util.c	Fri Oct 18 02:57:44 2002 +0300
+++ b/src/lib-imap/imap-util.c	Sat Oct 19 17:51:59 2002 +0300
@@ -4,11 +4,14 @@
 #include "temp-string.h"
 #include "imap-util.h"
 
-const char *imap_write_flags(MailFlags flags, const char *custom_flags[])
+const char *imap_write_flags(MailFlags flags, const char *custom_flags[],
+			     unsigned int custom_flags_count)
 {
 	TempString *str;
 	const char *sysflags, *name;
-	int i;
+	unsigned int i;
+
+	i_assert(custom_flags_count <= MAIL_CUSTOM_FLAGS_COUNT);
 
 	if (flags == 0)
 		return "";
@@ -31,7 +34,7 @@
 	str = t_string_new(256);
 	t_string_append(str, sysflags);
 
-	for (i = 0; i < MAIL_CUSTOM_FLAGS_COUNT; i++) {
+	for (i = 0; i < custom_flags_count; i++) {
 		if (flags & (1 << (i + MAIL_CUSTOM_FLAG_1_BIT))) {
 			name = custom_flags[i];
 			if (name != NULL && *name != '\0') {
--- a/src/lib-imap/imap-util.h	Fri Oct 18 02:57:44 2002 +0300
+++ b/src/lib-imap/imap-util.h	Sat Oct 19 17:51:59 2002 +0300
@@ -32,7 +32,8 @@
 
 /* Return flags as a space separated string. custom_flags[] is a list of
    names for custom flags, flags having NULL or "" entry are ignored. */
-const char *imap_write_flags(MailFlags flags, const char *custom_flags[]);
+const char *imap_write_flags(MailFlags flags, const char *custom_flags[],
+			     unsigned int custom_flags_count);
 
 /* Escape the string */
 const char *imap_escape(const char *str);
--- a/src/lib-index/mail-custom-flags.c	Fri Oct 18 02:57:44 2002 +0300
+++ b/src/lib-index/mail-custom-flags.c	Sat Oct 19 17:51:59 2002 +0300
@@ -35,17 +35,19 @@
 
 	unsigned int syncing:1;
 	unsigned int noupdate:1;
+	unsigned int changed:1;
 };
 
 static int lock_file(MailCustomFlags *mcf, int type);
 
-static void index_cf_set_syscall_error(MailCustomFlags *mcf,
+static int index_cf_set_syscall_error(MailCustomFlags *mcf,
 				       const char *function)
 {
 	i_assert(function != NULL);
 
 	index_set_error(mcf->index, "%s failed with custom flags file %s: %m",
 			function, mcf->filepath);
+	return FALSE;
 }
 
 static int update_mmap(MailCustomFlags *mcf)
@@ -53,8 +55,7 @@
 	mcf->mmap_base = mmap_rw_file(mcf->fd, &mcf->mmap_length);
 	if (mcf->mmap_base == MAP_FAILED) {
 		mcf->mmap_base = NULL;
-		index_cf_set_syscall_error(mcf, "mmap()");
-		return FALSE;
+		return index_cf_set_syscall_error(mcf, "mmap()");
 	}
 
 	(void)madvise(mcf->mmap_base, mcf->mmap_length, MADV_SEQUENTIAL);
@@ -191,6 +192,7 @@
 	}
 
 	custom_flags_sync(mcf);
+	mcf->changed = TRUE;
 	return TRUE;
 }
 
@@ -199,10 +201,8 @@
 	if (mcf->lock_type == type)
 		return TRUE;
 
-	if (file_wait_lock(mcf->fd, type) < 0) {
-		index_cf_set_syscall_error(mcf, "file_wait_lock()");
-		return FALSE;
-	}
+	if (file_wait_lock(mcf->fd, type) < 0)
+		return index_cf_set_syscall_error(mcf, "file_wait_lock()");
 
 	mcf->lock_type = type;
 
@@ -290,10 +290,8 @@
 {
 	int i;
 
-	if (lseek(mcf->fd, 0, SEEK_SET) < 0) {
-		index_cf_set_syscall_error(mcf, "lseek()");
-		return FALSE;
-	}
+	if (lseek(mcf->fd, 0, SEEK_SET) < 0)
+		return index_cf_set_syscall_error(mcf, "lseek()");
 
 	for (i = COUNTER_SIZE-1; i >= 0; i--) {
 		if (mcf->sync_counter[i] == '9') {
@@ -310,11 +308,10 @@
 		}
 	}
 
-	if (write_full(mcf->fd, mcf->sync_counter, COUNTER_SIZE) < 0) {
-		index_cf_set_syscall_error(mcf, "write_full()");
-		return FALSE;
-	}
+	if (write_full(mcf->fd, mcf->sync_counter, COUNTER_SIZE) < 0)
+		return index_cf_set_syscall_error(mcf, "write_full()");
 
+	mcf->changed = TRUE;
 	return TRUE;
 }
 
@@ -332,10 +329,8 @@
 
 	/* add the flag */
 	pos = lseek(mcf->fd, 0, SEEK_END);
-	if (pos < 0) {
-		index_cf_set_syscall_error(mcf, "lseek()");
-		return FALSE;
-	}
+	if (pos < 0)
+		return index_cf_set_syscall_error(mcf, "lseek()");
 
 	if (pos != (off_t)mcf->mmap_length) {
 		index_set_error(mcf->index, "Custom flags file %s was "
@@ -353,10 +348,8 @@
 		len--;
 	}
 
-	if (write_full(mcf->fd, buf, len) < 0) {
-		index_cf_set_syscall_error(mcf, "write_full()");
-		return FALSE;
-	}
+	if (write_full(mcf->fd, buf, len) < 0)
+		return index_cf_set_syscall_error(mcf, "write_full()");
 
 	if (!update_mmap(mcf))
 		return FALSE;
@@ -484,6 +477,8 @@
 	if (mcf->lock_type != F_WRLCK) {
 		if (!lock_file(mcf, F_UNLCK) || !lock_file(mcf, F_WRLCK))
 			return -1;
+
+		// FIXME: sync and check it again
 	}
 
 	/* new flag, add it. first find the first free flag, note that
@@ -559,3 +554,13 @@
 
 	mcf->custom_flags_refcount--;
 }
+
+int mail_custom_flags_has_changes(MailCustomFlags *mcf)
+{
+	if (!mcf->changed)
+		return FALSE;
+	else {
+		mcf->changed = FALSE;
+		return TRUE;
+	}
+}
--- a/src/lib-index/mail-custom-flags.h	Fri Oct 18 02:57:44 2002 +0300
+++ b/src/lib-index/mail-custom-flags.h	Sat Oct 19 17:51:59 2002 +0300
@@ -21,4 +21,8 @@
 /* Call this after you've done with the flags list above */
 void mail_custom_flags_list_unref(MailCustomFlags *mcf);
 
+/* Returns TRUE if there's been any changes since this function was
+   called last time, or since open if this is the first call. */
+int mail_custom_flags_has_changes(MailCustomFlags *mcf);
+
 #endif
--- a/src/lib-index/mail-index.c	Fri Oct 18 02:57:44 2002 +0300
+++ b/src/lib-index/mail-index.c	Sat Oct 19 17:51:59 2002 +0300
@@ -338,6 +338,33 @@
 		return FALSE;
 	}
 
+	if (index->header->flags & MAIL_INDEX_FLAG_FSCK) {
+		/* someone just partially updated the index, need to fsck it */
+		if (lock_type == MAIL_LOCK_SHARED) {
+			/* we need exclusive lock so fsck()'s set_lock() won't
+			   get us back here */
+			if (!mail_index_lock_remove(index))
+				return FALSE;
+
+			if (file_wait_lock(index->fd, MAIL_LOCK_EXCLUSIVE) < 0)
+				return index_set_syscall_error(index,
+							"file_wait_lock()");
+			index->lock_type = MAIL_LOCK_EXCLUSIVE;
+		}
+
+		/* check again, in case it was already fscked while we had
+		   it unlocked for a while */
+		if (index->header->flags & MAIL_INDEX_FLAG_FSCK) {
+			if (!index->fsck(index))
+				return FALSE;
+		}
+
+		if (lock_type == MAIL_LOCK_SHARED) {
+			/* drop exclusive lock */
+			return mail_index_set_lock(index, lock_type);
+		}
+	}
+
 	if (lock_type == MAIL_LOCK_EXCLUSIVE) {
 		/* while holding exclusive lock, keep the FSCK flag on.
 		   when the lock is released, the FSCK flag will also be
--- a/src/lib-index/mail-index.h	Fri Oct 18 02:57:44 2002 +0300
+++ b/src/lib-index/mail-index.h	Sat Oct 19 17:51:59 2002 +0300
@@ -150,7 +150,11 @@
         (SIZEOF_MAIL_INDEX_DATA + (rec)->full_field_size)
 
 struct _MailIndex {
-	/* If fast is TRUE, compressing and cache updates are not performed. */
+	/* If fast is TRUE, compressing and cache updates are not performed.
+	   Note that opening same index twice in the same process is a bad
+	   idea since they share the same file locks. As soon one of the
+	   indexes is closed, the locks in second index are dropped which
+	   especially hurts modify log since it keeps locks all the time. */
 	int (*open)(MailIndex *index, int update_recent, int fast);
 	int (*open_or_create)(MailIndex *index, int update_recent, int fast);
 
@@ -184,7 +188,8 @@
 	int (*rebuild)(MailIndex *index);
 
 	/* Verify that the index is valid. If anything invalid is found,
-	   rebuild() is called. Same locking issues as with rebuild(). */
+	   index is set inconsistent and to be rebuilt at next open.
+	   Same locking issues as with rebuild(). */
 	int (*fsck)(MailIndex *index);
 
 	/* Synchronize the index with the mailbox. Same locking issues as
--- a/src/lib-index/mail-modifylog.c	Fri Oct 18 02:57:44 2002 +0300
+++ b/src/lib-index/mail-modifylog.c	Sat Oct 19 17:51:59 2002 +0300
@@ -98,7 +98,7 @@
 
 	if (ret == 0) {
 		/* shouldn't happen */
-		index_set_error(log->index, "file_lock(F_WRLCK -> F_RDLCK) "
+		index_set_error(log->index, "file_try_lock(F_WRLCK -> F_RDLCK) "
 				"failed with file %s", log->filepath);
 		return -1;
 	}
@@ -807,7 +807,7 @@
 	expunges = arr = t_malloc((max_records+1) * sizeof(unsigned int));
 
 	before = 0;
-	while (rec < end_rec) {
+	for (; rec < end_rec; rec++) {
 		if (rec->type != RECORD_TYPE_EXPUNGE)
 			continue;
 
@@ -825,7 +825,6 @@
 			/* before our range */
 			before++;
 		}
-		rec++;
 	}
 	*arr = 0;
 
--- a/src/lib-storage/index/index-expunge.c	Fri Oct 18 02:57:44 2002 +0300
+++ b/src/lib-storage/index/index-expunge.c	Sat Oct 19 17:51:59 2002 +0300
@@ -43,7 +43,7 @@
 	return TRUE;
 }
 
-int index_storage_expunge(Mailbox *box)
+int index_storage_expunge(Mailbox *box, int notify)
 {
 	IndexMailbox *ibox = (IndexMailbox *) box;
 	int failed;
@@ -53,12 +53,20 @@
 		return FALSE;
 	}
 
+	if (!index_storage_sync_index_if_possible(ibox))
+		return FALSE;
+
 	if (!ibox->index->set_lock(ibox->index, MAIL_LOCK_EXCLUSIVE))
 		return mail_storage_set_index_error(ibox);
 
-	failed = !ibox->expunge_locked(ibox, NULL, NULL);
+	/* modifylog must be marked synced before expunging anything new */
+	failed = !index_storage_sync_modifylog(ibox);
+
+	if (!failed)
+		failed = !ibox->expunge_locked(ibox, notify);
 
-	if (!ibox->index->set_lock(ibox->index, MAIL_LOCK_UNLOCK) || failed)
+	if (!ibox->index->set_lock(ibox->index, MAIL_LOCK_UNLOCK))
 		return mail_storage_set_index_error(ibox);
-	return TRUE;
+
+	return !failed;
 }
--- a/src/lib-storage/index/index-fetch.c	Fri Oct 18 02:57:44 2002 +0300
+++ b/src/lib-storage/index/index-fetch.c	Sat Oct 19 17:51:59 2002 +0300
@@ -91,7 +91,8 @@
 		flags |= MAIL_SEEN;
 
 	t_string_printfa(ctx->str, "FLAGS (%s) ",
-			 imap_write_flags(flags, ctx->custom_flags));
+			 imap_write_flags(flags, ctx->custom_flags,
+					  ctx->custom_flags_count));
 }
 
 static void index_fetch_internaldate(MailIndexRecord *rec, FetchContext *ctx)
@@ -348,7 +349,7 @@
 	MailFetchBodyData *sect;
 	int ret;
 
-	if (!index_storage_sync_if_possible(ibox))
+	if (!index_storage_sync_index_if_possible(ibox))
 		return FALSE;
 
 	memset(&ctx, 0, sizeof(ctx));
@@ -388,6 +389,7 @@
 	ctx.index = ibox->index;
 	ctx.custom_flags =
 		mail_custom_flags_list_get(ibox->index->custom_flags);
+        ctx.custom_flags_count = MAIL_CUSTOM_FLAGS_COUNT;
 
 	ctx.fetch_data = fetch_data;
 	ctx.outbuf = outbuf;
--- a/src/lib-storage/index/index-fetch.h	Fri Oct 18 02:57:44 2002 +0300
+++ b/src/lib-storage/index/index-fetch.h	Sat Oct 19 17:51:59 2002 +0300
@@ -6,7 +6,9 @@
 	MailStorage *storage;
 	ImapMessageCache *cache;
 	MailIndex *index;
+
 	const char **custom_flags;
+	unsigned int custom_flags_count;
 
 	MailFetchData *fetch_data;
 	OBuffer *outbuf;
--- a/src/lib-storage/index/index-messageset.c	Fri Oct 18 02:57:44 2002 +0300
+++ b/src/lib-storage/index/index-messageset.c	Sat Oct 19 17:51:59 2002 +0300
@@ -135,11 +135,6 @@
 							     messageset, NULL);
 					return -2;
 				}
-
-				if (seq2 > messages_count) {
-					/* too large .. ignore silently */
-					seq2 = messages_count;
-				}
 			} else {
 				seq2 = messages_count;
 				input++;
@@ -155,18 +150,24 @@
 			return -2;
 		}
 
-		if (seq > messages_count) {
-			/* too large .. ignore silently */
-		} else {
-			t_push();
-			ret = mail_index_foreach(index, seq, seq2,
-						 func, context, error);
-			t_pop();
-			if (ret <= 0)
-				return ret;
-			if (ret == 2)
-				all_found = FALSE;
+		if (seq > messages_count || seq2 > messages_count) {
+			/* non-existent messages requested */
+			if (seq <= messages_count)
+				seq = seq2;
+			*error = t_strdup_printf("Message sequence %u "
+						 "larger than mailbox size %u",
+						 seq, messages_count);
+			return -2;
 		}
+
+		t_push();
+		ret = mail_index_foreach(index, seq, seq2,
+					 func, context, error);
+		t_pop();
+		if (ret <= 0)
+			return ret;
+		if (ret == 2)
+			all_found = FALSE;
 	}
 
 	return all_found ? 1 : 2;
--- a/src/lib-storage/index/index-search.c	Fri Oct 18 02:57:44 2002 +0300
+++ b/src/lib-storage/index/index-search.c	Sat Oct 19 17:51:59 2002 +0300
@@ -879,7 +879,7 @@
 	IndexMailbox *ibox = (IndexMailbox *) box;
 	int failed;
 
-	if (!index_storage_sync_if_possible(ibox))
+	if (!index_storage_sync_index_if_possible(ibox))
 		return FALSE;
 
 	if (!ibox->index->set_lock(ibox->index, MAIL_LOCK_SHARED))
--- a/src/lib-storage/index/index-status.c	Fri Oct 18 02:57:44 2002 +0300
+++ b/src/lib-storage/index/index-status.c	Sat Oct 19 17:51:59 2002 +0300
@@ -49,15 +49,17 @@
 }
 
 static void
-get_custom_flags(MailCustomFlags *mcf,
-		 const char *result[MAIL_CUSTOM_FLAGS_COUNT])
+get_custom_flags(MailCustomFlags *mcf, MailboxStatus *status)
 {
 	const char **flags;
-	int i;
+	unsigned int i;
+
+	status->custom_flags_count = MAIL_CUSTOM_FLAGS_COUNT;
+	status->custom_flags = t_new(const char *, MAIL_CUSTOM_FLAGS_COUNT);
 
 	flags = mail_custom_flags_list_get(mcf);
 	for (i = 0; i < MAIL_CUSTOM_FLAGS_COUNT; i++)
-		result[i] = t_strdup(flags[i]);
+		status->custom_flags[i] = t_strdup(flags[i]);
 	mail_custom_flags_list_unref(mcf);
 }
 
@@ -69,12 +71,20 @@
 
 	memset(status, 0, sizeof(MailboxStatus));
 
-	if (!index_storage_sync_if_possible(ibox))
+	/* if we're doing STATUS for selected mailbox, we have to sync it
+	   first or STATUS reply may give different data */
+	if (!index_storage_sync_index_if_possible(ibox))
 		return FALSE;
 
 	if (!ibox->index->set_lock(ibox->index, MAIL_LOCK_SHARED))
 		return mail_storage_set_index_error(ibox);
 
+	if (!index_storage_sync_modifylog(ibox)) {
+		if (!ibox->index->set_lock(ibox->index, MAIL_LOCK_UNLOCK))
+			return mail_storage_set_index_error(ibox);
+		return FALSE;
+	}
+
 	/* we can get most of the status items without any trouble */
 	hdr = mail_index_get_header(ibox->index);
 	status->messages = hdr->messages_count;
@@ -91,13 +101,8 @@
 	if (items & STATUS_RECENT)
 		status->recent = index_storage_get_recent_count(ibox->index);
 
-	if (items & STATUS_CUSTOM_FLAGS) {
-		get_custom_flags(ibox->index->custom_flags,
-				 status->custom_flags);
-	}
-
-	/* STATUS sends EXISTS, so we've synced it */
-	ibox->synced_messages_count = hdr->messages_count;
+	if (items & STATUS_CUSTOM_FLAGS)
+		get_custom_flags(ibox->index->custom_flags, status);
 
 	if (!ibox->index->set_lock(ibox->index, MAIL_LOCK_UNLOCK))
 		return mail_storage_set_index_error(ibox);
--- a/src/lib-storage/index/index-storage.c	Fri Oct 18 02:57:44 2002 +0300
+++ b/src/lib-storage/index/index-storage.c	Sat Oct 19 17:51:59 2002 +0300
@@ -6,32 +6,120 @@
 #include "mail-custom-flags.h"
 #include "index-storage.h"
 
+#include <unistd.h>
+#include <sys/stat.h>
+
+typedef struct _IndexList IndexList;
+
+struct _IndexList {
+	IndexList *next;
+
+	MailIndex *index;
+	int refcount;
+};
+
+static IndexList *indexes = NULL;
+
+void index_storage_add(MailIndex *index)
+{
+	IndexList *list;
+
+	list = i_new(IndexList, 1);
+	list->refcount = 1;
+	list->index = index;
+
+	list->next = indexes;
+	indexes = list;
+}
+
+MailIndex *index_storage_lookup_ref(const char *path)
+{
+	IndexList *list;
+	struct stat st1, st2;
+
+	if (stat(path, &st1) < 0)
+		return NULL;
+
+	/* compare inodes so we don't break even with symlinks */
+	for (list = indexes; list != NULL; list = list->next) {
+		if (stat(list->index->dir, &st2) == 0) {
+			if (st1.st_ino == st2.st_ino &&
+			    st1.st_dev == st2.st_dev) {
+				list->refcount++;
+				return list->index;
+			}
+		}
+	}
+
+	return NULL;
+}
+
+void index_storage_unref(MailIndex *index)
+{
+	IndexList **list, *rec;
+
+	for (list = &indexes; *list != NULL; list = &(*list)->next) {
+		rec = *list;
+
+		if (rec->index == index) {
+			if (rec->refcount == 0) {
+				index->free(index);
+				*list = rec->next;
+				i_free(rec);
+			}
+			return;
+		}
+	}
+
+	/* shouldn't get here */
+	i_assert(0);
+}
+
 IndexMailbox *index_storage_init(MailStorage *storage, Mailbox *box,
 				 MailIndex *index, const char *name,
 				 int readonly, int fast)
 {
 	IndexMailbox *ibox;
+	MailIndexHeader *hdr;
+	unsigned int messages;
 
 	i_assert(name != NULL);
 
-	/* open the index first */
-	if (!index->open_or_create(index, !readonly, fast)) {
-		mail_storage_set_internal_error(storage);
-		index->free(index);
-		return NULL;
-	}
+	do {
+		if (!index->opened) {
+			/* open the index first */
+			if (!index->open_or_create(index, !readonly, fast))
+				break;
+		}
+
+		/* Get the synced messages count */
+		if (!index->set_lock(index, MAIL_LOCK_SHARED))
+			break;
+
+		hdr = mail_index_get_header(index);
+		messages = hdr->messages_count;
+
+		if (!index->set_lock(index, MAIL_LOCK_UNLOCK))
+			break;
 
-	ibox = i_new(IndexMailbox, 1);
-	ibox->box = *box;
+		ibox = i_new(IndexMailbox, 1);
+		ibox->box = *box;
+
+		ibox->box.storage = storage;
+		ibox->box.name = i_strdup(name);
+		ibox->box.readonly = readonly;
+		ibox->box.allow_custom_flags = TRUE;
 
-	ibox->box.storage = storage;
-	ibox->box.name = i_strdup(name);
-	ibox->box.readonly = readonly;
+		ibox->index = index;
+		ibox->cache = imap_msgcache_alloc(&index_msgcache_iface);
+		ibox->synced_messages_count = messages;
 
-	ibox->index = index;
-	ibox->cache = imap_msgcache_alloc(&index_msgcache_iface);
+		return ibox;
+	} while (0);
 
-	return ibox;
+	mail_storage_set_internal_error(storage);
+	index_storage_unref(index);
+	return NULL;
 }
 
 void index_storage_close(Mailbox *box)
@@ -39,11 +127,21 @@
 	IndexMailbox *ibox = (IndexMailbox *) box;
 
 	imap_msgcache_free(ibox->cache);
-	ibox->index->free(ibox->index);
+	index_storage_unref(ibox->index);
 	i_free(box->name);
 	i_free(box);
 }
 
+void index_storage_set_sync_callbacks(Mailbox *box,
+				      MailboxSyncCallbacks *callbacks,
+				      void *context)
+{
+	IndexMailbox *ibox = (IndexMailbox *) box;
+
+	memcpy(&ibox->sync_callbacks, callbacks, sizeof(MailboxSyncCallbacks));
+	ibox->sync_context = context;
+}
+
 int mail_storage_set_index_error(IndexMailbox *ibox)
 {
 	ibox->box.inconsistent =
--- a/src/lib-storage/index/index-storage.h	Fri Oct 18 02:57:44 2002 +0300
+++ b/src/lib-storage/index/index-storage.h	Sat Oct 19 17:51:59 2002 +0300
@@ -12,24 +12,33 @@
 
 	/* expunge messages marked as deleted, requires index to be
 	   exclusively locked */
-	int (*expunge_locked)(IndexMailbox *ibox,
-			      MailExpungeFunc expunge_func, void *context);
+	int (*expunge_locked)(IndexMailbox *ibox, int notify);
+
+        MailboxSyncCallbacks sync_callbacks;
+	void *sync_context;
 
 	MailIndex *index;
 	ImapMessageCache *cache;
 	unsigned int synced_messages_count;
+
+	unsigned int sent_diskspace_warning:1;
 };
 
 extern ImapMessageCacheIface index_msgcache_iface;
 
 int mail_storage_set_index_error(IndexMailbox *ibox);
 
+void index_storage_add(MailIndex *index);
+MailIndex *index_storage_lookup_ref(const char *path);
+void index_storage_unref(MailIndex *index);
+
 IndexMailbox *index_storage_init(MailStorage *storage, Mailbox *box,
 				 MailIndex *index, const char *name,
 				 int readonly, int fast);
 void index_storage_close(Mailbox *box);
 
-int index_storage_sync_if_possible(IndexMailbox *ibox);
+int index_storage_sync_index_if_possible(IndexMailbox *ibox);
+int index_storage_sync_modifylog(IndexMailbox *ibox);
 
 int index_mailbox_fix_custom_flags(IndexMailbox *ibox, MailFlags *flags,
                                    const char *custom_flags[]);
@@ -45,20 +54,18 @@
 void *index_msgcache_get_context(MailIndex *index, MailIndexRecord *rec);
 
 /* Mailbox methods: */
+void index_storage_set_sync_callbacks(Mailbox *box,
+				      MailboxSyncCallbacks *callbacks,
+				      void *context);
 int index_storage_copy(Mailbox *box, Mailbox *destbox,
 		       const char *messageset, int uidset);
-int index_storage_expunge(Mailbox *box);
+int index_storage_expunge(Mailbox *box, int notify);
 int index_storage_get_status(Mailbox *box, MailboxStatusItems items,
 			     MailboxStatus *status);
-int index_storage_sync(Mailbox *box, int expunge,
-		       unsigned int *messages, unsigned int *recent,
-		       MailExpungeFunc expunge_func,
-		       MailFlagUpdateFunc flag_func,
-		       void *context);
+int index_storage_sync(Mailbox *box, int sync_expunges);
 int index_storage_update_flags(Mailbox *box, const char *messageset, int uidset,
 			       MailFlags flags, const char *custom_flags[],
-			       ModifyType modify_type,
-			       MailFlagUpdateFunc func, void *context,
+			       ModifyType modify_type, int notify,
 			       int *all_found);
 int index_storage_fetch(Mailbox *box, MailFetchData *fetch_data,
 			OBuffer *outbuf, int *all_found);
--- a/src/lib-storage/index/index-sync.c	Fri Oct 18 02:57:44 2002 +0300
+++ b/src/lib-storage/index/index-sync.c	Sat Oct 19 17:51:59 2002 +0300
@@ -7,122 +7,137 @@
 #include "mail-custom-flags.h"
 
 /* may leave the index locked */
-int index_storage_sync_if_possible(IndexMailbox *ibox)
+int index_storage_sync_index_if_possible(IndexMailbox *ibox)
 {
-	if (!ibox->index->sync(ibox->index)) {
+	unsigned int messages, recent;
+	const char **custom_flags;
+
+	if (ibox->index->sync(ibox->index)) {
+		/* reset every time it has worked */
+		ibox->sent_diskspace_warning = FALSE;
+	} else {
 		if (!ibox->index->is_diskspace_error(ibox->index)) {
 			(void)ibox->index->set_lock(ibox->index,
 						    MAIL_LOCK_UNLOCK);
 			return mail_storage_set_index_error(ibox);
 		}
 
-		/* not enough disk space to sync. can't do much about it
-		   though, giving error message would just make it impossible
-		   to delete messages. */
+		/* notify client once about it */
+		if (!ibox->sent_diskspace_warning) {
+			ibox->sent_diskspace_warning = TRUE;
+			ibox->sync_callbacks.alert_no_diskspace(
+						&ibox->box, ibox->sync_context);
+		}
+
 		index_reset_error(ibox->index);
 	}
 
+	/* notify about changes in mailbox size. */
+	if (ibox->index->lock_type == MAIL_LOCK_UNLOCK)
+		return TRUE; /* no changes - must be no new mail either */
+
+	messages = ibox->index->get_header(ibox->index)->messages_count;
+	messages += mail_modifylog_get_expunge_count(ibox->index->modifylog);
+	if (messages != ibox->synced_messages_count) {
+		i_assert(messages > ibox->synced_messages_count);
+
+		/* new messages in mailbox */
+		recent = index_storage_get_recent_count(ibox->index);
+		ibox->sync_callbacks.new_messages(&ibox->box, messages, recent,
+						  ibox->sync_context);
+		ibox->synced_messages_count = messages;
+	}
+
+	/* notify changes in custom flags */
+	if (mail_custom_flags_has_changes(ibox->index->custom_flags)) {
+		custom_flags = mail_custom_flags_list_get(
+					ibox->index->custom_flags);
+		ibox->sync_callbacks.new_custom_flags(
+			&ibox->box, custom_flags, MAIL_CUSTOM_FLAGS_COUNT,
+			ibox->sync_context);
+		mail_custom_flags_list_unref(ibox->index->custom_flags);
+	}
+
 	return TRUE;
 }
 
-static int index_storage_sync_log(Mailbox *box, MailIndex *index,
-				  MailExpungeFunc expunge_func,
-				  MailFlagUpdateFunc flag_func,
-				  void *context)
+int index_storage_sync_modifylog(IndexMailbox *ibox)
 {
 	ModifyLogRecord *log;
 	MailIndexRecord *rec;
 	MailFlags flags;
+        MailboxSyncCallbacks *sc;
+	void *sc_context;
 	const char **custom_flags;
 	unsigned int count, seq;
 
 	/* show the log */
-	log = mail_modifylog_get_nonsynced(index->modifylog, &count);
+	log = mail_modifylog_get_nonsynced(ibox->index->modifylog, &count);
 	if (log == NULL)
-		return FALSE;
+		return mail_storage_set_index_error(ibox);
+
+	sc = &ibox->sync_callbacks;
+	sc_context = ibox->sync_context;
 
-	custom_flags = mail_custom_flags_list_get(index->custom_flags);
+	custom_flags = mail_custom_flags_list_get(ibox->index->custom_flags);
 	for (; count > 0; count--, log++) {
+		if (log->seq > ibox->synced_messages_count) {
+			/* client doesn't know about this message yet */
+			continue;
+		}
+
 		switch (log->type) {
 		case RECORD_TYPE_EXPUNGE:
-			if (expunge_func != NULL) {
-				expunge_func(box, log->seq,
-					     log->uid, context);
-			}
+			sc->expunge(&ibox->box, log->seq,
+				    log->uid, sc_context);
+                        ibox->synced_messages_count--;
 			break;
 		case RECORD_TYPE_FLAGS_CHANGED:
-			if (flag_func == NULL)
+			rec = ibox->index->lookup_uid_range(ibox->index,
+							    log->uid, log->uid,
+							    &seq);
+			if (rec == NULL)
 				break;
 
-			rec = index->lookup_uid_range(index,
-						      log->uid, log->uid, &seq);
-			if (rec != NULL) {
-				flags = rec->msg_flags;
-				if (rec->uid >= index->first_recent_uid)
-					flags |= MAIL_RECENT;
+			flags = rec->msg_flags;
+			if (rec->uid >= ibox->index->first_recent_uid)
+				flags |= MAIL_RECENT;
 
-				flag_func(box, log->seq, log->uid, flags,
-					  custom_flags, context);
-			}
+			sc->update_flags(&ibox->box, log->seq, log->uid, flags,
+					 custom_flags, MAIL_CUSTOM_FLAGS_COUNT,
+					 sc_context);
 			break;
 		}
 	}
-	mail_custom_flags_list_unref(index->custom_flags);
+	mail_custom_flags_list_unref(ibox->index->custom_flags);
 
 	/* mark synced */
-	return mail_modifylog_mark_synced(index->modifylog);
-}
-
-int index_storage_sync(Mailbox *box, int expunge,
-		       unsigned int *messages, unsigned int *recent,
-		       MailExpungeFunc expunge_func,
-		       MailFlagUpdateFunc flag_func,
-		       void *context)
-{
-	IndexMailbox *ibox = (IndexMailbox *) box;
-	unsigned int count;
-	int failed;
-
-	if (expunge && box->readonly) {
-		mail_storage_set_error(box->storage, "Mailbox is read-only");
-		return FALSE;
-	}
-
-	*messages = *recent = 0;
-
-	if (!index_storage_sync_if_possible(ibox))
-		return FALSE;
-
-	if (!ibox->index->set_lock(ibox->index, expunge ?
-				   MAIL_LOCK_EXCLUSIVE : MAIL_LOCK_SHARED))
+	if (!mail_modifylog_mark_synced(ibox->index->modifylog))
 		return mail_storage_set_index_error(ibox);
 
-	failed = FALSE;
+	return TRUE;
+}
+
+int index_storage_sync(Mailbox *box, int sync_expunges)
+{
+	IndexMailbox *ibox = (IndexMailbox *) box;
+	int failed;
 
-	if (expunge_func != NULL || flag_func != NULL) {
-		failed = !index_storage_sync_log(box, ibox->index, expunge_func,
-						 flag_func, context);
-	}
+	if (!index_storage_sync_index_if_possible(ibox))
+		return FALSE;
 
-	if (!failed && expunge) {
-		/* expunge messages */
-		failed = !ibox->expunge_locked(ibox, expunge_func, context);
+	if (!sync_expunges) {
+		/* FIXME: we could still send flag changes */
+		failed = FALSE;
+	} else {
+		if (!ibox->index->set_lock(ibox->index, MAIL_LOCK_SHARED))
+			return mail_storage_set_index_error(ibox);
+
+		failed = !index_storage_sync_modifylog(ibox);
 	}
 
-	/* get the messages count even if there was some failures.
-	   also it must be done after expunging messages */
-	count = ibox->index->get_header(ibox->index)->messages_count;
-	count += mail_modifylog_get_expunge_count(ibox->index->modifylog);
-	if (count != ibox->synced_messages_count) {
-		if (count > ibox->synced_messages_count) {
-			/* new messages in mailbox */
-			*messages = count;
-			*recent = index_storage_get_recent_count(ibox->index);
-		}
-		ibox->synced_messages_count = count;
-	}
+	if (!ibox->index->set_lock(ibox->index, MAIL_LOCK_UNLOCK))
+		return mail_storage_set_index_error(ibox);
 
-	if (!ibox->index->set_lock(ibox->index, MAIL_LOCK_UNLOCK) || failed)
-		return mail_storage_set_index_error(ibox);
-	return TRUE;
+	return !failed;
 }
--- a/src/lib-storage/index/index-update-flags.c	Fri Oct 18 02:57:44 2002 +0300
+++ b/src/lib-storage/index/index-update-flags.c	Sat Oct 19 17:51:59 2002 +0300
@@ -6,12 +6,10 @@
 #include "mail-custom-flags.h"
 
 typedef struct {
-	Mailbox *box;
-	MailCustomFlags *custom_flags;
+	IndexMailbox *ibox;
 	MailFlags flags;
 	ModifyType modify_type;
-	MailFlagUpdateFunc func;
-	void *context;
+	int notify;
 } UpdateContext;
 
 static int update_func(MailIndex *index, MailIndexRecord *rec,
@@ -20,6 +18,7 @@
 {
 	UpdateContext *ctx = context;
 	MailFlags flags;
+	const char **custom_flags;
 
 	switch (ctx->modify_type) {
 	case MODIFY_ADD:
@@ -39,22 +38,33 @@
 	if (!index->update_flags(index, rec, idx_seq, flags, FALSE))
 		return FALSE;
 
-	if (rec->uid >= index->first_recent_uid)
-		flags |= MAIL_RECENT;
+	if (mail_custom_flags_has_changes(index->custom_flags)) {
+                custom_flags = mail_custom_flags_list_get(index->custom_flags);
+		ctx->ibox->sync_callbacks.new_custom_flags(
+			&ctx->ibox->box, custom_flags, MAIL_CUSTOM_FLAGS_COUNT,
+			ctx->ibox->sync_context);
+		mail_custom_flags_list_unref(index->custom_flags);
+	}
 
-	if (ctx->func != NULL) {
-		ctx->func(ctx->box, client_seq, rec->uid, flags,
-			  mail_custom_flags_list_get(ctx->custom_flags),
-			  ctx->context);
-		mail_custom_flags_list_unref(ctx->custom_flags);
+	if (ctx->notify) {
+		if (rec->uid >= index->first_recent_uid)
+			flags |= MAIL_RECENT;
+
+                custom_flags = mail_custom_flags_list_get(index->custom_flags);
+		ctx->ibox->sync_callbacks.update_flags(&ctx->ibox->box,
+						       client_seq, rec->uid,
+						       flags, custom_flags,
+						       MAIL_CUSTOM_FLAGS_COUNT,
+						       ctx->ibox->sync_context);
+		mail_custom_flags_list_unref(index->custom_flags);
 	}
+
 	return TRUE;
 }
 
 int index_storage_update_flags(Mailbox *box, const char *messageset, int uidset,
 			       MailFlags flags, const char *custom_flags[],
-			       ModifyType modify_type,
-			       MailFlagUpdateFunc func, void *context,
+			       ModifyType modify_type, int notify,
 			       int *all_found)
 {
 	IndexMailbox *ibox = (IndexMailbox *) box;
@@ -66,7 +76,7 @@
 		return FALSE;
 	}
 
-	if (!index_storage_sync_if_possible(ibox))
+	if (!index_storage_sync_index_if_possible(ibox))
 		return FALSE;
 
 	if (!index_mailbox_fix_custom_flags(ibox, &flags, custom_flags))
@@ -75,12 +85,10 @@
 	if (!ibox->index->set_lock(ibox->index, MAIL_LOCK_EXCLUSIVE))
 		return mail_storage_set_index_error(ibox);
 
-	ctx.box = box;
+	ctx.ibox = ibox;
 	ctx.flags = flags & ~MAIL_RECENT; /* \Recent can't be changed */
-	ctx.custom_flags = ibox->index->custom_flags;
 	ctx.modify_type = modify_type;
-	ctx.func = func;
-	ctx.context = context;
+	ctx.notify = notify;
 
 	ret = index_messageset_foreach(ibox, messageset, uidset,
 				       update_func, &ctx);
--- a/src/lib-storage/index/maildir/maildir-expunge.c	Fri Oct 18 02:57:44 2002 +0300
+++ b/src/lib-storage/index/maildir/maildir-expunge.c	Sat Oct 19 17:51:59 2002 +0300
@@ -30,8 +30,7 @@
 
 }
 
-int maildir_expunge_locked(IndexMailbox *ibox,
-			   MailExpungeFunc expunge_func, void *context)
+int maildir_expunge_locked(IndexMailbox *ibox, int notify)
 {
 	MailIndexRecord *rec;
 	unsigned int seq, uid;
@@ -47,8 +46,12 @@
 			if (!expunge_msg(ibox, rec, seq))
 				return FALSE;
 
-			if (expunge_func != NULL)
-				expunge_func(&ibox->box, seq, uid, context);
+			if (notify) {
+				ibox->sync_callbacks.expunge(
+						&ibox->box, seq, uid,
+						ibox->sync_context);
+			}
+			ibox->synced_messages_count--;
 			seq--;
 		}
 		rec = ibox->index->next(ibox->index, rec);
--- a/src/lib-storage/index/maildir/maildir-storage.c	Fri Oct 18 02:57:44 2002 +0300
+++ b/src/lib-storage/index/maildir/maildir-storage.c	Sat Oct 19 17:51:59 2002 +0300
@@ -126,13 +126,19 @@
 			     int readonly, int fast)
 {
 	IndexMailbox *ibox;
+	MailIndex *index;
 	const char *path;
 
 	path = t_strconcat(storage->dir, "/.", name, NULL);
 
-	ibox = index_storage_init(storage, &maildir_mailbox,
-				  maildir_index_alloc(path), name, readonly,
-				  fast);
+	index = index_storage_lookup_ref(path);
+	if (index == NULL) {
+		index = maildir_index_alloc(path);
+		index_storage_add(index);
+	}
+
+	ibox = index_storage_init(storage, &maildir_mailbox, index, name,
+				  readonly, fast);
 	if (ibox != NULL)
 		ibox->expunge_locked = maildir_expunge_locked;
 	return (Mailbox *) ibox;
@@ -394,6 +400,7 @@
 	NULL, /* storage */
 
 	index_storage_close,
+	index_storage_set_sync_callbacks,
 	index_storage_get_status,
 	index_storage_sync,
 	index_storage_expunge,
@@ -405,5 +412,6 @@
 	mail_storage_is_inconsistency_error,
 
 	FALSE,
+	FALSE,
 	FALSE
 };
--- a/src/lib-storage/index/maildir/maildir-storage.h	Fri Oct 18 02:57:44 2002 +0300
+++ b/src/lib-storage/index/maildir/maildir-storage.h	Sat Oct 19 17:51:59 2002 +0300
@@ -14,7 +14,6 @@
 int maildir_find_subscribed(MailStorage *storage, const char *mask,
 			    MailboxFunc func, void *context);
 
-int maildir_expunge_locked(IndexMailbox *ibox,
-			   MailExpungeFunc expunge_func, void *context);
+int maildir_expunge_locked(IndexMailbox *ibox, int notify);
 
 #endif
--- a/src/lib-storage/index/mbox/mbox-expunge.c	Fri Oct 18 02:57:44 2002 +0300
+++ b/src/lib-storage/index/mbox/mbox-expunge.c	Sat Oct 19 17:51:59 2002 +0300
@@ -12,7 +12,7 @@
 
 static int expunge_real(IndexMailbox *ibox, MailIndexRecord *rec,
 			unsigned int seq, IBuffer *inbuf, OBuffer *outbuf,
-			MailExpungeFunc expunge_func, void *context)
+			int notify)
 {
 	uoff_t offset, end_offset, from_offset, copy_size, old_limit;
 	unsigned int uid;
@@ -53,8 +53,12 @@
 						  seq, FALSE))
 				return FALSE;
 
-			if (expunge_func != NULL)
-				expunge_func(&ibox->box, seq, uid, context);
+			if (notify) {
+				ibox->sync_callbacks.expunge(
+						&ibox->box, seq, uid,
+						ibox->sync_context);
+			}
+			ibox->synced_messages_count--;
 			seq--;
 
 			if (!expunges) {
@@ -106,8 +110,7 @@
 	return o_buffer_send_ibuffer(outbuf, inbuf) >= 0;
 }
 
-int mbox_expunge_locked(IndexMailbox *ibox,
-			MailExpungeFunc expunge_func, void *context)
+int mbox_expunge_locked(IndexMailbox *ibox, int notify)
 {
 	MailIndexRecord *rec;
 	IBuffer *inbuf;
@@ -136,8 +139,7 @@
 	outbuf = o_buffer_create_file(ibox->index->mbox_fd, data_stack_pool,
 				      4096, IO_PRIORITY_DEFAULT, FALSE);
 
-	failed = !expunge_real(ibox, rec, seq, inbuf, outbuf,
-			       expunge_func, context);
+	failed = !expunge_real(ibox, rec, seq, inbuf, outbuf, notify);
 
 	if (failed && outbuf->offset > 0) {
 		/* we moved some of the data. move the rest as well so there
--- a/src/lib-storage/index/mbox/mbox-storage.c	Fri Oct 18 02:57:44 2002 +0300
+++ b/src/lib-storage/index/mbox/mbox-storage.c	Sat Oct 19 17:51:59 2002 +0300
@@ -153,6 +153,7 @@
 			  int readonly, int fast)
 {
 	IndexMailbox *ibox;
+	MailIndex *index;
 	const char *path, *index_dir;
 
 	/* name = "foo/bar"
@@ -161,8 +162,13 @@
 	path = t_strconcat(storage->dir, "/", name, NULL);
 	index_dir = mbox_get_index_dir(path);
 
-	ibox = index_storage_init(storage, &mbox_mailbox,
-				  mbox_index_alloc(index_dir, path),
+	index = index_storage_lookup_ref(index_dir);
+	if (index == NULL) {
+		index = mbox_index_alloc(index_dir, path);
+		index_storage_add(index);
+	}
+
+	ibox = index_storage_init(storage, &mbox_mailbox, index,
 				  name, readonly, fast);
 	if (ibox != NULL)
 		ibox->expunge_locked = mbox_expunge_locked;
@@ -408,6 +414,7 @@
 	NULL, /* storage */
 
 	mbox_storage_close,
+	index_storage_set_sync_callbacks,
 	index_storage_get_status,
 	index_storage_sync,
 	index_storage_expunge,
@@ -419,5 +426,6 @@
 	mail_storage_is_inconsistency_error,
 
 	FALSE,
+	FALSE,
 	FALSE
 };
--- a/src/lib-storage/index/mbox/mbox-storage.h	Fri Oct 18 02:57:44 2002 +0300
+++ b/src/lib-storage/index/mbox/mbox-storage.h	Sat Oct 19 17:51:59 2002 +0300
@@ -13,7 +13,6 @@
 int mbox_find_subscribed(MailStorage *storage, const char *mask,
 			 MailboxFunc func, void *context);
 
-int mbox_expunge_locked(IndexMailbox *ibox,
-			MailExpungeFunc expunge_func, void *context);
+int mbox_expunge_locked(IndexMailbox *ibox, int notify);
 
 #endif
--- a/src/lib-storage/mail-storage.h	Fri Oct 18 02:57:44 2002 +0300
+++ b/src/lib-storage/mail-storage.h	Sat Oct 19 17:51:59 2002 +0300
@@ -40,6 +40,7 @@
 typedef struct _MailStorage MailStorage;
 typedef struct _Mailbox Mailbox;
 typedef struct _MailboxStatus MailboxStatus;
+typedef struct _MailboxSyncCallbacks MailboxSyncCallbacks;
 typedef struct _MailFetchData MailFetchData;
 typedef struct _MailFetchBodyData MailFetchBodyData;
 typedef struct _MailSearchArg MailSearchArg;
@@ -47,13 +48,6 @@
 typedef void (*MailboxFunc)(MailStorage *storage, const char *name,
 			    MailboxFlags flags, void *context);
 
-typedef void (*MailExpungeFunc)(Mailbox *mailbox, unsigned int seq,
-				unsigned int uid, void *context);
-typedef void (*MailFlagUpdateFunc)(Mailbox *mailbox, unsigned int seq,
-				   unsigned int uid, MailFlags flags,
-				   const char *custom_flags[],
-				   void *context);
-
 /* All methods returning int return either TRUE or FALSE. */
 struct _MailStorage {
 	char *name;
@@ -73,13 +67,17 @@
 	/* Open a mailbox. If readonly is TRUE, mailbox must not be
 	   modified in any way even when it's asked. If fast is TRUE,
 	   any extra time consuming operations shouldn't be performed
-	   (eg. when opening mailbox just for STATUS). */
+	   (eg. when opening mailbox just for STATUS).
+
+	   Note that append and copy may open the selected mailbox again
+	   with possibly different readonly-state. */
 	Mailbox *(*open_mailbox)(MailStorage *storage, const char *name,
 				 int readonly, int fast);
 
 	/* name is allowed to contain multiple new hierarchy levels */
 	int (*create_mailbox)(MailStorage *storage, const char *name);
 	int (*delete_mailbox)(MailStorage *storage, const char *name);
+
 	/* If the name has inferior hierarchical names, then the inferior
 	   hierarchical names MUST also be renamed (ie. foo -> bar renames
 	   also foo/bar -> bar/bar).
@@ -124,35 +122,29 @@
 	/* Close the box */
 	void (*close)(Mailbox *box);
 
+	/* Set synchronization callback functions to use. */
+	void (*set_sync_callbacks)(Mailbox *box,
+				   MailboxSyncCallbacks *callbacks,
+				   void *context);
+
 	/* Gets the mailbox status information. */
 	int (*get_status)(Mailbox *box, MailboxStatusItems items,
 			  MailboxStatus *status);
 
-	/* Synchronize the mailbox by reading all expunges and flag changes.
-	   If new mail has been added to mailbox, messages contains the total
-	   number of messages in mailbox and recent is set, otherwise 0.
-
-	   If both functions are NULL, only the message counts are returned
-	   and mailbox isn't marked synchronized.
+	/* Synchronize the mailbox. If sync_expunges is FALSE, everything
+	   but expunges are synced. */
+	int (*sync)(Mailbox *box, int sync_expunges);
 
-	   If expunge is TRUE, deleted mails are expunged as well. Difference
-	   to expunge() function is that expunge_func is also called. */
-	int (*sync)(Mailbox *box, int expunge,
-		    unsigned int *messages, unsigned int *recent,
-		    MailExpungeFunc expunge_func, MailFlagUpdateFunc flag_func,
-		    void *context);
+	/* Expunge all mails with \Deleted flag. If notify is TRUE, call
+	   expunge callbacks. Also always does full syncing. */
+	int (*expunge)(Mailbox *box, int notify);
 
-	/* Expunge all mails with \Deleted flag. */
-	int (*expunge)(Mailbox *box);
-
-	/* Update mail flags. func may be NULL. */
+	/* Update mail flags, calling update_flags callbacks. */
 	int (*update_flags)(Mailbox *box, const char *messageset, int uidset,
 			    MailFlags flags, const char *custom_flags[],
-			    ModifyType modify_type,
-			    MailFlagUpdateFunc func, void *context,
-			    int *all_found);
+			    ModifyType modify_type, int notify, int *all_found);
 
-	/* Copy mails to another mailbox */
+	/* Copy mails to another mailbox. */
 	int (*copy)(Mailbox *box, Mailbox *destbox,
 		    const char *messageset, int uidset);
 
@@ -177,8 +169,10 @@
 	   do forced CLOSE. */
 	int (*is_inconsistency_error)(Mailbox *box);
 
+/* public: */
+	unsigned int readonly:1;
+	unsigned int allow_custom_flags:1;
 /* private: */
-	unsigned int readonly:1;
 	unsigned int inconsistent:1;
 };
 
@@ -195,7 +189,31 @@
 	unsigned int diskspace_full:1;
 
 	/* may be allocated from data stack */
-	const char *custom_flags[MAIL_CUSTOM_FLAGS_COUNT];
+	unsigned int custom_flags_count;
+	const char **custom_flags;
+};
+
+struct _MailboxSyncCallbacks {
+	/* Alert: Not enough disk space */
+	void (*alert_no_diskspace)(Mailbox *mailbox, void *context);
+
+	/* EXPUNGE */
+	void (*expunge)(Mailbox *mailbox, unsigned int seq,
+			unsigned int uid, void *context);
+	/* FETCH FLAGS */
+	void (*update_flags)(Mailbox *mailbox, unsigned int seq,
+			     unsigned int uid, MailFlags flags,
+			     const char *custom_flags[],
+			     unsigned int custom_flags_count, void *context);
+
+	/* EXISTS, RECENT */
+	void (*new_messages)(Mailbox *mailbox, unsigned int messages_count,
+			     unsigned int recent_count, void *context);
+	/* FLAGS, PERMANENTFLAGS */
+	void (*new_custom_flags)(Mailbox *mailbox, const char *custom_flags[],
+				 unsigned int custom_flags_count,
+				 void *context);
+
 };
 
 struct _MailFetchData {