Mercurial > dovecot > core-2.2

changeset 14599:dbd42f7198eb

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
shared mailboxes: Per-user flags can now be stored in private index files. This can be enabled by adding e.g.: mail_location = mdbox:/var/shared/mdbox:INDEXPVT=~/mdbox/shared
author Timo Sirainen <tss@iki.fi>
date Mon, 11 Jun 2012 18:22:06 +0300
parents 7d0ce7586c36
children 27f769be0f85
files src/doveadm/doveadm-dump-log.c src/lib-index/mail-index-private.h src/lib-index/mail-index-transaction-export.c src/lib-index/mail-index-transaction-finish.c src/lib-index/mail-index-transaction-private.h src/lib-index/mail-index-transaction-update.c src/lib-index/mail-index-transaction-view.c src/lib-index/mail-index.h src/lib-index/mail-transaction-log.h src/lib-index/test-mail-index-transaction-finish.c src/lib-index/test-mail-index-transaction-update.c src/lib-storage/index/Makefile.am src/lib-storage/index/index-mail.c src/lib-storage/index/index-storage.c src/lib-storage/index/index-storage.h src/lib-storage/index/index-sync-private.h src/lib-storage/index/index-sync-pvt.c src/lib-storage/index/index-sync.c src/lib-storage/index/index-transaction.c src/lib-storage/index/shared/shared-list.c src/lib-storage/list/mailbox-list-fs.c src/lib-storage/list/mailbox-list-maildir.c src/lib-storage/mail-storage-private.h src/lib-storage/mail-storage.c src/lib-storage/mailbox-list-private.h src/lib-storage/mailbox-list.c src/lib-storage/mailbox-list.h
diffstat 27 files changed, 466 insertions(+), 64 deletions(-) [+]
line wrap: on
line diff
--- a/src/doveadm/doveadm-dump-log.c	Tue Jun 05 00:34:32 2012 +0300
+++ b/src/doveadm/doveadm-dump-log.c	Mon Jun 11 18:22:06 2012 +0300
@@ -284,8 +284,8 @@
 		const struct mail_transaction_flag_update *u = data;
 
 		for (; size > 0; size -= sizeof(*u), u++) {
-			printf(" - uids=%u-%u (flags +%x-%x)\n",
-			       u->uid1, u->uid2, u->add_flags, u->remove_flags);
+			printf(" - uids=%u-%u (flags +%x-%x, modseq_inc_flag=%d)\n",
+			       u->uid1, u->uid2, u->add_flags, u->remove_flags, u->modseq_inc_flag);
 		}
 		break;
 	}
--- a/src/lib-index/mail-index-private.h	Tue Jun 05 00:34:32 2012 +0300
+++ b/src/lib-index/mail-index-private.h	Mon Jun 11 18:22:06 2012 +0300
@@ -39,8 +39,8 @@
 	 PTR_OFFSET((map)->rec_map->records, (idx) * (map)->hdr.record_size))
 
 #define MAIL_TRANSACTION_FLAG_UPDATE_IS_INTERNAL(u) \
-	((((u)->add_flags | (u)->remove_flags) & \
-	  MAIL_INDEX_FLAGS_MASK) == 0)
+	((((u)->add_flags | (u)->remove_flags) & MAIL_INDEX_FLAGS_MASK) == 0 && \
+	 (u)->modseq_inc_flag == 0)
 
 #define MAIL_INDEX_EXT_KEYWORDS "keywords"
 
--- a/src/lib-index/mail-index-transaction-export.c	Tue Jun 05 00:34:32 2012 +0300
+++ b/src/lib-index/mail-index-transaction-export.c	Mon Jun 11 18:22:06 2012 +0300
@@ -20,6 +20,34 @@
 					buf->data, buf->used);
 }
 
+static void log_append_flag_updates(struct mail_index_export_context *ctx,
+				    struct mail_index_transaction *t)
+{
+	ARRAY_DEFINE(log_updates, struct mail_transaction_flag_update);
+	const struct mail_index_flag_update *updates;
+	struct mail_transaction_flag_update *log_update;
+	unsigned int i, count;
+
+	updates = array_get(&t->updates, &count);
+	if (count == 0)
+		return;
+
+	i_array_init(&log_updates, count);
+
+	for (i = 0; i < count; i++) {
+		log_update = array_append_space(&log_updates);
+		log_update->uid1 = updates[i].uid1;
+		log_update->uid2 = updates[i].uid2;
+		log_update->add_flags = updates[i].add_flags & 0xff;
+		log_update->remove_flags = updates[i].remove_flags & 0xff;
+		if ((updates[i].add_flags & MAIL_INDEX_MAIL_FLAG_UPDATE_MODSEQ) != 0)
+			log_update->modseq_inc_flag = 1;
+	}
+	log_append_buffer(ctx, log_updates.arr.buffer,
+			  MAIL_TRANSACTION_FLAG_UPDATE);
+	array_free(&log_updates);
+}
+
 static const buffer_t *
 log_get_hdr_update_buffer(struct mail_index_transaction *t, bool prepend)
 {
@@ -373,8 +401,7 @@
 
 	if (array_is_created(&t->updates)) {
 		change_mask |= MAIL_INDEX_SYNC_TYPE_FLAGS;
-		log_append_buffer(&ctx, t->updates.arr.buffer, 
-				  MAIL_TRANSACTION_FLAG_UPDATE);
+		log_append_flag_updates(&ctx, t);
 	}
 
 	if (array_is_created(&t->ext_rec_updates)) {
--- a/src/lib-index/mail-index-transaction-finish.c	Tue Jun 05 00:34:32 2012 +0300
+++ b/src/lib-index/mail-index-transaction-finish.c	Mon Jun 11 18:22:06 2012 +0300
@@ -80,7 +80,7 @@
 
 static unsigned int
 mail_transaction_drop_range(struct mail_index_transaction *t,
-			    struct mail_transaction_flag_update update,
+			    struct mail_index_flag_update update,
 			    unsigned int update_idx,
 			    ARRAY_TYPE(seq_range) *keeps)
 {
@@ -109,7 +109,7 @@
 static void
 mail_index_transaction_finish_flag_updates(struct mail_index_transaction *t)
 {
-	const struct mail_transaction_flag_update *updates, *u;
+	const struct mail_index_flag_update *updates, *u;
 	const struct mail_index_record *rec;
 	unsigned int i, count;
 	ARRAY_TYPE(seq_range) keeps;
--- a/src/lib-index/mail-index-transaction-private.h	Tue Jun 05 00:34:32 2012 +0300
+++ b/src/lib-index/mail-index-transaction-private.h	Mon Jun 11 18:22:06 2012 +0300
@@ -29,6 +29,12 @@
 	struct mail_index_module_register *reg;
 };
 
+struct mail_index_flag_update {
+	uint32_t uid1, uid2;
+	uint16_t add_flags;
+	uint16_t remove_flags;
+};
+
 struct mail_index_transaction {
 	int refcount;
 
@@ -47,7 +53,7 @@
 
 	ARRAY_DEFINE(modseq_updates, struct mail_transaction_modseq_update);
 	ARRAY_DEFINE(expunges, struct mail_transaction_expunge_guid);
-	ARRAY_DEFINE(updates, struct mail_transaction_flag_update);
+	ARRAY_DEFINE(updates, struct mail_index_flag_update);
 	size_t last_update_idx;
 
 	unsigned char pre_hdr_change[sizeof(struct mail_index_header)];
--- a/src/lib-index/mail-index-transaction-update.c	Tue Jun 05 00:34:32 2012 +0300
+++ b/src/lib-index/mail-index-transaction-update.c	Mon Jun 11 18:22:06 2012 +0300
@@ -388,7 +388,7 @@
 					   unsigned int right_idx,
 					   uint32_t seq)
 {
-	const struct mail_transaction_flag_update *updates;
+	const struct mail_index_flag_update *updates;
 	unsigned int idx, count;
 
 	updates = array_get(&t->updates, &count);
@@ -414,10 +414,10 @@
 
 static void
 mail_index_insert_flag_update(struct mail_index_transaction *t,
-			      struct mail_transaction_flag_update u,
+			      struct mail_index_flag_update u,
 			      unsigned int idx)
 {
-	struct mail_transaction_flag_update *updates, tmp_update;
+	struct mail_index_flag_update *updates, tmp_update;
 	unsigned int count, first_idx, max;
 
 	updates = array_get_modifiable(&t->updates, &count);
@@ -543,7 +543,7 @@
 				   enum mail_flags flags)
 {
 	struct mail_index_record *rec;
-	struct mail_transaction_flag_update u, *last_update;
+	struct mail_index_flag_update u, *last_update;
 	unsigned int idx, first_idx, count;
 
 	update_minmax_flagupdate_seq(t, seq1, seq2);
@@ -1108,7 +1108,7 @@
 bool mail_index_cancel_flag_updates(struct mail_index_transaction *t,
 				    uint32_t seq)
 {
-	struct mail_transaction_flag_update *updates, tmp_update;
+	struct mail_index_flag_update *updates, tmp_update;
 	unsigned int i, count;
 
 	if (!array_is_created(&t->updates))
--- a/src/lib-index/mail-index-transaction-view.c	Tue Jun 05 00:34:32 2012 +0300
+++ b/src/lib-index/mail-index-transaction-view.c	Mon Jun 11 18:22:06 2012 +0300
@@ -85,7 +85,7 @@
 			 const struct mail_index_record *rec, uint32_t seq)
 {
 	struct mail_index_transaction *t = tview->t;
-	const struct mail_transaction_flag_update *updates;
+	const struct mail_index_flag_update *updates;
 	struct mail_index_record *trec;
 	unsigned int idx, count;
 
--- a/src/lib-index/mail-index.h	Tue Jun 05 00:34:32 2012 +0300
+++ b/src/lib-index/mail-index.h	Mon Jun 11 18:22:06 2012 +0300
@@ -42,9 +42,11 @@
 
 enum mail_index_mail_flags {
 	/* For private use by backend. Replacing flags doesn't change this. */
-	MAIL_INDEX_MAIL_FLAG_BACKEND	= 0x40,
+	MAIL_INDEX_MAIL_FLAG_BACKEND		= 0x40,
 	/* Message flags haven't been written to backend */
-	MAIL_INDEX_MAIL_FLAG_DIRTY	= 0x80
+	MAIL_INDEX_MAIL_FLAG_DIRTY		= 0x80,
+	/* Force updating this message's modseq via a flag update record */
+	MAIL_INDEX_MAIL_FLAG_UPDATE_MODSEQ	= 0x100
 };
 
 #define MAIL_INDEX_FLAGS_MASK \
--- a/src/lib-index/mail-transaction-log.h	Tue Jun 05 00:34:32 2012 +0300
+++ b/src/lib-index/mail-transaction-log.h	Mon Jun 11 18:22:06 2012 +0300
@@ -90,7 +90,8 @@
 	uint32_t uid1, uid2;
 	uint8_t add_flags;
 	uint8_t remove_flags;
-	uint16_t padding;
+	uint8_t modseq_inc_flag;
+	uint8_t padding;
 };
 
 struct mail_transaction_keyword_update {
--- a/src/lib-index/test-mail-index-transaction-finish.c	Tue Jun 05 00:34:32 2012 +0300
+++ b/src/lib-index/test-mail-index-transaction-finish.c	Mon Jun 11 18:22:06 2012 +0300
@@ -60,8 +60,8 @@
 static void test_mail_index_transaction_finish_flag_updates(void)
 {
 	struct mail_index_transaction *t;
-	const struct mail_transaction_flag_update *updates;
-	struct mail_transaction_flag_update u;
+	const struct mail_index_flag_update *updates;
+	struct mail_index_flag_update u;
 	unsigned int count;
 
 	t = t_new(struct mail_index_transaction, 1);
--- a/src/lib-index/test-mail-index-transaction-update.c	Tue Jun 05 00:34:32 2012 +0300
+++ b/src/lib-index/test-mail-index-transaction-update.c	Mon Jun 11 18:22:06 2012 +0300
@@ -139,7 +139,7 @@
 static void test_mail_index_flag_update_fastpath(void)
 {
 	struct mail_index_transaction *t;
-	const struct mail_transaction_flag_update *updates;
+	const struct mail_index_flag_update *updates;
 	unsigned int count;
 
 	hdr.messages_count = 20;
@@ -168,12 +168,10 @@
 	test_assert(updates[0].add_flags == MAIL_DELETED);
 	test_assert(updates[0].remove_flags ==
 		    (MAIL_ANSWERED | MAIL_FLAGGED | MAIL_SEEN | MAIL_DRAFT));
-	test_assert(updates[0].padding == 0);
 	test_assert(updates[1].uid1 == 16);
 	test_assert(updates[1].uid2 == 16);
 	test_assert(updates[1].add_flags == MAIL_DELETED);
 	test_assert(updates[1].remove_flags == 0);
-	test_assert(updates[1].padding == 0);
 	test_assert(!t->log_updates);
 	test_end();
 }
@@ -181,7 +179,7 @@
 static void test_mail_index_flag_update_simple_merges(void)
 {
 	struct mail_index_transaction *t;
-	const struct mail_transaction_flag_update *updates;
+	const struct mail_index_flag_update *updates;
 	unsigned int count;
 
 	hdr.messages_count = 20;
@@ -224,7 +222,7 @@
 static void test_mail_index_flag_update_complex_merges(void)
 {
 	struct mail_index_transaction *t;
-	const struct mail_transaction_flag_update *updates;
+	const struct mail_index_flag_update *updates;
 	unsigned int count;
 
 	hdr.messages_count = 20;
@@ -283,7 +281,7 @@
 flags_array_check(struct mail_index_transaction *t,
 		  const enum mail_flags *flags, unsigned int msg_count)
 {
-	const struct mail_transaction_flag_update *updates;
+	const struct mail_index_flag_update *updates;
 	unsigned int i, count, seq;
 
 	if (array_is_created(&t->updates))
@@ -357,7 +355,7 @@
 static void test_mail_index_cancel_flag_updates(void)
 {
 	struct mail_index_transaction *t;
-	const struct mail_transaction_flag_update *updates;
+	const struct mail_index_flag_update *updates;
 	unsigned int count;
 
 	hdr.messages_count = 20;
@@ -390,7 +388,7 @@
 {
 	struct mail_index_transaction *t;
 	const struct mail_index_record *appends;
-	const struct mail_transaction_flag_update *updates;
+	const struct mail_index_flag_update *updates;
 	unsigned int count;
 	uint32_t seq;
 
--- a/src/lib-storage/index/Makefile.am	Tue Jun 05 00:34:32 2012 +0300
+++ b/src/lib-storage/index/Makefile.am	Mon Jun 11 18:22:06 2012 +0300
@@ -26,6 +26,7 @@
 	index-storage.c \
 	index-sync.c \
 	index-sync-changes.c \
+	index-sync-pvt.c \
 	index-sync-search.c \
 	index-thread.c \
 	index-thread-finish.c \
--- a/src/lib-storage/index/index-mail.c	Tue Jun 05 00:34:32 2012 +0300
+++ b/src/lib-storage/index/index-mail.c	Mon Jun 11 18:22:06 2012 +0300
@@ -138,18 +138,45 @@
 					  size_r, sizeof(*size_r));
 }
 
-enum mail_flags index_mail_get_flags(struct mail *mail)
+static bool index_mail_get_pvt(struct mail *_mail)
 {
+	struct mail_private *mail = (struct mail_private *)_mail;
+
+	if (mail->seq_pvt != 0)
+		return TRUE;
+	if (_mail->box->view_pvt == NULL) {
+		/* no private view (set by view syncing) -> no private flags */
+		return FALSE;
+	}
+
+	index_transaction_init_pvt(_mail->transaction);
+	if (!mail_index_lookup_seq(_mail->transaction->view_pvt, _mail->uid,
+				   &mail->seq_pvt))
+		mail->seq_pvt = 0;
+	return mail->seq_pvt != 0;
+}
+
+enum mail_flags index_mail_get_flags(struct mail *_mail)
+{
+	struct mail_private *mail = (struct mail_private *)_mail;
 	const struct mail_index_record *rec;
-	enum mail_flags flags;
+	enum mail_flags flags, pvt_flags_mask;
 
-	rec = mail_index_lookup(mail->transaction->view, mail->seq);
+	rec = mail_index_lookup(_mail->transaction->view, _mail->seq);
 	flags = rec->flags & (MAIL_FLAGS_NONRECENT |
 			      MAIL_INDEX_MAIL_FLAG_BACKEND);
 
-	if (index_mailbox_is_recent(mail->box, mail->uid))
+	if (index_mailbox_is_recent(_mail->box, _mail->uid))
 		flags |= MAIL_RECENT;
 
+	if (index_mail_get_pvt(_mail)) {
+		/* mailbox has private flags */
+		pvt_flags_mask = mailbox_get_private_flags_mask(_mail->box);
+		flags &= ~pvt_flags_mask;
+		rec = mail_index_lookup(_mail->transaction->view_pvt,
+					mail->seq_pvt);
+		flags |= rec->flags & pvt_flags_mask;
+	}
 	return flags;
 }
 
@@ -1223,6 +1250,7 @@
 
 	mail->mail.mail.seq = 0;
 	mail->mail.mail.uid = 0;
+	mail->mail.seq_pvt = 0;
 	mail->mail.mail.expunged = FALSE;
 	mail->mail.mail.has_nuls = FALSE;
 	mail->mail.mail.has_no_nuls = FALSE;
@@ -1599,15 +1627,43 @@
 	}
 }
 
-void index_mail_update_flags(struct mail *mail, enum modify_type modify_type,
+void index_mail_update_flags(struct mail *_mail, enum modify_type modify_type,
 			     enum mail_flags flags)
 {
+	struct mail_private *mail = (struct mail_private *)_mail;
+	enum mail_flags pvt_flags_mask, pvt_flags = 0;
+	bool update_modseq = FALSE;
+
 	if ((flags & MAIL_RECENT) == 0 &&
-	    index_mailbox_is_recent(mail->box, mail->uid))
-		index_mail_drop_recent_flag(mail);
+	    index_mailbox_is_recent(_mail->box, _mail->uid))
+		index_mail_drop_recent_flag(_mail);
+	flags &= MAIL_FLAGS_NONRECENT | MAIL_INDEX_MAIL_FLAG_BACKEND;
 
-	flags &= MAIL_FLAGS_NONRECENT | MAIL_INDEX_MAIL_FLAG_BACKEND;
-	mail_index_update_flags(mail->transaction->itrans, mail->seq,
+	if (_mail->box->view_pvt != NULL) {
+		/* mailbox has private flags */
+		pvt_flags_mask = mailbox_get_private_flags_mask(_mail->box);
+		pvt_flags = flags & pvt_flags_mask;
+		flags &= ~pvt_flags_mask;
+		if (index_mail_get_pvt(_mail) &&
+		    (pvt_flags != 0 || modify_type == MODIFY_REPLACE)) {
+			mail_index_update_flags(_mail->transaction->itrans_pvt,
+						mail->seq_pvt,
+						modify_type, pvt_flags);
+			update_modseq = TRUE;
+		}
+	}
+
+	if (!update_modseq) {
+		/* no forced modseq update */
+	} else if (modify_type == MODIFY_REMOVE) {
+		/* add the modseq update separately */
+		mail_index_update_flags(_mail->transaction->itrans, _mail->seq,
+			MODIFY_ADD, (enum mail_flags )MAIL_INDEX_MAIL_FLAG_UPDATE_MODSEQ);
+	} else {
+		/* add as part of the flag updates */
+		flags |= MAIL_INDEX_MAIL_FLAG_UPDATE_MODSEQ;
+	}
+	mail_index_update_flags(_mail->transaction->itrans, _mail->seq,
 				modify_type, flags);
 }
 
--- a/src/lib-storage/index/index-storage.c	Tue Jun 05 00:34:32 2012 +0300
+++ b/src/lib-storage/index/index-storage.c	Mon Jun 11 18:22:06 2012 +0300
@@ -332,6 +332,10 @@
 	if (box->input != NULL)
 		i_stream_unref(&box->input);
 
+	if (box->view_pvt != NULL)
+		mail_index_view_close(&box->view_pvt);
+	if (box->index_pvt != NULL)
+		mail_index_close(box->index_pvt);
 	mail_index_view_close(&box->view);
 	mail_index_close(box->index);
 	box->cache = NULL;
@@ -349,6 +353,8 @@
 
 void index_storage_mailbox_free(struct mailbox *box)
 {
+	if (box->index_pvt != NULL)
+		mail_index_alloc_cache_unref(&box->index_pvt);
 	if (box->index != NULL)
 		mail_index_alloc_cache_unref(&box->index);
 }
--- a/src/lib-storage/index/index-storage.h	Tue Jun 05 00:34:32 2012 +0300
+++ b/src/lib-storage/index/index-storage.h	Mon Jun 11 18:22:06 2012 +0300
@@ -129,6 +129,7 @@
 void index_transaction_init(struct mailbox_transaction_context *t,
 			    struct mailbox *box,
 			    enum mailbox_transaction_flags flags);
+void index_transaction_init_pvt(struct mailbox_transaction_context *t);
 int index_transaction_commit(struct mailbox_transaction_context *t,
 			     struct mail_transaction_commit_changes *changes_r);
 void index_transaction_rollback(struct mailbox_transaction_context *t);
--- a/src/lib-storage/index/index-sync-private.h	Tue Jun 05 00:34:32 2012 +0300
+++ b/src/lib-storage/index/index-sync-private.h	Mon Jun 11 18:22:06 2012 +0300
@@ -22,4 +22,6 @@
 void index_sync_search_results_update(struct index_mailbox_sync_context *ctx);
 void index_sync_search_results_expunge(struct index_mailbox_sync_context *ctx);
 
+int index_storage_mailbox_sync_pvt(struct mailbox *box);
+
 #endif
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/src/lib-storage/index/index-sync-pvt.c	Mon Jun 11 18:22:06 2012 +0300
@@ -0,0 +1,221 @@
+/* Copyright (c) 2012 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "mail-index-alloc-cache.h"
+#include "mailbox-list-private.h"
+#include "index-sync-private.h"
+
+static int index_storage_mailbox_alloc_index_pvt(struct mailbox *box)
+{
+	const char *index_dir;
+
+	if (box->index_pvt != NULL)
+		return 1;
+
+	index_dir = mailbox_list_get_path(box->list, box->name,
+					  MAILBOX_LIST_PATH_TYPE_INDEX_PRIVATE);
+	if (index_dir == NULL) {
+		/* no private indexes */
+		return 0;
+	}
+
+	if (mailbox_list_create_missing_index_pvt_dir(box->list, box->name) < 0) {
+		mail_storage_set_internal_error(box->storage);
+		return -1;
+	}
+
+	box->index_pvt = mail_index_alloc_cache_get(NULL, index_dir,
+		t_strconcat(box->index_prefix, ".pvt", NULL));
+	mail_index_set_fsync_mode(box->index_pvt,
+				  box->storage->set->parsed_fsync_mode, 0);
+	mail_index_set_lock_method(box->index_pvt,
+		box->storage->set->parsed_lock_method,
+		mail_storage_get_lock_timeout(box->storage, -1U));
+	return 1;
+}
+
+static int index_storage_mailbox_open_index_pvt(struct mailbox *box)
+{
+	int ret;
+
+	if (box->view_pvt != NULL)
+		return 1;
+
+	if ((ret = index_storage_mailbox_alloc_index_pvt(box)) <= 0)
+		return ret;
+	if (mail_index_open(box->index_pvt, MAIL_INDEX_OPEN_FLAG_CREATE) < 0)
+		return -1;
+	box->view_pvt = mail_index_view_open(box->index_pvt);
+	return 1;
+}
+
+static int sync_pvt_expunges(struct mailbox *box,
+			     struct mail_index_view *view_pvt,
+			     struct mail_index_transaction *trans_pvt,
+			     struct mail_index_view *view_shared)
+{
+	uint32_t seq_shared, seq_pvt, count_shared, count_pvt;
+	uint32_t uid_shared, uid_pvt;
+
+	count_shared = mail_index_view_get_messages_count(view_shared);
+	count_pvt = mail_index_view_get_messages_count(view_pvt);
+	seq_shared = seq_pvt = 1;
+	while (seq_pvt <= count_pvt && seq_shared <= count_shared) {
+		mail_index_lookup_uid(view_pvt, seq_pvt, &uid_pvt);
+		mail_index_lookup_uid(view_shared, seq_shared, &uid_shared);
+		if (uid_pvt == uid_shared) {
+			seq_pvt++;
+			seq_shared++;
+		} else if (uid_pvt < uid_shared) {
+			/* message expunged */
+			mail_index_expunge(trans_pvt, seq_pvt);
+			seq_pvt++;
+		} else {
+			mail_storage_set_critical(box->storage,
+				"%s: Message UID=%u unexpectedly inserted to mailbox",
+				box->index_pvt->filepath, uid_shared);
+			return -1;
+		}
+	}
+	return 0;
+}
+
+static void
+sync_pvt_copy_self_flags(struct mailbox *box,
+			 struct mail_index_view *view,
+			 struct mail_index_transaction *trans,
+			 ARRAY_TYPE(keyword_indexes) *keywords,
+			 uint32_t seq_old, uint32_t seq_new)
+{
+	const struct mail_index_record *old_rec;
+
+	old_rec = mail_index_lookup(view, seq_old);
+	mail_index_lookup_keywords(view, seq_old, keywords);
+	if (old_rec->flags != 0) {
+		mail_index_update_flags(trans, seq_new,
+					MODIFY_ADD, old_rec->flags);
+	}
+	if (array_count(keywords) > 0) {
+		struct mail_keywords *kw;
+
+		kw = mail_index_keywords_create_from_indexes(box->index_pvt,
+							     keywords);
+		mail_index_update_keywords(trans, seq_new, MODIFY_ADD, kw);
+		mail_index_keywords_unref(&kw);
+	}
+}
+
+static int
+index_storage_mailbox_sync_pvt_index(struct mailbox *box)
+{
+	struct mail_index_sync_ctx *sync_ctx;
+	struct mail_index_view *view_pvt;
+	struct mail_index_transaction *trans_pvt;
+	const struct mail_index_header *hdr_shared, *hdr_pvt;
+	struct mail_index_view *view_shared;
+	ARRAY_TYPE(keyword_indexes) keywords;
+	uint32_t seq_shared, seq_pvt, seq_old_pvt, seq2, count_shared, uid;
+	bool reset = FALSE, preserve_old_flags = FALSE;
+	int ret;
+
+	/* open a view for the latest version of the index */
+	if (mail_index_refresh(box->index) < 0) {
+		mail_storage_set_index_error(box);
+		return -1;
+	}
+	view_shared = mail_index_view_open(box->index);
+	hdr_shared = mail_index_get_header(view_shared);
+	if (hdr_shared->uid_validity == 0) {
+		/* the mailbox hasn't been fully created yet,
+		   no need for a private index yet */
+		mail_index_view_close(&view_shared);
+		return 0;
+	}
+	hdr_pvt = mail_index_get_header(box->view_pvt);
+	if (hdr_pvt->next_uid == hdr_shared->next_uid) {
+		/* no new mails, don't bother syncing */
+		mail_index_view_close(&view_shared);
+		return 0;
+	}
+
+	if (mail_index_sync_begin(box->index_pvt, &sync_ctx,
+				  &view_pvt, &trans_pvt, 0) < 0) {
+		mail_storage_set_index_error(box);
+		mail_index_view_close(&view_shared);
+		return -1;
+	}
+	/* get an updated private header */
+	hdr_pvt = mail_index_get_header(view_pvt);
+
+	if (hdr_shared->uid_validity == hdr_pvt->uid_validity) {
+		/* same mailbox. expunge messages from private index that
+		   no longer exist. */
+		if (sync_pvt_expunges(box, view_pvt, trans_pvt, view_shared) < 0) {
+			reset = TRUE;
+			preserve_old_flags = TRUE;
+			t_array_init(&keywords, 32);
+		}
+	} else if (hdr_pvt->uid_validity == 0 || hdr_pvt->uid_validity != 0) {
+		/* mailbox created/recreated */
+		reset = TRUE;
+	}
+
+	count_shared = mail_index_view_get_messages_count(view_shared);
+	if (!reset) {
+		if (!mail_index_lookup_seq_range(view_shared, hdr_pvt->next_uid,
+						 hdr_shared->next_uid,
+						 &seq_shared, &seq2)) {
+			/* no new messages */
+			seq_shared = count_shared+1;
+		}
+	} else {
+		mail_index_reset(trans_pvt);
+		mail_index_update_header(trans_pvt,
+			offsetof(struct mail_index_header, uid_validity),
+			&hdr_shared->uid_validity,
+			sizeof(hdr_shared->uid_validity), TRUE);
+		seq_shared = 1;
+	}
+
+	for (; seq_shared <= count_shared; seq_shared++) {
+		mail_index_lookup_uid(view_shared, seq_shared, &uid);
+		mail_index_append(trans_pvt, uid, &seq_pvt);
+		if (preserve_old_flags &&
+		    mail_index_lookup_seq(view_pvt, uid, &seq_old_pvt)) {
+			/* copy flags from the original index */
+			sync_pvt_copy_self_flags(box, view_pvt, trans_pvt,
+						 &keywords,
+						 seq_old_pvt, seq_pvt);
+		}
+	}
+	if ((ret = mail_index_sync_commit(&sync_ctx)) < 0)
+		mail_storage_set_index_error(box);
+	mail_index_view_close(&view_shared);
+	return ret;
+}
+
+int index_storage_mailbox_sync_pvt(struct mailbox *box)
+{
+	struct mail_index_view_sync_ctx *view_sync_ctx;
+	enum mail_flags pvt_flags;
+	bool delayed_expunges;
+	int ret;
+
+	pvt_flags = mailbox_get_private_flags_mask(box);
+	if (pvt_flags == 0)
+		return 0;
+
+	if ((ret = index_storage_mailbox_open_index_pvt(box)) <= 0)
+		return ret;
+
+	/* sync private index against shared index by adding/removing mails */
+	if (index_storage_mailbox_sync_pvt_index(box) < 0)
+		return -1;
+
+	/* sync the private view */
+	view_sync_ctx = mail_index_view_sync_begin(box->view_pvt,
+		MAIL_INDEX_VIEW_SYNC_FLAG_FIX_INCONSISTENT);
+	if (mail_index_view_sync_commit(&view_sync_ctx, &delayed_expunges) < 0)
+		return -1;
+	return 0;
+}
--- a/src/lib-storage/index/index-sync.c	Tue Jun 05 00:34:32 2012 +0300
+++ b/src/lib-storage/index/index-sync.c	Mon Jun 11 18:22:06 2012 +0300
@@ -184,6 +184,9 @@
 	ctx->ctx.box = box;
 	ctx->ctx.flags = flags;
 
+	/* sync private index if needed */
+	(void)index_storage_mailbox_sync_pvt(box);
+
 	if (failed) {
 		ctx->failed = TRUE;
 		return &ctx->ctx;
--- a/src/lib-storage/index/index-transaction.c	Tue Jun 05 00:34:32 2012 +0300
+++ b/src/lib-storage/index/index-transaction.c	Mon Jun 11 18:22:06 2012 +0300
@@ -7,6 +7,8 @@
 
 static void index_transaction_free(struct mailbox_transaction_context *t)
 {
+	if (t->view_pvt != NULL)
+		mail_index_view_close(&t->view_pvt);
 	mail_cache_view_close(&t->cache_view);
 	mail_index_view_close(&t->view);
 	array_free(&t->module_contexts);
@@ -67,26 +69,48 @@
 	index_transaction_free(t);
 }
 
+static enum mail_index_transaction_flags
+index_transaction_flags_get(enum mailbox_transaction_flags flags)
+{
+	enum mail_index_transaction_flags itrans_flags;
+
+	itrans_flags = MAIL_INDEX_TRANSACTION_FLAG_AVOID_FLAG_UPDATES;
+	if ((flags & MAILBOX_TRANSACTION_FLAG_HIDE) != 0)
+		itrans_flags |= MAIL_INDEX_TRANSACTION_FLAG_HIDE;
+	if ((flags & MAILBOX_TRANSACTION_FLAG_EXTERNAL) != 0)
+		itrans_flags |= MAIL_INDEX_TRANSACTION_FLAG_EXTERNAL;
+	if ((flags & MAILBOX_TRANSACTION_FLAG_SYNC) != 0)
+		itrans_flags |= MAIL_INDEX_TRANSACTION_FLAG_SYNC;
+	return itrans_flags;
+}
+
+void index_transaction_init_pvt(struct mailbox_transaction_context *t)
+{
+	enum mail_index_transaction_flags itrans_flags;
+
+	if (t->box->view_pvt == NULL || t->itrans_pvt != NULL)
+		return;
+
+	itrans_flags = index_transaction_flags_get(t->flags);
+	t->itrans_pvt = mail_index_transaction_begin(t->box->view_pvt,
+						     itrans_flags);
+	t->view_pvt = mail_index_transaction_open_updated_view(t->itrans_pvt);
+}
+
 void index_transaction_init(struct mailbox_transaction_context *t,
 			    struct mailbox *box,
 			    enum mailbox_transaction_flags flags)
 {
-	enum mail_index_transaction_flags trans_flags;
+	enum mail_index_transaction_flags itrans_flags;
 
 	i_assert(box->opened);
 
-	trans_flags = MAIL_INDEX_TRANSACTION_FLAG_AVOID_FLAG_UPDATES;
-	if ((flags & MAILBOX_TRANSACTION_FLAG_HIDE) != 0)
-		trans_flags |= MAIL_INDEX_TRANSACTION_FLAG_HIDE;
-	if ((flags & MAILBOX_TRANSACTION_FLAG_EXTERNAL) != 0)
-		trans_flags |= MAIL_INDEX_TRANSACTION_FLAG_EXTERNAL;
-	if ((flags & MAILBOX_TRANSACTION_FLAG_SYNC) != 0)
-		trans_flags |= MAIL_INDEX_TRANSACTION_FLAG_SYNC;
+	itrans_flags = index_transaction_flags_get(flags);
 	if ((flags & MAILBOX_TRANSACTION_FLAG_REFRESH) != 0)
 		(void)mail_index_refresh(box->index);
 
 	t->box = box;
-	t->itrans = mail_index_transaction_begin(box->view, trans_flags);
+	t->itrans = mail_index_transaction_begin(box->view, itrans_flags);
 	t->view = mail_index_transaction_open_updated_view(t->itrans);
 
 	array_create(&t->module_contexts, default_pool,
@@ -123,7 +147,7 @@
 	struct mailbox *box = t->box;
 	struct mail_index_transaction *itrans = t->itrans;
 	struct mail_index_transaction_commit_result result;
-	int ret;
+	int ret = 0;
 
 	memset(changes_r, 0, sizeof(*changes_r));
 	changes_r->pool = pool_alloconly_create(MEMPOOL_GROWING
@@ -131,7 +155,10 @@
 	p_array_init(&changes_r->saved_uids, changes_r->pool, 32);
 	t->changes = changes_r;
 
-	ret = mail_index_transaction_commit_full(&itrans, &result);
+	if (t->itrans_pvt != NULL)
+		ret = mail_index_transaction_commit(&t->itrans_pvt);
+	if (mail_index_transaction_commit_full(&itrans, &result) < 0)
+		ret = -1;
 	t = NULL;
 
 	if (ret < 0 && mail_index_is_deleted(box->index))
@@ -145,5 +172,7 @@
 {
 	struct mail_index_transaction *itrans = t->itrans;
 
+	if (t->itrans_pvt != NULL)
+		mail_index_transaction_rollback(&t->itrans_pvt);
 	mail_index_transaction_rollback(&itrans);
 }
--- a/src/lib-storage/index/shared/shared-list.c	Tue Jun 05 00:34:32 2012 +0300
+++ b/src/lib-storage/index/shared/shared-list.c	Mon Jun 11 18:22:06 2012 +0300
@@ -99,6 +99,7 @@
 		case MAILBOX_LIST_PATH_TYPE_MAILBOX:
 		case MAILBOX_LIST_PATH_TYPE_ALT_MAILBOX:
 		case MAILBOX_LIST_PATH_TYPE_CONTROL:
+		case MAILBOX_LIST_PATH_TYPE_INDEX_PRIVATE:
 			break;
 		case MAILBOX_LIST_PATH_TYPE_INDEX:
 			/* we can safely say we don't use indexes */
--- a/src/lib-storage/list/mailbox-list-fs.c	Tue Jun 05 00:34:32 2012 +0300
+++ b/src/lib-storage/list/mailbox-list-fs.c	Mon Jun 11 18:22:06 2012 +0300
@@ -193,6 +193,11 @@
 					       set->mailbox_dir_name, name);
 		}
 		break;
+	case MAILBOX_LIST_PATH_TYPE_INDEX_PRIVATE:
+		if (set->index_pvt_dir == NULL)
+			return NULL;
+		return t_strdup_printf("%s/%s%s", set->index_pvt_dir,
+				       set->mailbox_dir_name, name);
 	}
 
 	if (type == MAILBOX_LIST_PATH_TYPE_ALT_DIR ||
--- a/src/lib-storage/list/mailbox-list-maildir.c	Tue Jun 05 00:34:32 2012 +0300
+++ b/src/lib-storage/list/mailbox-list-maildir.c	Mon Jun 11 18:22:06 2012 +0300
@@ -225,6 +225,11 @@
 						_list->set.index_dir, name);
 		}
 		break;
+	case MAILBOX_LIST_PATH_TYPE_INDEX_PRIVATE:
+		if (_list->set.index_pvt_dir == NULL)
+			return NULL;
+		return maildir_list_get_dirname_path(_list,
+					_list->set.index_pvt_dir, name);
 	}
 
 	if (type == MAILBOX_LIST_PATH_TYPE_ALT_DIR ||
--- a/src/lib-storage/mail-storage-private.h	Tue Jun 05 00:34:32 2012 +0300
+++ b/src/lib-storage/mail-storage-private.h	Mon Jun 11 18:22:06 2012 +0300
@@ -220,6 +220,11 @@
 	struct mail_index *index;
 	struct mail_index_view *view;
 	struct mail_cache *cache;
+	/* Private per-user index/view for shared mailboxes. These are synced
+	   against the primary index and used to store per-user flags.
+	   These are non-NULL only when mailbox has per-user flags. */
+	struct mail_index *index_pvt;
+	struct mail_index_view *view_pvt;
 	/* Filled lazily by mailbox_get_permissions() */
 	struct mailbox_permissions _perm;
 	/* Filled lazily by mailbox_get_path() */
@@ -345,6 +350,8 @@
 	struct mail mail;
 	struct mail_vfuncs v, *vlast;
 
+	uint32_t seq_pvt;
+
 	/* initial wanted fields/headers, set by mail_alloc(): */
 	enum mail_fetch_field wanted_fields;
 	struct mailbox_header_lookup_ctx *wanted_headers;
@@ -387,6 +394,10 @@
 	/* view contains all changes done within this transaction */
 	struct mail_index_view *view;
 
+	/* for private index updates: */
+	struct mail_index_transaction *itrans_pvt;
+	struct mail_index_view *view_pvt;
+
 	struct mail_cache_view *cache_view;
 	struct mail_cache_transaction_ctx *cache_trans;
 
--- a/src/lib-storage/mail-storage.c	Tue Jun 05 00:34:32 2012 +0300
+++ b/src/lib-storage/mail-storage.c	Mon Jun 11 18:22:06 2012 +0300
@@ -1301,10 +1301,12 @@
 
 enum mail_flags mailbox_get_private_flags_mask(struct mailbox *box)
 {
-	if (box->v.get_private_flags_mask == NULL)
+	if (box->v.get_private_flags_mask != NULL)
+		return box->v.get_private_flags_mask(box);
+	else if (box->list->set.index_pvt_dir != NULL)
+		return MAIL_SEEN; /* FIXME */
+	else
 		return 0;
-	else
-		return box->v.get_private_flags_mask(box);
 }
 
 struct mailbox_sync_context *
--- a/src/lib-storage/mailbox-list-private.h	Tue Jun 05 00:34:32 2012 +0300
+++ b/src/lib-storage/mailbox-list-private.h	Mon Jun 11 18:22:06 2012 +0300
@@ -196,6 +196,8 @@
 					const char **name);
 int mailbox_list_create_missing_index_dir(struct mailbox_list *list,
 					  const char *name);
+int mailbox_list_create_missing_index_pvt_dir(struct mailbox_list *list,
+					      const char *name);
 
 void mailbox_list_add_change(struct mailbox_list *list,
 			     enum mailbox_log_record_type type,
--- a/src/lib-storage/mailbox-list.c	Tue Jun 05 00:34:32 2012 +0300
+++ b/src/lib-storage/mailbox-list.c	Mon Jun 11 18:22:06 2012 +0300
@@ -149,6 +149,9 @@
 		list->set.index_dir = set->index_dir == NULL ||
 			strcmp(set->index_dir, set->root_dir) == 0 ? NULL :
 			p_strdup(list->pool, set->index_dir);
+		list->set.index_pvt_dir = set->index_pvt_dir == NULL ||
+			strcmp(set->index_pvt_dir, set->root_dir) == 0 ? NULL :
+			p_strdup(list->pool, set->index_pvt_dir);
 		list->set.control_dir = set->control_dir == NULL ||
 			strcmp(set->control_dir, set->root_dir) == 0 ? NULL :
 			p_strdup(list->pool, set->control_dir);
@@ -175,10 +178,11 @@
 	list->set.utf8 = set->utf8;
 
 	if (ns->mail_set->mail_debug) {
-		i_debug("%s: root=%s, index=%s, control=%s, inbox=%s, alt=%s",
+		i_debug("%s: root=%s, index=%s, indexpvt=%s, control=%s, inbox=%s, alt=%s",
 			list->name,
 			list->set.root_dir == NULL ? "" : list->set.root_dir,
 			list->set.index_dir == NULL ? "" : list->set.index_dir,
+			list->set.index_pvt_dir == NULL ? "" : list->set.index_pvt_dir,
 			list->set.control_dir == NULL ?
 			"" : list->set.control_dir,
 			list->set.inbox_path == NULL ?
@@ -289,6 +293,8 @@
 			dest = &set_r->inbox_path;
 		else if (strcmp(key, "INDEX") == 0)
 			dest = &set_r->index_dir;
+		else if (strcmp(key, "INDEXPVT") == 0)
+			dest = &set_r->index_pvt_dir;
 		else if (strcmp(key, "CONTROL") == 0)
 			dest = &set_r->control_dir;
 		else if (strcmp(key, "ALT") == 0)
@@ -967,6 +973,8 @@
 	case MAILBOX_LIST_PATH_TYPE_INDEX:
 		return set->index_dir != NULL ?
 			set->index_dir : set->root_dir;
+	case MAILBOX_LIST_PATH_TYPE_INDEX_PRIVATE:
+		return set->index_pvt_dir;
 	}
 	i_unreached();
 }
@@ -1363,19 +1371,18 @@
 	return mailbox_list_mkdir(list, mailbox, t_strdup_until(path, p));
 }
 
-int mailbox_list_create_missing_index_dir(struct mailbox_list *list,
-					  const char *name)
+static int mailbox_list_create_missing_dir(struct mailbox_list *list,
+					   const char *name,
+					   enum mailbox_list_path_type type)
 {
 	const char *root_dir, *index_dir, *parent_dir, *p, *error;
 	struct mailbox_permissions perm;
 	unsigned int n = 0;
 
-	list->index_root_dir_created = TRUE;
 	root_dir = mailbox_list_get_path(list, name,
 					 MAILBOX_LIST_PATH_TYPE_MAILBOX);
-	index_dir = mailbox_list_get_path(list, name,
-					  MAILBOX_LIST_PATH_TYPE_INDEX);
-	if (*index_dir == '\0')
+	index_dir = mailbox_list_get_path(list, name, type);
+	if (index_dir == NULL || *index_dir == '\0')
 		return 0;
 	if (strcmp(index_dir, root_dir) == 0) {
 		if ((list->props & MAILBOX_LIST_PROP_AUTOCREATE_DIRS) == 0)
@@ -1384,8 +1391,7 @@
 	}
 
 	if (name == NULL) {
-		if (mailbox_list_mkdir_root(list, index_dir,
-					    MAILBOX_LIST_PATH_TYPE_INDEX,
+		if (mailbox_list_mkdir_root(list, index_dir, type,
 					    &error) < 0) {
 			mailbox_list_set_critical(list,
 				"Couldn't create index root dir %s: %s",
@@ -1410,8 +1416,7 @@
 		}
 		/* create the parent directory first */
 		parent_dir = t_strdup_until(index_dir, p);
-		if (mailbox_list_mkdir_root(list, parent_dir,
-					    MAILBOX_LIST_PATH_TYPE_INDEX,
+		if (mailbox_list_mkdir_root(list, parent_dir, type,
 					    &error) < 0) {
 			mailbox_list_set_critical(list,
 				"Couldn't create index dir %s: %s",
@@ -1422,6 +1427,21 @@
 	return 0;
 }
 
+int mailbox_list_create_missing_index_dir(struct mailbox_list *list,
+					  const char *name)
+{
+	list->index_root_dir_created = TRUE;
+	return mailbox_list_create_missing_dir(list, name,
+		MAILBOX_LIST_PATH_TYPE_INDEX);
+}
+
+int mailbox_list_create_missing_index_pvt_dir(struct mailbox_list *list,
+					      const char *name)
+{
+	return mailbox_list_create_missing_dir(list, name,
+		MAILBOX_LIST_PATH_TYPE_INDEX_PRIVATE);
+}
+
 const char *mailbox_list_get_last_error(struct mailbox_list *list,
 					enum mail_error *error_r)
 {
--- a/src/lib-storage/mailbox-list.h	Tue Jun 05 00:34:32 2012 +0300
+++ b/src/lib-storage/mailbox-list.h	Mon Jun 11 18:22:06 2012 +0300
@@ -101,8 +101,10 @@
 	MAILBOX_LIST_PATH_TYPE_ALT_MAILBOX,
 	/* Return control directory */
 	MAILBOX_LIST_PATH_TYPE_CONTROL,
-	/* Return index file directory */
-	MAILBOX_LIST_PATH_TYPE_INDEX
+	/* Return index directory ("" for in-memory) */
+	MAILBOX_LIST_PATH_TYPE_INDEX,
+	/* Return the private index directory (NULL if none) */
+	MAILBOX_LIST_PATH_TYPE_INDEX_PRIVATE
 };
 
 enum mailbox_list_file_type {
@@ -117,6 +119,7 @@
 	const char *layout; /* FIXME: shouldn't be here */
 	const char *root_dir;
 	const char *index_dir;
+	const char *index_pvt_dir;
 	const char *control_dir;
 	const char *alt_dir; /* FIXME: dbox-specific.. */