--- a/src/deliver/deliver.c	Sun Nov 26 00:12:11 2006 +0200
+++ b/src/deliver/deliver.c	Sun Nov 26 00:17:39 2006 +0200
@@ -40,8 +40,6 @@
 struct deliver_settings *deliver_set;
 deliver_mail_func_t *deliver_mail = NULL;
 
-void (*hook_mail_storage_created)(struct mail_storage *storage) = NULL;
-
 static struct module *modules;
 static struct ioloop *ioloop;
 
@@ -58,12 +56,11 @@
 {
 	struct mailbox_sync_context *ctx;
         struct mailbox_sync_rec sync_rec;
-	struct mailbox_status status;
 
 	ctx = mailbox_sync_init(box, 0);
 	while (mailbox_sync_next(ctx, &sync_rec) > 0)
 		;
-	return mailbox_sync_deinit(&ctx, &status);
+	return mailbox_sync_deinit(&ctx, 0, NULL);
 }
 
 static struct mailbox *
@@ -567,9 +564,6 @@
 			destination, mail_env == NULL ? "(null)" : mail_env);
 	}
 
-	if (hook_mail_storage_created != NULL)
-		hook_mail_storage_created(storage);
-
 	mbox_storage = mail_storage_create("mbox", "/tmp", destination, 0,
 					   MAIL_STORAGE_LOCK_FCNTL);
 	input = create_mbox_stream(0, envelope_sender);

--- a/src/imap/Makefile.am	Sun Nov 26 00:12:11 2006 +0200
+++ b/src/imap/Makefile.am	Sun Nov 26 00:17:39 2006 +0200
@@ -21,8 +21,8 @@
 
 libs = \
 	../lib-storage/register/libstorage-register.a \
+	../lib-storage/list/libstorage_list.a \
 	$(STORAGE_LIBS) \
-	../lib-storage/list/libstorage_list.a \
 	../lib-storage/libstorage.a \
 	../lib-imap/libimap.a \
 	../lib-mail/libmail.a \

--- a/src/imap/cmd-list.c	Sun Nov 26 00:12:11 2006 +0200
+++ b/src/imap/cmd-list.c	Sun Nov 26 00:17:39 2006 +0200
@@ -33,13 +33,6 @@
 {
 	const char *str;
 
-	if (flags & MAILBOX_PLACEHOLDER) {
-		i_assert((flags & ~MAILBOX_CHILDREN) == MAILBOX_PLACEHOLDER);
-
-		if ((list_flags & _MAILBOX_LIST_ITER_LISTEXT) == 0)
-			flags = MAILBOX_NOSELECT;
-		flags |= MAILBOX_CHILDREN;
-	}
 	if ((flags & MAILBOX_NONEXISTENT) != 0 &&
 	    (list_flags & _MAILBOX_LIST_ITER_LISTEXT) == 0) {
 		flags |= MAILBOX_NOSELECT;
@@ -52,7 +45,6 @@
 	str = t_strconcat(
 		(flags & MAILBOX_NOSELECT) ? " \\Noselect" : "",
 		(flags & MAILBOX_NONEXISTENT) ? " \\NonExistent" : "",
-		(flags & MAILBOX_PLACEHOLDER) ? " \\PlaceHolder" : "",
 		(flags & MAILBOX_CHILDREN) ? " \\HasChildren" : "",
 		(flags & MAILBOX_NOCHILDREN) ? " \\HasNoChildren" : "",
 		(flags & MAILBOX_NOINFERIORS) ? " \\NoInferiors" : "",
@@ -318,7 +310,7 @@
                         enum mailbox_info_flags flags;
 			string_t *str = t_str_new(128);
 
-			flags = MAILBOX_PLACEHOLDER;
+			flags = MAILBOX_NONEXISTENT | MAILBOX_CHILDREN;
 			str_printfa(str, "* LIST (%s) \"%s\" ",
 				    mailbox_flags2str(flags, ctx->list_flags),
 				    ns->sep_str);
@@ -386,7 +378,8 @@
 	cur_mask = namespace_fix_sep(ns, cur_mask);
 
 	list = mail_storage_get_list(ns->storage);
-	ctx->list_iter = mailbox_list_iter_init(list, cur_ref, cur_mask,
+	cur_mask = mailbox_list_join_refmask(list, cur_ref, cur_mask);
+	ctx->list_iter = mailbox_list_iter_init(list, cur_mask,
 						ctx->list_flags);
 }
 

--- a/src/imap/common.h	Sun Nov 26 00:12:11 2006 +0200
+++ b/src/imap/common.h	Sun Nov 26 00:17:39 2006 +0200
@@ -37,7 +37,6 @@
 
 extern string_t *capability_string;
 
-extern void (*hook_mail_storage_created)(struct mail_storage *storage);
 extern void (*hook_client_created)(struct client **client);
 
 #endif

--- a/src/imap/imap-sync.c	Sun Nov 26 00:12:11 2006 +0200
+++ b/src/imap/imap-sync.c	Sun Nov 26 00:17:39 2006 +0200
@@ -63,7 +63,9 @@
 
 	mail_free(&ctx->mail);
 
-	if (mailbox_sync_deinit(&ctx->sync_ctx, &status) < 0 || ctx->failed) {
+	if (mailbox_sync_deinit(&ctx->sync_ctx,
+				STATUS_MESSAGES | STATUS_RECENT, &status) < 0 ||
+	    ctx->failed) {
 		mailbox_transaction_rollback(&ctx->t);
 		i_free(ctx);
 		return -1;
@@ -190,12 +192,11 @@
 {
 	struct mailbox_sync_context *ctx;
         struct mailbox_sync_rec sync_rec;
-	struct mailbox_status status;
 
 	ctx = mailbox_sync_init(box, flags);
 	while (mailbox_sync_next(ctx, &sync_rec) > 0)
 		;
-	return mailbox_sync_deinit(&ctx, &status);
+	return mailbox_sync_deinit(&ctx, 0, NULL);
 }
 
 static bool cmd_sync_continue(struct client_command_context *cmd)

--- a/src/imap/imap-thread.c	Sun Nov 26 00:12:11 2006 +0200
+++ b/src/imap/imap-thread.c	Sun Nov 26 00:17:39 2006 +0200
@@ -2017,17 +2017,42 @@
 					    tbox, TRUE);
 }
 
+static struct mailbox_sync_context *
+imap_thread_sync_init(struct mailbox *box, enum mailbox_sync_flags flags)
+{
+	struct imap_thread_mailbox *tbox = IMAP_THREAD_CONTEXT(box);
+	struct mailbox_sync_context *ctx;
+
+	ctx = tbox->super.sync_init(box, flags);
+	if (box->opened) {
+		imap_thread_hash_init(box, FALSE);
+		/* we don't want to get back here */
+		box->v.sync_init = tbox->super.sync_init;
+	}
+	return ctx;
+}
+
 static void imap_thread_mailbox_opened(struct mailbox *box)
 {
 	struct imap_thread_mailbox *tbox;
 
+	if (next_hook_mailbox_opened != NULL)
+		next_hook_mailbox_opened(box);
+
 	tbox = i_new(struct imap_thread_mailbox, 1);
 	tbox->super = box->v;
 
 	array_idx_set(&box->module_contexts,
 		      imap_thread_storage_module_id, &tbox);
 
-	imap_thread_hash_init(box, FALSE);
+	if (box->opened)
+		imap_thread_hash_init(box, FALSE);
+	else {
+		/* delayed opening used. we want to try to open the hash
+		   anyway, because if syncing expunges anything and we didn't
+		   notice it, we would have to rebuild the hash */
+		box->v.sync_init = imap_thread_sync_init;
+	}
 }
 
 void imap_thread_init(void)

--- a/src/imap/main.c	Sun Nov 26 00:12:11 2006 +0200
+++ b/src/imap/main.c	Sun Nov 26 00:17:39 2006 +0200
@@ -48,7 +48,6 @@
 static char log_prefix[128]; /* syslog() needs this to be permanent */
 static pool_t namespace_pool;
 
-void (*hook_mail_storage_created)(struct mail_storage *storage) = NULL;
 void (*hook_client_created)(struct client **client) = NULL;
 
 string_t *capability_string;

--- a/src/imap/namespace.c	Sun Nov 26 00:12:11 2006 +0200
+++ b/src/imap/namespace.c	Sun Nov 26 00:17:39 2006 +0200
@@ -20,9 +20,6 @@
 	} else {
 		ns->sep_str[0] = ns->sep;
 	}
-
-	if (hook_mail_storage_created != NULL)
-		hook_mail_storage_created(ns->storage);
 }
 
 static struct namespace *

--- a/src/lib-index/Makefile.am	Sun Nov 26 00:12:11 2006 +0200
+++ b/src/lib-index/Makefile.am	Sun Nov 26 00:17:39 2006 +0200
@@ -28,7 +28,9 @@
         mail-transaction-log.c \
         mail-transaction-log-append.c \
         mail-transaction-log-view.c \
-        mail-transaction-util.c
+        mail-transaction-util.c \
+        mailbox-list-index.c \
+        mailbox-list-index-sync.c
 
 noinst_HEADERS = \
 	mail-cache.h \
@@ -41,4 +43,6 @@
 	mail-index-view-private.h \
         mail-transaction-log.h \
 	mail-transaction-log-private.h \
-        mail-transaction-util.h
+        mail-transaction-util.h \
+        mailbox-list-index.h \
+        mailbox-list-index-private.h

--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/src/lib-index/mailbox-list-index-private.h	Sun Nov 26 00:17:39 2006 +0200
@@ -0,0 +1,79 @@
+#ifndef __MAILBOX_LIST_INDEX_PRIVATE_H
+#define __MAILBOX_LIST_INDEX_PRIVATE_H
+
+#include "mailbox-list-index.h"
+
+#define MAILBOX_LIST_INDEX_MAJOR_VERSION 1
+#define MAILBOX_LIST_INDEX_MINOR_VERSION 0
+
+struct mailbox_list_index_header {
+	uint8_t major_version;
+	uint8_t minor_version;
+	uint8_t unused[2];
+
+	uint32_t header_size;
+	uint32_t uid_validity;
+
+	/* locking required to access the fields below: */
+	uint32_t next_uid;
+
+	uint32_t used_space;
+	uint32_t deleted_space;
+};
+
+struct mailbox_list_dir_record {
+	/* If non-zero, contains a pointer to updated directory list.
+	   Stored using mail_index_uint32_to_offset(). */
+	uint32_t next_offset;
+
+	uint32_t count;
+	/* The records are sorted by their name_hash */
+	/* struct mailbox_list_record records[count]; */
+};
+
+struct mailbox_list_record {
+	/* CRC32 hash of the name */
+	uint32_t name_hash;
+	uint32_t uid:31;
+	/* Set when this record has been marked as deleted. It will be removed
+	   permanently the next time a new record is added to this directory
+	   or on the next index compression. */
+	uint32_t deleted:1;
+
+	/* Points to a NUL-terminated record name */
+	uint32_t name_offset;
+	/* the dir offset is stored using mail_index_uint32_to_offset()
+	   since it may change while we're reading */
+	uint32_t dir_offset;
+};
+
+struct mailbox_list_index {
+	char *filepath;
+	char separator;
+	struct mail_index *mail_index;
+
+	int fd;
+
+	void *mmap_base;
+	size_t mmap_size;
+	const struct mailbox_list_index_header *hdr;
+};
+
+#define MAILBOX_LIST_RECORDS(dir) \
+	((struct mailbox_list_record *)(dir + 1))
+#define MAILBOX_LIST_RECORD_IDX(dir, rec) \
+	((rec) - MAILBOX_LIST_RECORDS(dir))
+
+int mailbox_list_index_set_syscall_error(struct mailbox_list_index *index,
+					 const char *function);
+
+int mailbox_list_index_dir_lookup_rec(struct mailbox_list_index *index,
+				      const struct mailbox_list_dir_record *dir,
+				      const char *name,
+				      const struct mailbox_list_record **rec_r);
+int mailbox_list_index_get_dir(struct mailbox_list_index *index,
+			       uint32_t *offset,
+			       const struct mailbox_list_dir_record **dir_r);
+int mailbox_list_index_map(struct mailbox_list_index *index);
+
+#endif

--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/src/lib-index/mailbox-list-index-sync.c	Sun Nov 26 00:17:39 2006 +0200
@@ -0,0 +1,737 @@
+/* Copyright (C) 2006 Timo Sirainen */
+
+#include "lib.h"
+#include "array.h"
+#include "bsearch-insert-pos.h"
+#include "crc32.h"
+#include "file-set-size.h"
+#include "mmap-util.h"
+#include "mail-index-private.h"
+#include "mailbox-list-index-private.h"
+
+#include <stddef.h>
+
+#define ROOT_INIT_COUNT 128
+#define DIR_ALLOC_MORE_COUNT 4
+#define MAILBOX_LIST_INDEX_GROW_PERCENTAGE 10
+#define MAILBOX_LIST_INDEX_MIN_SIZE 512
+
+struct mailbox_list_sync_record {
+	uint32_t name_hash;
+	uint32_t seq;
+	uint32_t uid;
+	const char *name;
+
+	/* dir is used if it's non-NULL, otherwise dir_offset is used */
+	struct mailbox_list_sync_dir *dir;
+	uint32_t dir_offset;
+
+	uint32_t created:1;
+	uint32_t seen:1;
+};
+
+struct mailbox_list_sync_dir {
+	/* The records are sorted by their name_hash */
+	ARRAY_DEFINE(records, struct mailbox_list_sync_record);
+
+	/* Offset to the original location in the index, or 0 for new dirs */
+	uint32_t offset;
+	unsigned int seen_records_count;
+	unsigned int new_records_count;
+};
+
+struct mailbox_list_index_sync_ctx {
+	struct mailbox_list_index *index;
+	pool_t pool;
+
+	enum mailbox_list_sync_flags flags;
+	const char *sync_path;
+	struct mail_index_sync_ctx *mail_sync_ctx;
+	struct mail_index_view *view;
+	struct mail_index_transaction *trans;
+
+	struct mailbox_list_index_header hdr;
+	struct mailbox_list_sync_dir *root, *sync_root;
+
+	unsigned int failed:1;
+	unsigned int partial:1;
+	unsigned int seen_sync_root:1;
+};
+
+struct mailbox_list_sync_lookup_key {
+	uint32_t name_hash;
+	const char *name;
+	bool *match;
+};
+
+static struct mailbox_list_sync_dir *
+mailbox_list_alloc_sync_dir(struct mailbox_list_index_sync_ctx *ctx,
+			    unsigned int initial_count)
+{
+	struct mailbox_list_sync_dir *sync_dir;
+
+	sync_dir = p_new(ctx->pool, struct mailbox_list_sync_dir, 1);
+	p_array_init(&sync_dir->records, ctx->pool, initial_count);
+	return sync_dir;
+}
+
+static int
+mailbox_list_copy_sync_dir(struct mailbox_list_index_sync_ctx *ctx,
+			   uint32_t offset,
+			   struct mailbox_list_sync_dir **sync_dir_r)
+{
+	const struct mailbox_list_dir_record *dir;
+	const struct mailbox_list_record *recs;
+	struct mailbox_list_sync_dir *sync_dir;
+	struct mailbox_list_sync_record *sync_rec;
+	const char *name;
+	size_t max_len;
+	unsigned int i;
+
+	if (mailbox_list_index_get_dir(ctx->index, &offset, &dir) < 0)
+		return -1;
+
+	sync_dir = mailbox_list_alloc_sync_dir(ctx, dir->count +
+					       DIR_ALLOC_MORE_COUNT);
+	sync_dir->offset = offset;
+
+	recs = MAILBOX_LIST_RECORDS(dir);
+	for (i = 0; i < dir->count; i++) {
+		sync_rec = array_append_space(&sync_dir->records);
+		sync_rec->name_hash = recs[i].name_hash;
+		sync_rec->uid = recs[i].uid;
+		sync_rec->dir_offset =
+			mail_index_offset_to_uint32(recs[i].dir_offset);
+
+		max_len = ctx->index->mmap_size - recs[i].name_offset;
+		name = CONST_PTR_OFFSET(ctx->index->mmap_base,
+					recs[i].name_offset);
+
+		sync_rec->name = p_strndup(ctx->pool, name, max_len);
+	}
+
+	*sync_dir_r = sync_dir;
+	return 0;
+}
+
+static int mailbox_list_sync_record_cmp(const void *_key, const void *_rec)
+{
+	const struct mailbox_list_sync_lookup_key *key = _key;
+	const struct mailbox_list_sync_record *rec = _rec;
+	int ret;
+
+	if (key->name_hash < rec->name_hash)
+		return -1;
+	if (key->name_hash > rec->name_hash)
+		return 1;
+
+	ret = strcmp(key->name, rec->name);
+	if (ret == 0)
+		*key->match = TRUE;
+	return ret;
+}
+
+static struct mailbox_list_sync_record *
+mailbox_list_sync_dir_lookup(struct mailbox_list_sync_dir *dir,
+			     const char *name, unsigned int *idx_r)
+{
+	struct mailbox_list_sync_lookup_key key;
+	const struct mailbox_list_sync_record *recs;
+	struct mailbox_list_sync_record *rec;
+	unsigned int count;
+	bool match;
+
+	/* binary search the current hierarchy level name. the values are
+	   sorted primarily by their hash value and secondarily by the actual
+	   name */
+	match = FALSE;
+	key.name = name;
+	key.name_hash = crc32_str(name);
+	key.match = &match;
+
+	recs = array_get(&dir->records, &count);
+	rec = bsearch_insert_pos(&key, recs, count, sizeof(*rec),
+				 mailbox_list_sync_record_cmp);
+	*idx_r = rec - recs;
+	return match ? rec : NULL;
+}
+
+static struct mailbox_list_sync_record *
+mailbox_list_alloc_add_record(struct mailbox_list_index_sync_ctx *ctx,
+			      struct mailbox_list_sync_dir *dir,
+			      const char *name, unsigned int idx)
+{
+	struct mailbox_list_sync_record *rec;
+
+	rec = array_insert_space(&dir->records, idx);
+	rec->name_hash = crc32_str(name);
+	rec->name = p_strdup(ctx->pool, name);
+	rec->uid = ctx->hdr.next_uid++;
+	rec->created = TRUE;
+	mail_index_append(ctx->trans, rec->uid, &rec->seq);
+
+	dir->new_records_count++;
+	return rec;
+}
+
+static int
+mailbox_list_index_sync_get_seq(struct mailbox_list_index_sync_ctx *ctx,
+				struct mailbox_list_sync_record *rec)
+{
+	if (rec->uid == 0) {
+		return mailbox_list_index_set_corrupted(ctx->index,
+							"Record with UID=0");
+	}
+	if (mail_index_lookup_uid_range(ctx->view, rec->uid, rec->uid,
+					&rec->seq, &rec->seq) < 0)
+		return -1;
+
+	if (rec->seq == 0) {
+		return mailbox_list_index_set_corrupted(ctx->index,
+			"Desync: Record expunged from mail index");
+	}
+	return 0;
+}
+
+static int
+mailbox_list_index_sync_int(struct mailbox_list_index_sync_ctx *ctx,
+			    const char *name,
+			    struct mailbox_list_sync_dir **dir_r,
+			    uint32_t *seq_r)
+{
+	const char *p, *hier_name;
+	struct mailbox_list_sync_dir *dir;
+	struct mailbox_list_sync_record *rec = NULL;
+	unsigned int idx;
+
+	if (ctx->failed)
+		return -1;
+
+	dir = ctx->sync_root;
+
+	t_push();
+	for (;;) {
+		p = strchr(name, ctx->index->separator);
+		hier_name = p == NULL ? name : t_strdup_until(name, p);
+
+		if (*hier_name == '\0') {
+			if (p == NULL) {
+				/* name ended with a separator */
+				break;
+			}
+			/* two separators adjacently, skip this */
+			name = p + 1;
+			continue;
+		}
+
+		if (rec != NULL) {
+			mail_index_update_flags(ctx->trans, rec->seq,
+				MODIFY_REPLACE,
+				MAILBOX_LIST_INDEX_FLAG_NONEXISTENT |
+				MAILBOX_LIST_INDEX_FLAG_CHILDREN);
+		}
+
+		rec = mailbox_list_sync_dir_lookup(dir, hier_name, &idx);
+		if (rec == NULL) {
+			/* new record */
+			rec = mailbox_list_alloc_add_record(ctx, dir,
+							    hier_name, idx);
+		} else if (rec->seq == 0) {
+			/* this record was copied from existing index.
+			   the uid is known, but the sequence isn't. */
+			if (mailbox_list_index_sync_get_seq(ctx, rec) < 0) {
+				ctx->failed = TRUE;
+				break;
+			}
+		}
+		*seq_r = rec->seq;
+
+		/* remember that we've seen this record */
+		if (!rec->seen) {
+			rec->seen = TRUE;
+			dir->seen_records_count++;
+		}
+
+		if (p == NULL) {
+			/* leaf */
+			break;
+		}
+
+		if (rec->dir == NULL) {
+			if (rec->dir_offset != 0) {
+				if (mailbox_list_copy_sync_dir(ctx,
+							       rec->dir_offset,
+							       &rec->dir) < 0) {
+					ctx->failed = TRUE;
+					break;
+				}
+			} else {
+				rec->dir = mailbox_list_alloc_sync_dir(ctx,
+						1 + DIR_ALLOC_MORE_COUNT);
+			}
+		}
+
+		name = p + 1;
+		dir = rec->dir;
+	}
+	t_pop();
+
+	i_assert(dir != NULL);
+	*dir_r = dir;
+	return ctx->failed ? -1 : 0;
+}
+
+static int mailbox_list_index_get_root(struct mailbox_list_index_sync_ctx *ctx)
+{
+	uint32_t seq;
+
+	i_assert(ctx->index->mmap_size > 0);
+
+	if (ctx->index->mmap_size == sizeof(*ctx->index->hdr)) {
+		/* root doesn't exist in the file yet */
+		ctx->root = mailbox_list_alloc_sync_dir(ctx,
+							ROOT_INIT_COUNT);
+	} else {
+		if (mailbox_list_copy_sync_dir(ctx, sizeof(*ctx->index->hdr),
+					       &ctx->root) < 0)
+			return -1;
+	}
+
+	/* keep sync_root=root until we've built the sync_root path. */
+	ctx->sync_root = ctx->root;
+
+	if (*ctx->sync_path != '\0') {
+		if (mailbox_list_index_sync_more(ctx, ctx->sync_path, &seq) < 0)
+			return -1;
+	}
+
+	return mailbox_list_index_sync_int(ctx, ctx->sync_path,
+					   &ctx->sync_root, &seq);
+}
+
+static int sync_init_mail_sync(struct mailbox_list_index_sync_ctx *ctx)
+{
+	struct mail_index_sync_rec sync_rec;
+	const struct mail_index_header *hdr;
+
+	if (mail_index_sync_begin(ctx->index->mail_index, &ctx->mail_sync_ctx,
+				  &ctx->view, (uint32_t)-1, 0,
+				  FALSE, FALSE) < 0)
+		return -1;
+
+	/* we should have only external transactions in here, for which we
+	   don't need to do anything but write them to the index */
+	while (mail_index_sync_next(ctx->mail_sync_ctx, &sync_rec) > 0)
+		;
+
+	hdr = mail_index_get_header(ctx->view);
+	if (hdr->uid_validity != 0) {
+		if (hdr->uid_validity != ctx->hdr.uid_validity) {
+			return mailbox_list_index_set_corrupted(ctx->index,
+				"Desync: uid_validity changed");
+		}
+	}
+
+	ctx->trans = mail_index_transaction_begin(ctx->view, FALSE, TRUE);
+	if (hdr->uid_validity == 0) {
+		mail_index_update_header(ctx->trans,
+			offsetof(struct mail_index_header, uid_validity),
+			&ctx->hdr.uid_validity, sizeof(ctx->hdr.uid_validity),
+			TRUE);
+	}
+
+	return mailbox_list_index_get_root(ctx);
+}
+
+int mailbox_list_index_sync_init(struct mailbox_list_index *index,
+				 const char *path,
+				 enum mailbox_list_sync_flags flags,
+				 struct mailbox_list_index_sync_ctx **ctx_r)
+{
+	struct mailbox_list_index_sync_ctx *ctx;
+	pool_t pool;
+	size_t len;
+
+	/* add separator to end of path if it isn't there */
+	len = strlen(path);
+	if (len > 0 && path[len-1] != index->separator)
+		path = t_strdup_printf("%s%c", path, index->separator);
+
+	pool = pool_alloconly_create("mailbox list index sync", 1024*32);
+
+	ctx = p_new(pool, struct mailbox_list_index_sync_ctx, 1);
+	ctx->pool = pool;
+	ctx->index = index;
+	ctx->sync_path = p_strdup(pool, path);
+	ctx->flags = flags;
+	ctx->hdr = *index->hdr;
+
+	/* mail index syncing acts as the only locking for us */
+	if (sync_init_mail_sync(ctx) < 0) {
+		mailbox_list_index_sync_commit(&ctx);
+		return -1;
+	}
+
+	*ctx_r = ctx;
+	return 0;
+}
+
+struct mail_index_view *
+mailbox_list_index_sync_get_view(struct mailbox_list_index_sync_ctx *ctx)
+{
+	return ctx->view;
+}
+
+struct mail_index_transaction *
+mailbox_list_index_sync_get_transaction(struct mailbox_list_index_sync_ctx *ctx)
+{
+	return ctx->trans;
+}
+
+int mailbox_list_index_sync_more(struct mailbox_list_index_sync_ctx *ctx,
+				 const char *name, uint32_t *seq_r)
+{
+	struct mailbox_list_sync_dir *dir;
+
+	return mailbox_list_index_sync_int(ctx, name, &dir, seq_r);
+}
+
+static int
+mailbox_list_index_sync_grow(struct mailbox_list_index_sync_ctx *ctx,
+			     uint32_t size)
+{
+	struct mailbox_list_index *index = ctx->index;
+	uoff_t new_fsize, grow_size;
+
+	new_fsize = ctx->hdr.used_space + size;
+	grow_size = new_fsize / 100 * MAILBOX_LIST_INDEX_GROW_PERCENTAGE;
+	if (grow_size < MAILBOX_LIST_INDEX_MIN_SIZE)
+		grow_size = MAILBOX_LIST_INDEX_MIN_SIZE;
+	new_fsize += grow_size;
+	new_fsize &= ~(512-1);
+
+	i_assert(new_fsize >= ctx->hdr.used_space + size);
+
+	if (file_set_size(index->fd, (off_t)new_fsize) < 0) {
+		mailbox_list_index_set_syscall_error(index, "file_set_size()");
+		return -1;
+	}
+
+	return mailbox_list_index_map(index);
+}
+
+static int
+mailbox_list_index_sync_alloc_space(struct mailbox_list_index_sync_ctx *ctx,
+				    uint32_t size, void **base_r,
+				    uint32_t *base_offset_r)
+{
+	size_t pos = ctx->hdr.used_space;
+
+	/* all allocations must be 32bit aligned */
+	pos = (pos + 3) & ~3;
+
+	if (pos + size > ctx->index->mmap_size) {
+		if (mailbox_list_index_sync_grow(ctx, size + 3) < 0)
+			return -1;
+
+		i_assert(pos + size < ctx->index->mmap_size);
+	}
+
+	*base_offset_r = pos;
+	*base_r = PTR_OFFSET(ctx->index->mmap_base, *base_offset_r);
+	ctx->hdr.used_space = pos + size;
+	return 0;
+}
+
+static int
+mailbox_list_index_sync_recreate_dir(struct mailbox_list_index_sync_ctx *ctx,
+				     struct mailbox_list_sync_dir *sync_dir,
+				     uint32_t offset_pos, bool partial)
+{
+	struct mailbox_list_dir_record *dir, *new_dir;
+	struct mailbox_list_record *recs, *new_recs;
+	struct mailbox_list_sync_record *sync_recs;
+	unsigned int src, dest, orig, count, nondeleted_count;
+	unsigned int name_space_needed, deleted_space;
+	uint32_t base_offset, name_pos, size;
+	void *base;
+
+	i_assert((offset_pos % sizeof(uint32_t)) == 0);
+	i_assert(offset_pos < ctx->index->mmap_size);
+
+	/* count how much space we need and how much we wasted for deleted
+	   records */
+	nondeleted_count = 0; name_space_needed = 0; deleted_space = 0;
+	sync_recs = array_get_modifiable(&sync_dir->records, &count);
+	for (src = 0; src < count; src++) {
+		if (sync_recs[src].seen || partial) {
+			nondeleted_count++;
+			if (sync_recs[src].created) {
+				/* new record */
+				name_space_needed +=
+					strlen(sync_recs[src].name) + 1;
+			}
+		} else {
+			deleted_space += sizeof(*new_recs) +
+				strlen(sync_recs[src].name) + 1;
+		}
+	}
+
+	/* @UNSAFE */
+	name_space_needed += sizeof(*dir) +
+		nondeleted_count * sizeof(*new_recs);
+	if (mailbox_list_index_sync_alloc_space(ctx, name_space_needed,
+						&base, &base_offset) < 0)
+		return -1;
+	/* NOTE: any pointers to the index file may have been invalidated
+	   as a result of growing the the memory area */
+
+	if (sync_dir->offset == 0) {
+		dir = NULL;
+		recs = NULL;
+	} else {
+		/* the offset should have been verified already to be valid */
+		i_assert(sync_dir->offset == offset_pos);
+		i_assert(sync_dir->offset < ctx->index->mmap_size);
+		dir = PTR_OFFSET(ctx->index->mmap_base, sync_dir->offset);
+		recs = MAILBOX_LIST_RECORDS(dir);
+	}
+
+	new_dir = base;
+	new_dir->count = nondeleted_count;
+
+	new_recs = MAILBOX_LIST_RECORDS(new_dir);
+	name_pos = (const char *)(new_recs + nondeleted_count) -
+		(const char *)base;
+	for (src = dest = 0; src < count; src++) {
+		if (!sync_recs[src].seen && !partial) {
+			/* expunge from mail index */
+			uint32_t seq;
+
+			if (mail_index_lookup_uid_range(ctx->view,
+							sync_recs[src].uid,
+							sync_recs[src].uid,
+							&seq, &seq) < 0)
+				return -1;
+
+			if (seq != 0)
+				mail_index_expunge(ctx->trans, seq);
+			// FIXME: expunge also NONEXISTENT parents
+			continue;
+		}
+
+		new_recs[dest].name_hash = sync_recs[src].name_hash;
+		new_recs[dest].dir_offset =
+			mail_index_uint32_to_offset(sync_recs[src].dir_offset);
+		if (sync_recs[src].created) {
+			/* new record */
+			new_recs[dest].uid = sync_recs[src].uid;
+			new_recs[dest].name_offset = base_offset + name_pos;
+			size = strlen(sync_recs[src].name) + 1;
+			memcpy(PTR_OFFSET(base, name_pos), sync_recs[src].name,
+			       size);
+			name_pos += size;
+		} else {
+			/* existing record. need to find its name_offset */
+			for (orig = 0; orig < dir->count; orig++) {
+				if (recs[orig].uid == sync_recs[src].uid)
+					break;
+			}
+			i_assert(orig < dir->count);
+
+			new_recs[dest].uid = sync_recs[src].uid;
+			new_recs[dest].name_offset = recs[orig].name_offset;
+		}
+		dest++;
+	}
+	i_assert(dest == nondeleted_count);
+	i_assert(name_pos == name_space_needed);
+
+	if (offset_pos == 0) {
+		/* we're writing the root directory */
+		i_assert(base_offset == sizeof(*ctx->index->hdr));
+	} else {
+		/* add a link to this newly created directory. */
+		uint32_t *pos;
+
+		pos = PTR_OFFSET(ctx->index->mmap_base, offset_pos);
+		i_assert(mail_index_offset_to_uint32(*pos) == 0);
+		*pos = mail_index_uint32_to_offset(base_offset);
+	}
+
+	sync_dir->offset = base_offset;
+	return 0;
+}
+
+static int
+mailbox_list_index_sync_update_dir(struct mailbox_list_index_sync_ctx *ctx,
+				   struct mailbox_list_sync_dir *sync_dir)
+{
+	const struct mailbox_list_dir_record *dir;
+	struct mailbox_list_record *recs;
+	const struct mailbox_list_sync_record *sync_recs;
+	unsigned int i, count;
+
+	i_assert(sync_dir->offset != 0);
+
+	if (mailbox_list_index_get_dir(ctx->index, &sync_dir->offset, &dir) < 0)
+		return -1;
+
+	sync_recs = array_get(&sync_dir->records, &count);
+	i_assert(dir->count == count);
+	i_assert(sync_dir->seen_records_count < count);
+
+	recs = MAILBOX_LIST_RECORDS(dir);
+	for (i = 0; i < dir->count; i++) {
+		if (!sync_recs[i].seen)
+			recs[i].deleted = TRUE;
+	}
+	return 0;
+}
+
+static int
+mailbox_list_index_sync_write_dir(struct mailbox_list_index_sync_ctx *ctx,
+				  struct mailbox_list_sync_dir *sync_dir,
+				  uint32_t offset_pos, bool partial)
+{
+	const struct mailbox_list_dir_record *dir;
+	const struct mailbox_list_record *recs;
+	const struct mailbox_list_sync_record *sync_recs;
+	uint32_t child_offset_pos;
+	unsigned int i, j, count;
+
+	if (!ctx->seen_sync_root && ctx->sync_root == sync_dir) {
+		i_assert(partial);
+		ctx->seen_sync_root = TRUE;
+		partial = (ctx->flags & MAILBOX_LIST_SYNC_FLAG_PARTIAL) != 0;
+	}
+
+	if (sync_dir->offset != 0) {
+		/* point to latest dir entry's next_offset */
+		offset_pos = sync_dir->offset +
+			offsetof(struct mailbox_list_dir_record, next_offset);
+	}
+
+	if (sync_dir->new_records_count > 0) {
+		/* need to recreate the dir record */
+		if (mailbox_list_index_sync_recreate_dir(ctx, sync_dir,
+							 offset_pos,
+							 partial) < 0)
+			return -1;
+		/* NOTE: index may have been remaped here */
+	} else if (sync_dir->seen_records_count !=
+		   array_count(&sync_dir->records) && !partial) {
+		/* just mark the records deleted */
+		if (mailbox_list_index_sync_update_dir(ctx, sync_dir) < 0)
+			return -1;
+	}
+
+	if (!partial && (ctx->flags & MAILBOX_LIST_SYNC_FLAG_RECURSIVE) == 0) {
+		/* we're doing a full sync only for the root */
+		partial = TRUE;
+	}
+
+	/* update child mailboxes */
+	sync_recs = array_get(&sync_dir->records, &count);
+	if (count == 0)
+		return 0;
+
+	i_assert(sync_dir->offset != 0 &&
+		 sync_dir->offset < ctx->index->mmap_size);
+	for (i = j = 0; i < count; i++) {
+		if (sync_recs[i].dir == NULL)
+			continue;
+
+		/* these may change after each sync_write_dir() call */
+		dir = CONST_PTR_OFFSET(ctx->index->mmap_base, sync_dir->offset);
+		recs = MAILBOX_LIST_RECORDS(dir);
+
+		/* child_offset_pos needs to point to record's dir_offset */
+		for (; j < dir->count; j++) {
+			if (recs[j].uid == sync_recs[i].uid)
+				break;
+		}
+		i_assert(j < dir->count);
+
+		child_offset_pos = (const char *)&recs[j].dir_offset -
+			(const char *)ctx->index->mmap_base;
+		if (mailbox_list_index_sync_write_dir(ctx, sync_recs[i].dir,
+						      child_offset_pos,
+						      partial) < 0)
+			return -1;
+	}
+	return 0;
+}
+
+static int
+mailbox_list_index_sync_write(struct mailbox_list_index_sync_ctx *ctx)
+{
+	struct mailbox_list_index_header *hdr;
+	bool partial;
+
+	if (ctx->sync_root == ctx->root) {
+		ctx->seen_sync_root = TRUE;
+		partial = (ctx->flags & MAILBOX_LIST_SYNC_FLAG_PARTIAL) != 0;
+	} else {
+		/* until we've seen the sync root, we're doing only partial
+		   syncing */
+		partial = TRUE;
+	}
+
+	if (mailbox_list_index_sync_write_dir(ctx, ctx->root, 0, partial) < 0)
+		return -1;
+
+	/* update header */
+	hdr = ctx->index->mmap_base;
+	hdr->next_uid = ctx->hdr.next_uid;
+	hdr->used_space = ctx->hdr.used_space;
+	hdr->deleted_space = ctx->hdr.deleted_space;
+
+	if (msync(ctx->index->mmap_base, hdr->used_space, MS_SYNC) < 0) {
+		mailbox_list_index_set_syscall_error(ctx->index, "msync()");
+		return -1;
+	}
+	return 0;
+}
+
+int mailbox_list_index_sync_commit(struct mailbox_list_index_sync_ctx **_ctx)
+{
+	struct mailbox_list_index_sync_ctx *ctx = *_ctx;
+	int ret = ctx->failed ? -1 : 0;
+
+	*_ctx = NULL;
+
+	if (!ctx->failed) {
+		/* write all the changes to the index */
+		ret = mailbox_list_index_sync_write(ctx);
+	}
+
+	if (ctx->mail_sync_ctx != NULL) {
+		if (ret < 0)
+			mail_index_transaction_rollback(&ctx->trans);
+		else {
+			uint32_t seq;
+			uoff_t offset;
+
+			if (mail_index_transaction_commit(&ctx->trans,
+							  &seq, &offset) < 0)
+				ret = -1;
+		}
+
+		if (ret < 0)
+			mail_index_sync_rollback(&ctx->mail_sync_ctx);
+		else {
+			if (mail_index_sync_commit(&ctx->mail_sync_ctx) < 0)
+				ret = -1;
+		}
+	}
+
+	pool_unref(ctx->pool);
+	return ret;
+}
+
+void mailbox_list_index_sync_rollback(struct mailbox_list_index_sync_ctx **ctx)
+{
+	(*ctx)->failed = TRUE;
+	(void)mailbox_list_index_sync_commit(ctx);
+}

--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/src/lib-index/mailbox-list-index.c	Sun Nov 26 00:17:39 2006 +0200
@@ -0,0 +1,605 @@
+/* Copyright (C) 2006 Timo Sirainen */
+
+#include "lib.h"
+#include "array.h"
+#include "crc32.h"
+#include "ioloop.h"
+#include "str.h"
+#include "file-dotlock.h"
+#include "mmap-util.h"
+#include "write-full.h"
+#include "mail-index-private.h"
+#include "mailbox-list-index-private.h"
+
+#include <stdlib.h>
+#include <unistd.h>
+#include <fcntl.h>
+#include <sys/stat.h>
+
+struct mailbox_list_iter_path {
+	const struct mailbox_list_dir_record *dir;
+	unsigned int pos;
+	unsigned int name_path_len;
+};
+
+struct mailbox_list_iter_ctx {
+	struct mailbox_list_index *index;
+
+	unsigned int recurse_level;
+
+	struct mailbox_list_iter_path cur;
+	ARRAY_DEFINE(path, struct mailbox_list_iter_path);
+	string_t *name_path;
+
+	unsigned int failed:1;
+};
+
+const struct dotlock_settings dotlock_set = {
+	MEMBER(temp_prefix) NULL,
+	MEMBER(lock_suffix) NULL,
+
+	MEMBER(timeout) 60,
+	MEMBER(stale_timeout) 30,
+
+	MEMBER(callback) NULL,
+	MEMBER(context) NULL,
+
+	MEMBER(use_excl_lock) FALSE
+};
+
+int mailbox_list_index_set_syscall_error(struct mailbox_list_index *index,
+					 const char *function)
+{
+	i_error("%s failed with file %s: %m", index->filepath, function);
+	return -1;
+}
+
+static void mailbox_list_index_unmap(struct mailbox_list_index *index)
+{
+	if (index->mmap_base != NULL) {
+		if (munmap(index->mmap_base, index->mmap_size) < 0)
+			mailbox_list_index_set_syscall_error(index, "munmap()");
+		index->mmap_base = NULL;
+		index->mmap_size = 0;
+	}
+
+	index->hdr = NULL;
+}
+
+static void mailbox_list_index_file_close(struct mailbox_list_index *index)
+{
+	mailbox_list_index_unmap(index);
+
+	if (index->fd != -1) {
+		if (close(index->fd) < 0)
+			mailbox_list_index_set_syscall_error(index, "close()");
+	}
+}
+
+int mailbox_list_index_set_corrupted(struct mailbox_list_index *index,
+				     const char *str)
+{
+	(void)unlink(index->filepath);
+	// FIXME: reopen or something
+
+	i_error("Corrupted mailbox list index file %s: %s",
+		index->filepath, str);
+	return -1;
+}
+
+static int
+mailbox_list_index_check_header(struct mailbox_list_index *index,
+				const struct mailbox_list_index_header *hdr)
+{
+	if (hdr->major_version != MAILBOX_LIST_INDEX_MAJOR_VERSION)
+		return -1;
+
+	if (hdr->header_size < sizeof(*hdr)) {
+		return mailbox_list_index_set_corrupted(index,
+			"header_size is too small");
+	}
+	if (hdr->header_size > index->mmap_size) {
+		return mailbox_list_index_set_corrupted(index,
+			"header_size is too large");
+	}
+
+	if (hdr->uid_validity == 0) {
+		return mailbox_list_index_set_corrupted(index,
+							"uid_validity is 0");
+	}
+	if (hdr->next_uid == 0)
+		return mailbox_list_index_set_corrupted(index, "next_uid is 0");
+
+	if (hdr->uid_validity != index->mail_index->hdr->uid_validity &&
+	    index->mail_index->hdr->uid_validity != 0) {
+		mail_index_set_error(index->mail_index,
+			"uid_validity changed in file %s", index->filepath);
+		mail_index_mark_corrupted(index->mail_index);
+	}
+
+	return 0;
+}
+
+int mailbox_list_index_map(struct mailbox_list_index *index)
+{
+	const struct mailbox_list_index_header *hdr;
+
+	mailbox_list_index_unmap(index);
+
+	// FIXME: handle non-mmaps
+	index->mmap_base = mmap_rw_file(index->fd, &index->mmap_size);
+	if (index->mmap_base == MAP_FAILED) {
+		index->mmap_base = NULL;
+		return mailbox_list_index_set_syscall_error(index, "mmap()");
+	}
+
+	if (index->mmap_size < sizeof(*hdr)) {
+		mailbox_list_index_set_corrupted(index, "File too small");
+		return 0;
+	}
+
+	hdr = index->mmap_base;
+	if (mailbox_list_index_check_header(index, hdr) < 0)
+		return 0;
+
+	index->hdr = hdr;
+	return 1;
+}
+
+static void
+mailbox_list_index_init_header(struct mailbox_list_index_header *hdr)
+{
+	memset(hdr, 0, sizeof(*hdr));
+	hdr->major_version = MAILBOX_LIST_INDEX_MAJOR_VERSION;
+	hdr->minor_version = MAILBOX_LIST_INDEX_MINOR_VERSION;
+
+	hdr->header_size = sizeof(*hdr);
+	hdr->used_space = hdr->header_size;
+
+	hdr->uid_validity = ioloop_time;
+	hdr->next_uid = 1;
+}
+
+static int mailbox_list_index_is_recreated(struct mailbox_list_index *index)
+{
+	struct stat st1, st2;
+
+	if (stat(index->filepath, &st1) < 0) {
+		mailbox_list_index_set_syscall_error(index, "stat()");
+		return -1;
+	}
+	if (fstat(index->fd, &st2) < 0) {
+		mailbox_list_index_set_syscall_error(index, "fstat()");
+		return -1;
+	}
+
+	return st1.st_ino != st2.st_ino ||
+		!CMP_DEV_T(st1.st_dev, st2.st_dev);
+}
+
+static int
+mailbox_list_index_file_create(struct mailbox_list_index *index)
+{
+	struct mailbox_list_index_header hdr;
+	struct dotlock *dotlock;
+	int fd, ret;
+
+	fd = file_dotlock_open(&dotlock_set, index->filepath, 0, &dotlock);
+	if (fd == -1) {
+		mailbox_list_index_set_syscall_error(index,
+						     "file_dotlock_open()");
+		return -1;
+	}
+
+	if (index->fd != -1) {
+		/* if the file has been recreated by someone else,
+		   retry opening it */
+		ret = mailbox_list_index_is_recreated(index);
+		if (ret != 0) {
+			(void)file_dotlock_delete(&dotlock);
+			return ret < 0 ? -1 : 0;
+		}
+	}
+
+	mailbox_list_index_init_header(&hdr);
+	if (write_full(fd, &hdr, sizeof(hdr)) < 0) {
+		mailbox_list_index_set_syscall_error(index, "write_full()");
+		(void)file_dotlock_delete(&dotlock);
+		return -1;
+	}
+
+	if (file_dotlock_replace(&dotlock,
+				 DOTLOCK_REPLACE_FLAG_DONT_CLOSE_FD) < 0) {
+		mailbox_list_index_set_syscall_error(index,
+						     "file_dotlock_replace()");
+		(void)close(fd);
+		return -1;
+	}
+
+	if (index->fd != -1)
+		mailbox_list_index_file_close(index);
+	index->fd = fd;
+
+	ret = mailbox_list_index_map(index);
+	if (ret == 0) {
+		i_error("Self-created mailbox list index file %s was corrupted",
+			index->filepath);
+		return -1;
+	}
+	return ret;
+}
+
+static int
+mailbox_list_index_file_try_open_or_create(struct mailbox_list_index *index)
+{
+	int ret;
+
+	i_assert(index->fd == -1);
+
+	index->fd = open(index->filepath, O_RDWR);
+	if (index->fd == -1) {
+		if (errno != ENOENT) {
+			mailbox_list_index_set_syscall_error(index, "open()");
+			return -1;
+		}
+	} else {
+		ret = mailbox_list_index_map(index);
+		if (ret != 0) {
+			if (ret < 0)
+				mailbox_list_index_file_close(index);
+			return ret;
+		}
+	}
+
+	ret = mailbox_list_index_file_create(index);
+	if (ret <= 0)
+		mailbox_list_index_file_close(index);
+	return ret;
+}
+
+int mailbox_list_index_open_or_create(struct mailbox_list_index *index)
+{
+	int ret;
+
+	while ((ret = mailbox_list_index_file_try_open_or_create(index)) == 0) {
+		/* file was recreated by someone else, try reopening */
+	}
+	return ret < 0 ? -1 : 0;
+}
+
+struct mailbox_list_index *
+mailbox_list_index_alloc(const char *path, char separator,
+			 struct mail_index *mail_index)
+{
+	struct mailbox_list_index *index;
+
+	index = i_new(struct mailbox_list_index, 1);
+	index->filepath = i_strdup(path);
+	index->separator = separator;
+	index->mail_index = mail_index;
+	index->fd = -1;
+	return index;
+}
+
+void mailbox_list_index_free(struct mailbox_list_index **_index)
+{
+	struct mailbox_list_index *index = *_index;
+
+	*_index = NULL;
+
+	i_free(index->filepath);
+	i_free(index);
+}
+
+struct mailbox_list_index_lookup_key {
+	uint32_t name_hash;
+
+	struct mailbox_list_index *index;
+	const char *name;
+
+	bool *failed;
+};
+
+static int
+mailbox_list_get_name(struct mailbox_list_index *index, pool_t pool,
+		      const struct mailbox_list_record *rec,
+		      const char **name_r)
+{
+	size_t max_len;
+	const char *name;
+
+	if (rec->name_offset >= index->mmap_size) {
+		mailbox_list_index_set_corrupted(index,
+			"record name_offset points outside file");
+		return -1;
+	}
+	max_len = index->mmap_size - rec->name_offset;
+	name = CONST_PTR_OFFSET(index->mmap_base, rec->name_offset);
+	/* get name length. don't bother checking if it's not NUL-terminated,
+	   because practically it always is even if the file is corrupted.
+	   just make sure we don't crash if it happens. */
+	*name_r = p_strndup(pool, name, max_len);
+	return 0;
+}
+
+static int mailbox_list_record_cmp(const void *_key, const void *_rec)
+{
+	const struct mailbox_list_index_lookup_key *key = _key;
+	const struct mailbox_list_record *rec = _rec;
+	const char *name;
+	int ret;
+
+	if (key->name_hash < rec->name_hash)
+		return -1;
+	if (key->name_hash > rec->name_hash)
+		return 1;
+
+	t_push();
+	if (mailbox_list_get_name(key->index, unsafe_data_stack_pool,
+				  rec, &name) < 0) {
+		*key->failed = TRUE;
+		ret = -1;
+	} else {
+		ret = strcmp(key->name, name);
+	}
+	t_pop();
+	return ret;
+}
+
+int mailbox_list_index_get_dir(struct mailbox_list_index *index,
+			       uint32_t *offset,
+			       const struct mailbox_list_dir_record **dir_r)
+{
+	const struct mailbox_list_dir_record *dir;
+	uint32_t next_offset, cur_offset = *offset;
+
+	i_assert(index->mmap_size > 0);
+
+	do {
+		if (cur_offset >= index->mmap_size - sizeof(*dir)) {
+			return mailbox_list_index_set_corrupted(index,
+				"dir_offset points outside file");
+		}
+		if ((cur_offset % 4) != 0) {
+			return mailbox_list_index_set_corrupted(index,
+				"dir_offset not 32bit aligned");
+		}
+
+		dir = CONST_PTR_OFFSET(index->mmap_base, cur_offset);
+		next_offset = mail_index_offset_to_uint32(dir->next_offset);
+		if (next_offset != 0 && next_offset <= cur_offset) {
+			return mailbox_list_index_set_corrupted(index,
+				"next_offset points backwards");
+		}
+		cur_offset = next_offset;
+	} while (cur_offset != 0);
+
+	cur_offset = (const char *)dir - (const char *)index->mmap_base;
+	if (dir->count > INT_MAX/sizeof(struct mailbox_list_record) ||
+	    dir->count * sizeof(struct mailbox_list_record) >
+	    index->mmap_size - cur_offset) {
+		mailbox_list_index_set_corrupted(index, "dir count too large");
+		return -1;
+	}
+
+	*offset = cur_offset;
+	*dir_r = dir;
+	return 0;
+}
+
+int mailbox_list_index_dir_lookup_rec(struct mailbox_list_index *index,
+				      const struct mailbox_list_dir_record *dir,
+				      const char *name,
+				      const struct mailbox_list_record **rec_r)
+{
+	const struct mailbox_list_record *rec;
+	struct mailbox_list_index_lookup_key key;
+	bool failed = FALSE;
+
+	/* binary search the current hierarchy level name. the values are
+	   sorted primarily by their hash value and secondarily by the actual
+	   name */
+	memset(&key, 0, sizeof(key));
+	key.index = index;
+	key.name = name;
+	key.name_hash = crc32_str(name);
+	key.failed = &failed;
+
+	rec = bsearch(&key, MAILBOX_LIST_RECORDS(dir), dir->count, sizeof(*rec),
+		      mailbox_list_record_cmp);
+	if (failed)
+		return -1;
+	if (rec == NULL)
+		return 0;
+
+	*rec_r = rec;
+	return 1;
+}
+
+static int
+mailbox_list_index_lookup_rec(struct mailbox_list_index *index,
+			      uint32_t dir_offset, const char *name,
+			      const struct mailbox_list_record **rec_r)
+{
+	const struct mailbox_list_dir_record *dir;
+	const char *p, *hier_name;
+	int ret;
+
+	if (dir_offset == index->mmap_size &&
+	    dir_offset == sizeof(*index->hdr)) {
+		/* root doesn't exist in the file yet */
+		return 0;
+	}
+
+	if (mailbox_list_index_get_dir(index, &dir_offset, &dir) < 0)
+		return -1;
+
+	p = strchr(name, index->separator);
+	hier_name = p == NULL ? name : t_strdup_until(name, p);
+
+	ret = mailbox_list_index_dir_lookup_rec(index, dir, hier_name, rec_r);
+	if (ret <= 0)
+		return ret;
+
+	if (p == NULL) {
+		/* found it */
+		return 1;
+	}
+
+	/* recurse to children */
+	dir_offset = mail_index_offset_to_uint32((*rec_r)->dir_offset);
+	if (dir_offset == 0)
+		return 0;
+
+	return mailbox_list_index_lookup_rec(index, dir_offset, p + 1, rec_r);
+}
+
+static int mailbox_list_index_refresh(struct mailbox_list_index *index)
+{
+	int ret;
+
+	if ((ret = mailbox_list_index_is_recreated(index)) <= 0)
+		return ret;
+
+	mailbox_list_index_file_close(index);
+	return mailbox_list_index_open_or_create(index);
+}
+
+int mailbox_list_index_lookup(struct mailbox_list_index *index,
+			      const char *name, uint32_t *uid_r)
+{
+	const struct mailbox_list_record *rec;
+	uint32_t offset = sizeof(*index->hdr);
+	int ret;
+
+	ret = mailbox_list_index_lookup_rec(index, offset, name, &rec);
+	if (ret == 0) {
+		/* not found, see if it's found after a refresh */
+		if ((ret = mailbox_list_index_refresh(index)) <= 0)
+			return ret;
+
+		ret = mailbox_list_index_lookup_rec(index, offset, name, &rec);
+	}
+
+	*uid_r = ret <= 0 ? 0 : rec->uid;
+	return ret;
+}
+
+struct mailbox_list_iter_ctx *
+mailbox_list_index_iterate_init(struct mailbox_list_index *index,
+				const char *path, int recurse_level)
+{
+	struct mailbox_list_iter_ctx *ctx;
+	const struct mailbox_list_record *rec;
+	uint32_t offset = sizeof(*index->hdr);
+	int ret;
+
+	ctx = i_new(struct mailbox_list_iter_ctx, 1);
+	ctx->index = index;
+	ctx->recurse_level = recurse_level < 0 ? (unsigned int)-1 :
+		(unsigned int)recurse_level;
+	ctx->name_path = str_new(default_pool, 512);
+
+	if (*path != '\0') {
+		ret = mailbox_list_index_lookup_rec(index, offset, path, &rec);
+		if (ret < 0)
+			ctx->failed = TRUE;
+		else {
+			offset = ret == 0 ? 0 :
+				mail_index_offset_to_uint32(rec->dir_offset);
+		}
+	}
+	if (!ctx->failed && offset != 0) {
+		if (mailbox_list_index_get_dir(index, &offset,
+					       &ctx->cur.dir) < 0)
+			ctx->failed = TRUE;
+	}
+	i_array_init(&ctx->path, I_MIN(ctx->recurse_level, 16));
+	return ctx;
+}
+
+int mailbox_list_index_iterate_next(struct mailbox_list_iter_ctx *ctx,
+				    struct mailbox_list_index_info *info_r)
+{
+	const struct mailbox_list_iter_path *cur;
+	const struct mailbox_list_record *recs;
+	const char *name;
+	uint32_t dir_offset;
+	unsigned int count;
+
+	if (ctx->failed)
+		return -1;
+
+	if (ctx->cur.dir == NULL) {
+		/* no mailboxes */
+		i_assert(array_count(&ctx->path) == 0);
+		return 0;
+	}
+
+	while (ctx->cur.pos == ctx->cur.dir->count) {
+		count = array_count(&ctx->path);
+		if (count == 0) {
+			/* we're done */
+			return 0;
+		}
+
+		/* go back to parent path */
+		cur = array_idx(&ctx->path, count-1);
+		ctx->cur = *cur;
+		array_delete(&ctx->path, count-1, 1);
+
+		ctx->cur.pos++;
+	}
+
+	recs = MAILBOX_LIST_RECORDS(ctx->cur.dir);
+	recs += ctx->cur.pos;
+
+	if (recs->deleted) {
+		ctx->cur.pos++;
+		return mailbox_list_index_iterate_next(ctx, info_r);
+	}
+
+	t_push();
+	if (mailbox_list_get_name(ctx->index, unsafe_data_stack_pool,
+				  recs, &name) < 0) {
+		ctx->failed = TRUE;
+		t_pop();
+		return -1;
+	}
+	str_truncate(ctx->name_path, ctx->cur.name_path_len);
+	if (ctx->cur.name_path_len > 0)
+		str_append_c(ctx->name_path, ctx->index->separator);
+	str_append(ctx->name_path, name);
+	t_pop();
+
+	dir_offset = mail_index_offset_to_uint32(recs->dir_offset);
+	if (dir_offset != 0 && array_count(&ctx->path) < ctx->recurse_level) {
+		/* recurse into children */
+		array_append(&ctx->path, &ctx->cur, 1);
+
+		ctx->cur.name_path_len = str_len(ctx->name_path);
+		ctx->cur.pos = 0;
+		if (mailbox_list_index_get_dir(ctx->index, &dir_offset,
+					       &ctx->cur.dir) < 0) {
+			ctx->failed = TRUE;
+			return -1;
+		}
+	} else {
+		ctx->cur.pos++;
+	}
+
+	info_r->name = str_c(ctx->name_path);
+	info_r->uid = recs->uid;
+	info_r->has_children = dir_offset != 0;
+	return 1;
+}
+
+void mailbox_list_index_iterate_deinit(struct mailbox_list_iter_ctx **_ctx)
+{
+	struct mailbox_list_iter_ctx *ctx = *_ctx;
+
+	*_ctx = NULL;
+	array_free(&ctx->path);
+	str_free(&ctx->name_path);
+	i_free(ctx);
+}

--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/src/lib-index/mailbox-list-index.h	Sun Nov 26 00:17:39 2006 +0200
@@ -0,0 +1,82 @@
+#ifndef __MAILBOX_LIST_INDEX_H
+#define __MAILBOX_LIST_INDEX_H
+
+struct mailbox_list_index_sync_ctx;
+
+/* Mailbox list index contains UID <-> mailbox name mapping. It also takes in
+   a mail_index index which contains UID -> metadata information for the
+   mailboxes. The mmap, in-memory and lock settings are taken from the
+   mail_index. */
+
+enum mailbox_list_index_flags {
+	/* Mailbox has children. They may not be indexed however, so
+	   mailbox_list_index_info.has_children=FALSE is possible. */
+	MAILBOX_LIST_INDEX_FLAG_CHILDREN	= 0x01,
+	/* Mailbox has no children. mailbox_list_index_info.has_children
+	   should be FALSE. */
+	MAILBOX_LIST_INDEX_FLAG_NOCHILDREN	= 0x02,
+	/* The mailbox isn't selectable (eg. a directory) */
+	MAILBOX_LIST_INDEX_FLAG_NOSELECT	= 0x04,
+	/* The mailbox doesn't exist at all. This is only a placeholder for
+	   a child mailbox. When the children are deleted, this mailbox will
+	   be automatically deleted as well. */
+	MAILBOX_LIST_INDEX_FLAG_NONEXISTENT	= 0x08
+};
+
+
+enum mailbox_list_sync_flags {
+	/* All the child mailboxes are also being synced */
+	MAILBOX_LIST_SYNC_FLAG_RECURSIVE	= 0x01,
+	/* New mailboxes may be added, but none are removed */
+	MAILBOX_LIST_SYNC_FLAG_PARTIAL		= 0x02
+};
+
+struct mailbox_list_index_info {
+	const char *name;
+	uint32_t uid;
+	bool has_children;
+};
+
+struct mailbox_list_index *
+mailbox_list_index_alloc(const char *path, char separator,
+			 struct mail_index *mail_index);
+void mailbox_list_index_free(struct mailbox_list_index **index);
+
+/* Open or create mailbox list index. */
+int mailbox_list_index_open_or_create(struct mailbox_list_index *index);
+
+/* Synchronize the index with the backend. */
+int mailbox_list_index_sync_init(struct mailbox_list_index *index,
+				 const char *path,
+				 enum mailbox_list_sync_flags flags,
+				 struct mailbox_list_index_sync_ctx **ctx_r);
+struct mail_index_view *
+mailbox_list_index_sync_get_view(struct mailbox_list_index_sync_ctx *ctx);
+struct mail_index_transaction *
+mailbox_list_index_sync_get_transaction(struct mailbox_list_index_sync_ctx*ctx);
+int mailbox_list_index_sync_more(struct mailbox_list_index_sync_ctx *ctx,
+				 const char *name, uint32_t *seq_r);
+int mailbox_list_index_sync_commit(struct mailbox_list_index_sync_ctx **ctx);
+void mailbox_list_index_sync_rollback(struct mailbox_list_index_sync_ctx **ctx);
+
+/* Get mailbox UID for a given name. Returns 1 if found, 0 if not,
+   -1 if error */
+int mailbox_list_index_lookup(struct mailbox_list_index *index,
+			      const char *name, uint32_t *uid_r);
+
+/* Iterate through all the mailboxes. If recurse_level is -1, all the child
+   mailboxes are returned, otherwise it's the number of levels to return
+   (0 = only the mailboxes directly under the path). Returned mailbox names
+   are allocated from name_pool. */
+struct mailbox_list_iter_ctx *
+mailbox_list_index_iterate_init(struct mailbox_list_index *index,
+				const char *path, int recurse_level);
+/* Returns 1 if mailbox was returned, 0 at the end of iteration, -1 if error */
+int mailbox_list_index_iterate_next(struct mailbox_list_iter_ctx *ctx,
+				    struct mailbox_list_index_info *info_r);
+void mailbox_list_index_iterate_deinit(struct mailbox_list_iter_ctx **ctx);
+
+int mailbox_list_index_set_corrupted(struct mailbox_list_index *index,
+				     const char *str);
+
+#endif

--- a/src/lib-storage/index/dbox/dbox-storage.c	Sun Nov 26 00:12:11 2006 +0200
+++ b/src/lib-storage/index/dbox/dbox-storage.c	Sun Nov 26 00:17:39 2006 +0200
@@ -93,6 +93,8 @@
 
 	if (dbox_get_list_settings(&list_set, data, flags) < 0)
 		return NULL;
+	list_set.mail_storage_flags = &flags;
+	list_set.mail_storage_lock_method = &lock_method;
 
 	if (mkdir_parents(list_set.root_dir, CREATE_MODE) < 0 &&
 	    errno != EEXIST) {

--- a/src/lib-storage/index/dbox/dbox-sync.c	Sun Nov 26 00:12:11 2006 +0200
+++ b/src/lib-storage/index/dbox/dbox-sync.c	Sun Nov 26 00:17:39 2006 +0200
@@ -593,6 +593,11 @@
 	struct dbox_mailbox *mbox = (struct dbox_mailbox *)box;
 	int ret = 0;
 
+	if (!box->opened) {
+		if (index_storage_mailbox_open(&mbox->ibox) < 0)
+			return index_mailbox_sync_init(box, 0, TRUE);
+	}
+
 	if ((flags & MAILBOX_SYNC_FLAG_FAST) == 0 ||
 	    mbox->ibox.sync_last_check + MAILBOX_FULL_SYNC_INTERVAL <=
 	    ioloop_time)

--- a/src/lib-storage/index/dbox/dbox-transaction.c	Sun Nov 26 00:12:11 2006 +0200
+++ b/src/lib-storage/index/dbox/dbox-transaction.c	Sun Nov 26 00:17:39 2006 +0200
@@ -66,7 +66,8 @@
 {
 	struct mailbox *box = MAIL_STORAGE_INDEX(t->view->index);
 
-	if (strcmp(box->storage->name, DBOX_STORAGE_NAME) == 0) {
+	/* index can be for mailbox list index, in which case box=NULL */
+	if (box != NULL && strcmp(box->storage->name, DBOX_STORAGE_NAME) == 0) {
 		struct dbox_mailbox *dbox = (struct dbox_mailbox *)box;
 		struct dbox_transaction_context *mt;
 

--- a/src/lib-storage/index/index-status.c	Sun Nov 26 00:12:11 2006 +0200
+++ b/src/lib-storage/index/index-status.c	Sun Nov 26 00:17:39 2006 +0200
@@ -40,6 +40,11 @@
 	struct index_mailbox *ibox = (struct index_mailbox *)box;
 	int ret;
 
+	if (!box->opened) {
+		if (index_storage_mailbox_open(ibox) < 0)
+			return -1;
+	}
+
 	ret = index_storage_get_status_locked(ibox, items, status);
 	mail_index_view_unlock(ibox->view);
 	return ret;

--- a/src/lib-storage/index/index-storage.c	Sun Nov 26 00:12:11 2006 +0200
+++ b/src/lib-storage/index/index-storage.c	Sun Nov 26 00:17:39 2006 +0200
@@ -314,24 +314,16 @@
 	ibox->last_notify_type = MAILBOX_LOCK_NOTIFY_NONE;
 }
 
-int index_storage_mailbox_init(struct index_mailbox *ibox,
-			       struct mail_index *index, const char *name,
-			       enum mailbox_open_flags flags,
-			       bool move_to_memory)
+int index_storage_mailbox_open(struct index_mailbox *ibox)
 {
 	struct mail_storage *storage = &ibox->storage->storage;
 	enum mail_index_open_flags index_flags;
 	enum mail_index_lock_method lock_method = 0;
 	int ret;
 
-	i_assert(name != NULL);
+	i_assert(!ibox->box.opened);
 
-	ibox->box.storage = storage;
-	ibox->box.name = p_strdup(ibox->box.pool, name);
-	array_create(&ibox->box.module_contexts,
-		     ibox->box.pool, sizeof(void *), 5);
-
-	index_flags = move_to_memory ? 0 : MAIL_INDEX_OPEN_FLAG_CREATE;
+	index_flags = ibox->move_to_memory ? 0 : MAIL_INDEX_OPEN_FLAG_CREATE;
 	if ((storage->flags & MAIL_STORAGE_FLAG_MMAP_DISABLE) != 0)
 		index_flags |= MAIL_INDEX_OPEN_FLAG_MMAP_DISABLE;
 #ifndef MMAP_CONFLICTS_WRITE
@@ -351,23 +343,13 @@
 		break;
 	}
 
-	ibox->open_flags = flags;
-	ibox->readonly = (flags & MAILBOX_OPEN_READONLY) != 0;
-	ibox->keep_recent = (flags & MAILBOX_OPEN_KEEP_RECENT) != 0;
-	ibox->keep_locked = (flags & MAILBOX_OPEN_KEEP_LOCKED) != 0;
-	ibox->index = index;
-
-	ibox->next_lock_notify = time(NULL) + LOCK_NOTIFY_INTERVAL;
-	ibox->commit_log_file_seq = 0;
-	ibox->mail_read_mmaped = (storage->flags &
-				  MAIL_STORAGE_FLAG_MMAP_MAILS) != 0;
-
-	ret = mail_index_open(index, index_flags, lock_method);
-	if (ret <= 0 || move_to_memory) {
-		if (mail_index_move_to_memory(index) < 0) {
+	ret = mail_index_open(ibox->index, index_flags, lock_method);
+	if (ret <= 0 || ibox->move_to_memory) {
+		if (mail_index_move_to_memory(ibox->index) < 0) {
 			/* try opening once more. it should be created
 			   directly into memory now. */
-			ret = mail_index_open(index, index_flags, lock_method);
+			ret = mail_index_open(ibox->index, index_flags,
+					      lock_method);
 			if (ret <= 0) {
 				mail_storage_set_index_error(ibox);
 				index_storage_mailbox_free(&ibox->box);
@@ -376,17 +358,49 @@
 		}
 	}
 
+	ibox->cache = mail_index_get_cache(ibox->index);
+	index_cache_register_defaults(ibox);
+	ibox->view = mail_index_view_open(ibox->index);
+	ibox->keyword_names = mail_index_get_keywords(ibox->index);
+
+	ibox->box.opened = TRUE;
+	return 0;
+}
+
+int index_storage_mailbox_init(struct index_mailbox *ibox,
+			       struct mail_index *index, const char *name,
+			       enum mailbox_open_flags flags,
+			       bool move_to_memory)
+{
+	struct mail_storage *storage = &ibox->storage->storage;
+
+	i_assert(name != NULL);
+
+	ibox->box.storage = storage;
+	ibox->box.name = p_strdup(ibox->box.pool, name);
+	array_create(&ibox->box.module_contexts,
+		     ibox->box.pool, sizeof(void *), 5);
+
+	ibox->open_flags = flags;
+	ibox->readonly = (flags & MAILBOX_OPEN_READONLY) != 0;
+	ibox->keep_recent = (flags & MAILBOX_OPEN_KEEP_RECENT) != 0;
+	ibox->keep_locked = (flags & MAILBOX_OPEN_KEEP_LOCKED) != 0;
+	ibox->move_to_memory = move_to_memory;
+	ibox->index = index;
+
+	ibox->next_lock_notify = time(NULL) + LOCK_NOTIFY_INTERVAL;
+	ibox->commit_log_file_seq = 0;
+	ibox->mail_read_mmaped =
+		(storage->flags & MAIL_STORAGE_FLAG_MMAP_MAILS) != 0;
+
 	ibox->md5hdr_ext_idx =
 		mail_index_ext_register(index, "header-md5", 0, 16, 1);
 
-	ibox->cache = mail_index_get_cache(index);
-	index_cache_register_defaults(ibox);
-	ibox->view = mail_index_view_open(index);
-	ibox->keyword_names = mail_index_get_keywords(index);
-
 	array_idx_set(&index->mail_index_module_contexts,
 		      mail_storage_mail_index_module_id, &ibox);
-	return 0;
+
+	return (flags & MAILBOX_OPEN_FAST) != 0 ? 0 :
+		index_storage_mailbox_open(ibox);
 }
 
 void index_storage_mailbox_free(struct mailbox *box)

--- a/src/lib-storage/index/index-storage.h	Sun Nov 26 00:12:11 2006 +0200
+++ b/src/lib-storage/index/index-storage.h	Sun Nov 26 00:17:39 2006 +0200
@@ -76,6 +76,7 @@
 	unsigned int sent_readonly_flags_warning:1;
 	unsigned int notify_pending:1;
 	unsigned int mail_read_mmaped:1;
+	unsigned int move_to_memory:1;
 };
 
 struct index_transaction_context {
@@ -117,6 +118,7 @@
 			       struct mail_index *index, const char *name,
 			       enum mailbox_open_flags flags,
 			       bool move_to_memory);
+int index_storage_mailbox_open(struct index_mailbox *ibox);
 void index_storage_mailbox_free(struct mailbox *box);
 
 bool index_storage_is_readonly(struct mailbox *box);
@@ -144,6 +146,7 @@
 int index_mailbox_sync_next(struct mailbox_sync_context *ctx,
 			    struct mailbox_sync_rec *sync_rec_r);
 int index_mailbox_sync_deinit(struct mailbox_sync_context *ctx,
+			      enum mailbox_status_items status_items,
 			      struct mailbox_status *status_r);
 
 int index_storage_sync(struct mailbox *box, enum mailbox_sync_flags flags);

--- a/src/lib-storage/index/index-sync.c	Sun Nov 26 00:12:11 2006 +0200
+++ b/src/lib-storage/index/index-sync.c	Sun Nov 26 00:17:39 2006 +0200
@@ -294,11 +294,8 @@
 	return ret;
 }
 
-#define SYNC_STATUS_FLAGS \
-	(STATUS_MESSAGES | STATUS_RECENT | STATUS_UIDNEXT | \
-	 STATUS_UIDVALIDITY | STATUS_UNSEEN | STATUS_KEYWORDS)
-
 int index_mailbox_sync_deinit(struct mailbox_sync_context *_ctx,
+			      enum mailbox_status_items status_items,
 			      struct mailbox_status *status_r)
 {
 	struct index_mailbox_sync_context *ctx =
@@ -319,9 +316,9 @@
 		}
 		ibox->synced_recent_count = ibox->recent_flags_count;
 
-		ret = index_storage_get_status_locked(ctx->ibox,
-						      SYNC_STATUS_FLAGS,
-						      status_r);
+		ret = status_items == 0 ? 0 :
+			index_storage_get_status_locked(ctx->ibox, status_items,
+							status_r);
 	}
 
 	mail_index_view_unlock(ctx->ibox->view);

--- a/src/lib-storage/index/maildir/maildir-copy.c	Sun Nov 26 00:12:11 2006 +0200
+++ b/src/lib-storage/index/maildir/maildir-copy.c	Sun Nov 26 00:17:39 2006 +0200
@@ -7,6 +7,7 @@
 #include "maildir-storage.h"
 #include "maildir-uidlist.h"
 #include "maildir-keywords.h"
+#include "maildir-sync.h"
 #include "index-mail.h"
 #include "mail-copy.h"
 

--- a/src/lib-storage/index/maildir/maildir-save.c	Sun Nov 26 00:12:11 2006 +0200
+++ b/src/lib-storage/index/maildir/maildir-save.c	Sun Nov 26 00:17:39 2006 +0200
@@ -12,6 +12,7 @@
 #include "index-mail.h"
 #include "maildir-storage.h"
 #include "maildir-uidlist.h"
+#include "maildir-sync.h"
 
 #include <stdio.h>
 #include <stdlib.h>

--- a/src/lib-storage/index/maildir/maildir-storage.c	Sun Nov 26 00:12:11 2006 +0200
+++ b/src/lib-storage/index/maildir/maildir-storage.c	Sun Nov 26 00:17:39 2006 +0200
@@ -10,6 +10,7 @@
 #include "maildir-storage.h"
 #include "maildir-uidlist.h"
 #include "maildir-keywords.h"
+#include "maildir-sync.h"
 #include "index-mail.h"
 
 #include <stdio.h>
@@ -147,6 +148,8 @@
 
 	if (maildir_get_list_settings(&list_set, data, flags) < 0)
 		return NULL;
+	list_set.mail_storage_flags = &flags;
+	list_set.mail_storage_lock_method = &lock_method;
 
 	pool = pool_alloconly_create("storage", 512);
 	storage = p_new(pool, struct maildir_storage, 1);
@@ -727,7 +730,7 @@
 
 	mask = t_strdup_printf("%s%c*", oldname,
 			       mailbox_list_get_hierarchy_sep(storage->list));
-	iter = mailbox_list_iter_init(storage->list, "", mask,
+	iter = mailbox_list_iter_init(storage->list, mask,
 				      MAILBOX_LIST_ITER_FAST_FLAGS);
 	while ((info = mailbox_list_iter_next(iter)) != NULL) {
 		const char *name;

--- a/src/lib-storage/index/maildir/maildir-storage.h	Sun Nov 26 00:12:11 2006 +0200
+++ b/src/lib-storage/index/maildir/maildir-storage.h	Sun Nov 26 00:17:39 2006 +0200
@@ -42,8 +42,6 @@
 struct timeval;
 struct maildir_save_context;
 struct maildir_copy_context;
-struct maildir_keywords_sync_ctx;
-struct maildir_index_sync_context;
 
 struct maildir_storage {
 	struct index_storage storage;
@@ -55,6 +53,11 @@
 	unsigned int stat_dirs:1;
 };
 
+enum maildir_dirty_flags {
+	MAILDIR_DIRTY_NEW = 0x01,
+	MAILDIR_DIRTY_CUR = 0x02
+};
+
 struct maildir_mailbox {
 	struct index_mailbox ibox;
 	struct maildir_storage *storage;
@@ -64,8 +67,9 @@
 	/* maildir sync: */
 	struct maildir_uidlist *uidlist;
 	struct maildir_keywords *keywords;
-	time_t last_new_mtime, last_cur_mtime, last_new_sync_time;
+	time_t last_new_mtime, last_cur_mtime;
 	time_t dirty_cur_time;
+	enum maildir_dirty_flags last_dirty_flags;
 
         mode_t mail_create_mode;
 	unsigned int private_flags_mask;
@@ -91,19 +95,6 @@
 		       mode_t mode, const char **fname_r);
 bool maildir_filename_get_size(const char *fname, char type, uoff_t *size_r);
 
-int maildir_sync_is_synced(struct maildir_mailbox *mbox);
-
-struct mailbox_sync_context *
-maildir_storage_sync_init(struct mailbox *box, enum mailbox_sync_flags flags);
-int maildir_storage_sync_force(struct maildir_mailbox *mbox);
-
-int maildir_sync_index_begin(struct maildir_mailbox *mbox,
-			     struct maildir_index_sync_context **ctx_r);
-int maildir_sync_index(struct maildir_index_sync_context *sync_ctx,
-		       bool partial);
-int maildir_sync_index_finish(struct maildir_index_sync_context **sync_ctx,
-			      bool failed, bool cancel);
-
 void maildir_transaction_created(struct mail_index_transaction *t);
 void maildir_transaction_class_init(void);
 void maildir_transaction_class_deinit(void);
@@ -136,17 +127,6 @@
 int maildir_transaction_copy_commit(struct maildir_copy_context *ctx);
 void maildir_transaction_copy_rollback(struct maildir_copy_context *ctx);
 
-int maildir_sync_last_commit(struct maildir_mailbox *mbox);
-
-int maildir_filename_get_flags(struct maildir_keywords_sync_ctx *ctx,
-			       const char *fname, enum mail_flags *flags_r,
-			       ARRAY_TYPE(keyword_indexes) *keywords);
-struct maildir_keywords_sync_ctx *
-maildir_sync_get_keywords_sync_ctx(struct maildir_index_sync_context *ctx);
-const char *maildir_filename_set_flags(struct maildir_keywords_sync_ctx *ctx,
-				       const char *fname, enum mail_flags flags,
-				       ARRAY_TYPE(keyword_indexes) *keywords);
-
 unsigned int maildir_hash(const void *p);
 int maildir_cmp(const void *p1, const void *p2);
 

--- a/src/lib-storage/index/maildir/maildir-sync.c	Sun Nov 26 00:12:11 2006 +0200
+++ b/src/lib-storage/index/maildir/maildir-sync.c	Sun Nov 26 00:17:39 2006 +0200
@@ -179,6 +179,7 @@
 #include "maildir-storage.h"
 #include "maildir-uidlist.h"
 #include "maildir-keywords.h"
+#include "maildir-sync.h"
 
 #include <stdio.h>
 #include <stddef.h>
@@ -186,8 +187,6 @@
 #include <dirent.h>
 #include <sys/stat.h>
 
-#define MAILDIR_SYNC_SECS 1
-
 #define MAILDIR_FILENAME_FLAG_FOUND 128
 
 /* When rename()ing many files from new/ to cur/, it's possible that next
@@ -774,17 +773,29 @@
 }
 
 static void
-maildir_sync_update_from_header(struct maildir_mailbox *mbox)
+maildir_sync_update_from_header(struct maildir_mailbox *mbox,
+				struct mail_index_header *hdr_r)
 {
-	uint64_t value;
+	struct mail_index_view *view;
+	const struct mail_index_header *hdr;
+
+	/* open a new view so we get the latest header */
+	view = mail_index_view_open(mbox->ibox.index);
+	hdr = mail_index_get_header(view);
 
 	/* FIXME: ugly, replace with extension header */
-	value = mail_index_get_header(mbox->ibox.view)->sync_size;
-	mbox->last_new_mtime = value & 0xffffffff;
-	mbox->last_new_sync_time = value >> 32;
+	mbox->last_new_mtime = hdr->sync_size & 0xffffffff;
+	mbox->last_dirty_flags = (hdr->sync_size >> 32) &
+		(MAILDIR_DIRTY_NEW | MAILDIR_DIRTY_CUR);
+
+	mbox->last_cur_mtime = hdr->sync_stamp;
 
-	mbox->last_cur_mtime =
-		mail_index_get_header(mbox->ibox.view)->sync_stamp;
+	if ((mbox->last_dirty_flags & MAILDIR_DIRTY_CUR) != 0 &&
+	    mbox->dirty_cur_time < mbox->last_cur_mtime)
+		mbox->dirty_cur_time = mbox->last_cur_mtime;
+
+	*hdr_r = *hdr;
+	mail_index_view_close(&view);
 }
 
 static int
@@ -793,6 +804,7 @@
 			 bool *new_changed_r, bool *cur_changed_r)
 {
 	struct index_mailbox *ibox = &mbox->ibox;
+	struct mail_index_header hdr;
 	struct stat st;
 	time_t new_mtime, cur_mtime;
 
@@ -817,7 +829,7 @@
 
 	   FIXME: For now we're using sync_size field as the new/ dir's stamp.
 	   Pretty ugly.. */
-        maildir_sync_update_from_header(mbox);
+        maildir_sync_update_from_header(mbox, &hdr);
 	if ((mbox->dirty_cur_time == 0 && cur_mtime != mbox->last_cur_mtime) ||
 	    (new_mtime != mbox->last_new_mtime)) {
 		/* check if the index has been updated.. */
@@ -826,18 +838,22 @@
 			return -1;
 		}
 
-		maildir_sync_update_from_header(mbox);
+		maildir_sync_update_from_header(mbox, &hdr);
 	}
 
 	/* If we're removing recent flags, always sync new/ directory if
 	   it has mails. */
 	if (new_mtime != mbox->last_new_mtime ||
-	    new_mtime >= mbox->last_new_sync_time - MAILDIR_SYNC_SECS ||
-	    (!ibox->keep_recent &&
-	     mail_index_get_header(ibox->view)->recent_messages_count > 0)) {
+	    ((mbox->last_dirty_flags & MAILDIR_DIRTY_NEW) != 0 &&
+	     new_mtime < ioloop_time - MAILDIR_SYNC_SECS) ||
+	    (!ibox->keep_recent && hdr.recent_messages_count > 0)) {
 		*new_changed_r = TRUE;
 		mbox->last_new_mtime = new_mtime;
-		mbox->last_new_sync_time = ioloop_time;
+
+		if (new_mtime < ioloop_time - MAILDIR_SYNC_SECS)
+			mbox->last_dirty_flags &= ~MAILDIR_DIRTY_NEW;
+		else
+			mbox->last_dirty_flags |= MAILDIR_DIRTY_NEW;
 	}
 
 	if (cur_mtime != mbox->last_cur_mtime ||
@@ -847,9 +863,13 @@
 		*cur_changed_r = TRUE;
 		mbox->last_cur_mtime = cur_mtime;
 
-		mbox->dirty_cur_time =
-			cur_mtime >= ioloop_time - MAILDIR_SYNC_SECS ?
-			cur_mtime : 0;
+		if (cur_mtime < ioloop_time - MAILDIR_SYNC_SECS) {
+			mbox->last_dirty_flags &= ~MAILDIR_DIRTY_CUR;
+			mbox->dirty_cur_time = 0;
+		} else {
+			mbox->last_dirty_flags |= MAILDIR_DIRTY_CUR;
+			mbox->dirty_cur_time = cur_mtime;
+		}
 	}
 
 	return 0;
@@ -947,7 +967,6 @@
 	ARRAY_TYPE(keyword_indexes) idx_keywords;
 	uint32_t uid_validity, next_uid;
 	uint64_t value;
-	time_t old_new_sync_time;
 	int ret = 0;
 	bool full_rescan = FALSE;
 
@@ -1179,8 +1198,10 @@
 		mbox->syncing_commit = FALSE;
 	}
 
-	if (mbox->dirty_cur_time == 0 &&
-	    mbox->last_cur_mtime != (time_t)hdr->sync_stamp) {
+	if (mbox->dirty_cur_time != 0)
+		mbox->last_dirty_flags |= MAILDIR_DIRTY_CUR;
+
+	if (mbox->last_cur_mtime != (time_t)hdr->sync_stamp) {
 		uint32_t sync_stamp = mbox->last_cur_mtime;
 
 		mail_index_update_header(trans,
@@ -1189,20 +1210,12 @@
 	}
 
 	/* FIXME: use a header extension instead of sync_size.. */
-	value = mbox->last_new_mtime;
-	old_new_sync_time = hdr->sync_size >> 32;
-	if (mbox->last_new_mtime >= old_new_sync_time - MAILDIR_SYNC_SECS) {
-		value |= (uint64_t)mbox->last_new_sync_time << 32;
-	} else {
-		value |= (uint64_t)old_new_sync_time << 32;
-	}
+	value = mbox->last_new_mtime |
+		((uint64_t)mbox->last_dirty_flags << 32);
 	if (value != hdr->sync_size) {
-		uint64_t sync_stamp = mbox->last_new_mtime |
-			((uint64_t)mbox->last_new_sync_time << 32);
-
 		mail_index_update_header(trans,
 			offsetof(struct mail_index_header, sync_size),
-			&sync_stamp, sizeof(sync_stamp), TRUE);
+			&value, sizeof(value), TRUE);
 	}
 
 	if (hdr->uid_validity == 0) {
@@ -1384,6 +1397,11 @@
 	struct maildir_sync_context *ctx;
 	int ret = 0;
 
+	if (!box->opened) {
+		if (index_storage_mailbox_open(&mbox->ibox) < 0)
+			return index_mailbox_sync_init(box, 0, TRUE);
+	}
+
 	if ((flags & MAILBOX_SYNC_FLAG_FAST) == 0 ||
 	    mbox->ibox.sync_last_check + MAILBOX_FULL_SYNC_INTERVAL <=
 	    ioloop_time) {

--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/src/lib-storage/index/maildir/maildir-sync.h	Sun Nov 26 00:17:39 2006 +0200
@@ -0,0 +1,35 @@
+#ifndef __MAILDIR_SYNC_H
+#define __MAILDIR_SYNC_H
+
+#define MAILDIR_SYNC_SECS 1
+
+struct maildir_mailbox;
+
+struct maildir_keywords_sync_ctx;
+struct maildir_index_sync_context;
+
+int maildir_sync_is_synced(struct maildir_mailbox *mbox);
+
+struct mailbox_sync_context *
+maildir_storage_sync_init(struct mailbox *box, enum mailbox_sync_flags flags);
+int maildir_storage_sync_force(struct maildir_mailbox *mbox);
+
+int maildir_sync_index_begin(struct maildir_mailbox *mbox,
+			     struct maildir_index_sync_context **ctx_r);
+int maildir_sync_index(struct maildir_index_sync_context *sync_ctx,
+		       bool partial);
+int maildir_sync_index_finish(struct maildir_index_sync_context **sync_ctx,
+			      bool failed, bool cancel);
+
+int maildir_sync_last_commit(struct maildir_mailbox *mbox);
+
+int maildir_filename_get_flags(struct maildir_keywords_sync_ctx *ctx,
+			       const char *fname, enum mail_flags *flags_r,
+			       ARRAY_TYPE(keyword_indexes) *keywords);
+struct maildir_keywords_sync_ctx *
+maildir_sync_get_keywords_sync_ctx(struct maildir_index_sync_context *ctx);
+const char *maildir_filename_set_flags(struct maildir_keywords_sync_ctx *ctx,
+				       const char *fname, enum mail_flags flags,
+				       ARRAY_TYPE(keyword_indexes) *keywords);
+
+#endif

--- a/src/lib-storage/index/maildir/maildir-transaction.c	Sun Nov 26 00:12:11 2006 +0200
+++ b/src/lib-storage/index/maildir/maildir-transaction.c	Sun Nov 26 00:17:39 2006 +0200
@@ -3,6 +3,7 @@
 #include "lib.h"
 #include "array.h"
 #include "maildir-storage.h"
+#include "maildir-sync.h"
 
 static void (*next_hook_mail_index_transaction_created)
 	(struct mail_index_transaction *t) = NULL;
@@ -54,7 +55,9 @@
 {
 	struct mailbox *box = MAIL_STORAGE_INDEX(t->view->index);
 
-	if (strcmp(box->storage->name, MAILDIR_STORAGE_NAME) == 0) {
+	/* index can be for mailbox list index, in which case box=NULL */
+	if (box != NULL &&
+	    strcmp(box->storage->name, MAILDIR_STORAGE_NAME) == 0) {
 		struct maildir_mailbox *mbox = (struct maildir_mailbox *)box;
 		struct maildir_transaction_context *mt;
 

--- a/src/lib-storage/index/maildir/maildir-util.c	Sun Nov 26 00:12:11 2006 +0200
+++ b/src/lib-storage/index/maildir/maildir-util.c	Sun Nov 26 00:17:39 2006 +0200
@@ -7,6 +7,7 @@
 #include "maildir-storage.h"
 #include "maildir-uidlist.h"
 #include "maildir-keywords.h"
+#include "maildir-sync.h"
 
 #include <unistd.h>
 #include <fcntl.h>

--- a/src/lib-storage/index/mbox/mbox-storage.c	Sun Nov 26 00:12:11 2006 +0200
+++ b/src/lib-storage/index/mbox/mbox-storage.c	Sun Nov 26 00:17:39 2006 +0200
@@ -358,6 +358,8 @@
 
 	if (mbox_get_list_settings(&list_set, data, flags) < 0)
 		return NULL;
+	list_set.mail_storage_flags = &flags;
+	list_set.mail_storage_lock_method = &lock_method;
 
 	pool = pool_alloconly_create("storage", 512);
 	storage = p_new(pool, struct mbox_storage, 1);

--- a/src/lib-storage/index/mbox/mbox-sync.c	Sun Nov 26 00:12:11 2006 +0200
+++ b/src/lib-storage/index/mbox/mbox-sync.c	Sun Nov 26 00:17:39 2006 +0200
@@ -1798,6 +1798,11 @@
 	enum mbox_sync_flags mbox_sync_flags = 0;
 	int ret = 0;
 
+	if (!box->opened) {
+		if (index_storage_mailbox_open(&mbox->ibox) < 0)
+			return index_mailbox_sync_init(box, 0, TRUE);
+	}
+
 	if ((flags & MAILBOX_SYNC_FLAG_FAST) == 0 ||
 	    mbox->ibox.sync_last_check + MAILBOX_FULL_SYNC_INTERVAL <=
 	    ioloop_time) {

--- a/src/lib-storage/index/mbox/mbox-transaction.c	Sun Nov 26 00:12:11 2006 +0200
+++ b/src/lib-storage/index/mbox/mbox-transaction.c	Sun Nov 26 00:17:39 2006 +0200
@@ -79,7 +79,8 @@
 {
 	struct mailbox *box = MAIL_STORAGE_INDEX(t->view->index);
 
-	if (strcmp(box->storage->name, MBOX_STORAGE_NAME) == 0) {
+	/* index can be for mailbox list index, in which case box=NULL */
+	if (box != NULL && strcmp(box->storage->name, MBOX_STORAGE_NAME) == 0) {
 		struct mbox_mailbox *mbox = (struct mbox_mailbox *)box;
 		struct mbox_transaction_context *mt;
 

--- a/src/lib-storage/list/Makefile.am	Sun Nov 26 00:12:11 2006 +0200
+++ b/src/lib-storage/list/Makefile.am	Sun Nov 26 00:17:39 2006 +0200
@@ -2,10 +2,15 @@
 
 AM_CPPFLAGS = \
 	-I$(top_srcdir)/src/lib \
+	-I$(top_srcdir)/src/lib-mail \
+	-I$(top_srcdir)/src/lib-index \
 	-I$(top_srcdir)/src/lib-imap \
-	-I$(top_srcdir)/src/lib-storage
+	-I$(top_srcdir)/src/lib-storage \
+	-I$(top_srcdir)/src/lib-storage/index
 
 libstorage_list_a_SOURCES = \
+	index-mailbox-list.c \
+	index-mailbox-list-sync.c \
 	mailbox-list-fs.c \
 	mailbox-list-fs-iter.c \
 	mailbox-list-maildir.c \
@@ -13,6 +18,7 @@
 	subscription-file.c
 
 noinst_HEADERS = \
+	index-mailbox-list.h \
 	mailbox-list-fs.h \
 	mailbox-list-maildir.h \
 	subscription-file.h

--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/src/lib-storage/list/index-mailbox-list-sync.c	Sun Nov 26 00:17:39 2006 +0200
@@ -0,0 +1,508 @@
+/* Copyright (C) 2006 Timo Sirainen */
+
+#include "lib.h"
+#include "ioloop.h"
+#include "array.h"
+#include "index-storage.h"
+#include "mailbox-list-index.h"
+#include "index-mailbox-list.h"
+#include "maildir/maildir-sync.h"
+
+#include <sys/stat.h>
+
+#define INDEX_LIST_STORAGE_CONTEXT(obj) \
+	*((void **)array_idx_modifiable(&(obj)->module_contexts, \
+					index_list_storage_module_id))
+
+#define CACHED_STATUS_ITEMS \
+	(STATUS_MESSAGES | STATUS_UNSEEN | STATUS_RECENT | \
+	 STATUS_UIDNEXT | STATUS_UIDVALIDITY)
+
+struct index_list_mailbox {
+	struct mailbox_vfuncs super;
+
+	uint32_t log_seq;
+	uoff_t log_offset;
+};
+
+struct index_list_map {
+	const char *name;
+	unsigned int eid_offset;
+	unsigned int status_offset;
+};
+#undef DEF
+#define DEF(a, b, c) \
+	{ a, offsetof(struct index_mailbox_list, b), \
+	  offsetof(struct mailbox_status, c) }
+static struct index_list_map index_list_map[] = {
+	DEF("msgs", eid_messages, messages),
+	DEF("unseen", eid_unseen, unseen),
+	DEF("recent", eid_recent, recent),
+	DEF("uid_validity", eid_uid_validity, uidvalidity),
+	DEF("uidnext", eid_uidnext, uidnext),
+	{ NULL, 0, 0 }
+};
+
+static void (*index_list_next_hook_mailbox_created)(struct mailbox *box);
+
+static unsigned int index_list_storage_module_id = 0;
+static bool index_list_storage_module_id_set = FALSE;
+
+static int index_list_box_close(struct mailbox *box)
+{
+	struct index_list_mailbox *ibox = INDEX_LIST_STORAGE_CONTEXT(box);
+
+	return ibox->super.close(box);
+}
+
+static int index_list_update_mail_index(struct index_mailbox_list *ilist,
+					struct mailbox *box)
+{
+	struct index_list_mailbox *ibox = INDEX_LIST_STORAGE_CONTEXT(box);
+	struct mail_index_sync_ctx *mail_sync_ctx;
+	struct mail_index_view *view;
+	struct mail_index_sync_rec sync_rec;
+
+	if (ibox->log_seq == 0)
+		return 0;
+
+	if (mail_index_sync_begin(ilist->mail_index, &mail_sync_ctx,
+				  &view, ibox->log_seq, ibox->log_offset,
+				  FALSE, FALSE) < 0)
+		return -1;
+
+	/* we should have only external transactions in here, for which we
+	   don't need to do anything but write them to the index */
+	while (mail_index_sync_next(mail_sync_ctx, &sync_rec) > 0)
+		;
+
+	return mail_index_sync_commit(&mail_sync_ctx);
+}
+
+static int
+index_list_lookup_stamps(struct index_mailbox_list *ilist,
+			 struct mail_index_view *view, uint32_t seq,
+			 time_t *new_stamp_r, time_t *cur_stamp_r,
+			 uint8_t *dirty_flags_r)
+{
+	const void *data;
+
+	if (mail_index_lookup_ext(view, seq, ilist->eid_new_sync_stamp,
+				  &data) <= 0)
+		return -1;
+	*new_stamp_r = data == NULL ? 0 : *(const uint32_t *)data;
+
+	if (mail_index_lookup_ext(view, seq, ilist->eid_cur_sync_stamp,
+				  &data) <= 0)
+		return -1;
+	*cur_stamp_r = data == NULL ? 0 : *(const uint32_t *)data;
+
+	if (mail_index_lookup_ext(view, seq, ilist->eid_dirty_flags,
+				  &data) <= 0)
+		return -1;
+	*dirty_flags_r = data == NULL ? 0 : *(const uint8_t *)data;
+	return 0;
+}
+
+static int
+index_list_has_mailbox_changed(struct mailbox *box,
+			       struct mail_index_view *view, uint32_t seq)
+{
+	/* FIXME: this function shouldn't be maildir-specific */
+	struct index_mailbox_list *ilist;
+	struct mailbox_list *list;
+	const char *root_dir, *new_dir, *cur_dir;
+	struct stat st;
+	time_t idx_new_stamp, idx_cur_stamp, max_stamp;
+	uint8_t idx_dirty_flags;
+
+	list = mail_storage_get_list(box->storage);
+	ilist = INDEX_LIST_CONTEXT(list);
+
+	if (index_list_lookup_stamps(ilist, view, seq, &idx_new_stamp,
+				     &idx_cur_stamp, &idx_dirty_flags) < 0)
+		return -1;
+
+	/* if there are dirty flags and the timestamp is old enough,
+	   do a resync in any case */
+	max_stamp = I_MAX(idx_new_stamp, idx_cur_stamp);
+	if (idx_dirty_flags != 0 &&
+	    ioloop_time - max_stamp >= MAILDIR_SYNC_SECS)
+		return 1;
+
+	/* check if new/ changed */
+	root_dir = mailbox_list_get_path(list, box->name,
+					 MAILBOX_LIST_PATH_TYPE_MAILBOX);
+	new_dir = t_strconcat(root_dir, "/new", NULL);
+	if (stat(new_dir, &st) < 0) {
+		mail_storage_set_critical(box->storage,
+					  "stat(%s) failed: %m", new_dir);
+		return -1;
+	}
+	if (idx_new_stamp != st.st_mtime)
+		return 1;
+
+	/* check if cur/ changed */
+	cur_dir = t_strconcat(root_dir, "/cur", NULL);
+	if (stat(cur_dir, &st) < 0) {
+		mail_storage_set_critical(box->storage,
+					  "stat(%s) failed: %m", cur_dir);
+		return -1;
+	}
+	if (idx_cur_stamp != st.st_mtime)
+		return 1;
+
+	return 0;
+}
+
+static int
+index_list_mailbox_open_unchanged_view(struct mailbox *box,
+				       struct mail_index_view **view_r,
+				       uint32_t *seq_r)
+{
+	struct mailbox_list *list;
+	struct index_mailbox_list *ilist;
+	struct mail_index_view *view;
+	uint32_t uid, seq;
+	int ret;
+
+	list = mail_storage_get_list(box->storage);
+	ilist = INDEX_LIST_CONTEXT(list);
+
+	ret = mailbox_list_index_lookup(ilist->list_index, box->name, &uid);
+	if (ret <= 0)
+		return ret;
+
+	/* make sure we're synced */
+	if (index_list_update_mail_index(ilist, box) < 0)
+		return -1;
+
+	/* found from list index. lookup the mail index record for it */
+	view = mail_index_view_open(ilist->mail_index);
+	ret = mail_index_lookup_uid_range(view, uid, uid, &seq, &seq);
+	if (ret < 0 || seq == 0) {
+		mail_index_view_close(&view);
+		return ret;
+	}
+
+	t_push();
+	ret = index_list_has_mailbox_changed(box, view, seq);
+	t_pop();
+	if (ret != 0) {
+		/* error / mailbox has changed. we'll need to sync it. */
+		mail_index_view_close(&view);
+		return ret < 0 ? -1 : 0;
+	}
+
+	*view_r = view;
+	*seq_r = seq;
+	return 1;
+}
+
+static int
+index_list_get_cached_status(struct mailbox *box, struct mailbox_status *status)
+{
+	struct mailbox_list *list;
+	struct index_mailbox_list *ilist;
+	struct mail_index_view *view;
+	const void *data;
+	uint32_t seq, *ext_id_p, *counter_p;
+	unsigned int i;
+	int ret;
+
+	memset(status, 0, sizeof(*status));
+
+	ret = index_list_mailbox_open_unchanged_view(box, &view, &seq);
+	if (ret <= 0)
+		return ret;
+
+	list = mail_storage_get_list(box->storage);
+	ilist = INDEX_LIST_CONTEXT(list);
+	for (i = 0; index_list_map[i].name != NULL; i++) {
+		ext_id_p = PTR_OFFSET(ilist, index_list_map[i].eid_offset);
+		ret = mail_index_lookup_ext(view, seq, *ext_id_p, &data);
+		if (ret <= 0 || data == NULL)
+			break;
+
+		counter_p = PTR_OFFSET(status, index_list_map[i].status_offset);
+		*counter_p = *(const uint32_t *)data;
+	}
+
+	mail_index_view_close(&view);
+	return 1;
+}
+
+static int
+index_list_get_status(struct mailbox *box, enum mailbox_status_items items,
+		      struct mailbox_status *status)
+{
+	struct index_list_mailbox *ibox = INDEX_LIST_STORAGE_CONTEXT(box);
+
+	if ((items & ~CACHED_STATUS_ITEMS) == 0) {
+		if (index_list_get_cached_status(box, status) > 0)
+			return 0;
+		/* nonsynced / error, fallback to doing it the slow way */
+	}
+
+	return ibox->super.get_status(box, items, status);
+}
+
+static int index_list_lookup_or_create(struct index_mailbox_list *ilist,
+				       struct mailbox *box, uint32_t *uid_r)
+{
+	struct mailbox_list_index_sync_ctx *sync_ctx;
+	int ret;
+
+	ret = mailbox_list_index_lookup(ilist->list_index, box->name, uid_r);
+	if (ret > 0) {
+		/* we'll need the mailbox synced since we're updating its
+		   contents based on what it already contains */
+		if (index_list_update_mail_index(ilist, box) < 0)
+			return -1;
+		return 1;
+	} else if (ret < 0)
+		return -1;
+
+	/* create the mailbox by doing a partial sync with the mailbox name
+	   as the sync root path */
+	if (mailbox_list_index_sync_init(ilist->list_index, box->name,
+					 MAILBOX_LIST_SYNC_FLAG_PARTIAL,
+					 &sync_ctx) < 0)
+		return -1;
+	if (mailbox_list_index_sync_commit(&sync_ctx) < 0)
+		return -1;
+
+	ret = mailbox_list_index_lookup(ilist->list_index, box->name, uid_r);
+	if (ret != 0)
+		return ret < 0 ? -1 : 0;
+
+	mail_storage_set_critical(box->storage,
+		"mailbox index: Created mailbox %s not found", box->name);
+	return -1;
+}
+
+static int
+index_list_update_sync_stamps(struct index_mailbox_list *ilist,
+			      struct mailbox *box,
+			      struct mail_index_transaction *trans,
+			      struct mail_index_view *view, uint32_t seq)
+{
+	struct index_mailbox *ibox = (struct index_mailbox *)box;
+	const struct mail_index_header *hdr;
+	time_t hdr_new_stamp, hdr_cur_stamp;
+	time_t idx_new_stamp, idx_cur_stamp;
+	uint8_t hdr_dirty_flags, idx_dirty_flags;
+
+	hdr = mail_index_get_header(ibox->view);
+	hdr_cur_stamp = hdr->sync_stamp;
+	hdr_new_stamp = hdr->sync_size & 0xffffffff;
+	hdr_dirty_flags = hdr->sync_size >> 32;
+
+	if (index_list_lookup_stamps(ilist, view, seq, &idx_new_stamp,
+				     &idx_cur_stamp, &idx_dirty_flags) < 0)
+		return -1;
+
+	if (idx_new_stamp != hdr_new_stamp) {
+		mail_index_update_ext(trans, seq, ilist->eid_new_sync_stamp,
+				      &hdr_new_stamp, NULL);
+	}
+	if (idx_cur_stamp != hdr_cur_stamp) {
+		mail_index_update_ext(trans, seq, ilist->eid_cur_sync_stamp,
+				      &hdr_cur_stamp, NULL);
+	}
+	if (idx_dirty_flags != hdr_dirty_flags) {
+		mail_index_update_ext(trans, seq, ilist->eid_dirty_flags,
+				      &hdr_dirty_flags, NULL);
+	}
+	return 0;
+}
+
+static int
+index_list_update(struct index_mailbox_list *ilist, struct mailbox *box,
+		  struct mail_index_view *view, uint32_t seq,
+		  const struct mailbox_status *status)
+{
+	struct index_list_mailbox *ibox = INDEX_LIST_STORAGE_CONTEXT(box);
+	struct mail_index_transaction *trans;
+	const void *data;
+	const uint32_t *counter_p;
+	uint32_t *ext_id_p;
+	unsigned int i;
+	int ret = 1;
+
+	trans = mail_index_transaction_begin(view, FALSE, TRUE);
+
+	/* update counters */
+	for (i = 0; index_list_map[i].name != NULL; i++) {
+		ext_id_p = PTR_OFFSET(ilist, index_list_map[i].eid_offset);
+		ret = mail_index_lookup_ext(view, seq, *ext_id_p, &data);
+		if (ret <= 0)
+			break;
+
+		counter_p = CONST_PTR_OFFSET(status,
+					     index_list_map[i].status_offset);
+		if (data == NULL ||
+		    *(const uint32_t *)data != *counter_p) {
+			mail_index_update_ext(trans, seq, *ext_id_p,
+					      counter_p, NULL);
+		}
+	}
+	if (index_list_update_sync_stamps(ilist, box, trans, view, seq) < 0)
+		ret = -1;
+	if (ret <= 0) {
+		mail_index_transaction_rollback(&trans);
+		return -1;
+	}
+
+	return mail_index_transaction_commit(&trans, &ibox->log_seq,
+					     &ibox->log_offset);
+}
+
+static struct mailbox_sync_context *
+index_list_sync_init(struct mailbox *box, enum mailbox_sync_flags flags)
+{
+	struct index_list_mailbox *ibox = INDEX_LIST_STORAGE_CONTEXT(box);
+	struct mailbox_sync_context *ctx;
+
+	/* clear any cached log seq/offset */
+	ibox->log_seq = (uint32_t)-1;
+	ibox->log_offset = 0;
+
+	if (!box->opened) {
+		/* check using the mailbox list index if the mailbox has
+		   changed. if not, we don't need to open the mailbox yet. */
+		struct mail_index_view *view;
+		uint32_t seq;
+		int ret;
+
+		ret = index_list_mailbox_open_unchanged_view(box, &view, &seq);
+		if (ret > 0) {
+			ctx = i_new(struct mailbox_sync_context, 1);
+			ctx->box = box;
+			mail_index_view_close(&view);
+
+			/* no changes, so don't bother checking again before
+			   next sync */
+			ibox->log_seq = 0;
+			return ctx;
+		}
+	}
+
+	return ibox->super.sync_init(box, flags);
+}
+
+static int index_list_sync_next(struct mailbox_sync_context *ctx,
+				struct mailbox_sync_rec *sync_rec_r)
+{
+	struct index_list_mailbox *ibox = INDEX_LIST_STORAGE_CONTEXT(ctx->box);
+
+	if (!ctx->box->opened)
+		return 0;
+
+	return ibox->super.sync_next(ctx, sync_rec_r);
+}
+
+static int index_list_sync_deinit(struct mailbox_sync_context *ctx,
+				  enum mailbox_status_items status_items,
+				  struct mailbox_status *status_r)
+{
+	struct mailbox *box = ctx->box;
+	struct index_list_mailbox *ibox = INDEX_LIST_STORAGE_CONTEXT(box);
+	struct mailbox_list *list;
+	struct index_mailbox_list *ilist;
+	struct mail_index_view *view;
+	struct mailbox_status tmp_status, *status;
+	uint32_t uid, seq;
+
+	if (!box->opened) {
+		/* nothing synced. just return the status. */
+		i_free(ctx);
+
+		return status_items == 0 ? 0 :
+			index_list_get_status(box, status_items, status_r);
+	}
+
+	/* if status_items == 0, the status_r may be NULL. we really want to
+	   know the status anyway, so save it elsewhere then */
+	status = status_items == 0 ? &tmp_status : status_r;
+	status_items |= CACHED_STATUS_ITEMS;
+
+	if (ibox->super.sync_deinit(ctx, status_items, status) < 0)
+		return -1;
+	ctx = NULL;
+
+	/* sync mailbox list index */
+	list = mail_storage_get_list(box->storage);
+	ilist = INDEX_LIST_CONTEXT(list);
+
+	if (index_list_lookup_or_create(ilist, box, &uid) < 0) {
+		/* just ignore the error */
+		return 0;
+	}
+
+	view = mail_index_view_open(ilist->mail_index);
+	if (mail_index_lookup_uid_range(view, uid, uid, &seq, &seq) == 0 &&
+	    seq > 0)
+		(void)index_list_update(ilist, box, view, seq, status);
+	mail_index_view_close(&view);
+	return 0;
+}
+
+static void index_list_mail_mailbox_opened(struct mailbox *box)
+{
+	struct index_list_mailbox *ibox;
+
+	if (index_list_next_hook_mailbox_created != NULL)
+		index_list_next_hook_mailbox_created(box);
+
+	/* FIXME: maildir-only for now */
+	if (strcmp(box->storage->name, "maildir") != 0)
+		return;
+
+	ibox = p_new(box->pool, struct index_list_mailbox, 1);
+	ibox->super = box->v;
+	box->v.close = index_list_box_close;
+	box->v.get_status = index_list_get_status;
+	box->v.sync_init = index_list_sync_init;
+	box->v.sync_next = index_list_sync_next;
+	box->v.sync_deinit = index_list_sync_deinit;
+
+	if (!index_list_storage_module_id_set) {
+		index_list_storage_module_id = mail_storage_module_id++;
+		index_list_storage_module_id_set = TRUE;
+	}
+
+	array_idx_set(&box->module_contexts,
+		      index_list_storage_module_id, &ibox);
+}
+
+void index_mailbox_list_sync_init_list(struct mailbox_list *list)
+{
+	struct index_mailbox_list *ilist = INDEX_LIST_CONTEXT(list);
+	unsigned int i;
+	uint32_t *ext_id_p;
+
+	for (i = 0; index_list_map[i].name != NULL; i++) {
+		ext_id_p = PTR_OFFSET(ilist, index_list_map[i].eid_offset);
+		*ext_id_p = mail_index_ext_register(ilist->mail_index,
+					index_list_map[i].name, 0,
+					sizeof(uint32_t), sizeof(uint32_t));
+	}
+
+	/* FIXME: maildir-only: */
+	ilist->eid_cur_sync_stamp =
+		mail_index_ext_register(ilist->mail_index, "sync-cur", 0,
+					sizeof(uint32_t), sizeof(uint32_t));
+	ilist->eid_new_sync_stamp =
+		mail_index_ext_register(ilist->mail_index, "sync-new", 0,
+					sizeof(uint32_t), sizeof(uint32_t));
+	ilist->eid_dirty_flags =
+		mail_index_ext_register(ilist->mail_index, "sync-dirty", 0,
+					sizeof(uint8_t), sizeof(uint8_t));
+}
+
+void index_mailbox_list_sync_init(void)
+{
+	index_list_next_hook_mailbox_created = hook_mailbox_opened;
+	hook_mailbox_opened = index_list_mail_mailbox_opened;
+}

--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/src/lib-storage/list/index-mailbox-list.c	Sun Nov 26 00:17:39 2006 +0200
@@ -0,0 +1,410 @@
+/* Copyright (C) 2006 Timo Sirainen */
+
+#include "lib.h"
+#include "ioloop.h"
+#include "array.h"
+#include "imap-match.h"
+#include "mail-index.h"
+#include "mail-storage.h"
+#include "mailbox-list-index.h"
+#include "index-mailbox-list.h"
+
+#include <time.h>
+#include <sys/stat.h>
+
+/* min 2 seconds */
+#define MAILBOX_LIST_SYNC_SECS 2
+
+unsigned int index_mailbox_list_module_id = 0;
+
+static bool index_mailbox_list_module_id_set = FALSE;
+static void (*index_next_hook_mailbox_list_created)(struct mailbox_list *list);
+
+static int
+index_mailbox_list_is_synced(struct index_mailbox_list_iterate_context *ctx)
+{
+	const struct mail_index_header *hdr;
+	struct stat st;
+	const char *path = ctx->ctx.list->set.root_dir;
+
+	/* FIXME: single sync_stamp works only with maildir++ */
+	if (stat(path, &st) < 0) {
+		mailbox_list_set_critical(ctx->ctx.list,
+					  "stat(%s) failed: %m", path);
+		return -1;
+	}
+	/*
+	   if mtime is older than 2 secs, we set the first bit on
+	   if mtime is 0-2 secs old, we set the first bit off.
+
+	   this way we'll always do a resync later when syncing a recently
+	   changed directory. if the directory changes while we're syncing it
+	   we'll resync it again later.
+
+	   this would work with 1 second difference if we didn't store the
+	   dirtyness flag in the stamp's first bit.
+	*/
+	if (st.st_mtime < ioloop_time - MAILBOX_LIST_SYNC_SECS)
+		st.st_mtime |= 1;
+	else
+		st.st_mtime &= ~1;
+
+	ctx->sync_stamp = st.st_mtime;
+
+	hdr = mail_index_get_header(ctx->view);
+	return hdr->sync_stamp == ctx->sync_stamp;
+}
+
+static void mask_parse(struct mailbox_list *list, const char *mask,
+		       const char **prefix_r, int *recurse_level_r)
+{
+	char sep = list->hierarchy_sep;
+	const char *prefix_start, *prefix_end;
+	bool seen_wildcards = FALSE;
+	int recurse_level = 0;
+
+	prefix_start = prefix_end = mask;
+	for (; *mask != '\0'; mask++) {
+		if (*mask == '%')
+			seen_wildcards = TRUE;
+		else if (*mask == '*') {
+			recurse_level = -1;
+			break;
+		}
+
+		if (*mask == sep) {
+			if (!seen_wildcards)
+				prefix_end = mask;
+			recurse_level++;
+		}
+	}
+
+	*prefix_r = prefix_start == prefix_end ? "" :
+		t_strdup_until(prefix_start, prefix_end);
+	*recurse_level_r = recurse_level;
+}
+
+static struct mailbox_list_iterate_context *
+index_mailbox_list_iter_init(struct mailbox_list *list, const char *mask,
+			     enum mailbox_list_iter_flags flags)
+{
+	struct index_mailbox_list *ilist = INDEX_LIST_CONTEXT(list);
+	struct index_mailbox_list_iterate_context *ctx;
+	enum mailbox_list_sync_flags sync_flags;
+	const char *prefix;
+	int recurse_level;
+
+	ctx = i_new(struct index_mailbox_list_iterate_context, 1);
+	ctx->ctx.list = list;
+	ctx->ctx.flags = flags;
+	ctx->glob = imap_match_init(default_pool, mask, TRUE,
+				    list->hierarchy_sep);
+
+	ctx->view = mail_index_view_open(ilist->mail_index);
+	if (index_mailbox_list_is_synced(ctx) > 0) {
+		/* synced, list from index */
+		mask_parse(list, mask, &prefix, &recurse_level);
+
+		ctx->info_pool =
+			pool_alloconly_create("mailbox name pool", 128);
+		ctx->iter_ctx =
+			mailbox_list_index_iterate_init(ilist->list_index,
+							prefix, recurse_level);
+		ctx->recurse_level = recurse_level;
+		ctx->prefix = *prefix == '\0' ? i_strdup("") :
+			i_strdup_printf("%s%c", prefix, list->hierarchy_sep);
+	} else {
+		/* FIXME: this works nicely with maildir++, but not others */
+		sync_flags = MAILBOX_LIST_SYNC_FLAG_RECURSIVE;
+		mask = "*";
+		prefix = "";
+
+		if (mailbox_list_index_sync_init(ilist->list_index, prefix,
+						 sync_flags,
+						 &ctx->sync_ctx) == 0) {
+			ctx->trans =
+				mailbox_list_index_sync_get_transaction(ctx->sync_ctx);
+		}
+
+		ctx->backend_ctx = ilist->super.iter_init(list, mask, flags);
+	}
+	return &ctx->ctx;
+}
+
+static enum mailbox_info_flags
+index_mailbox_list_index_flags_translate(enum mailbox_list_index_flags flags)
+{
+	enum mailbox_info_flags info_flags = 0;
+
+	if ((flags & MAILBOX_LIST_INDEX_FLAG_CHILDREN) != 0)
+		info_flags |= MAILBOX_CHILDREN;
+	if ((flags & MAILBOX_LIST_INDEX_FLAG_NOCHILDREN) != 0)
+		info_flags |= MAILBOX_NOCHILDREN;
+
+	if ((flags & MAILBOX_LIST_INDEX_FLAG_NONEXISTENT) != 0)
+		info_flags |= MAILBOX_NONEXISTENT;
+	if ((flags & MAILBOX_LIST_INDEX_FLAG_NOSELECT) != 0)
+		info_flags |= MAILBOX_NOSELECT;
+	return info_flags;
+}
+
+static enum mailbox_list_index_flags
+index_mailbox_list_info_flags_translate(enum mailbox_info_flags info_flags)
+{
+	enum mailbox_list_index_flags flags = 0;
+
+	if ((info_flags & MAILBOX_CHILDREN) != 0)
+		flags |= MAILBOX_LIST_INDEX_FLAG_CHILDREN;
+	if ((info_flags & MAILBOX_NOCHILDREN) != 0)
+		flags |= MAILBOX_LIST_INDEX_FLAG_NOCHILDREN;
+
+	if ((info_flags & MAILBOX_NONEXISTENT) != 0)
+		flags |= MAILBOX_LIST_INDEX_FLAG_NONEXISTENT;
+	if ((info_flags & MAILBOX_NOSELECT) != 0)
+		flags |= MAILBOX_LIST_INDEX_FLAG_NOSELECT;
+	return flags;
+}
+
+/* skip nonexistent mailboxes when finding with "*" */
+#define info_flags_match(ctx, info) \
+	(((info)->flags & MAILBOX_NONEXISTENT) == 0 || \
+	 (ctx)->recurse_level >= 0)
+
+static int iter_next_nonsync(struct index_mailbox_list_iterate_context *ctx,
+			     struct mailbox_info **info_r)
+{
+	struct index_mailbox_list *ilist = INDEX_LIST_CONTEXT(ctx->ctx.list);
+	struct mailbox_list_index_info iinfo;
+	const struct mail_index_record *rec;
+	uint32_t seq;
+	int ret;
+
+	/* find the next matching mailbox */
+	do {
+		p_clear(ctx->info_pool);
+		ret = mailbox_list_index_iterate_next(ctx->iter_ctx, &iinfo);
+		if (ret <= 0) {
+			*info_r = NULL;
+			return ret;
+		}
+
+		ctx->info.name = *ctx->prefix == '\0' ? iinfo.name :
+			p_strconcat(ctx->info_pool, ctx->prefix,
+				    iinfo.name, NULL);
+	} while (imap_match(ctx->glob, ctx->info.name) != IMAP_MATCH_YES);
+
+	/* get the mailbox's flags */
+	if (mail_index_lookup_uid_range(ctx->view, iinfo.uid, iinfo.uid,
+					&seq, &seq) < 0)
+		return -1;
+	if (seq == 0) {
+		mailbox_list_index_set_corrupted(ilist->list_index,
+			"Desynced: Record expunged from mail index");
+		return -1;
+	}
+
+	if (mail_index_lookup(ctx->view, seq, &rec) < 0)
+		return -1;
+	ctx->info.flags = index_mailbox_list_index_flags_translate(rec->flags);
+
+	/* do some sanity checks to the flags */
+	if ((ctx->info.flags & MAILBOX_CHILDREN) != 0 &&
+	    (ctx->info.flags & MAILBOX_NOCHILDREN) != 0) {
+		mailbox_list_index_set_corrupted(ilist->list_index,
+			"Mail index has both children and nochildren flags");
+		return -1;
+	}
+	if ((ctx->info.flags & MAILBOX_NOCHILDREN) != 0 &&
+	    iinfo.has_children) {
+		mailbox_list_index_set_corrupted(ilist->list_index,
+			"Desynced: Children flags wrong in mail index");
+	}
+
+	if (!info_flags_match(ctx, &ctx->info))
+		return iter_next_nonsync(ctx, info_r);
+
+	*info_r = &ctx->info;
+	return 0;
+}
+
+static struct mailbox_info *
+index_mailbox_list_iter_next(struct mailbox_list_iterate_context *_ctx)
+{
+	struct index_mailbox_list_iterate_context *ctx =
+		(struct index_mailbox_list_iterate_context *)_ctx;
+	struct index_mailbox_list *ilist = INDEX_LIST_CONTEXT(_ctx->list);
+	struct mailbox_info *info;
+	uint32_t seq, flags;
+
+	if (ctx->iter_ctx != NULL) {
+		if (iter_next_nonsync(ctx, &info) < 0) {
+			ctx->failed = TRUE;
+			return NULL;
+		}
+		return info;
+	}
+
+	do {
+		info = ilist->super.iter_next(ctx->backend_ctx);
+		if (info == NULL || ctx->sync_ctx == NULL)
+			return info;
+
+		/* if the sync fails, just ignore it. we don't require synced
+		   indexes to return valid output. */
+		if (mailbox_list_index_sync_more(ctx->sync_ctx, info->name,
+						 &seq) < 0)
+			return info;
+
+		flags = index_mailbox_list_info_flags_translate(info->flags);
+		mail_index_update_flags(ctx->trans, seq, MODIFY_REPLACE, flags);
+	} while (imap_match(ctx->glob, info->name) != IMAP_MATCH_YES ||
+		 !info_flags_match(ctx, info));
+
+	return info;
+}
+
+static int
+index_mailbox_list_iter_deinit(struct mailbox_list_iterate_context *_ctx)
+{
+	struct index_mailbox_list_iterate_context *ctx =
+		(struct index_mailbox_list_iterate_context *)_ctx;
+	struct index_mailbox_list *ilist = INDEX_LIST_CONTEXT(_ctx->list);
+	int ret = ctx->failed ? -1 : 0;
+
+	if (ctx->iter_ctx != NULL) {
+		mailbox_list_index_iterate_deinit(&ctx->iter_ctx);
+		pool_unref(ctx->info_pool);
+	}
+
+	if (ctx->view != NULL)
+		mail_index_view_close(&ctx->view);
+
+	if (ctx->backend_ctx != NULL) {
+		/* FIXME: single sync_stamp works only with maildir++ */
+		mail_index_update_header(ctx->trans,
+			offsetof(struct mail_index_header, sync_stamp),
+			&ctx->sync_stamp, sizeof(ctx->sync_stamp), TRUE);
+
+		if ((ret = ilist->super.iter_deinit(ctx->backend_ctx)) < 0)
+			mailbox_list_index_sync_rollback(&ctx->sync_ctx);
+		else {
+			/* index updates aren't that important. if the commit
+			   fails, we've still returned full output. */
+			(void)mailbox_list_index_sync_commit(&ctx->sync_ctx);
+		}
+	}
+
+	imap_match_deinit(&ctx->glob);
+	i_free(ctx->prefix);
+	i_free(ctx);
+	return ret;
+}
+
+static void index_mailbox_list_deinit(struct mailbox_list *list)
+{
+	struct index_mailbox_list *ilist = INDEX_LIST_CONTEXT(list);
+
+	mailbox_list_index_free(&ilist->list_index);
+	mail_index_free(&ilist->mail_index);
+
+	ilist->super.deinit(list);
+}
+
+static void index_mailbox_list_created(struct mailbox_list *list)
+{
+	struct index_mailbox_list *ilist;
+	struct mail_index *mail_index;
+	struct mailbox_list_index *list_index;
+	enum mail_index_open_flags index_flags;
+	enum mail_index_lock_method lock_method;
+	enum mail_storage_flags storage_flags;
+	const char *dir, *path;
+	int ret;
+
+	/* FIXME: for now we only work with maildir++ */
+	if (strcmp(list->name, "maildir++") != 0)
+		return;
+
+	/* FIXME: a bit ugly way to get the flags, but this will do for now.. */
+	index_flags = MAIL_INDEX_OPEN_FLAG_CREATE;
+	storage_flags = *list->set.mail_storage_flags;
+	if ((storage_flags & MAIL_STORAGE_FLAG_MMAP_DISABLE) != 0) {
+		index_flags |= MAIL_INDEX_OPEN_FLAG_MMAP_DISABLE;
+		return; /* FIXME: we currently don't support mmap_disable */
+	}
+#ifndef MMAP_CONFLICTS_WRITE
+	if ((storage_flags & MAIL_STORAGE_FLAG_MMAP_NO_WRITE) != 0)
+#endif
+		index_flags |= MAIL_INDEX_OPEN_FLAG_MMAP_NO_WRITE;
+
+	switch (*list->set.mail_storage_lock_method) {
+	case MAIL_STORAGE_LOCK_FCNTL:
+		lock_method = MAIL_INDEX_LOCK_FCNTL;
+		break;
+	case MAIL_STORAGE_LOCK_FLOCK:
+		lock_method = MAIL_INDEX_LOCK_FLOCK;
+		break;
+	case MAIL_STORAGE_LOCK_DOTLOCK:
+		lock_method = MAIL_INDEX_LOCK_DOTLOCK;
+		break;
+	}
+
+	dir = mailbox_list_get_path(list, NULL, MAILBOX_LIST_PATH_TYPE_INDEX);
+	path = t_strconcat(dir, "/"MAILBOX_LIST_INDEX_NAME, NULL);
+
+	mail_index = mail_index_alloc(dir, MAIL_INDEX_PREFIX);
+	if (mail_index_open(mail_index, index_flags, lock_method) < 0) {
+		if (mail_index_move_to_memory(mail_index) < 0) {
+			/* try opening once more. it should be created
+			   directly into memory now. */
+			ret = mail_index_open(mail_index, index_flags,
+					      lock_method);
+			if (ret <= 0) {
+				/* everything failed. there's a bug in the
+				   code, but just work around it by disabling
+				   the index completely */
+				mail_index_free(&mail_index);
+				return;
+			}
+		}
+	}
+
+	list_index = mailbox_list_index_alloc(path, list->hierarchy_sep,
+					      mail_index);
+	if (mailbox_list_index_open_or_create(list_index) < 0) {
+		/* skip indexing */
+		mailbox_list_index_free(&list_index);
+		mail_index_free(&mail_index);
+		return;
+	}
+
+	ilist = p_new(list->pool, struct index_mailbox_list, 1);
+	ilist->super = list->v;
+	ilist->mail_index = mail_index;
+	ilist->list_index = list_index;
+
+	list->v.deinit = index_mailbox_list_deinit;
+	list->v.iter_init = index_mailbox_list_iter_init;
+	list->v.iter_deinit = index_mailbox_list_iter_deinit;
+	list->v.iter_next = index_mailbox_list_iter_next;
+
+	if (!index_mailbox_list_module_id_set) {
+		index_mailbox_list_module_id = mailbox_list_module_id++;
+		index_mailbox_list_module_id_set = TRUE;
+	}
+
+	array_idx_set(&list->module_contexts,
+		      index_mailbox_list_module_id, &ilist);
+
+	index_mailbox_list_sync_init_list(list);
+}
+
+void index_mailbox_list_init(void); /* called in mailbox-list-register.c */
+
+void index_mailbox_list_init(void)
+{
+	index_next_hook_mailbox_list_created = hook_mailbox_list_created;
+	hook_mailbox_list_created = index_mailbox_list_created;
+
+	index_mailbox_list_sync_init();
+}

--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/src/lib-storage/list/index-mailbox-list.h	Sun Nov 26 00:17:39 2006 +0200
@@ -0,0 +1,51 @@
+#ifndef __INDEX_MAILBOX_LIST_H
+#define __INDEX_MAILBOX_LIST_H
+
+#include "mailbox-list-private.h"
+
+#define MAIL_INDEX_PREFIX "dovecot.list.index"
+#define MAILBOX_LIST_INDEX_NAME MAIL_INDEX_PREFIX".uidmap"
+
+#define INDEX_LIST_CONTEXT(obj) \
+	*((void **)array_idx_modifiable(&(obj)->module_contexts, \
+					index_mailbox_list_module_id))
+
+struct index_mailbox_list {
+	struct mailbox_list_vfuncs super;
+
+	struct mail_index *mail_index;
+	struct mailbox_list_index *list_index;
+
+	uint32_t eid_messages, eid_unseen, eid_recent;
+	uint32_t eid_uid_validity, eid_uidnext;
+
+	uint32_t eid_cur_sync_stamp, eid_new_sync_stamp, eid_dirty_flags;
+};
+
+struct index_mailbox_list_iterate_context {
+	struct mailbox_list_iterate_context ctx;
+
+	struct mailbox_list_iter_ctx *iter_ctx;
+	struct mailbox_list_index_sync_ctx *sync_ctx;
+	struct mailbox_list_iterate_context *backend_ctx;
+
+	struct mail_index_view *view;
+	struct mail_index_transaction *trans;
+
+	char *prefix;
+	int recurse_level;
+	struct imap_match_glob *glob;
+
+	pool_t info_pool;
+	struct mailbox_info info;
+	uint32_t sync_stamp;
+
+	unsigned int failed:1;
+};
+
+extern unsigned int index_mailbox_list_module_id;
+
+void index_mailbox_list_sync_init(void);
+void index_mailbox_list_sync_init_list(struct mailbox_list *list);
+
+#endif

--- a/src/lib-storage/list/mailbox-list-fs-iter.c	Sun Nov 26 00:12:11 2006 +0200
+++ b/src/lib-storage/list/mailbox-list-fs-iter.c	Sun Nov 26 00:17:39 2006 +0200
@@ -76,8 +76,7 @@
 }
 
 struct mailbox_list_iterate_context *
-fs_list_iter_init(struct mailbox_list *_list,
-		  const char *ref, const char *mask,
+fs_list_iter_init(struct mailbox_list *_list, const char *mask,
 		  enum mailbox_list_iter_flags flags)
 {
 	struct fs_mailbox_list *list =
@@ -95,20 +94,12 @@
         ctx->next = fs_list_next;
 
 	/* check that we're not trying to do any "../../" lists */
-	if (!mailbox_list_is_valid_mask(_list, ref) ||
-	    !mailbox_list_is_valid_mask(_list, mask)) {
+	if (!mailbox_list_is_valid_mask(_list, mask)) {
 		mailbox_list_set_error(_list, "Invalid mask");
 		ctx->ctx.failed = TRUE;
 		return &ctx->ctx;
 	}
 
-	if (*mask == '/' || *mask == '~') {
-		/* mask overrides reference */
-	} else if (*ref != '\0') {
-		/* merge reference and mask */
-		mask = t_strconcat(ref, mask, NULL);
-	}
-
 	if ((flags & MAILBOX_LIST_ITER_SUBSCRIBED) != 0) {
 		ctx->next = fs_list_subs;
 
@@ -313,7 +304,7 @@
 
 	if (match == IMAP_MATCH_PARENT) {
 		/* placeholder */
-		ctx->info.flags = MAILBOX_PLACEHOLDER;
+		ctx->info.flags = MAILBOX_NONEXISTENT | MAILBOX_CHILDREN;
 		while ((p = strrchr(name, '/')) != NULL) {
 			name = t_strdup_until(name, p);
 			if (imap_match(ctx->glob, name) > 0) {

--- a/src/lib-storage/list/mailbox-list-fs.c	Sun Nov 26 00:12:11 2006 +0200
+++ b/src/lib-storage/list/mailbox-list-fs.c	Sun Nov 26 00:17:39 2006 +0200
@@ -247,6 +247,19 @@
 	return list->temp_prefix;
 }
 
+static const char *
+fs_list_join_refmask(struct mailbox_list *_list __attr_unused__,
+		     const char *ref, const char *mask)
+{
+	if (*mask == '/' || *mask == '~') {
+		/* mask overrides reference */
+	} else if (*ref != '\0') {
+		/* merge reference and mask */
+		mask = t_strconcat(ref, mask, NULL);
+	}
+	return mask;
+}
+
 static int fs_list_set_subscribed(struct mailbox_list *_list,
 				  const char *name, bool set)
 {
@@ -277,6 +290,7 @@
 		fs_list_get_path,
 		fs_list_get_mailbox_name_status,
 		fs_list_get_temp_prefix,
+		fs_list_join_refmask,
 		fs_list_iter_init,
 		fs_list_iter_next,
 		fs_list_iter_deinit,

--- a/src/lib-storage/list/mailbox-list-fs.h	Sun Nov 26 00:12:11 2006 +0200
+++ b/src/lib-storage/list/mailbox-list-fs.h	Sun Nov 26 00:17:39 2006 +0200
@@ -14,8 +14,7 @@
 };
 
 struct mailbox_list_iterate_context *
-fs_list_iter_init(struct mailbox_list *_list,
-		  const char *ref, const char *mask,
+fs_list_iter_init(struct mailbox_list *_list, const char *mask,
 		  enum mailbox_list_iter_flags flags);
 int fs_list_iter_deinit(struct mailbox_list_iterate_context *ctx);
 struct mailbox_info *

--- a/src/lib-storage/list/mailbox-list-maildir-iter.c	Sun Nov 26 00:12:11 2006 +0200
+++ b/src/lib-storage/list/mailbox-list-maildir-iter.c	Sun Nov 26 00:17:39 2006 +0200
@@ -33,11 +33,12 @@
 			node->flags |= MAILBOX_CHILDREN;
 			node->flags &= ~MAILBOX_NOCHILDREN;
 			maildir_nodes_fix(node->children, is_subs);
-		} else if ((node->flags & MAILBOX_PLACEHOLDER) != 0) {
+		} else if ((node->flags & MAILBOX_NONEXISTENT) != 0) {
 			if (!is_subs) {
-				node->flags &= ~MAILBOX_PLACEHOLDER;
+				node->flags &= ~MAILBOX_NONEXISTENT;
 				node->flags |= MAILBOX_NOSELECT;
 			}
+			node->flags |= MAILBOX_CHILDREN;
 		}
 		node = node->next;
 	}
@@ -126,7 +127,7 @@
 						 mailbox_c, &created);
 			if (node != NULL) {
 				if (created)
-					node->flags = MAILBOX_PLACEHOLDER;
+					node->flags = MAILBOX_NONEXISTENT;
 
 				node->flags |= MAILBOX_CHILDREN |
 					MAILBOX_FLAG_MATCHED;
@@ -144,8 +145,7 @@
 			if (node != NULL) {
 				if (created)
 					node->flags = MAILBOX_NOCHILDREN;
-				node->flags &= ~(MAILBOX_PLACEHOLDER |
-						 MAILBOX_NONEXISTENT);
+				node->flags &= ~MAILBOX_NONEXISTENT;
 				node->flags |= MAILBOX_FLAG_MATCHED;
 			}
 		}
@@ -165,7 +165,7 @@
 		if (created)
 			node->flags = MAILBOX_NOCHILDREN;
 		else
-			node->flags &= ~MAILBOX_PLACEHOLDER;
+			node->flags &= ~MAILBOX_NONEXISTENT;
 
 		switch (imap_match(glob, "INBOX")) {
 		case IMAP_MATCH_YES:
@@ -218,7 +218,7 @@
 			i_assert(p != NULL);
 
 			node = mailbox_tree_get(ctx->tree_ctx, name, &created);
-			if (created) node->flags = MAILBOX_PLACEHOLDER;
+			if (created) node->flags = MAILBOX_NONEXISTENT;
 			node->flags |= MAILBOX_FLAG_MATCHED | MAILBOX_CHILDREN;
 			node->flags &= ~MAILBOX_NOCHILDREN;
 			break;
@@ -231,8 +231,7 @@
 }
 
 struct mailbox_list_iterate_context *
-maildir_list_iter_init(struct mailbox_list *_list,
-		       const char *ref, const char *mask,
+maildir_list_iter_init(struct mailbox_list *_list, const char *mask,
 		       enum mailbox_list_iter_flags flags)
 {
 	struct maildir_mailbox_list *list =
@@ -251,11 +250,6 @@
 	ctx->pool = pool;
 	ctx->tree_ctx = mailbox_tree_init(_list->hierarchy_sep);
 
-	if (*ref != '\0') {
-		/* join reference + mask */
-		mask = t_strconcat(ref, mask, NULL);
-	}
-
 	glob = imap_match_init(pool, mask, TRUE, _list->hierarchy_sep);
 
 	ctx->dir = _list->set.root_dir;

--- a/src/lib-storage/list/mailbox-list-maildir.c	Sun Nov 26 00:12:11 2006 +0200
+++ b/src/lib-storage/list/mailbox-list-maildir.c	Sun Nov 26 00:17:39 2006 +0200
@@ -239,6 +239,17 @@
 	return list->temp_prefix;
 }
 
+static const char *
+maildir_list_join_refmask(struct mailbox_list *_list __attr_unused__,
+			  const char *ref, const char *mask)
+{
+	if (*ref != '\0') {
+		/* merge reference and mask */
+		mask = t_strconcat(ref, mask, NULL);
+	}
+	return mask;
+}
+
 static int maildir_list_set_subscribed(struct mailbox_list *_list,
 				       const char *name, bool set)
 {
@@ -270,6 +281,7 @@
 		maildir_list_get_path,
 		maildir_list_get_mailbox_name_status,
 		maildir_list_get_temp_prefix,
+		maildir_list_join_refmask,
 		maildir_list_iter_init,
 		maildir_list_iter_next,
 		maildir_list_iter_deinit,

--- a/src/lib-storage/list/mailbox-list-maildir.h	Sun Nov 26 00:12:11 2006 +0200
+++ b/src/lib-storage/list/mailbox-list-maildir.h	Sun Nov 26 00:17:39 2006 +0200
@@ -14,8 +14,7 @@
 };
 
 struct mailbox_list_iterate_context *
-maildir_list_iter_init(struct mailbox_list *_list,
-		       const char *ref, const char *mask,
+maildir_list_iter_init(struct mailbox_list *_list, const char *mask,
 		       enum mailbox_list_iter_flags flags);
 int maildir_list_iter_deinit(struct mailbox_list_iterate_context *ctx);
 struct mailbox_info *

--- a/src/lib-storage/mail-storage-private.h	Sun Nov 26 00:12:11 2006 +0200
+++ b/src/lib-storage/mail-storage-private.h	Sun Nov 26 00:17:39 2006 +0200
@@ -8,6 +8,8 @@
 #define MAIL_STORAGE_ERR_MAILBOX_NOT_FOUND "Mailbox doesn't exist: %s"
 #define MAIL_STORAGE_ERR_NO_PERMISSION "Permission denied"
 
+/* Called after mail storage has been created */
+extern void (*hook_mail_storage_created)(struct mail_storage *storage);
 /* Called after mailbox has been opened */
 extern void (*hook_mailbox_opened)(struct mailbox *box);
 
@@ -93,6 +95,7 @@
 	int (*sync_next)(struct mailbox_sync_context *ctx,
 			 struct mailbox_sync_rec *sync_rec_r);
 	int (*sync_deinit)(struct mailbox_sync_context *ctx,
+			   enum mailbox_status_items status_items,
 			   struct mailbox_status *status_r);
 
 	void (*notify_changes)(struct mailbox *box, unsigned int min_interval,
@@ -162,6 +165,10 @@
 
 	/* Module-specific contexts. See mail_storage_module_id. */
 	ARRAY_DEFINE(module_contexts, void);
+
+	/* When FAST open flag is used, the mailbox isn't actually opened until
+	   it's synced for the first time. */
+	unsigned int opened:1;
 };
 
 struct mail_vfuncs {

--- a/src/lib-storage/mail-storage.c	Sun Nov 26 00:12:11 2006 +0200
+++ b/src/lib-storage/mail-storage.c	Sun Nov 26 00:17:39 2006 +0200
@@ -19,6 +19,7 @@
 unsigned int mail_storage_module_id = 0;
 unsigned int mail_storage_mail_index_module_id = 0;
 
+void (*hook_mail_storage_created)(struct mail_storage *storage);
 void (*hook_mailbox_opened)(struct mailbox *box) = NULL;
 
 static ARRAY_DEFINE(storages, struct mail_storage *);
@@ -119,10 +120,13 @@
 	struct mail_storage *storage;
 
 	storage = mail_storage_find(driver);
-	if (storage != NULL)
-		return storage->v.create(data, user, flags, lock_method);
-	else
+	if (storage == NULL)
 		return NULL;
+
+	storage = storage->v.create(data, user, flags, lock_method);
+	if (hook_mail_storage_created != NULL && storage != NULL)
+		hook_mail_storage_created(storage);
+	return storage;
 }
 
 static struct mail_storage *
@@ -136,8 +140,11 @@
 	classes = array_get(&storages, &count);
 	for (i = 0; i < count; i++) {
 		storage = classes[i]->v.create(NULL, user, flags, lock_method);
-		if (storage != NULL)
+		if (storage != NULL) {
+			if (hook_mail_storage_created != NULL)
+				hook_mail_storage_created(storage);
 			return storage;
+		}
 	}
 	return NULL;
 }
@@ -192,6 +199,8 @@
 		}
 	}
 
+	if (hook_mail_storage_created != NULL && storage != NULL)
+		hook_mail_storage_created(storage);
 	return storage;
 }
 
@@ -458,12 +467,13 @@
 }
 
 int mailbox_sync_deinit(struct mailbox_sync_context **_ctx,
+			enum mailbox_status_items status_items,
 			struct mailbox_status *status_r)
 {
 	struct mailbox_sync_context *ctx = *_ctx;
 
 	*_ctx = NULL;
-	return ctx->box->v.sync_deinit(ctx, status_r);
+	return ctx->box->v.sync_deinit(ctx, status_items, status_r);
 }
 
 void mailbox_notify_changes(struct mailbox *box, unsigned int min_interval,

--- a/src/lib-storage/mail-storage.h	Sun Nov 26 00:12:11 2006 +0200
+++ b/src/lib-storage/mail-storage.h	Sun Nov 26 00:17:39 2006 +0200
@@ -303,6 +303,7 @@
 int mailbox_sync_next(struct mailbox_sync_context *ctx,
 		      struct mailbox_sync_rec *sync_rec_r);
 int mailbox_sync_deinit(struct mailbox_sync_context **ctx,
+			enum mailbox_status_items status_items,
 			struct mailbox_status *status_r);
 
 /* Call given callback function when something changes in the mailbox.

--- a/src/lib-storage/mailbox-list-private.h	Sun Nov 26 00:12:11 2006 +0200
+++ b/src/lib-storage/mailbox-list-private.h	Sun Nov 26 00:17:39 2006 +0200
@@ -22,10 +22,11 @@
 				       enum mailbox_name_status *status);
 
 	const char *(*get_temp_prefix)(struct mailbox_list *list);
+	const char *(*join_refmask)(struct mailbox_list *list,
+				    const char *ref, const char *mask);
 
 	struct mailbox_list_iterate_context *
-		(*iter_init)(struct mailbox_list *list,
-			     const char *ref, const char *mask,
+		(*iter_init)(struct mailbox_list *list, const char *mask,
 			     enum mailbox_list_iter_flags flags);
 	struct mailbox_info *
 		(*iter_next)(struct mailbox_list_iterate_context *ctx);
@@ -75,5 +76,6 @@
 void mailbox_list_set_error(struct mailbox_list *list, const char *error);
 void mailbox_list_set_critical(struct mailbox_list *list, const char *fmt, ...)
 	__attr_format__(2, 3);
+void mailbox_list_set_internal_error(struct mailbox_list *list);
 
 #endif

--- a/src/lib-storage/mailbox-list.c	Sun Nov 26 00:12:11 2006 +0200
+++ b/src/lib-storage/mailbox-list.c	Sun Nov 26 00:17:39 2006 +0200
@@ -111,6 +111,9 @@
 		p_strdup(list->pool, set->subscription_fname);
 	list->set.maildir_name = p_strdup(list->pool, set->maildir_name);
 
+	list->set.mail_storage_flags = set->mail_storage_flags;
+	list->set.mail_storage_lock_method = set->mail_storage_lock_method;
+
 	if ((flags & MAILBOX_LIST_FLAG_DEBUG) != 0) {
 		i_info("%s: root=%s, index=%s, control=%s, inbox=%s",
 		       driver, list->set.root_dir,
@@ -126,6 +129,9 @@
 	if (hook_mailbox_list_created != NULL)
 		hook_mailbox_list_created(list);
 
+	list->set.mail_storage_flags = NULL;
+	list->set.mail_storage_lock_method = NULL;
+
 	*list_r = list;
 	return 0;
 }
@@ -175,6 +181,12 @@
 	return list->v.get_temp_prefix(list);
 }
 
+const char *mailbox_list_join_refmask(struct mailbox_list *list,
+				      const char *ref, const char *mask)
+{
+	return list->v.join_refmask(list, ref, mask);
+}
+
 int mailbox_list_get_mailbox_name_status(struct mailbox_list *list,
 					 const char *name,
 					 enum mailbox_name_status *status)
@@ -183,11 +195,10 @@
 }
 
 struct mailbox_list_iterate_context *
-mailbox_list_iter_init(struct mailbox_list *list,
-		       const char *ref, const char *mask,
+mailbox_list_iter_init(struct mailbox_list *list, const char *mask,
 		       enum mailbox_list_iter_flags flags)
 {
-	return list->v.iter_init(list, ref, mask, flags);
+	return list->v.iter_init(list, mask, flags);
 }
 
 struct mailbox_info *
@@ -284,7 +295,7 @@
 	list->temporary_error = FALSE;
 }
 
-static void mailbox_list_set_internal_error(struct mailbox_list *list)
+void mailbox_list_set_internal_error(struct mailbox_list *list)
 {
 	struct tm *tm;
 	char str[256];