Mercurial > dovecot > original-hg > dovecot-1.2

changeset 6600:416d9ee66047 HEAD

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Support for reading maildir files. This makes it possible to do a fast maildir to dbox conversion by only renaming a few files.
author Timo Sirainen <tss@iki.fi>
date Sun, 21 Oct 2007 18:58:00 +0300
parents a6f51026b969
children 835daa872b0a
files src/lib-storage/index/dbox/Makefile.am src/lib-storage/index/dbox/dbox-file-maildir.c src/lib-storage/index/dbox/dbox-file-maildir.h src/lib-storage/index/dbox/dbox-file.c src/lib-storage/index/dbox/dbox-file.h src/lib-storage/index/dbox/dbox-index.c src/lib-storage/index/dbox/dbox-index.h src/lib-storage/index/dbox/dbox-mail.c src/lib-storage/index/dbox/dbox-storage.h src/lib-storage/index/dbox/dbox-sync-rebuild.c
diffstat 10 files changed, 330 insertions(+), 32 deletions(-) [+]
line wrap: on
line diff
--- a/src/lib-storage/index/dbox/Makefile.am	Sun Oct 21 18:54:32 2007 +0300
+++ b/src/lib-storage/index/dbox/Makefile.am	Sun Oct 21 18:58:00 2007 +0300
@@ -10,6 +10,7 @@
 
 libstorage_dbox_a_SOURCES = \
 	dbox-file.c \
+	dbox-file-maildir.c \
 	dbox-index.c \
 	dbox-mail.c \
 	dbox-save.c \
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/src/lib-storage/index/dbox/dbox-file-maildir.c	Sun Oct 21 18:58:00 2007 +0300
@@ -0,0 +1,85 @@
+/* Copyright (c) 2007-2007 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "array.h"
+#include "str.h"
+#include "dbox-storage.h"
+#include "../maildir/maildir-storage.h"
+#include "../maildir/maildir-filename.h"
+#include "dbox-file.h"
+#include "dbox-file-maildir.h"
+
+static const char *
+dbox_file_maildir_get_flags(struct dbox_file *file, enum dbox_metadata_key key)
+{
+	ARRAY_TYPE(keyword_indexes) keyword_indexes;
+	struct mail_keywords *keywords;
+	enum mail_flags flags;
+	string_t *str;
+	const char *fname;
+
+	if (file->mbox->maildir_sync_keywords == NULL)
+		return NULL;
+
+	fname = strrchr(file->path, '/');
+	i_assert(fname != NULL);
+	fname++;
+
+	t_array_init(&keyword_indexes, 32);
+	maildir_filename_get_flags(file->mbox->maildir_sync_keywords,
+				   fname, &flags, &keyword_indexes);
+	str = t_str_new(64);
+	if (key == DBOX_METADATA_FLAGS)
+		dbox_mail_metadata_flags_append(str, flags);
+	else {
+		keywords = mail_index_keywords_create_from_indexes(
+			file->mbox->ibox.index, &keyword_indexes);
+		dbox_mail_metadata_keywords_append(file->mbox, str, keywords);
+		mail_index_keywords_free(&keywords);
+	}
+	return str_c(str);
+}
+
+const char *dbox_file_maildir_metadata_get(struct dbox_file *file,
+					   enum dbox_metadata_key key)
+{
+	const char *fname;
+	struct stat st;
+	uoff_t size;
+
+	switch (key) {
+	case DBOX_METADATA_FLAGS:
+	case DBOX_METADATA_KEYWORDS:
+		return dbox_file_maildir_get_flags(file, key);
+	case DBOX_METADATA_RECEIVED_TIME:
+	case DBOX_METADATA_SAVE_TIME:
+		if (file->fd != -1) {
+			if (fstat(file->fd, &st) < 0) {
+				dbox_file_set_syscall_error(file, "fstat");
+				return NULL;
+			}
+		} else {
+			if (stat(file->path, &st) < 0) {
+				if (errno == ENOENT)
+					return NULL;
+				dbox_file_set_syscall_error(file, "stat");
+				return NULL;
+			}
+		}
+		if (key == DBOX_METADATA_RECEIVED_TIME)
+			return dec2str(st.st_mtime);
+		else
+			return dec2str(st.st_ctime);
+	case DBOX_METADATA_VIRTUAL_SIZE:
+		fname = strrchr(file->path, '/');
+		i_assert(fname != NULL);
+		maildir_filename_get_size(fname + 1, MAILDIR_EXTRA_VIRTUAL_SIZE,
+					  &size);
+		return dec2str(size);
+	case DBOX_METADATA_EXPUNGED:
+	case DBOX_METADATA_EXT_REF:
+	case DBOX_METADATA_SPACE:
+		break;
+	}
+	return NULL;
+}
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/src/lib-storage/index/dbox/dbox-file-maildir.h	Sun Oct 21 18:58:00 2007 +0300
@@ -0,0 +1,7 @@
+#ifndef DBOX_FILE_MAILDIR_H
+#define DBOX_FILE_MAILDIR_H
+
+const char *dbox_file_maildir_metadata_get(struct dbox_file *file,
+					   enum dbox_metadata_key key);
+
+#endif
--- a/src/lib-storage/index/dbox/dbox-file.c	Sun Oct 21 18:54:32 2007 +0300
+++ b/src/lib-storage/index/dbox/dbox-file.c	Sun Oct 21 18:58:00 2007 +0300
@@ -12,6 +12,7 @@
 #include "dbox-storage.h"
 #include "dbox-index.h"
 #include "dbox-file.h"
+#include "dbox-file-maildir.h"
 
 #include <stdio.h>
 #include <stdlib.h>
@@ -128,16 +129,29 @@
 }
 
 static char *
-dbox_file_id_get_path(struct dbox_mailbox *mbox, unsigned int file_id)
+dbox_file_id_get_path(struct dbox_mailbox *mbox, unsigned int file_id,
+		      bool *maildir_file_r)
 {
+	struct dbox_index_record *rec;
+	const char *p;
+
+	*maildir_file_r = FALSE;
 	if ((file_id & DBOX_FILE_ID_FLAG_UID) != 0) {
 		file_id &= ~DBOX_FILE_ID_FLAG_UID;
 		return i_strdup_printf("%s/"DBOX_MAIL_FILE_UID_FORMAT,
 				       mbox->path, file_id);
-	} else {
-		return i_strdup_printf("%s/"DBOX_MAIL_FILE_MULTI_FORMAT,
-				       mbox->path, file_id);
 	}
+
+	rec = dbox_index_record_lookup(mbox->dbox_index, file_id);
+	if (rec != NULL && rec->status == DBOX_INDEX_FILE_STATUS_MAILDIR) {
+		/* data contains <uid> <filename> */
+		*maildir_file_r = TRUE;
+		p = strchr(rec->data, ' ');
+		return i_strdup_printf("%s/%s", mbox->path, p + 1);
+	}
+
+	return i_strdup_printf("%s/"DBOX_MAIL_FILE_MULTI_FORMAT,
+			       mbox->path, file_id);
 }
 
 struct dbox_file *
@@ -145,6 +159,7 @@
 {
 	struct dbox_file *file;
 	unsigned int count;
+	bool maildir;
 
 	file = file_id == 0 ? NULL :
 		dbox_find_and_move_open_file(mbox, file_id);
@@ -162,7 +177,8 @@
 	file->mbox = mbox;
 	if (file_id != 0) {
 		file->file_id = file_id;
-		file->path = dbox_file_id_get_path(mbox, file_id);
+		file->path = dbox_file_id_get_path(mbox, file_id, &maildir);
+		file->maildir_file = maildir;
 	} else {
 		file->path = dbox_generate_tmp_filename(mbox->path);
 	}
@@ -173,24 +189,38 @@
 	return file;
 }
 
+struct dbox_file *
+dbox_file_init_new_maildir(struct dbox_mailbox *mbox, const char *fname)
+{
+	struct dbox_file *file;
+
+	file = dbox_file_init(mbox, 0);
+	file->maildir_file = TRUE;
+	file->path = i_strdup_printf("%s/%s", mbox->path, fname);
+	return file;
+}
+
 int dbox_file_assign_id(struct dbox_file *file, unsigned int file_id)
 {
 	char *new_path;
+	bool maildir;
 
 	i_assert(file->file_id == 0);
 	i_assert(file_id != 0);
 
-	new_path = dbox_file_id_get_path(file->mbox, file_id);
-	if (rename(file->path, new_path) < 0) {
-		mail_storage_set_critical(file->mbox->ibox.box.storage,
-			"rename(%s, %s) failed: %m", file->path, new_path);
-		i_free(new_path);
-		return -1;
+	if (!file->maildir_file) {
+		new_path = dbox_file_id_get_path(file->mbox, file_id, &maildir);
+		if (rename(file->path, new_path) < 0) {
+			mail_storage_set_critical(file->mbox->ibox.box.storage,
+						  "rename(%s, %s) failed: %m",
+						  file->path, new_path);
+			i_free(new_path);
+			return -1;
+		}
+		i_free(file->path);
+		file->path = new_path;
 	}
 
-	i_free(file->path);
-	file->path = new_path;
-
 	file->file_id = file_id;
 	array_append(&file->mbox->open_files, &file, 1);
 	return 0;
@@ -252,7 +282,7 @@
 bool dbox_file_can_append(struct dbox_file *file, uoff_t mail_size)
 {
 	if (file->nonappendable)
-		return 0;
+		return FALSE;
 
 	if (file->append_offset == 0) {
 		/* messages have been expunged */
@@ -352,7 +382,8 @@
 	}
 
 	file->input = i_stream_create_fd(file->fd, MAIL_READ_BLOCK_SIZE, FALSE);
-	return !read_header ? 1 : dbox_file_read_header(file);
+	return !read_header || file->maildir_file ? 1 :
+		dbox_file_read_header(file);
 }
 
 static int dbox_file_create(struct dbox_file *file)
@@ -406,6 +437,30 @@
 		return dbox_file_open(file, read_header, deleted_r);
 }
 
+static int
+dbox_file_get_maildir_data(struct dbox_file *file, uint32_t *uid_r,
+			   uoff_t *physical_size_r)
+{
+	struct dbox_index_record *rec;
+	struct stat st;
+
+	if (fstat(file->fd, &st) < 0) {
+		dbox_file_set_syscall_error(file, "fstat");
+		return -1;
+	}
+
+	rec = dbox_index_record_lookup(file->mbox->dbox_index, file->file_id);
+	if (rec == NULL) {
+		/* should happen only when we're rebuilding the index */
+		*uid_r = 0;
+	} else {
+		i_assert(rec->status == DBOX_INDEX_FILE_STATUS_MAILDIR);
+		*uid_r = strtoul(rec->data, NULL, 10);
+	}
+	*physical_size_r = st.st_size;
+	return 1;
+}
+
 static int dbox_file_read_mail_header(struct dbox_file *file, uint32_t *uid_r,
 				      uoff_t *physical_size_r)
 {
@@ -414,6 +469,9 @@
 	size_t size;
 	int ret;
 
+	if (file->maildir_file)
+		return dbox_file_get_maildir_data(file, uid_r, physical_size_r);
+
 	ret = i_stream_read_data(file->input, &data, &size,
 				 file->msg_header_size - 1);
 	if (ret <= 0) {
@@ -627,6 +685,9 @@
 				     uoff_t physical_size)
 {
 	if (offset == 0) {
+		if (file->maildir_file)
+			return 0;
+
 		i_assert(file->file_header_size != 0);
 		offset = file->file_header_size;
 	}
@@ -683,6 +744,12 @@
 	}
 	file->metadata_read_offset = 0;
 
+	if (file->maildir_file) {
+		/* no metadata in maildir files, but we do later some kludging
+		   to return metadata when needed. */
+		return 0;
+	}
+
 	if (file->input == NULL) {
 		if ((ret = dbox_file_open(file, TRUE, &deleted)) <= 0)
 			return ret;
@@ -749,6 +816,9 @@
 	const char *const *metadata;
 	unsigned int i, count;
 
+	if (file->maildir_file)
+		return dbox_file_maildir_metadata_get(file, key);
+
 	metadata = array_get(&file->metadata, &count);
 	for (i = 0; i < count; i++) {
 		if (*metadata[i] == (char)key)
--- a/src/lib-storage/index/dbox/dbox-file.h	Sun Oct 21 18:54:32 2007 +0300
+++ b/src/lib-storage/index/dbox/dbox-file.h	Sun Oct 21 18:58:00 2007 +0300
@@ -102,7 +102,7 @@
 	unsigned int append_offset_header_pos;
 
 	unsigned int append_count;
-	uint32_t last_append_uid;
+	uint32_t last_append_uid, maildir_append_seq;
 
 	uoff_t append_offset;
 	time_t create_time;
@@ -122,6 +122,7 @@
 	/* Includes the trailing LF that shouldn't be used */
 	unsigned int metadata_len;
 
+	unsigned int maildir_file:1;
 	unsigned int nonappendable:1;
 	unsigned int deleted:1;
 };
@@ -131,6 +132,8 @@
 
 struct dbox_file *
 dbox_file_init(struct dbox_mailbox *mbox, unsigned int file_id);
+struct dbox_file *
+dbox_file_init_new_maildir(struct dbox_mailbox *mbox, const char *fname);
 void dbox_file_unref(struct dbox_file **file);
 
 /* Free all currently opened files. */
--- a/src/lib-storage/index/dbox/dbox-index.c	Sun Oct 21 18:54:32 2007 +0300
+++ b/src/lib-storage/index/dbox/dbox-index.c	Sun Oct 21 18:58:00 2007 +0300
@@ -28,6 +28,7 @@
 	uint32_t uid_validity, next_uid;
 	unsigned int next_file_id;
 
+	pool_t record_data_pool;
 	ARRAY_DEFINE(records, struct dbox_index_record);
 };
 
@@ -55,6 +56,8 @@
 	index->next_uid = 1;
 	index->next_file_id = 1;
 	i_array_init(&index->records, 128);
+	index->record_data_pool =
+		pool_alloconly_create("dbox index record data", 256);
 	return index;
 }
 
@@ -77,10 +80,25 @@
 
 	dbox_index_close(index);
 	array_free(&index->records);
+	pool_unref(&index->record_data_pool);
 	i_free(index->path);
 	i_free(index);
 }
 
+static int dbox_index_parse_maildir(struct dbox_index *index, const char *line,
+				    struct dbox_index_record *rec)
+{
+	char *p;
+	unsigned long uid;
+
+	uid = strtoul(line, &p, 10);
+	if (*p++ != ' ' || *p == '\0' || uid == 0 || uid >= (uint32_t)-1)
+		return -1;
+
+	rec->data = p_strdup(index->record_data_pool, p);
+	return 0;
+}
+
 static int dbox_index_parse_line(struct dbox_index *index, const char *line,
 				 unsigned int offset)
 {
@@ -109,6 +127,10 @@
 	rec.dirty = line[2] != '0';
 
 	line += 3;
+	if (rec.status == DBOX_INDEX_FILE_STATUS_MAILDIR) {
+		if (dbox_index_parse_maildir(index, line, &rec) < 0)
+			return -1;
+	}
 	array_append(&index->records, &rec, 1);
 	return 0;
 }
@@ -201,6 +223,7 @@
 		return -1;
 	}
 
+	p_clear(index->record_data_pool);
 	array_clear(&index->records);
 	input = index->input = i_stream_create_fd(index->fd, 1024, FALSE);
 
@@ -479,7 +502,8 @@
 	case DBOX_INDEX_FILE_STATUS_SINGLE_MESSAGE:
 		break;
 	case DBOX_INDEX_FILE_STATUS_MAILDIR:
-		/* FIXME */
+		str_append_c(str, ' ');
+		str_append(str, rec->data);
 		break;
 	}
 	str_append_c(str, '\n');
@@ -697,6 +721,13 @@
 	return 0;
 }
 
+void dbox_index_append_file(struct dbox_index_append_context *ctx,
+			    struct dbox_file *file)
+{
+	file->refcount++;
+	array_append(&ctx->files, &file, 1);
+}
+
 static int dbox_index_append_commit_new(struct dbox_index_append_context *ctx,
 					struct dbox_file *file, string_t *str)
 {
@@ -707,7 +738,8 @@
 
 	i_assert(file->append_count > 0);
 
-	if (file->append_count == 1 && !dbox_file_can_append(file, 0)) {
+	if (file->append_count == 1 && !file->maildir_file &&
+	    !dbox_file_can_append(file, 0)) {
 		/* single UID message file */
 		i_assert(file->last_append_uid != 0);
 		file_id = file->last_append_uid | DBOX_FILE_ID_FLAG_UID;
@@ -740,9 +772,17 @@
 	memset(&rec, 0, sizeof(rec));
 	rec.file_id = file_id;
 	rec.file_offset = ctx->output_offset + str_len(str);
-	rec.status = dbox_file_can_append(file, 0) ?
-		DBOX_INDEX_FILE_STATUS_APPENDABLE :
-		DBOX_INDEX_FILE_STATUS_NONAPPENDABLE;
+	if (file->maildir_file) {
+		rec.status = DBOX_INDEX_FILE_STATUS_MAILDIR;
+		rec.data = p_strdup_printf(ctx->index->record_data_pool,
+					   "%u %s", file->last_append_uid,
+					   strrchr(file->path, '/') + 1);
+
+	} else {
+		rec.status = dbox_file_can_append(file, 0) ?
+			DBOX_INDEX_FILE_STATUS_APPENDABLE :
+			DBOX_INDEX_FILE_STATUS_NONAPPENDABLE;
+	}
 
 	array_append(&ctx->index->records, &rec, 1);
 	dbox_index_append_record(&rec, str);
@@ -791,7 +831,7 @@
 		ret = -1;
 	}
 	dbox_index_unlock_range(ctx->index, ctx->output_offset, str_len(str));
-	return ret;
+	return ret < 0 ? -1 : 0;
 }
 
 static int dbox_index_write_header(struct dbox_index *index)
--- a/src/lib-storage/index/dbox/dbox-index.h	Sun Oct 21 18:54:32 2007 +0300
+++ b/src/lib-storage/index/dbox/dbox-index.h	Sun Oct 21 18:58:00 2007 +0300
@@ -80,6 +80,8 @@
 	unsigned int file_offset;
 
 	enum dbox_index_file_status status;
+	const char *data;
+
 	unsigned int expunges:1;
 	unsigned int dirty:1;
 	unsigned int locked:1;
@@ -105,8 +107,6 @@
 /* Try to lock index file for recreating. Returns 1 if ok, 0 if file already
    contains locks, -1 if error. */
 int dbox_index_try_lock_recreate(struct dbox_index *index);
-/* Lock index file for syncing. Returns 0 if ok, -1 if error. */
-int dbox_index_lock_sync(struct dbox_index *index);
 
 struct dbox_index_append_context *
 dbox_index_append_begin(struct dbox_index *index);
@@ -117,6 +117,8 @@
 			   uoff_t mail_size,
 			   struct dbox_file **file_r,
 			   struct ostream **output_r);
+void dbox_index_append_file(struct dbox_index_append_context *ctx,
+			    struct dbox_file *file);
 /* Assign file_ids to all appended files. */
 int dbox_index_append_assign_file_ids(struct dbox_index_append_context *ctx);
 /* Returns 0 if ok, -1 if error. */
--- a/src/lib-storage/index/dbox/dbox-mail.c	Sun Oct 21 18:54:32 2007 +0300
+++ b/src/lib-storage/index/dbox/dbox-mail.c	Sun Oct 21 18:58:00 2007 +0300
@@ -187,7 +187,7 @@
 	struct dbox_mail *mail = (struct dbox_mail *)_mail;
 	struct index_mail_data *data = &mail->imail.data;
 	struct istream *input;
-	uoff_t offset;
+	uoff_t offset, size;
 	uint32_t uid;
 	bool expunged;
 	int ret;
@@ -197,8 +197,7 @@
 			return -1;
 
 		ret = dbox_file_get_mail_stream(mail->open_file, offset, &uid,
-						&data->physical_size, &input,
-						&expunged);
+						&size, &input, &expunged);
 		if (ret < 0)
 			return -1;
 		if (ret > 0 && expunged) {
@@ -211,6 +210,7 @@
 				i_stream_unref(&input);
 			return -1;
 		}
+		data->physical_size = size;
 		data->stream = input;
 	}
 
--- a/src/lib-storage/index/dbox/dbox-storage.h	Sun Oct 21 18:54:32 2007 +0300
+++ b/src/lib-storage/index/dbox/dbox-storage.h	Sun Oct 21 18:58:00 2007 +0300
@@ -43,6 +43,8 @@
 	uint32_t dbox_ext_id, dbox_hdr_ext_id;
 	/* timestamp when the mailbox was last modified interactively */
 	time_t last_interactive_change;
+	/* set while rebuilding indexes with converted maildir files */
+	struct maildir_keywords_sync_ctx *maildir_sync_keywords;
 
 	uoff_t rotate_size, rotate_min_size;
 	unsigned int rotate_days;
--- a/src/lib-storage/index/dbox/dbox-sync-rebuild.c	Sun Oct 21 18:54:32 2007 +0300
+++ b/src/lib-storage/index/dbox/dbox-sync-rebuild.c	Sun Oct 21 18:58:00 2007 +0300
@@ -1,7 +1,10 @@
 /* Copyright (c) 2007 Dovecot authors, see the included COPYING file */
 
 #include "lib.h"
+#include "array.h"
 #include "dbox-storage.h"
+#include "../maildir/maildir-uidlist.h"
+#include "../maildir/maildir-keywords.h"
 #include "dbox-index.h"
 #include "dbox-file.h"
 #include "dbox-sync.h"
@@ -11,7 +14,11 @@
 
 struct dbox_sync_rebuild_context {
 	struct dbox_mailbox *mbox;
+	struct dbox_index_append_context *append_ctx;
 	struct mail_index_transaction *trans;
+
+	struct maildir_uidlist *maildir_uidlist;
+	struct maildir_keywords *mk;
 };
 
 static int dbox_sync_set_uidvalidity(struct dbox_sync_rebuild_context *ctx)
@@ -62,6 +69,7 @@
 {
 	uint32_t seq, uid;
 	uoff_t metadata_offset, physical_size;
+	const char *fname;
 	bool expunged;
 	int ret;
 
@@ -83,6 +91,18 @@
 		i_warning("%s: Header contains wrong UID %u", file->path, uid);
 		return 0;
 	}
+	if (file->maildir_file) {
+		i_assert(uid == 0);
+		fname = strrchr(file->path, '/');
+		i_assert(fname != NULL);
+		if (!maildir_uidlist_get_uid(ctx->maildir_uidlist, fname + 1,
+					     &uid)) {
+			/* FIXME: not in uidlist, give it an uid */
+			return 0;
+		}
+		file->append_count = 1;
+		file->last_append_uid = uid;
+	}
 
 	metadata_offset =
 		dbox_file_get_metadata_offset(file, *offset, physical_size);
@@ -95,6 +115,7 @@
 	}
 	if (!expunged) {
 		mail_index_append(ctx->trans, uid, &seq);
+		file->maildir_append_seq = seq;
 		dbox_sync_index_metadata(ctx, file, seq);
 	}
 	return 1;
@@ -132,12 +153,39 @@
 	return 0;
 }
 
+static int
+dbox_sync_index_maildir_file(struct dbox_sync_rebuild_context *ctx,
+			     const char *fname)
+{
+	struct dbox_file *file;
+	uoff_t offset = 0;
+	int ret;
+
+	if (ctx->mbox->maildir_sync_keywords == NULL) {
+		ctx->maildir_uidlist =
+			maildir_uidlist_init_readonly(&ctx->mbox->ibox);
+		ctx->mk = maildir_keywords_init_readonly(&ctx->mbox->ibox.box);
+		ctx->mbox->maildir_sync_keywords =
+			maildir_keywords_sync_init(ctx->mk,
+						   ctx->mbox->ibox.index);
+
+		if (maildir_uidlist_refresh(ctx->maildir_uidlist) < 0)
+			return -1;
+	}
+
+	file = dbox_file_init_new_maildir(ctx->mbox, fname);
+	if ((ret = dbox_sync_index_file_next(ctx, file, &offset)) > 0)
+		dbox_index_append_file(ctx->append_ctx, file);
+	dbox_file_unref(&file);
+	return ret;
+}
+
 static int dbox_sync_index_rebuild_ctx(struct dbox_sync_rebuild_context *ctx)
 {
 	struct mail_storage *storage = ctx->mbox->ibox.box.storage;
 	DIR *dir;
 	struct dirent *d;
-	int ret = 0;
+	int ret = 1;
 
 	if (dbox_sync_set_uidvalidity(ctx) < 0)
 		return -1;
@@ -153,13 +201,15 @@
 		return -1;
 	}
 	errno = 0;
-	for (; ret == 0 && (d = readdir(dir)) != NULL; errno = 0) {
+	for (; ret > 0 && (d = readdir(dir)) != NULL; errno = 0) {
 		if (strncmp(d->d_name, DBOX_MAIL_FILE_UID_PREFIX,
 			    sizeof(DBOX_MAIL_FILE_UID_PREFIX)-1) == 0)
 			ret = dbox_sync_index_uid_file(ctx, d->d_name);
 		else if (strncmp(d->d_name, DBOX_MAIL_FILE_MULTI_PREFIX,
 				 sizeof(DBOX_MAIL_FILE_MULTI_PREFIX)-1) == 0)
 			ret = dbox_sync_index_multi_file(ctx, d->d_name);
+		else if (strstr(d->d_name, ":2,") != NULL)
+			ret = dbox_sync_index_maildir_file(ctx, d->d_name);
 	}
 	if (errno != 0) {
 		mail_storage_set_critical(storage,
@@ -175,6 +225,25 @@
 	return ret;
 }
 
+static void dbox_sync_update_maildir_ids(struct dbox_sync_rebuild_context *ctx)
+{
+	struct dbox_mail_index_record rec;
+	struct dbox_file *const *files;
+	unsigned int i, count;
+
+	memset(&rec, 0, sizeof(rec));
+	files = array_get(&ctx->mbox->open_files, &count);
+	for (i = 0; i < count; i++) {
+		if (!files[i]->maildir_file)
+			continue;
+
+		i_assert(files[i]->file_id != 0);
+		rec.file_id = files[i]->file_id;
+		mail_index_update_ext(ctx->trans, files[i]->maildir_append_seq,
+				      ctx->mbox->dbox_ext_id, &rec, NULL);
+	}
+}
+
 int dbox_sync_index_rebuild(struct dbox_mailbox *mbox)
 {
 	struct dbox_sync_rebuild_context ctx;
@@ -185,6 +254,7 @@
 
 	memset(&ctx, 0, sizeof(ctx));
 	ctx.mbox = mbox;
+	ctx.append_ctx = dbox_index_append_begin(mbox->dbox_index);
 	view = mail_index_view_open(mbox->ibox.index);
 	ctx.trans = mail_index_transaction_begin(view,
 					MAIL_INDEX_TRANSACTION_FLAG_EXTERNAL);
@@ -192,8 +262,26 @@
 
 	if ((ret = dbox_sync_index_rebuild_ctx(&ctx)) < 0)
 		mail_index_transaction_rollback(&ctx.trans);
+	else {
+		ret = dbox_index_append_assign_file_ids(ctx.append_ctx);
+		if (ret == 0) {
+			dbox_sync_update_maildir_ids(&ctx);
+			ret = mail_index_transaction_commit(&ctx.trans,
+							    &seq, &offset);
+		}
+	}
+	mail_index_view_close(&view);
+
+	if (ret == 0)
+		ret = dbox_index_append_commit(&ctx.append_ctx);
 	else
-		ret = mail_index_transaction_commit(&ctx.trans, &seq, &offset);
-	mail_index_view_close(&view);
+		dbox_index_append_rollback(&ctx.append_ctx);
+
+	if (mbox->maildir_sync_keywords != NULL)
+		maildir_keywords_sync_deinit(&mbox->maildir_sync_keywords);
+	if (ctx.mk != NULL)
+		maildir_keywords_deinit(&ctx.mk);
+	if (ctx.maildir_uidlist != NULL)
+		maildir_uidlist_deinit(&ctx.maildir_uidlist);
 	return ret;
 }