--- a/src/lib-storage/index/dbox/Makefile.am	Sat Sep 01 01:36:10 2007 +0300
+++ b/src/lib-storage/index/dbox/Makefile.am	Sat Sep 01 03:04:02 2007 +0300
@@ -10,23 +10,20 @@
 
 libstorage_dbox_a_SOURCES = \
 	dbox-file.c \
-	dbox-keywords.c \
+	dbox-index.c \
 	dbox-mail.c \
 	dbox-save.c \
 	dbox-sync.c \
-	dbox-sync-expunge.c \
-	dbox-sync-full.c \
+	dbox-sync-file.c \
+	dbox-sync-rebuild.c \
 	dbox-storage.c \
-	dbox-transaction.c \
-	dbox-uidlist.c
+	dbox-transaction.c
 
 headers = \
 	dbox-file.h \
-	dbox-format.h \
-	dbox-keywords.h \
+	dbox-index.h \
 	dbox-storage.h \
-	dbox-sync.h \
-	dbox-uidlist.h
+	dbox-sync.h
 
 if INSTALL_HEADERS
   pkginc_libdir=$(pkgincludedir)/src/lib-storage/index/dbox

--- a/src/lib-storage/index/dbox/dbox-file.c	Sat Sep 01 01:36:10 2007 +0300
+++ b/src/lib-storage/index/dbox/dbox-file.c	Sat Sep 01 03:04:02 2007 +0300
@@ -1,311 +1,1053 @@
-/* Copyright (C) 2005-2006 Timo Sirainen */
+/* Copyright (C) 2007 Timo Sirainen */
 
 #include "lib.h"
+#include "ioloop.h"
 #include "array.h"
-#include "bsearch-insert-pos.h"
 #include "hex-dec.h"
+#include "hostpid.h"
 #include "istream.h"
 #include "ostream.h"
-#include "read-full.h"
+#include "write-full.h"
+#include "str.h"
 #include "dbox-storage.h"
+#include "dbox-index.h"
 #include "dbox-file.h"
 
-int dbox_file_lookup_offset(struct dbox_mailbox *mbox,
-			    struct mail_index_view *view, uint32_t seq,
-			    uint32_t *file_seq_r, uoff_t *offset_r)
-{
-	const void *data1, *data2;
-	bool expunged;
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <fcntl.h>
+
+enum mail_flags dbox_mail_flags_map[DBOX_METADATA_FLAGS_COUNT] = {
+	MAIL_ANSWERED,
+	MAIL_FLAGGED,
+	MAIL_DELETED,
+	MAIL_SEEN,
+	MAIL_DRAFT
+};
+
+char dbox_mail_flag_chars[DBOX_METADATA_FLAGS_COUNT] = {
+	'A', 'F', 'D', 'S', 'T'
+};
 
-	mail_index_lookup_ext(view, seq, mbox->dbox_file_ext_idx,
-			      &data1, &expunged);
-	if (expunged)
-		return 0;
-	mail_index_lookup_ext(view, seq, mbox->dbox_offset_ext_idx,
-			      &data2, &expunged);
-	if (expunged)
-		return 0;
+static int dbox_file_metadata_skip_header(struct dbox_file *file);
+
+static char *dbox_generate_tmp_filename(const char *path)
+{
+	static unsigned int create_count = 0;
 
-	if (data1 == NULL || data2 == NULL) {
-		*file_seq_r = 0;
-		return 1;
-	}
+	return i_strdup_printf("%s/temp.%s.P%sQ%uM%s.%s",
+			       path, dec2str(ioloop_timeval.tv_sec), my_pid,
+			       create_count++,
+			       dec2str(ioloop_timeval.tv_usec), my_hostname);
+}
 
-	/* success */
-	*file_seq_r = *((const uint32_t *)data1);
-	*offset_r = *((const uint64_t *)data2);
-	return 1;
+void dbox_file_set_syscall_error(struct dbox_file *file, const char *function)
+{
+	mail_storage_set_critical(file->mbox->ibox.box.storage,
+				  "%s(%s) failed: %m", function, file->path);
 }
 
-void dbox_file_close(struct dbox_file *file)
+static void
+dbox_file_set_corrupted(struct dbox_file *file, const char *reason)
+{
+	mail_storage_set_critical(file->mbox->ibox.box.storage,
+				  "%s corrupted: %s", file->path, reason);
+}
+
+
+static struct dbox_file *
+dbox_find_and_move_open_file(struct dbox_mailbox *mbox, unsigned int file_id)
 {
-	if (array_is_created(&file->file_idx_keywords)) {
-		array_free(&file->idx_file_keywords);
-		array_free(&file->file_idx_keywords);
+	struct dbox_file *const *files, *file;
+	unsigned int i, count;
+
+	files = array_get(&mbox->open_files, &count);
+	for (i = 0; i < count; i++) {
+		if (files[i]->file_id == file_id) {
+			/* move to last in the array */
+			file = files[i];
+			array_delete(&mbox->open_files, i, 1);
+			array_append(&mbox->open_files, &file, 1);
+			return file;
+		}
 	}
+	return NULL;
+}
 
+static void dbox_file_free(struct dbox_file *file)
+{
+	i_assert(file->refcount == 0);
+
+	if (file->metadata_pool != NULL)
+		pool_unref(file->metadata_pool);
 	if (file->input != NULL)
-		i_stream_destroy(&file->input);
+		i_stream_unref(&file->input);
+	if (file->output != NULL)
+		o_stream_unref(&file->output);
 	if (file->fd != -1) {
 		if (close(file->fd) < 0)
-			i_error("close(dbox) failed: %m");
+			dbox_file_set_syscall_error(file, "close");
+		file->fd = -1;
 	}
-	i_free(file->seeked_keywords);
 	i_free(file->path);
 	i_free(file);
 }
 
-static int
-dbox_file_read_mail_header(struct dbox_mailbox *mbox, struct dbox_file *file,
-			   uoff_t offset)
+void dbox_files_free(struct dbox_mailbox *mbox)
+{
+	struct dbox_file *const *files;
+	unsigned int i, count;
+
+	files = array_get(&mbox->open_files, &count);
+	for (i = 0; i < count; i++)
+		dbox_file_free(files[i]);
+	array_clear(&mbox->open_files);
+}
+
+static void
+dbox_close_open_files(struct dbox_mailbox *mbox, unsigned int close_count)
+{
+	struct dbox_file *const *files;
+	unsigned int i, count;
+
+	files = array_get(&mbox->open_files, &count);
+	for (i = 0; i < count;) {
+		if (files[i]->refcount == 0) {
+			dbox_file_free(files[i]);
+			array_delete(&mbox->open_files, i, 1);
+
+			if (--close_count == 0)
+				break;
+
+			files = array_get(&mbox->open_files, &count);
+		} else {
+			i++;
+		}
+	}
+}
+
+static char *
+dbox_file_id_get_path(struct dbox_mailbox *mbox, unsigned int file_id)
+{
+	if ((file_id & DBOX_FILE_ID_FLAG_UID) != 0) {
+		file_id &= ~DBOX_FILE_ID_FLAG_UID;
+		return i_strdup_printf("%s/"DBOX_MAIL_FILE_UID_FORMAT,
+				       mbox->path, file_id);
+	} else {
+		return i_strdup_printf("%s/"DBOX_MAIL_FILE_MULTI_FORMAT,
+				       mbox->path, file_id);
+	}
+}
+
+struct dbox_file *
+dbox_file_init(struct dbox_mailbox *mbox, unsigned int file_id)
+{
+	struct dbox_file *file;
+	unsigned int count;
+
+	file = file_id == 0 ? NULL :
+		dbox_find_and_move_open_file(mbox, file_id);
+	if (file != NULL) {
+		file->refcount++;
+		return file;
+	}
+
+	count = array_count(&mbox->open_files);
+	if (count > mbox->max_open_files)
+		dbox_close_open_files(mbox, count - mbox->max_open_files);
+
+	file = i_new(struct dbox_file, 1);
+	file->refcount = 1;
+	file->mbox = mbox;
+	if (file_id != 0) {
+		file->file_id = file_id;
+		file->path = dbox_file_id_get_path(mbox, file_id);
+	} else {
+		file->path = dbox_generate_tmp_filename(mbox->path);
+	}
+	file->fd = -1;
+
+	if (file_id != 0)
+		array_append(&file->mbox->open_files, &file, 1);
+	return file;
+}
+
+int dbox_file_assign_id(struct dbox_file *file, unsigned int file_id)
+{
+	char *new_path;
+
+	i_assert(file->file_id == 0);
+	i_assert(file_id != 0);
+
+	new_path = dbox_file_id_get_path(file->mbox, file_id);
+	if (rename(file->path, new_path) < 0) {
+		mail_storage_set_critical(file->mbox->ibox.box.storage,
+			"rename(%s, %s) failed: %m", file->path, new_path);
+		i_free(new_path);
+		return -1;
+	}
+
+	i_free(file->path);
+	file->path = new_path;
+
+	file->file_id = file_id;
+	array_append(&file->mbox->open_files, &file, 1);
+	return 0;
+}
+
+void dbox_file_unref(struct dbox_file **_file)
 {
-	const struct dbox_mail_header *hdr;
-	const unsigned char *data;
-	size_t size;
+	struct dbox_file *file = *_file;
+	struct dbox_file *const *files;
+	unsigned int i, count;
+
+	*_file = NULL;
+
+	i_assert(file->refcount > 0);
+	if (--file->refcount > 0)
+		return;
+
+	/* don't cache metadata seeks while file isn't being referenced */
+	file->metadata_read_offset = 0;
+
+	if (file->file_id != 0) {
+		files = array_get(&file->mbox->open_files, &count);
+		if (!file->deleted && count <= file->mbox->max_open_files) {
+			/* we can leave this file open for now */
+			return;
+		}
+
+		for (i = 0; i < count; i++) {
+			if (files[i] == file)
+				break;
+		}
+		i_assert(i != count);
+		array_delete(&file->mbox->open_files, i, 1);
+	}
+
+	dbox_file_free(file);
+}
+
+static time_t day_begin_stamp(unsigned int days)
+{
+	struct tm tm;
+	time_t stamp;
+
+	if (days == 0)
+		return 0;
+
+	/* get beginning of today */
+	tm = *localtime(&ioloop_time);
+	tm.tm_hour = 0;
+	tm.tm_min = 0;
+	tm.tm_sec = 0;
+	stamp = mktime(&tm);
+	if (stamp == (time_t)-1)
+		i_panic("mktime(today) failed");
+
+	return stamp - (3600*24 * (days-1));
+}
+
+bool dbox_file_can_append(struct dbox_file *file, uoff_t mail_size)
+{
+	if (file->nonappendable)
+		return 0;
+
+	if (file->append_offset == 0) {
+		/* messages have been expunged */
+		return FALSE;
+	}
 
-	/* read the header */
-	i_stream_seek(file->input, offset);
-	(void)i_stream_read_data(file->input, &data, &size,
-				 file->mail_header_size-1);
-	if (size < file->mail_header_size) {
+	if (file->append_offset < file->mbox->rotate_min_size ||
+	    file->append_offset == file->file_header_size)
+		return TRUE;
+	if (file->append_offset + mail_size >= file->mbox->rotate_size)
+		return FALSE;
+	return file->create_time >= day_begin_stamp(file->mbox->rotate_days);
+}
+
+static int dbox_file_parse_header(struct dbox_file *file, const char *line)
+{
+	const char *const *tmp, *value;
+	unsigned int pos;
+	enum dbox_header_key key;
+
+	if (*line - '0' != DBOX_VERSION || line[1] != ' ') {
+		dbox_file_set_corrupted(file, "Invalid dbox version");
+		return -1;
+	}
+	line += 2;
+	pos = 2;
+
+	file->append_offset = 0;
+	file->msg_header_size = 0;
+
+	t_push();
+	for (tmp = t_strsplit(line, " "); *tmp != NULL; tmp++) {
+		key = **tmp;
+		value = *tmp + 1;
+
+		switch (key) {
+		case DBOX_HEADER_APPEND_OFFSET:
+			file->append_offset_header_pos = pos + 1;
+			file->append_offset = *value == 'X' ? 0 :
+				strtoull(value, NULL, 16);
+			break;
+		case DBOX_HEADER_MSG_HEADER_SIZE:
+			file->msg_header_size = strtoul(value, NULL, 16);
+			break;
+		case DBOX_HEADER_CREATE_STAMP:
+			file->create_time = strtoul(value, NULL, 16);
+			break;
+		}
+		pos += strlen(value) + 2;
+	}
+	t_pop();
+
+	if (file->msg_header_size == 0) {
+		dbox_file_set_corrupted(file, "Missing message header size");
+		return -1;
+	}
+
+	if (!file->nonappendable)
+		file->nonappendable = !dbox_file_can_append(file, 0);
+	return 0;
+}
+
+static int dbox_file_read_header(struct dbox_file *file)
+{
+	const char *line;
+
+	i_stream_seek(file->input, 0);
+	line = i_stream_read_next_line(file->input);
+	if (line == NULL) {
 		if (file->input->stream_errno == 0)
 			return 0;
 
-		errno = file->input->stream_errno;
-		mail_storage_set_critical(&mbox->storage->storage,
-					  "read(%s) failed: %m", file->path);
+		dbox_file_set_syscall_error(file, "read");
+		return -1;
+	}
+	file->file_header_size = file->input->v_offset;
+	return dbox_file_parse_header(file, line) < 0 ? 0 : 1;
+}
+
+static int dbox_file_open(struct dbox_file *file, bool read_header,
+			  bool *deleted_r)
+{
+	i_assert(file->input == NULL);
+
+	*deleted_r = FALSE;
+
+	if (file->fd == -1)
+		file->fd = open(file->path, O_RDWR);
+	if (file->fd == -1) {
+		if (errno == ENOENT) {
+			*deleted_r = TRUE;
+			return 1;
+		}
+
+		dbox_file_set_syscall_error(file, "open");
 		return -1;
 	}
 
-	memcpy(&file->seeked_mail_header, data,
-	       sizeof(file->seeked_mail_header));
-	/* @UNSAFE */
-	memcpy(file->seeked_keywords, data + sizeof(file->seeked_mail_header),
-	       file->keyword_count);
-	file->seeked_offset = offset;
+	file->input = i_stream_create_fd(file->fd, MAIL_READ_BLOCK_SIZE, FALSE);
+	return !read_header ? 1 : dbox_file_read_header(file);
+}
+
+static int dbox_file_create(struct dbox_file *file)
+{
+	string_t *hdr;
+	const char *hdrsize;
+
+	i_assert(file->fd == -1);
+
+	file->fd = open(file->path, O_RDWR | O_CREAT | O_TRUNC, 0600);
+	if (file->fd == -1) {
+		mail_storage_set_critical(file->mbox->ibox.box.storage,
+			"open(%s, O_CREAT) failed: %m", file->path);
+		return -1;
+	}
+	file->output = o_stream_create_fd_file(file->fd, 0, FALSE);
 
-	/* parse the header */
-	hdr = &file->seeked_mail_header;
-	file->seeked_mail_size =
-		hex2dec(hdr->mail_size_hex, sizeof(hdr->mail_size_hex));
-	file->seeked_uid = hex2dec(hdr->uid_hex, sizeof(hdr->uid_hex));
+	hdr = t_str_new(128);
+	str_printfa(hdr, "%u %c%x %c%x %c", DBOX_VERSION,
+		    DBOX_HEADER_MSG_HEADER_SIZE,
+		    (unsigned int)sizeof(struct dbox_message_header),
+		    DBOX_HEADER_CREATE_STAMP, (unsigned int)ioloop_time,
+		    DBOX_HEADER_APPEND_OFFSET);
+	file->append_offset_header_pos = str_len(hdr);
+	str_printfa(hdr, "%08x\n", 0);
 
-	if (memcmp(hdr->magic, DBOX_MAIL_HEADER_MAGIC,
-		   sizeof(hdr->magic)) != 0) {
-		mail_storage_set_critical(&mbox->storage->storage,
-			"Corrupted mail header at %"PRIuUOFF_T
-			" in dbox file %s", offset, file->path);
+	file->file_header_size = str_len(hdr);
+	file->msg_header_size = sizeof(struct dbox_message_header);
+	file->append_offset = str_len(hdr);
+
+	hdrsize = t_strdup_printf("%08x", (unsigned int)file->append_offset);
+	buffer_write(hdr, file->append_offset_header_pos, hdrsize, 8);
+
+	if (o_stream_send(file->output, str_data(hdr), str_len(hdr)) < 0) {
+		dbox_file_set_syscall_error(file, "write");
 		return -1;
 	}
+	return 0;
+}
+
+int dbox_file_open_or_create(struct dbox_file *file, bool read_header,
+			     bool *deleted_r)
+{
+	*deleted_r = FALSE;
+
+	if (file->file_id == 0)
+		return dbox_file_create(file) < 0 ? -1 : 1;
+	else if (file->input != NULL)
+		return 1;
+	else
+		return dbox_file_open(file, read_header, deleted_r);
+}
+
+static int dbox_file_read_mail_header(struct dbox_file *file, uint32_t *uid_r,
+				      uoff_t *physical_size_r)
+{
+	struct dbox_message_header hdr;
+	const unsigned char *data;
+	size_t size;
+	int ret;
+
+	ret = i_stream_read_data(file->input, &data, &size,
+				 file->msg_header_size - 1);
+	if (ret <= 0) {
+		if (file->input->stream_errno == 0) {
+			/* EOF, broken offset */
+			return 0;
+		}
+		dbox_file_set_syscall_error(file, "read");
+		return -1;
+	}
+	if (data[file->msg_header_size-1] != '\n')
+		return 0;
+
+	memcpy(&hdr, data, I_MIN(sizeof(hdr), file->msg_header_size));
+	if (memcmp(hdr.magic_pre, DBOX_MAGIC_PRE, sizeof(hdr.magic_pre)) != 0) {
+		/* probably broken offset */
+		return 0;
+	}
+
+	*uid_r = hex2dec(hdr.uid_hex, sizeof(hdr.uid_hex));
+	*physical_size_r = hex2dec(hdr.message_size_hex,
+				   sizeof(hdr.message_size_hex));
 	return 1;
 }
 
-int dbox_file_seek(struct dbox_mailbox *mbox, uint32_t file_seq, uoff_t offset,
-		   bool ignore_zero_uid)
+int dbox_file_get_mail_stream(struct dbox_file *file, uoff_t offset,
+			      uint32_t *uid_r, uoff_t *physical_size_r,
+			      struct istream **stream_r, bool *expunged_r)
 {
 	int ret;
 
-	if (mbox->file != NULL && mbox->file->file_seq != file_seq) {
-		dbox_file_close(mbox->file);
-		mbox->file = NULL;
-	}
-
-	if (mbox->file == NULL) {
-		mbox->file = i_new(struct dbox_file, 1);
-		mbox->file->file_seq = file_seq;
-		mbox->file->fd = -1;
-
-		mbox->file->path =
-			i_strdup_printf("%s/"DBOX_MAIL_FILE_FORMAT,
-					mbox->path, file_seq);
-	}
+	*expunged_r = FALSE;
 
-	if (mbox->file->fd == -1) {
-		mbox->file->fd = open(mbox->file->path, O_RDWR);
-		if (mbox->file->fd == -1) {
-			if (errno == ENOENT)
-				return 0;
-			mail_storage_set_critical(&mbox->storage->storage,
-				"open(%s) failed: %m", mbox->file->path);
-			return -1;
-		}
-
-		mbox->file->input =
-			i_stream_create_fd(mbox->file->fd, 65536, FALSE);
-
-		if (dbox_file_read_header(mbox, mbox->file) < 0)
-			return -1;
-	} else {
-		/* make sure we're not caching outdated data */
-		i_stream_sync(mbox->file->input);
+	if (file->input == NULL) {
+		if ((ret = dbox_file_open(file, TRUE, expunged_r)) <= 0 ||
+		    *expunged_r)
+			return ret;
 	}
 
 	if (offset == 0)
-		offset = mbox->file->header_size;
+		offset = file->file_header_size;
 
-	if ((ret = dbox_file_read_mail_header(mbox, mbox->file, offset)) <= 0)
+	i_stream_seek(file->input, offset);
+	ret = dbox_file_read_mail_header(file, uid_r, physical_size_r);
+	if (ret <= 0)
 		return ret;
 
-	if (mbox->file->seeked_mail_size == 0 ||
-	    (mbox->file->seeked_uid == 0 && !ignore_zero_uid)) {
-		/* could be legitimately just not written yet. we're at EOF. */
-		return 0;
+	i_stream_skip(file->input, file->msg_header_size);
+	if (stream_r != NULL) {
+		*stream_r = i_stream_create_limit(file->input,
+						  file->input->v_offset,
+						  *physical_size_r);
 	}
 	return 1;
 }
 
-int dbox_file_seek_next_nonexpunged(struct dbox_mailbox *mbox)
+static int
+dbox_file_seek_next_at_metadata(struct dbox_file *file, uoff_t *offset,
+				uint32_t *uid_r, uoff_t *physical_size_r)
 {
-	const struct dbox_mail_header *hdr;
-	uoff_t offset;
+	const char *line;
 	int ret;
 
-	for (;;) {
-		offset = mbox->file->seeked_offset +
-			mbox->file->mail_header_size +
-			mbox->file->seeked_mail_size;
+	if ((ret = dbox_file_metadata_skip_header(file)) <= 0)
+		return ret;
 
-		ret = dbox_file_seek(mbox, mbox->file->file_seq, offset, FALSE);
-		if (ret <= 0)
-			return ret;
-
-		hdr = &mbox->file->seeked_mail_header;
-		if (hdr->expunged != '1') {
-			/* non-expunged mail found */
+	/* skip over the actual metadata */
+	while ((line = i_stream_read_next_line(file->input)) != NULL) {
+		if (*line == DBOX_METADATA_SPACE) {
+			/* end of metadata */
 			break;
 		}
 	}
+	*offset = file->input->v_offset;
 
+	(void)i_stream_read(file->input);
+	if (!i_stream_have_bytes_left(file->input)) {
+		*uid_r = 0;
+		*physical_size_r = 0;
+		return 1;
+	}
+
+	return dbox_file_read_mail_header(file, uid_r, physical_size_r);
+}
+
+int dbox_file_seek_next(struct dbox_file *file, uoff_t *offset,
+			uint32_t *uid_r, uoff_t *physical_size_r)
+{
+	uint32_t uid;
+	uoff_t size;
+	bool first = *offset == 0;
+	bool deleted;
+	int ret;
+
+	ret = dbox_file_get_mail_stream(file, *offset, &uid, &size, NULL,
+					&deleted);
+	if (ret <= 0)
+		return ret;
+
+	if (deleted) {
+		*uid_r = 0;
+		*physical_size_r = 0;
+		return 1;
+	}
+	if (first) {
+		*uid_r = uid;
+		*physical_size_r = size;
+		return 1;
+	}
+
+	i_stream_skip(file->input, size);
+	return dbox_file_seek_next_at_metadata(file, offset, uid_r,
+					       physical_size_r);
+}
+
+static int dbox_file_seek_append_pos(struct dbox_file *file, uoff_t mail_size)
+{
+	int ret;
+
+	if ((ret = dbox_file_read_header(file)) <= 0)
+		return ret;
+
+	if (file->append_offset == 0 ||
+	    file->msg_header_size != sizeof(struct dbox_message_header) ||
+	    !dbox_file_can_append(file, mail_size)) {
+		/* can't append */
+		return 0;
+	}
+
+	file->output = o_stream_create_fd_file(file->fd, (uoff_t)-2, FALSE);
+	o_stream_seek(file->output, file->append_offset);
+	return 1;
+}
+
+static int
+dbox_file_get_append_stream_int(struct dbox_file *file, uoff_t mail_size,
+				struct ostream **stream_r)
+{
+	bool deleted;
+	int ret;
+
+	if (file->fd == -1) {
+		i_assert(file->output == NULL);
+		if ((ret = dbox_file_open_or_create(file, FALSE,
+						    &deleted)) <= 0 || deleted)
+			return ret;
+	}
+
+	if (file->output == NULL) {
+		ret = dbox_file_seek_append_pos(file, mail_size);
+		if (ret <= 0)
+			return ret;
+	} else {
+		if (!dbox_file_can_append(file, mail_size))
+			return 0;
+	}
+
+	if (file->output->offset > (uint32_t)-1) {
+		/* we use 32bit offsets to messages */
+		return 0;
+	}
+
+	o_stream_ref(file->output);
+	*stream_r = file->output;
+	return 1;
+}
+
+int dbox_file_get_append_stream(struct dbox_file *file, uoff_t mail_size,
+				struct ostream **stream_r)
+{
+	int ret;
+
+	if (file->nonappendable)
+		return 0;
+
+	ret = dbox_file_get_append_stream_int(file, mail_size, stream_r);
+	if (ret == 0)
+		file->nonappendable = TRUE;
+	return ret;
+}
+
+uoff_t dbox_file_get_next_append_offset(struct dbox_file *file)
+{
+	i_assert(file->output_stream_offset != 0);
+	i_assert(file->output == NULL ||
+		 file->output_stream_offset == file->output->offset);
+
+	return file->output_stream_offset;
+}
+
+void dbox_file_cancel_append(struct dbox_file *file, uoff_t append_offset)
+{
+	if (ftruncate(file->fd, append_offset) < 0) {
+		dbox_file_set_syscall_error(file, "ftruncate");
+		file->append_offset = 0;
+		file->nonappendable = TRUE;
+	}
+
+	o_stream_seek(file->output, append_offset);
+	file->output_stream_offset = append_offset;
+}
+
+void dbox_file_finish_append(struct dbox_file *file)
+{
+	file->output_stream_offset = file->output->offset;
+	file->append_offset = file->output->offset;
+	file->append_count++;
+}
+
+uoff_t dbox_file_get_metadata_offset(struct dbox_file *file, uoff_t offset,
+				     uoff_t physical_size)
+{
+	if (offset == 0) {
+		i_assert(file->file_header_size != 0);
+		offset = file->file_header_size;
+	}
+	return offset + sizeof(struct dbox_message_header) + physical_size;
+}
+
+static int dbox_file_metadata_skip_header(struct dbox_file *file)
+{
+	struct dbox_metadata_header metadata_hdr;
+	const unsigned char *data;
+	size_t size;
+	int ret;
+
+	ret = i_stream_read_data(file->input, &data, &size,
+				 sizeof(metadata_hdr) - 1);
+	if (ret <= 0) {
+		if (file->input->stream_errno == 0) {
+			/* EOF, broken offset */
+			return 0;
+		}
+		dbox_file_set_syscall_error(file, "read");
+		return -1;
+	}
+	memcpy(&metadata_hdr, data, sizeof(metadata_hdr));
+	if (memcmp(metadata_hdr.magic_post, DBOX_MAGIC_POST,
+		   sizeof(metadata_hdr.magic_post)) != 0) {
+		/* probably broken offset */
+		return 0;
+	}
+	i_stream_skip(file->input, sizeof(metadata_hdr));
+	return 1;
+}
+
+int dbox_file_metadata_seek(struct dbox_file *file, uoff_t metadata_offset,
+			    bool *expunged_r)
+{
+	const char *line;
+	uoff_t metadata_data_offset, prev_offset;
+	bool deleted;
+	int ret;
+
+	*expunged_r = FALSE;
+
+	if (file->metadata_read_offset == metadata_offset)
+		return 1;
+
+	if (file->metadata_pool != NULL) {
+		if (array_is_created(&file->metadata_changes))
+			array_free(&file->metadata_changes);
+		p_clear(file->metadata_pool);
+	} else {
+		file->metadata_pool =
+			pool_alloconly_create("dbox metadata", 512);
+	}
+	file->metadata_read_offset = 0;
+
+	if (file->input == NULL) {
+		if ((ret = dbox_file_open(file, TRUE, &deleted)) <= 0)
+			return ret;
+		if (deleted) {
+			*expunged_r = TRUE;
+			return 1;
+		}
+	}
+
+	i_stream_seek(file->input, metadata_offset);
+	if ((ret = dbox_file_metadata_skip_header(file)) <= 0)
+		return ret;
+	metadata_data_offset = file->input->v_offset;
+
+	*expunged_r = TRUE;
+	p_array_init(&file->metadata, file->metadata_pool, 16);
+	for (;;) {
+		prev_offset = file->input->v_offset;
+		if ((line = i_stream_read_next_line(file->input)) == NULL)
+			break;
+
+		if (*line == DBOX_METADATA_SPACE || *line == '\0') {
+			/* end of metadata */
+			file->metadata_space_pos =
+				prev_offset - metadata_data_offset;
+			*expunged_r = FALSE;
+			break;
+		}
+		line = p_strdup(file->metadata_pool, line);
+		array_append(&file->metadata, &line, 1);
+	}
+	file->metadata_read_offset = metadata_offset;
+	file->metadata_len = file->input->v_offset - metadata_data_offset;
+	if (*expunged_r)
+		file->metadata_space_pos = file->metadata_len;
+	return 1;
+}
+
+int dbox_file_metadata_seek_mail_offset(struct dbox_file *file, uoff_t offset,
+					bool *expunged_r)
+{
+	uoff_t physical_size, metadata_offset;
+	uint32_t uid;
+	bool expunged1, expunged2;
+	int ret;
+
+	ret = dbox_file_get_mail_stream(file, offset, &uid, &physical_size,
+					NULL, &expunged1);
+	if (ret <= 0)
+		return ret;
+
+	metadata_offset =
+		dbox_file_get_metadata_offset(file, offset, physical_size);
+	ret = dbox_file_metadata_seek(file, metadata_offset, &expunged2);
+	if (ret <= 0)
+		return ret;
+	*expunged_r = expunged1 || expunged2;
 	return 1;
 }
 
-void dbox_file_header_init(struct dbox_file_header *hdr)
+const char *dbox_file_metadata_get(struct dbox_file *file,
+				   enum dbox_metadata_key key)
 {
-	uint16_t base_header_size = sizeof(*hdr);
-	uint32_t header_size =
-		base_header_size + DBOX_KEYWORD_NAMES_RESERVED_SPACE;
-	uint32_t append_offset = header_size;
-	uint16_t keyword_count = DBOX_KEYWORD_COUNT;
-	uint16_t mail_header_size =
-		sizeof(struct dbox_mail_header) + keyword_count;
-	uint32_t create_time = ioloop_time;
-
-	memset(hdr, '0', sizeof(*hdr));
-	DEC2HEX(hdr->base_header_size_hex, base_header_size);
-	DEC2HEX(hdr->header_size_hex, header_size);
-	DEC2HEX(hdr->append_offset_hex, append_offset);
-	DEC2HEX(hdr->create_time_hex, create_time);
-	DEC2HEX(hdr->mail_header_size_hex, mail_header_size);
-	DEC2HEX(hdr->keyword_list_offset_hex, base_header_size);
-	DEC2HEX(hdr->keyword_count_hex, keyword_count);
-}
-
-int dbox_file_header_parse(struct dbox_mailbox *mbox, struct dbox_file *file,
-			   const struct dbox_file_header *hdr)
-{
-	file->hdr = *hdr;
+	const char *const *metadata;
+	unsigned int i, count;
 
-	file->base_header_size = hex2dec(hdr->base_header_size_hex,
-					 sizeof(hdr->base_header_size_hex));
-	file->header_size = hex2dec(hdr->header_size_hex,
-				    sizeof(hdr->header_size_hex));
-	file->append_offset = hex2dec(hdr->append_offset_hex,
-				      sizeof(hdr->append_offset_hex));
-	file->create_time = hex2dec(hdr->create_time_hex,
-				    sizeof(hdr->create_time_hex));
-	file->mail_header_size = hex2dec(hdr->mail_header_size_hex,
-					 sizeof(hdr->mail_header_size_hex));
-	file->mail_header_align =
-		hex2dec(hdr->mail_header_align_hex,
-			sizeof(hdr->mail_header_align_hex));
-	file->keyword_count = hex2dec(hdr->keyword_count_hex,
-				      sizeof(hdr->keyword_count_hex));
-	file->keyword_list_offset =
-		hex2dec(hdr->keyword_list_offset_hex,
-			sizeof(hdr->keyword_list_offset_hex));
-
-	if (file->base_header_size == 0 ||
-	    file->header_size < file->base_header_size ||
-	    file->append_offset < file->header_size ||
-            file->keyword_list_offset < file->base_header_size ||
-	    file->mail_header_size < sizeof(struct dbox_mail_header) ||
-	    file->keyword_count > file->mail_header_size -
-	    sizeof(struct dbox_mail_header)) {
-		mail_storage_set_critical(&mbox->storage->storage,
-			"dbox %s: broken file header", file->path);
-		return -1;
+	metadata = array_get(&file->metadata, &count);
+	for (i = 0; i < count; i++) {
+		if (*metadata[i] == (char)key)
+			return metadata[i] + 1;
 	}
-
-	i_free(file->seeked_keywords);
-	file->seeked_keywords = file->keyword_count == 0 ? NULL :
-		i_malloc(file->keyword_count);
-	return 0;
+	return NULL;
 }
 
-int dbox_file_read_header(struct dbox_mailbox *mbox, struct dbox_file *file)
+void dbox_file_metadata_set(struct dbox_file *file, enum dbox_metadata_key key,
+			    const char *value)
 {
-	struct dbox_file_header hdr;
-	const unsigned char *data;
-	size_t size;
+	const char **changes, *data;
+	unsigned int i, count;
 
-	/* read the file header */
-	i_stream_seek(file->input, 0);
-	(void)i_stream_read_data(file->input, &data, &size, sizeof(hdr)-1);
-	if (size < sizeof(hdr)) {
-		if (file->input->stream_errno != 0) {
-			errno = file->input->stream_errno;
-			mail_storage_set_critical(&mbox->storage->storage,
-				"read(%s) failed: %m", file->path);
-			return -1;
-		}
+	data = dbox_file_metadata_get(file, key);
+	if (data != NULL && strcmp(data, value) == 0) {
+		/* value didn't change */
+		return;
+	}
 
-		mail_storage_set_critical(&mbox->storage->storage,
-			"dbox %s: unexpected end of file", file->path);
-		return -1;
+	if (file->metadata_pool == NULL) {
+		file->metadata_pool =
+			pool_alloconly_create("dbox metadata", 512);
 	}
-	memcpy(&hdr, data, sizeof(hdr));
+	data = p_strdup_printf(file->metadata_pool, "%c%s", (char)key, value);
 
-	/* parse the header */
-	if (dbox_file_header_parse(mbox, file, &hdr) < 0)
-		return -1;
+	if (!array_is_created(&file->metadata_changes))
+		p_array_init(&file->metadata_changes, file->metadata_pool, 16);
+	else {
+		/* see if we have already changed this metadata */
+		changes = array_get_modifiable(&file->metadata_changes, &count);
+		for (i = 0; i < count; i++) {
+			if (*changes[i] == (char)key) {
+				changes[i] = data;
+				return;
+			}
+		}
+	}
 
-	/* keywords may not be up to date anymore */
-	if (array_is_created(&file->idx_file_keywords)) {
-		array_free(&file->idx_file_keywords);
-		array_free(&file->file_idx_keywords);
-	}
-	return 0;
+	array_append(&file->metadata_changes, &data, 1);
 }
 
-int dbox_file_write_header(struct dbox_mailbox *mbox, struct dbox_file *file)
+static int dbox_file_metadata_is_at_eof(struct dbox_file *file)
 {
-	struct dbox_file_header hdr;
-	char buf[1024];
+	uoff_t size;
+	uint32_t uid;
+	uoff_t offset;
 	int ret;
 
-	dbox_file_header_init(&hdr);
-	ret = dbox_file_header_parse(mbox, file, &hdr);
-	i_assert(ret == 0);
+	if ((file->file_id & DBOX_FILE_ID_FLAG_UID) != 0)
+		return 1;
+
+	offset = file->metadata_read_offset;
+	ret = dbox_file_seek_next_at_metadata(file, &offset, &uid, &size);
+	return ret <= 0 ? ret : uid == 0;
+}
 
-	/* write header + LF to mark end-of-keywords list */
-	if (o_stream_send(file->output, &hdr, sizeof(hdr)) < 0 ||
-	    o_stream_send_str(file->output, "\n") < 0) {
-		mail_storage_set_critical(&mbox->storage->storage,
-			"write(%s) failed: %m", file->path);
-		return -1;
-	}
+static int dbox_file_write_empty_block(struct dbox_file *file, uoff_t offset,
+				       unsigned int len)
+{
+	char space[256];
 
-	/* fill the rest of the header with spaces */
-	memset(buf, ' ', sizeof(buf));
-	while (file->output->offset < file->header_size) {
-		unsigned int size = I_MIN(sizeof(buf), file->header_size -
-					  file->output->offset);
+	i_assert(len > 0);
 
-		if (o_stream_send(file->output, buf, size) < 0) {
-			mail_storage_set_critical(&mbox->storage->storage,
-				"write(%s) failed: %m", file->path);
+	len--;
+	memset(space, DBOX_METADATA_SPACE, I_MIN(sizeof(space), len));
+	while (len >= sizeof(space)) {
+		if (pwrite_full(file->fd, space, sizeof(space), offset) < 0) {
+			dbox_file_set_syscall_error(file, "pwrite");
 			return -1;
 		}
 	}
-	return 0;
+	/* @UNSAFE: last block ends with LF */
+	space[len++] = '\n';
+	if (pwrite_full(file->fd, space, len, offset) < 0) {
+		dbox_file_set_syscall_error(file, "pwrite");
+		return -1;
+	}
+	file->metadata_len += len;
+	return 1;
+}
+
+static int dbox_file_grow_metadata(struct dbox_file *file, unsigned int len)
+{
+	enum dbox_index_file_lock_status lock_status;
+	uoff_t offset;
+	int ret;
+
+	ret = dbox_index_try_lock_file(file->mbox->dbox_index, file->file_id,
+				       &lock_status);
+	if (ret <= 0 || (ret = dbox_file_metadata_is_at_eof(file)) <= 0)
+		return ret;
+
+	offset = file->metadata_read_offset +
+		sizeof(struct dbox_metadata_header) + file->metadata_len;
+	i_stream_seek(file->input, offset);
+	(void)i_stream_read(file->input);
+	if (!i_stream_have_bytes_left(file->input)) {
+		len = len - file->metadata_len + DBOX_EXTRA_SPACE;
+		ret = dbox_file_write_empty_block(file, offset, len);
+	} else {
+		i_error("%s: Metadata changed unexpectedly", file->path);
+		ret = 0;
+	}
+
+	dbox_index_unlock_file(file->mbox->dbox_index, file->file_id);
+	return ret;
 }
+
+int dbox_file_metadata_write(struct dbox_file *file)
+{
+	const char *const *metadata, *const *changes;
+	unsigned int i, j, count, changes_count, space_needed, skip_pos;
+	char space[DBOX_EXTRA_SPACE];
+	string_t *str;
+	uoff_t offset;
+	size_t last_change_len, orig_len;
+	int ret;
+
+	if (!array_is_created(&file->metadata_changes)) {
+		/* nothing to write */
+		return 1;
+	}
+
+	offset = file->metadata_read_offset +
+		sizeof(struct dbox_metadata_header);
+	metadata = array_get(&file->metadata, &count);
+	changes = array_get(&file->metadata_changes, &changes_count);
+
+	/* skip as many metadata fields from beginning as we can */
+	for (i = skip_pos = 0; i < count; i++) {
+		for (j = 0; j < changes_count; j++) {
+			if (*changes[j] == *metadata[i])
+				break;
+		}
+		if (j != changes_count)
+			break;
+		skip_pos += strlen(metadata[i]) + 1;
+	}
+
+	t_push();
+	str = t_str_new(512);
+	last_change_len = orig_len = 0;
+	/* overwrite existing metadata fields */
+	for (; i < count; i++) {
+		for (j = 0; j < changes_count; j++) {
+			if (*changes[j] == *metadata[i])
+				break;
+		}
+		if (j != changes_count) {
+			str_append(str, changes[j]);
+			str_append_c(str, '\n');
+			last_change_len = str_len(str);
+		} else {
+			str_append(str, metadata[i]);
+			str_append_c(str, '\n');
+			if (orig_len != str_len(str))
+				last_change_len = str_len(str);
+		}
+		orig_len += strlen(metadata[i]) + 1;
+	}
+	/* add new metadata */
+	for (j = 0; j < changes_count; j++) {
+		for (i = 0; i < count; i++) {
+			if (*changes[j] == *metadata[i])
+				break;
+		}
+		if (i == count) {
+			str_append(str, changes[j]);
+			str_append_c(str, '\n');
+			last_change_len = str_len(str);
+		}
+	}
+	str_truncate(str, last_change_len);
+	if (skip_pos + str_len(str) >= file->metadata_len) {
+		if ((ret = dbox_file_grow_metadata(file, skip_pos +
+						   str_len(str))) <= 0) {
+			t_pop();
+			return ret;
+		}
+	}
+
+	memset(space, DBOX_METADATA_SPACE, sizeof(space));
+	while (skip_pos + str_len(str) < file->metadata_space_pos) {
+		space_needed = file->metadata_space_pos -
+			(skip_pos + str_len(str));
+		str_append_n(str, space, I_MIN(sizeof(space), space_needed));
+	}
+	i_assert(skip_pos + str_len(str) <= file->metadata_len);
+
+	ret = pwrite_full(file->fd, str_data(str), str_len(str),
+			  offset + skip_pos);
+	if (ret < 0)
+		dbox_file_set_syscall_error(file, "pwrite");
+	t_pop();
+	return ret < 0 ? -1 : 1;
+}
+
+void dbox_file_metadata_write_to(struct dbox_file *file, struct ostream *output)
+{
+	struct dbox_metadata_header metadata_hdr;
+	char space[DBOX_EXTRA_SPACE];
+	const char *const *metadata, *const *changes;
+	unsigned int i, j, count, changes_count;
+
+	memset(&metadata_hdr, 0, sizeof(metadata_hdr));
+	memcpy(metadata_hdr.magic_post, DBOX_MAGIC_POST,
+	       sizeof(metadata_hdr.magic_post));
+	o_stream_send(output, &metadata_hdr, sizeof(metadata_hdr));
+
+	metadata = array_get(&file->metadata, &count);
+	if (!array_is_created(&file->metadata_changes)) {
+		for (i = 0; i < count; i++) {
+			o_stream_send_str(output, metadata[i]);
+			o_stream_send(output, "\n", 1);
+		}
+	} else {
+		changes = array_get(&file->metadata_changes, &changes_count);
+		/* write unmodified metadata */
+		for (i = 0; i < count; i++) {
+			for (j = 0; j < changes_count; j++) {
+				if (*changes[j] == *metadata[i])
+					break;
+			}
+			if (j == changes_count) {
+				o_stream_send_str(output, metadata[i]);
+				o_stream_send(output, "\n", 1);
+			}
+		}
+		/* write modified metadata */
+		for (i = 0; i < changes_count; i++) {
+			o_stream_send_str(output, metadata[i]);
+			o_stream_send(output, "\n", 1);
+		}
+	}
+
+	memset(space, ' ', sizeof(space));
+	o_stream_send(output, space, sizeof(space));
+	o_stream_send(output, "\n", 1);
+}
+
+bool dbox_file_lookup(struct dbox_mailbox *mbox, struct mail_index_view *view,
+		      uint32_t seq, uint32_t *file_id_r, uoff_t *offset_r)
+{
+	const struct dbox_mail_index_record *dbox_rec;
+	const void *data;
+	uint32_t uid;
+	bool expunged;
+
+	mail_index_lookup_ext(view, seq, mbox->dbox_ext_id, &data, &expunged);
+	if (expunged)
+		return FALSE;
+	if (data == NULL) {
+		mail_index_lookup_uid(view, seq, &uid);
+		if ((uid & DBOX_FILE_ID_FLAG_UID) != 0) {
+			/* something's broken, we can't handle this high UIDs */
+			return FALSE;
+		}
+		*file_id_r = DBOX_FILE_ID_FLAG_UID | uid;
+		*offset_r = 0;
+	} else {
+		dbox_rec = data;
+		*file_id_r = dbox_rec->file_id;
+		*offset_r = dbox_rec->offset;
+	}
+	return TRUE;
+}
+
+void dbox_mail_metadata_flags_append(string_t *str, enum mail_flags flags)
+{
+	unsigned int i;
+
+	for (i = 0; i < DBOX_METADATA_FLAGS_COUNT; i++) {
+		if ((flags & dbox_mail_flags_map[i]) != 0)
+			str_append_c(str, dbox_mail_flag_chars[i]);
+		else
+			str_append_c(str, '0');
+	}
+}
+
+void dbox_mail_metadata_keywords_append(struct dbox_mailbox *mbox,
+					string_t *str,
+					const struct mail_keywords *keywords)
+{
+	const ARRAY_TYPE(keywords) *keyword_names_list;
+	const char *const *keyword_names;
+	unsigned int i, keyword_names_count;
+
+	if (keywords == NULL || keywords->count == 0)
+		return;
+
+	keyword_names_list = mail_index_get_keywords(mbox->ibox.index);
+	keyword_names = array_get(keyword_names_list, &keyword_names_count);
+
+	for (i = 0; i < keywords->count; i++) {
+		i_assert(keywords->idx[i] < keyword_names_count);
+
+		str_append(str, keyword_names[keywords->idx[i]]);
+		str_append_c(str, ' ');
+	}
+	str_truncate(str, str_len(str)-1);
+}

--- a/src/lib-storage/index/dbox/dbox-file.h	Sat Sep 01 01:36:10 2007 +0300
+++ b/src/lib-storage/index/dbox/dbox-file.h	Sat Sep 01 03:04:02 2007 +0300
@@ -1,26 +1,216 @@
 #ifndef __DBOX_FILE_H
 #define __DBOX_FILE_H
 
-struct mail_index_view;
-struct dbox_mailbox;
-struct dbox_file;
-struct dbox_file_header;
+/* The file begins with a header followed by zero or more messages:
+
+   <dbox message header>
+   <LF>
+   <message body>
+   <metadata>
+
+   Metadata block begins with DBOX_MAGIC_POST, followed by zero or more lines
+   in format <key character><value><LF>. The block ends with a line containing
+   zero or more spaces. The spaces can be used for writing more headers.
+   Unknown metadata should be ignored, but preserved when copying.
+
+   There should be no duplicates for the current metadata, but future
+   extensions may need them so they should be preserved.
+*/
+#define DBOX_VERSION 1
+#define DBOX_MAGIC_PRE "\001\002"
+#define DBOX_MAGIC_POST "\n\001\003\n"
+
+#define DBOX_EXTRA_SPACE 64
+/* If file_id has this flag set, the file is a single file with file_id=UID. */
+#define DBOX_FILE_ID_FLAG_UID 0x80000000
+
+enum dbox_header_key {
+	/* Offset for appending next message. In %08x format so it can be
+	   updated without moving data in header. If messages have been
+	   expunged and file must not be appended anymore, the value is filled
+	   with 'X'. */
+	DBOX_HEADER_APPEND_OFFSET	= 'A',
+	/* Must be sizeof(struct dbox_message_header) when appending (hex) */
+	DBOX_HEADER_MSG_HEADER_SIZE	= 'M',
+	/* Creation UNIX timestamp (hex) */
+	DBOX_HEADER_CREATE_STAMP	= 'C'
+};
+
+enum dbox_metadata_flags {
+	DBOX_METADATA_FLAGS_ANSWERED = 0,
+	DBOX_METADATA_FLAGS_FLAGGED,
+	DBOX_METADATA_FLAGS_DELETED,
+	DBOX_METADATA_FLAGS_SEEN,
+	DBOX_METADATA_FLAGS_DRAFT,
+
+	DBOX_METADATA_FLAGS_COUNT
+};
 
-/* Returns -1 = error, 0 = expunged, 1 = ok */
-int dbox_file_lookup_offset(struct dbox_mailbox *mbox,
-			    struct mail_index_view *view, uint32_t seq,
-			    uint32_t *file_seq_r, uoff_t *offset_r);
+enum dbox_metadata_key {
+	/* Message is marked as expunged. '0' = no, '1' = yes */
+	DBOX_METADATA_EXPUNGED		= 'E',
+	/* Message flags in dbox_metadata_flags order. '0' = not set, anything
+	   else = set. Unknown flags should be preserved. */
+	DBOX_METADATA_FLAGS		= 'F',
+	/* Space separated list of keywords */
+	DBOX_METADATA_KEYWORDS		= 'K',
+	/* Pointer to external message data. Format is:
+	   1*(<start offset> <byte count> <ref>) */
+	DBOX_METADATA_EXT_REF		= 'P',
+	/* Received UNIX timestamp in hex */
+	DBOX_METADATA_RECEIVED_TIME	= 'R',
+	/* Saved UNIX timestamp in hex */
+	DBOX_METADATA_SAVE_TIME		= 'S',
+	/* Virtual message size in hex (line feeds counted as CRLF) */
+	DBOX_METADATA_VIRTUAL_SIZE	= 'V',
+
+	/* End of metadata block. The spaces can be used for writing more
+	   metadata. */
+	DBOX_METADATA_SPACE		= ' '
+};
+
+enum dbox_message_type {
+	/* Normal message */
+	DBOX_MESSAGE_TYPE_NORMAL	= 'N',
+	/* Parts of the message exists outside the following data.
+	   See the metadata for how to find them. */
+	DBOX_MESSAGE_TYPE_EXT_REFS	= 'E'
+};
+
+struct dbox_message_header {
+	unsigned char magic_pre[2];
+	unsigned char type;
+	unsigned char space1;
+	unsigned char uid_hex[8];
+	unsigned char space2;
+	unsigned char message_size_hex[16];
+	/* <space reserved for future extensions, LF is always last> */
+	unsigned char save_lf;
+};
+
+struct dbox_metadata_header {
+	unsigned char magic_post[sizeof(DBOX_MAGIC_POST)-1];
+};
+
+struct dbox_file {
+	struct dbox_mailbox *mbox;
+	int refcount;
+	unsigned int file_id;
+
+	unsigned int file_header_size;
+	unsigned int msg_header_size;
+	unsigned int append_offset_header_pos;
+
+	unsigned int append_count;
+	uint32_t last_append_uid;
 
-void dbox_file_close(struct dbox_file *file);
-/* Returns -1 = error, 0 = EOF (mail was just moved / file broken), 1 = ok */
-int dbox_file_seek(struct dbox_mailbox *mbox, uint32_t file_seq, uoff_t offset,
-		   bool ignore_zero_uid);
-int dbox_file_seek_next_nonexpunged(struct dbox_mailbox *mbox);
+	uoff_t append_offset;
+	time_t create_time;
+	uoff_t output_stream_offset;
+
+	char *path;
+	int fd;
+	struct istream *input;
+	struct ostream *output;
+
+	/* Metadata for the currently seeked metadata block. */
+	pool_t metadata_pool;
+	ARRAY_DEFINE(metadata, const char *);
+	ARRAY_DEFINE(metadata_changes, const char *);
+	uoff_t metadata_read_offset;
+	unsigned int metadata_space_pos;
+	/* Includes the trailing LF that shouldn't be used */
+	unsigned int metadata_len;
+
+	unsigned int nonappendable:1;
+	unsigned int deleted:1;
+};
+
+extern enum mail_flags dbox_mail_flags_map[DBOX_METADATA_FLAGS_COUNT];
+extern char dbox_mail_flag_chars[DBOX_METADATA_FLAGS_COUNT];
+
+struct dbox_file *
+dbox_file_init(struct dbox_mailbox *mbox, unsigned int file_id);
+void dbox_file_unref(struct dbox_file **file);
+
+/* Free all currently opened files. */
+void dbox_files_free(struct dbox_mailbox *mbox);
+
+/* Assign a newly created file (file_id=0) a new id. */
+int dbox_file_assign_id(struct dbox_file *file, unsigned int file_id);
+
+/* If file_id is 0, open the file, otherwise create it. Returns 1 if ok,
+   0 if read_header=TRUE and opened file was broken, -1 if error. If file is
+   deleted, deleted_r=TRUE and 1 is returned. */
+int dbox_file_open_or_create(struct dbox_file *file, bool read_header,
+			     bool *deleted_r);
+
+/* Seek to given offset in file and return the message's input stream, UID
+   and physical size. Returns 1 if ok, 0 if file/offset is corrupted,
+   -1 if I/O error. */
+int dbox_file_get_mail_stream(struct dbox_file *file, uoff_t offset,
+			      uint32_t *uid_r, uoff_t *physical_size_r,
+			      struct istream **stream_r, bool *expunged_r);
+/* Seek to next message after given offset, or to first message if offset=0.
+   If there are no more messages, uid_r is set to 0. Returns 1 if ok, 0 if
+   file/offset is corrupted, -1 if I/O error. */
+int dbox_file_seek_next(struct dbox_file *file, uoff_t *offset,
+			uint32_t *uid_r, uoff_t *physical_size_r);
 
-void dbox_file_header_init(struct dbox_file_header *hdr);
-int dbox_file_header_parse(struct dbox_mailbox *mbox, struct dbox_file *file,
-			   const struct dbox_file_header *hdr);
-int dbox_file_read_header(struct dbox_mailbox *mbox, struct dbox_file *file);
-int dbox_file_write_header(struct dbox_mailbox *mbox, struct dbox_file *file);
+/* Returns TRUE if mail_size bytes can be appended to the file. */
+bool dbox_file_can_append(struct dbox_file *file, uoff_t mail_size);
+/* Get output stream for appending a new message. Returns 1 if ok, 0 if
+   file can't be appended to (limits reached, expunges, corrupted) or
+   -1 if error. If 0 is returned, index is also updated. */
+int dbox_file_get_append_stream(struct dbox_file *file, uoff_t mail_size,
+				struct ostream **stream_r);
+/* Returns the next offset for append a message. dbox_file_get_append_stream()
+   must have been called for this file already at least once. */
+uoff_t dbox_file_get_next_append_offset(struct dbox_file *file);
+/* Truncate file to append_offset */
+void dbox_file_cancel_append(struct dbox_file *file, uoff_t append_offset);
+/* Finish appending the current mail. */
+void dbox_file_finish_append(struct dbox_file *file);
+
+/* Calculate offset to message's metadata. */
+uoff_t dbox_file_get_metadata_offset(struct dbox_file *file, uoff_t offset,
+				     uoff_t physical_size);
+/* Seek to given metadata block. Returns 1 if ok, 0 if file/offset is
+   corrupted, -1 if I/O error. If message has already been expunged,
+   expunged_r=TRUE and 1 is returned. */
+int dbox_file_metadata_seek(struct dbox_file *file, uoff_t metadata_offset,
+			    bool *expunged_r);
+/* Like dbox_file_metadata_seek(), but the offset points to beginning of the
+   message. The function internally reads the message header to find the
+   metadata offset. */
+int dbox_file_metadata_seek_mail_offset(struct dbox_file *file, uoff_t offset,
+					bool *expunged_r);
+
+/* Return wanted metadata value, or NULL if not found. */
+const char *dbox_file_metadata_get(struct dbox_file *file,
+				   enum dbox_metadata_key key);
+/* Add key=value metadata update (not written yet, not visible to _get()).
+   The changes are reset by dbox_file_metadata_seek() call. */
+void dbox_file_metadata_set(struct dbox_file *file, enum dbox_metadata_key key,
+			    const char *value);
+/* Write all metadata updates to disk. Returns 1 if ok, 0 if metadata doesn't
+   fit to its reserved space and message isn't last in file, -1 if I/O error. */
+int dbox_file_metadata_write(struct dbox_file *file);
+/* Write all metadata to output stream. Returns 0 if ok, -1 if I/O error. */
+void dbox_file_metadata_write_to(struct dbox_file *file,
+				 struct ostream *output);
+
+/* Get file/offset for wanted message. Returns TRUE if found. */
+bool dbox_file_lookup(struct dbox_mailbox *mbox, struct mail_index_view *view,
+		      uint32_t seq, uint32_t *file_id_r, uoff_t *offset_r);
+
+/* Append flags as metadata value to given string */
+void dbox_mail_metadata_flags_append(string_t *str, enum mail_flags flags);
+/* Append keywords as metadata value to given string */
+void dbox_mail_metadata_keywords_append(struct dbox_mailbox *mbox,
+					string_t *str,
+					const struct mail_keywords *keywords);
+
+void dbox_file_set_syscall_error(struct dbox_file *file, const char *function);
 
 #endif

--- a/src/lib-storage/index/dbox/dbox-format.h	Sat Sep 01 01:36:10 2007 +0300
+++ /dev/null	Thu Jan 01 00:00:00 1970 +0000
@@ -1,67 +0,0 @@
-#ifndef __DBOX_FORMAT_H
-#define __DBOX_FORMAT_H
-
-#define DBOX_SUBSCRIPTION_FILE_NAME "subscriptions"
-#define DBOX_INDEX_PREFIX "dovecot.index"
-#define DBOX_MAILDIR_NAME "dbox-Mails"
-#define DBOX_MAIL_FILE_PREFIX "msg."
-#define DBOX_MAIL_FILE_FORMAT DBOX_MAIL_FILE_PREFIX"%u"
-
-#define DBOX_KEYWORD_COUNT 64
-#define DBOX_KEYWORD_NAMES_RESERVED_SPACE (2048-sizeof(struct dbox_file_header))
-
-/* Default rotation settings */
-#define DBOX_DEFAULT_ROTATE_SIZE (2*1024*1024)
-#define DBOX_DEFAULT_ROTATE_MIN_SIZE (1024*16)
-#define DBOX_DEFAULT_ROTATE_DAYS 0
-
-struct dbox_file_header {
-	/* Size of the base header. sizeof(struct dbox_file_header) */
-	unsigned char base_header_size_hex[4];
-	/* Size of the full header, including keywords list and padding */
-	unsigned char header_size_hex[8];
-	/* Offset where to store the next mail. note that a mail may already
-	   have been fully written here and added to uidlist, but this offset
-	   just wasn't updated. In that case the append_offset should be
-	   updated instead of overwriting the mail. */
-	unsigned char append_offset_hex[16];
-	/* Initial file creation time as UNIX timestamp. */
-	unsigned char create_time_hex[8];
-	/* Size of each message's header. */
-	unsigned char mail_header_size_hex[4];
-	/* If set, mail headers start always at given alignmentation.
-	   Currently not supported. */
-	unsigned char mail_header_align_hex[4];
-	/* Number of keywords allocated for each mail (not necessarily used) */
-	unsigned char keyword_count_hex[4];
-	/* Offset for the keyword list inside the file header. */
-	unsigned char keyword_list_offset_hex[8];
-
-	/* Non-zero if some mails have been marked as expunged in the file. */
-	unsigned char have_expunged_mails;
-
-	/* space reserved for keyword list and possible other future
-	   extensions. */
-	/* unsigned char [header_size - header_base_size]; */
-};
-
-#define DBOX_MAIL_HEADER_MAGIC "\001\003"
-struct dbox_mail_header {
-	/* This field acts as kind of a verification marker to make sure that
-	   seeked offset is valid. So the magic value should be something that
-	   normally doesn't occur in mails. */
-	unsigned char magic[2];
-	unsigned char uid_hex[8];
-	unsigned char mail_size_hex[16];
-	unsigned char received_time_hex[8];
-	unsigned char save_time_hex[8];
-	unsigned char answered;
-	unsigned char flagged;
-	unsigned char deleted;
-	unsigned char seen;
-	unsigned char draft;
-	unsigned char expunged;
-	/* unsigned char keywords[keywords_count]; */
-};
-
-#endif

--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/src/lib-storage/index/dbox/dbox-index.c	Sat Sep 01 03:04:02 2007 +0300
@@ -0,0 +1,890 @@
+/* Copyright (C) 2007 Timo Sirainen */
+
+#include "lib.h"
+#include "array.h"
+#include "hex-dec.h"
+#include "str.h"
+#include "istream.h"
+#include "ostream.h"
+#include "write-full.h"
+#include "safe-mkstemp.h"
+#include "dbox-storage.h"
+#include "dbox-file.h"
+#include "dbox-index.h"
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <sys/stat.h>
+
+#define DBOX_INDEX_LOCK_RETRY_COUNT 10
+
+struct dbox_index {
+	struct dbox_mailbox *mbox;
+
+	struct istream *input;
+	char *path;
+	int fd;
+
+	uint32_t uid_validity, next_uid;
+	unsigned int next_file_id;
+
+	ARRAY_DEFINE(records, struct dbox_index_record);
+};
+
+struct dbox_index_append_context {
+	struct dbox_index *index;
+	ARRAY_DEFINE(files, struct dbox_file *);
+
+	uoff_t output_offset;
+	unsigned int new_record_idx;
+	unsigned int first_new_file_id;
+
+	unsigned int locked_header:1;
+};
+
+static int dbox_index_recreate(struct dbox_index *index, bool locked);
+
+struct dbox_index *dbox_index_init(struct dbox_mailbox *mbox)
+{
+	struct dbox_index *index;
+
+	index = i_new(struct dbox_index, 1);
+	index->mbox = mbox;
+	index->path = i_strdup_printf("%s/"DBOX_INDEX_NAME, mbox->path);
+	index->fd = -1;
+	index->next_uid = 1;
+	index->next_file_id = 1;
+	i_array_init(&index->records, 128);
+	return index;
+}
+
+static void dbox_index_close(struct dbox_index *index)
+{
+	if (index->input != NULL)
+		i_stream_unref(&index->input);
+	if (index->fd != -1) {
+		if (close(index->fd) < 0)
+			i_error("close(%s) failed: %m", index->path);
+		index->fd = -1;
+	}
+}
+
+void dbox_index_deinit(struct dbox_index **_index)
+{
+	struct dbox_index *index = *_index;
+
+	*_index = NULL;
+
+	dbox_index_close(index);
+	array_free(&index->records);
+	i_free(index->path);
+	i_free(index);
+}
+
+static int dbox_index_parse_line(struct dbox_index *index, const char *line,
+				 unsigned int offset)
+{
+	struct dbox_index_record rec;
+
+	memset(&rec, 0, sizeof(rec));
+	rec.file_offset = offset;
+
+	/* <file id> <status><expunges><dirty> [<status-specific data>] */
+	while (*line >= '0' && *line <= '9') {
+		rec.file_id = rec.file_id*10 + *line - '0';
+		line++;
+	}
+	if (*line++ != ' ')
+		return -1;
+
+	if ((rec.file_id & DBOX_FILE_ID_FLAG_UID) != 0) {
+		/* UID files shouldn't be listed in dbox.index */
+		return -1;
+	}
+
+	if (line[0] == '\0' || line[1] == '\0' || line[2] == '\0')
+		return -1;
+	rec.status = line[0];
+	rec.expunges = line[1] != '0';
+	rec.dirty = line[2] != '0';
+
+	line += 3;
+	array_append(&index->records, &rec, 1);
+	return 0;
+}
+
+static int
+dbox_index_set_corrupted(struct dbox_index *index, const char *reason)
+{
+	mail_storage_set_critical(index->mbox->ibox.box.storage,
+				  "dbox index %s corrupted: %s",
+				  index->path, reason);
+
+	if (unlink(index->path) < 0 && errno != ENOENT)
+		i_error("unlink(%s) failed: %m", index->path);
+	return -1;
+}
+
+static void dbox_index_header_init(struct dbox_index *index,
+				   struct dbox_index_file_header *hdr)
+{
+	if (index->uid_validity == 0) {
+		const struct mail_index_header *hdr;
+
+		hdr = mail_index_get_header(index->mbox->ibox.view);
+		index->uid_validity = hdr->uid_validity != 0 ?
+			hdr->uid_validity : (uint32_t)ioloop_time;
+	}
+
+	memset(hdr, ' ', sizeof(*hdr));
+	hdr->version = DBOX_INDEX_VERSION;
+	dec2hex(hdr->uid_validity_hex, index->uid_validity,
+		sizeof(hdr->uid_validity_hex));
+	dec2hex(hdr->next_uid_hex, index->next_uid, sizeof(hdr->next_uid_hex));
+	dec2hex(hdr->next_file_id_hex, index->next_file_id,
+		sizeof(hdr->next_file_id_hex));
+}
+
+static int dbox_index_parse_header(struct dbox_index *index, const char *line)
+{
+	struct dbox_index_file_header hdr;
+
+	if (strlen(line) < sizeof(hdr))
+		return dbox_index_set_corrupted(index, "Header too short");
+
+	memcpy(&hdr, line, sizeof(hdr));
+	if (hdr.version != DBOX_INDEX_VERSION)
+		return dbox_index_set_corrupted(index, "Invalid version");
+
+	index->uid_validity =
+		hex2dec(hdr.uid_validity_hex, sizeof(hdr.uid_validity_hex));
+	if (index->uid_validity == 0)
+		return dbox_index_set_corrupted(index, "uid_validity = 0");
+
+	index->next_uid = hex2dec(hdr.next_uid_hex, sizeof(hdr.next_uid_hex));
+	if (index->next_uid == 0)
+		return dbox_index_set_corrupted(index, "next_uid = 0");
+	index->next_file_id =
+		hex2dec(hdr.next_file_id_hex, sizeof(hdr.next_file_id_hex));
+	return 0;
+}
+
+static int dbox_index_read_header(struct dbox_index *index)
+{
+	const char *line;
+
+	i_stream_sync(index->input);
+	i_stream_seek(index->input, 0);
+
+	line = i_stream_read_next_line(index->input);
+	if (line == NULL)
+		return dbox_index_set_corrupted(index, "Missing header");
+	return dbox_index_parse_header(index, line);
+}
+
+static int dbox_index_read(struct dbox_index *index)
+{
+	struct istream *input;
+	const char *line;
+	uoff_t start_offset;
+	int ret;
+
+	if (index->fd != -1)
+		dbox_index_close(index);
+
+	index->fd = open(index->path, O_RDWR);
+	if (index->fd == -1) {
+		if (errno == ENOENT)
+			return 0;
+		mail_storage_set_critical(index->mbox->ibox.box.storage,
+					  "open(%s) failed: %m", index->path);
+		return -1;
+	}
+
+	array_clear(&index->records);
+	input = index->input = i_stream_create_fd(index->fd, 1024, FALSE);
+
+	ret = dbox_index_read_header(index);
+	start_offset = input->v_offset;
+	while ((line = i_stream_read_next_line(input)) != NULL) {
+		if (dbox_index_parse_line(index, line, start_offset) < 0) {
+			dbox_index_set_corrupted(index, "Corrupted record");
+			ret = -1;
+			break;
+		}
+		start_offset = input->v_offset;
+	}
+	return ret == 0 ? 1 :
+		(input->stream_errno == 0 ? 0 : -1);
+}
+
+static int dbox_index_read_or_create(struct dbox_index *index)
+{
+	unsigned int i;
+	int ret;
+
+	for (i = 0;; i++) {
+		if ((ret = dbox_index_read(index)) != 0)
+			return ret;
+
+		/* doesn't exist / corrupted */
+		if (i == DBOX_INDEX_LOCK_RETRY_COUNT)
+			break;
+
+		if (index->fd != -1)
+			dbox_index_close(index);
+
+		if (dbox_index_recreate(index, FALSE) < 0)
+			return -1;
+	}
+
+	mail_storage_set_critical(index->mbox->ibox.box.storage,
+		"dbox index recreation keeps failing: %s", index->path);
+	return -1;
+}
+
+static int dbox_index_refresh(struct dbox_index *index)
+{
+	struct stat st1, st2;
+
+	if (index->fd == -1) {
+		if (dbox_index_read_or_create(index) < 0)
+			return -1;
+		i_assert(index->fd != -1);
+		return 1;
+	}
+
+	if (fstat(index->fd, &st1) < 0) {
+		mail_storage_set_critical(index->mbox->ibox.box.storage,
+					  "fstat(%s) failed: %m", index->path);
+		return -1;
+	}
+	if (stat(index->path, &st2) < 0) {
+		mail_storage_set_critical(index->mbox->ibox.box.storage,
+					  "stat(%s) failed: %m", index->path);
+		return -1;
+	}
+
+	if (st1.st_ino != st2.st_ino || !CMP_DEV_T(st1.st_dev, st2.st_dev)) {
+		if (dbox_index_read(index) < 0)
+			return -1;
+		return 1;
+	}
+	return 0;
+}
+
+int dbox_index_get_uid_validity(struct dbox_index *index,
+				uint32_t *uid_validity_r)
+{
+	if (index->fd == -1) {
+		if (dbox_index_refresh(index) < 0)
+			return -1;
+	}
+	*uid_validity_r = index->uid_validity;
+	return 0;
+}
+
+static int dbox_index_record_cmp(const void *key, const void *data)
+{
+	const unsigned int *file_id = key;
+	const struct dbox_index_record *rec = data;
+
+	return *file_id - rec->file_id;
+}
+
+struct dbox_index_record *
+dbox_index_record_lookup(struct dbox_index *index, unsigned int file_id)
+{
+	struct dbox_index_record *records;
+	unsigned int count;
+
+	if ((file_id & DBOX_FILE_ID_FLAG_UID) != 0)
+		return NULL;
+
+	records = array_get_modifiable(&index->records, &count);
+	return bsearch(&file_id, records, count, sizeof(*records),
+		       dbox_index_record_cmp);
+}
+
+static int
+dbox_index_lock_range(struct dbox_index *index, int cmd, int lock_type,
+		      off_t start, off_t len)
+{
+	struct flock fl;
+
+	fl.l_type = lock_type;
+	fl.l_whence = SEEK_SET;
+	fl.l_start = start;
+	fl.l_len = len;
+	if (fcntl(index->fd, cmd, &fl) < 0) {
+		if ((errno == EACCES || errno == EAGAIN || errno == EINTR) &&
+		    cmd == F_SETLK)
+			return 0;
+		mail_storage_set_critical(index->mbox->ibox.box.storage,
+			"fcntl(%s, %s) failed: %m", index->path,
+			lock_type == F_UNLCK ? "F_UNLCK" : "F_WRLCK");
+		return -1;
+	}
+	return 1;
+}
+
+static void dbox_index_unlock_range(struct dbox_index *index,
+				    off_t start, off_t len)
+{
+	(void)dbox_index_lock_range(index, F_SETLK, F_UNLCK, start, len);
+}
+
+static int
+dbox_index_try_lock_once(struct dbox_index *index, unsigned int file_id,
+			 enum dbox_index_file_lock_status *lock_status_r)
+{
+	struct dbox_index_record *rec;
+	int ret;
+
+	i_assert((file_id & DBOX_FILE_ID_FLAG_UID) == 0);
+
+	rec = dbox_index_record_lookup(index, file_id);
+	if (rec == NULL || rec->status == DBOX_INDEX_FILE_STATUS_UNLINKED) {
+		*lock_status_r = DBOX_INDEX_FILE_LOCK_UNLINKED;
+		return 0;
+	}
+
+	if (rec->status != DBOX_INDEX_FILE_STATUS_APPENDABLE) {
+		*lock_status_r = DBOX_INDEX_FILE_LOCK_NOT_NEEDED;
+		return 1;
+	}
+
+	/* we'll need to try to lock this record */
+	ret = dbox_index_lock_range(index, F_SETLK, F_WRLCK,
+				    rec->file_offset, 1);
+	if (ret > 0) {
+		*lock_status_r = DBOX_INDEX_FILE_LOCKED;
+		rec->locked = TRUE;
+	} else if (ret == 0)
+		*lock_status_r = DBOX_INDEX_FILE_LOCK_TRY_AGAIN;
+	return ret;
+}
+
+int dbox_index_try_lock_file(struct dbox_index *index, unsigned int file_id,
+			     enum dbox_index_file_lock_status *lock_status_r)
+{
+	int i, ret;
+
+	if ((file_id & DBOX_FILE_ID_FLAG_UID) != 0) {
+		*lock_status_r = DBOX_INDEX_FILE_LOCK_NOT_NEEDED;
+		return 1;
+	}
+
+	if (index->fd == -1) {
+		if (dbox_index_refresh(index) < 0)
+			return 1;
+	}
+
+	for (i = 0; i < DBOX_INDEX_LOCK_RETRY_COUNT; i++) {
+		ret = dbox_index_try_lock_once(index, file_id, lock_status_r);
+		if (ret <= 0 || *lock_status_r != DBOX_INDEX_FILE_LOCKED)
+			return ret;
+
+		/* if file was recreated, reopen it and try again */
+		if ((ret = dbox_index_refresh(index)) <= 0)
+			return ret < 0 ? -1 : 1;
+	}
+
+	i_warning("dbox index keeps getting recreated: %s", index->path);
+	return 0;
+}
+
+void dbox_index_unlock_file(struct dbox_index *index, unsigned int file_id)
+{
+	struct dbox_index_record *rec;
+
+	rec = dbox_index_record_lookup(index, file_id);
+	if (rec == NULL || !rec->locked)
+		return;
+
+	dbox_index_unlock_range(index, rec->file_offset, 1);
+	rec->locked = FALSE;
+}
+
+int dbox_index_try_lock_recreate(struct dbox_index *index)
+{
+	int i, ret;
+
+	if (index->fd == -1) {
+		if (dbox_index_refresh(index) < 0)
+			return 1;
+	}
+
+	for (i = 0; i < DBOX_INDEX_LOCK_RETRY_COUNT; i++) {
+		/* lock the whole file */
+		ret = dbox_index_lock_range(index, F_SETLK, F_WRLCK, 0, 0);
+		if (ret <= 0)
+			return ret;
+		if ((ret = dbox_index_refresh(index)) <= 0)
+			return ret < 0 ? -1 : 1;
+	}
+
+	i_warning("dbox index keeps getting recreated: %s", index->path);
+	return 0;
+}
+
+static int dbox_index_lock_header(struct dbox_index *index)
+{
+	int i, ret;
+
+	if (index->fd == -1) {
+		if (dbox_index_refresh(index) < 0)
+			return 1;
+	}
+
+	for (i = 0; i < DBOX_INDEX_LOCK_RETRY_COUNT; i++) {
+		ret = dbox_index_lock_range(index, F_SETLKW, F_WRLCK, 0,
+					sizeof(struct dbox_index_file_header));
+		if (ret <= 0)
+			return -1;
+
+		/* if file was recreated, reopen it and try again */
+		if ((ret = dbox_index_refresh(index)) <= 0)
+			return ret < 0;
+	}
+
+	mail_storage_set_critical(index->mbox->ibox.box.storage,
+		"dbox index keeps getting recreated: %s", index->path);
+	return -1;
+}
+
+static void dbox_index_unlock_header(struct dbox_index *index)
+{
+	dbox_index_unlock_range(index, 0,
+				sizeof(struct dbox_index_file_header));
+}
+
+static void
+dbox_index_append_record(const struct dbox_index_record *rec, string_t *str)
+{
+	str_printfa(str, "%u %c%c%c",
+		    rec->file_id, rec->status,
+		    rec->expunges ? 'E' : '0',
+		    rec->dirty ? 'D' : '0');
+
+	switch (rec->status) {
+	case DBOX_INDEX_FILE_STATUS_APPENDABLE:
+		str_append(str, " 00000000");
+		break;
+	case DBOX_INDEX_FILE_STATUS_APPENDING:
+	case DBOX_INDEX_FILE_STATUS_UNLINKED:
+		i_unreached();
+		break;
+	case DBOX_INDEX_FILE_STATUS_NONAPPENDABLE:
+	case DBOX_INDEX_FILE_STATUS_SINGLE_MESSAGE:
+		break;
+	case DBOX_INDEX_FILE_STATUS_MAILDIR:
+		/* FIXME */
+		break;
+	}
+	str_append_c(str, '\n');
+}
+
+static int dbox_index_recreate(struct dbox_index *index, bool locked)
+{
+	struct mail_storage *storage = &index->mbox->storage->storage;
+	struct dbox_index_record *records;
+	struct ostream *output;
+	struct dbox_index_file_header hdr;
+	string_t *temp_path, *str;
+	unsigned int i, count;
+	int fd, ret = 0;
+
+	t_push();
+	temp_path = t_str_new(256);
+	str_append(temp_path, index->path);
+	if (locked) {
+		str_append(temp_path, ".tmp");
+		fd = open(str_c(temp_path), O_RDWR | O_CREAT | O_TRUNC, 0600);
+		if (fd == -1) {
+			mail_storage_set_critical(storage,
+				"open(%s, O_CREAT) failed: %m",
+				str_c(temp_path));
+			t_pop();
+			return -1;
+		}
+	} else {
+		str_append_c(temp_path, '.');
+		fd = safe_mkstemp_hostpid(temp_path, 0600,
+					  (uid_t)-1, (gid_t)-1);
+		if (fd == -1) {
+			mail_storage_set_critical(storage,
+				"safe_mkstemp_hostpid(%s) failed: %m",
+				str_c(temp_path));
+			t_pop();
+			return -1;
+		}
+	}
+
+	str = t_str_new(256);
+	output = o_stream_create_fd_file(fd, 0, FALSE);
+	o_stream_cork(output);
+
+	dbox_index_header_init(index, &hdr);
+	o_stream_send(output, &hdr, sizeof(hdr));
+	o_stream_send(output, "\n", 1);
+
+	records = array_get_modifiable(&index->records, &count);
+	for (i = 0; i < count; ) {
+		if (records[i].status == DBOX_INDEX_FILE_STATUS_UNLINKED) {
+			array_delete(&index->records, i, 1);
+			records = array_get_modifiable(&index->records, &count);
+		} else {
+			records[i].file_offset = output->offset;
+			str_truncate(str, 0);
+			dbox_index_append_record(&records[i], str);
+			o_stream_send(output, str_data(str), str_len(str));
+			i++;
+		}
+	}
+
+	if (o_stream_flush(output) < 0) {
+		mail_storage_set_critical(storage,
+			"write(%s) failed: %m", str_c(temp_path));
+		ret = -1;
+	}
+
+	o_stream_destroy(&output);
+	if (ret == 0 && index->mbox->ibox.fsync_disable) {
+		if (fdatasync(fd) < 0) {
+			mail_storage_set_critical(storage,
+				"fdatasync(%s) failed: %m", str_c(temp_path));
+			ret = -1;
+		}
+	}
+	if (close(fd) < 0) {
+		mail_storage_set_critical(storage,
+			"close(%s) failed: %m", str_c(temp_path));
+		ret = -1;
+	}
+	if (ret == 0) {
+		if (locked) {
+			if (rename(str_c(temp_path), index->path) < 0) {
+				mail_storage_set_critical(storage,
+					"rename(%s, %s) failed: %m",
+					str_c(temp_path), index->path);
+				ret = -1;
+			}
+		} else {
+			if (link(str_c(temp_path), index->path) < 0 &&
+			    errno != EEXIST) {
+				mail_storage_set_critical(storage,
+					"link(%s, %s) failed: %m",
+					str_c(temp_path), index->path);
+				ret = -1;
+			}
+		}
+	}
+	if (ret < 0 || !locked) {
+		if (unlink(str_c(temp_path)) < 0)
+			i_error("unlink(%s) failed: %m", str_c(temp_path));
+	}
+	t_pop();
+	return ret;
+}
+
+struct dbox_index_append_context *
+dbox_index_append_begin(struct dbox_index *index)
+{
+	struct dbox_index_append_context *ctx;
+	const void *data;
+	bool expunged;
+
+	ctx = i_new(struct dbox_index_append_context, 1);
+	ctx->index = index;
+	ctx->first_new_file_id = (unsigned int)-1;
+	i_array_init(&ctx->files, 64);
+
+	/* refresh the index now if there's a possibility of some appendable
+	   files existing */
+	if (mail_index_view_get_messages_count(index->mbox->ibox.view) > 0) {
+		mail_index_lookup_ext(index->mbox->ibox.view, 1,
+				      index->mbox->dbox_ext_id,
+				      &data, &expunged);
+		if (data != NULL)
+			(void)dbox_index_refresh(index);
+	}
+	return ctx;
+}
+
+static bool
+dbox_index_append_file_record(struct dbox_index_append_context *ctx,
+			      struct dbox_index_record *record,
+			      uoff_t mail_size, struct dbox_file **file_r,
+			      struct ostream **output_r)
+{
+	struct dbox_file *const *files, *file;
+	enum dbox_index_file_lock_status lock_status;
+	unsigned int i, count;
+
+	if (record->status != DBOX_INDEX_FILE_STATUS_APPENDABLE)
+		return FALSE;
+
+	if (record->expunges)
+		return FALSE;
+
+	/* if we already have it in our files list, we already checked that
+	   we can't append to it. */
+	files = array_get(&ctx->files, &count);
+	for (i = 0; i < count; i++) {
+		if (files[i]->file_id == record->file_id)
+			return FALSE;
+	}
+	i_assert(!record->locked);
+
+	if (dbox_index_try_lock_file(ctx->index, record->file_id,
+				     &lock_status) <= 0)
+		return FALSE;
+
+	/* open the file to see if we can append */
+	file = dbox_file_init(ctx->index->mbox, record->file_id);
+	if (dbox_file_get_append_stream(file, mail_size, output_r) <= 0) {
+		dbox_index_unlock_file(ctx->index, record->file_id);
+		dbox_file_unref(&file);
+		return FALSE;
+	}
+	*file_r = file;
+	return TRUE;
+}
+
+int dbox_index_append_next(struct dbox_index_append_context *ctx,
+			   uoff_t mail_size,
+			   struct dbox_file **file_r,
+			   struct ostream **output_r)
+{
+	struct dbox_file *const *files, *file = NULL;
+	struct dbox_index_record *records;
+	unsigned int i, count;
+	int ret;
+
+	/* first try to use files already used in this append */
+	files = array_get(&ctx->files, &count);
+	for (i = 0; i < count; i++) {
+		if (dbox_file_get_append_stream(files[i], mail_size,
+						output_r) > 0) {
+			*file_r = files[i];
+			return 0;
+		}
+	}
+
+	/* try to find an existing appendable file */
+	records = array_get_modifiable(&ctx->index->records, &count);
+	for (i = 0; i < count; i++) {
+		if (dbox_index_append_file_record(ctx, &records[i], mail_size,
+						  &file, output_r))
+			break;
+	}
+
+	if (file == NULL) {
+		/* create a new file */
+		file = dbox_file_init(ctx->index->mbox, 0);
+		if ((ret = dbox_file_get_append_stream(file, mail_size,
+						       output_r)) <= 0) {
+			i_assert(ret < 0);
+			(void)unlink(file->path);
+			dbox_file_unref(&file);
+			return -1;
+		}
+	}
+
+	*file_r = file;
+	array_append(&ctx->files, &file, 1);
+	return 0;
+}
+
+static int dbox_index_append_commit_new(struct dbox_index_append_context *ctx,
+					struct dbox_file *file, string_t *str)
+{
+	struct mail_storage *storage = &ctx->index->mbox->storage->storage;
+	struct dbox_index_record rec;
+	struct stat st;
+	unsigned int file_id;
+
+	i_assert(file->append_count > 0);
+
+	if (file->append_count == 1 && !dbox_file_can_append(file, 0)) {
+		/* single UID message file */
+		i_assert(file->last_append_uid != 0);
+		file_id = file->last_append_uid | DBOX_FILE_ID_FLAG_UID;
+		return dbox_file_assign_id(file, file_id);
+	}
+
+	if (!ctx->locked_header) {
+		if (dbox_index_lock_header(ctx->index) < 0)
+			return -1;
+		if (dbox_index_read_header(ctx->index) < 0) {
+			dbox_index_unlock_header(ctx->index);
+			return -1;
+		}
+		if (fstat(ctx->index->fd, &st) < 0) {
+			mail_storage_set_critical(storage,
+				"fstat(%s) failed: %m", ctx->index->path);
+			dbox_index_unlock_header(ctx->index);
+			return -1;
+		}
+		ctx->output_offset = st.st_size;
+		ctx->new_record_idx = array_count(&ctx->index->records);
+		ctx->first_new_file_id = ctx->index->next_file_id;
+		ctx->locked_header = TRUE;
+	}
+
+	file_id = ctx->index->next_file_id++;
+	if (dbox_file_assign_id(file, file_id) < 0)
+		return -1;
+
+	memset(&rec, 0, sizeof(rec));
+	rec.file_id = file_id;
+	rec.file_offset = ctx->output_offset + str_len(str);
+	rec.status = dbox_file_can_append(file, 0) ?
+		DBOX_INDEX_FILE_STATUS_APPENDABLE :
+		DBOX_INDEX_FILE_STATUS_NONAPPENDABLE;
+
+	array_append(&ctx->index->records, &rec, 1);
+	dbox_index_append_record(&rec, str);
+	return 0;
+}
+
+static void
+dbox_index_append_rollback_commit(struct dbox_index_append_context *ctx)
+{
+	struct dbox_file *const *files;
+	unsigned int i, count;
+
+	files = array_get(&ctx->files, &count);
+	for (i = 0; i < count; i++) {
+		if (files[i]->file_id >= ctx->first_new_file_id) {
+			if (unlink(files[i]->path) < 0) {
+				i_error("unlink(%s) failed: %m",
+					files[i]->path);
+			}
+			files[i]->deleted = TRUE;
+		} else {
+			/* FIXME: we should delete the appended mails.. */
+		}
+	}
+	array_delete(&ctx->index->records, ctx->new_record_idx,
+		     array_count(&ctx->index->records) - ctx->new_record_idx);
+}
+
+static int
+dbox_index_append_write_records(struct dbox_index_append_context *ctx,
+				string_t *str)
+{
+	int ret;
+
+	ret = dbox_index_lock_range(ctx->index, F_SETLKW, F_WRLCK,
+				    ctx->output_offset, str_len(str));
+	if (ret <= 0)
+		return -1;
+
+	if (pwrite_full(ctx->index->fd, str_data(str), str_len(str),
+			ctx->output_offset) < 0) {
+		mail_storage_set_critical(&ctx->index->mbox->storage->storage,
+			"pwrite(%s) failed: %m", ctx->index->path);
+		if (ftruncate(ctx->index->fd, ctx->output_offset) < 0)
+			i_error("ftruncate(%s) failed: %m", ctx->index->path);
+		ret = -1;
+	}
+	dbox_index_unlock_range(ctx->index, ctx->output_offset, str_len(str));
+	return ret;
+}
+
+static int dbox_index_write_header(struct dbox_index *index)
+{
+	struct dbox_index_file_header hdr;
+
+	dbox_index_header_init(index, &hdr);
+	if (pwrite_full(index->fd, &hdr, sizeof(hdr), 0) < 0) {
+		mail_storage_set_critical(&index->mbox->storage->storage,
+			"pwrite(%s) failed: %m", index->path);
+		return -1;
+	}
+	return 0;
+}
+
+int dbox_index_append_assign_file_ids(struct dbox_index_append_context *ctx)
+{
+	struct dbox_file *const *files, *file;
+	string_t *str;
+	unsigned int i, count;
+	int ret = 0;
+
+	str = t_str_new(1024);
+	files = array_get(&ctx->files, &count);
+	for (i = 0; i < count; i++) {
+		file = files[i];
+
+		if (file->file_id == 0) {
+			if (dbox_index_append_commit_new(ctx, file, str) < 0)
+				ret = -1;
+		}
+	}
+
+	if (ret == 0 && str_len(str) > 0) {
+		/* write the new records to index */
+		ret = dbox_index_append_write_records(ctx, str);
+	}
+	if (ret < 0 && str_len(str) > 0) {
+		/* we have to rollback changes we made */
+		dbox_index_append_rollback_commit(ctx);
+	}
+	return ret;
+}
+
+int dbox_index_append_commit(struct dbox_index_append_context **_ctx)
+{
+	struct dbox_index_append_context *ctx = *_ctx;
+	struct dbox_file **files;
+	unsigned int i, count;
+	int ret = 0;
+
+	*_ctx = NULL;
+
+	files = array_get_modifiable(&ctx->files, &count);
+	for (i = 0; i < count; i++) {
+		if (files[i]->file_id < ctx->first_new_file_id) {
+			// FIXME: update status
+			dbox_index_unlock_file(ctx->index, files[i]->file_id);
+		}
+		dbox_file_unref(&files[i]);
+	}
+
+	if (ctx->locked_header) {
+		if (dbox_index_write_header(ctx->index) < 0)
+			ret = -1;
+		dbox_index_unlock_header(ctx->index);
+	}
+
+	array_free(&ctx->files);
+	i_free(ctx);
+	return 0;
+}
+
+void dbox_index_append_rollback(struct dbox_index_append_context **_ctx)
+{
+	struct dbox_index_append_context *ctx = *_ctx;
+	struct dbox_file *const *files, *file;
+	unsigned int i, count;
+
+	*_ctx = NULL;
+
+	files = array_get(&ctx->files, &count);
+	for (i = 0; i < count; i++) {
+		file = files[i];
+
+		if (file->file_id != 0)
+			dbox_index_unlock_file(ctx->index, file->file_id);
+		else {
+			if (unlink(file->path) < 0)
+				i_error("unlink(%s) failed: %m", file->path);
+		}
+		dbox_file_unref(&file);
+	}
+	array_free(&ctx->files);
+	i_free(ctx);
+}

--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/src/lib-storage/index/dbox/dbox-index.h	Sat Sep 01 03:04:02 2007 +0300
@@ -0,0 +1,130 @@
+#ifndef __DBOX_INDEX_H
+#define __DBOX_INDEX_H
+
+/* The file begins with a header followed by zero or more records:
+
+   <file id> <status><expunges><dirty> [<status-specific data>]<LF>
+
+   <expunges> contains either '0' = no or 'E' = file contains messages marked
+   as expunged, which should be removed when possible.
+
+   <dirty> contains either '0' = no or 'D' = file contains messages that don't
+   have up-to-date metadata. When expunge copies message data to a new file,
+   the dirty state should be flushed for the copied messages (or the dirty
+   state should be copied).
+
+   <expunges> and <dirty> can be written without locking the record, so syncing
+   can update them even while messages are being appended to the file.
+
+   If status-specific data isn't specified for the given status, it should be
+   ignored. Especially 'U' status may contain different kinds of data.
+*/
+
+struct dbox_file;
+struct dbox_index_append_context;
+
+#define DBOX_INDEX_VERSION	'1'
+
+enum dbox_index_file_status {
+	/* File can be appended to as long as <expunges> is zero. It must be
+	   locked when expunging. status-specific data contains a %08x lock
+	   timestamp. */
+	DBOX_INDEX_FILE_STATUS_APPENDABLE	= '0',
+	/* File is currently being appended to. If this record can be locked,
+	   the append crashed and this file should be opened for fixing
+	   (truncate non-committed appends from the file). */
+	DBOX_INDEX_FILE_STATUS_APPENDING	= 'A',
+	/* File can't be appended to. */
+	DBOX_INDEX_FILE_STATUS_NONAPPENDABLE	= 'N',
+	/* File contains only a single message. It can't be appended to
+	   and it can be expunged by unlinking the file. */
+	DBOX_INDEX_FILE_STATUS_SINGLE_MESSAGE	= '1',
+	/* The file has already been unlinked, this record should be removed. */
+	DBOX_INDEX_FILE_STATUS_UNLINKED		= 'U',
+
+	/* File is a maildir file. Status-specific data contains
+	   <uid> <filename>. */
+	DBOX_INDEX_FILE_STATUS_MAILDIR		= 'M'
+};
+
+enum dbox_index_file_lock_status {
+	/* File was locked (ret=1) */
+	DBOX_INDEX_FILE_LOCKED,
+	/* File didn't have appendable status (ret=1) */
+	DBOX_INDEX_FILE_LOCK_NOT_NEEDED,
+	/* File was already locked by someone else (ret=0) */
+	DBOX_INDEX_FILE_LOCK_TRY_AGAIN,
+	/* File is already unlinked (ret=0) */
+	DBOX_INDEX_FILE_LOCK_UNLINKED
+};
+
+struct dbox_index_file_header {
+	/* DBOX_INDEX_VERSION */
+	unsigned char version;
+	unsigned char space_1;
+
+	/* Current UIDVALIDITY */
+	unsigned char uid_validity_hex[8];
+	unsigned char space_2;
+
+	/* Next available message UID */
+	unsigned char next_uid_hex[8];
+	unsigned char space_3;
+
+	/* Next available <file id> */
+	unsigned char next_file_id_hex[8];
+};
+
+struct dbox_index_record {
+	unsigned int file_id;
+	unsigned int file_offset;
+
+	enum dbox_index_file_status status;
+	unsigned int expunges:1;
+	unsigned int dirty:1;
+	unsigned int locked:1;
+};
+
+struct dbox_index *dbox_index_init(struct dbox_mailbox *mbox);
+void dbox_index_deinit(struct dbox_index **index);
+
+/* Get the current UIDVALIDITY. Returns 0 if ok, -1 if I/O error. */
+int dbox_index_get_uid_validity(struct dbox_index *index,
+				uint32_t *uid_validity_r);
+
+struct dbox_index_record *
+dbox_index_record_lookup(struct dbox_index *index, unsigned int file_id);
+
+/* Try to lock a file record. Only appendable files are actually locked.
+   Returns 1 if lock acquired or not needed, 0 if we failed to get a lock or
+   file is unlinked, -1 if error. lock_status_r is set if 0 or 1 is returned. */
+int dbox_index_try_lock_file(struct dbox_index *index, unsigned int file_id,
+			     enum dbox_index_file_lock_status *lock_status_r);
+void dbox_index_unlock_file(struct dbox_index *index, unsigned int file_id);
+
+/* Try to lock index file for recreating. Returns 1 if ok, 0 if file already
+   contains locks, -1 if error. */
+int dbox_index_try_lock_recreate(struct dbox_index *index);
+/* Lock index file for syncing. Returns 0 if ok, -1 if error. */
+int dbox_index_lock_sync(struct dbox_index *index);
+
+struct dbox_index_append_context *
+dbox_index_append_begin(struct dbox_index *index);
+/* Request file for saving a new message with given size. If an existing file
+   can be used, the record is locked and updated in index. Returns 0 if ok,
+   -1 if error. */
+int dbox_index_append_next(struct dbox_index_append_context *ctx,
+			   uoff_t mail_size,
+			   struct dbox_file **file_r,
+			   struct ostream **output_r);
+/* Assign file_ids to all appended files. */
+int dbox_index_append_assign_file_ids(struct dbox_index_append_context *ctx);
+/* Returns 0 if ok, -1 if error. */
+int dbox_index_append_commit(struct dbox_index_append_context **ctx);
+void dbox_index_append_rollback(struct dbox_index_append_context **ctx);
+
+/* Mark  */
+void dbox_index_mark_expunges(struct dbox_index *index, unsigned int file_id);
+void dbox_index_mark_dirty(struct dbox_index *index, unsigned int file_id);
+
+#endif

--- a/src/lib-storage/index/dbox/dbox-keywords.c	Sat Sep 01 01:36:10 2007 +0300
+++ /dev/null	Thu Jan 01 00:00:00 1970 +0000
@@ -1,214 +0,0 @@
-/* Copyright (C) 2006 Timo Sirainen */
-
-#include "lib.h"
-#include "array.h"
-#include "str.h"
-#include "istream.h"
-#include "write-full.h"
-#include "seq-range-array.h"
-#include "bsearch-insert-pos.h"
-#include "dbox-file.h"
-#include "dbox-storage.h"
-#include "dbox-keywords.h"
-
-#include <stdlib.h>
-
-static int dbox_keyword_map_compare(const void *p1, const void *p2)
-{
-	const struct keyword_map *map1 = p1, *map2 = p2;
-
-	return map1->index_idx < map2->index_idx ? -1 :
-		map1->index_idx > map2->index_idx ? 1 : 0;
-}
-
-int dbox_file_read_keywords(struct dbox_mailbox *mbox, struct dbox_file *file)
-{
-	struct keyword_map *map, kw;
-	const char *line;
-	unsigned int idx, count, insert_idx;
-	uoff_t last_offset;
-
-	if (array_is_created(&file->idx_file_keywords)) {
-		array_clear(&file->idx_file_keywords);
-		array_clear(&file->file_idx_keywords);
-	} else {
-		i_array_init(&file->idx_file_keywords, file->keyword_count);
-		i_array_init(&file->file_idx_keywords, file->keyword_count);
-	}
-
-	/* currently we assume that all extra space at the end of header
-	   belongs to keyword list. */
-	file->keyword_list_size_alloc =
-		file->header_size - file->keyword_list_offset;
-
-	i_stream_seek(file->input, file->keyword_list_offset);
-	idx = 0;
-	last_offset = file->input->v_offset;
-	while ((line = i_stream_read_next_line(file->input)) != NULL) {
-		if (*line == '\0') {
-			/* end of list */
-			break;
-		}
-		last_offset = file->input->v_offset;
-
-		/* set up map record for the keyword */
-		mail_index_keyword_lookup_or_create(mbox->ibox.index, line,
-						    &kw.index_idx);
-		kw.file_idx = idx;
-
-		/* look up the position where to insert it */
-		map = array_get_modifiable(&file->idx_file_keywords, &count);
-		if (idx == 0)
-			insert_idx = 0;
-		else {
-			bsearch_insert_pos(&kw, map, count, sizeof(*map),
-					   dbox_keyword_map_compare,
-					   &insert_idx);
-		}
-		array_insert(&file->idx_file_keywords, insert_idx, &kw, 1);
-		array_append(&file->file_idx_keywords, &kw.index_idx, 1);
-
-		if (++idx == file->keyword_count)
-			break;
-	}
-
-	if (line == NULL || file->input->v_offset > file->header_size) {
-		/* unexpected end of list, or list continues outside its
-		   allocated area */
-		mail_storage_set_critical(&mbox->storage->storage,
-			"Corrupted keyword list offset in dbox file %s",
-			file->path);
-		array_clear(&file->idx_file_keywords);
-		return 0;
-	}
-
-	file->keyword_list_size_used =
-		last_offset - file->keyword_list_offset;
-	return 1;
-}
-
-static int keyword_lookup_cmp(const void *key, const void *obj)
-{
-	const unsigned int *index_idx = key;
-	const struct keyword_map *map = obj;
-
-	return *index_idx < map->index_idx ? -1 :
-		*index_idx > map->index_idx ? 1 : 0;
-}
-
-bool dbox_file_lookup_keyword(struct dbox_mailbox *mbox, struct dbox_file *file,
-			      unsigned int index_idx, unsigned int *idx_r)
-{
-	const struct keyword_map *map, *pos;
-	unsigned int count;
-
-	if (!array_is_created(&file->idx_file_keywords)) {
-		/* Read the keywords, if there are any */
-		if (dbox_file_read_keywords(mbox, file) <= 0)
-			return FALSE;
-	}
-
-	map = array_get(&file->idx_file_keywords, &count);
-	pos = bsearch(&index_idx, map, count, sizeof(*map),
-		      keyword_lookup_cmp);
-	if (pos != NULL && idx_r != NULL)
-		*idx_r = pos->file_idx;
-	return pos != NULL;
-}
-
-int dbox_file_append_keywords(struct dbox_mailbox *mbox, struct dbox_file *file,
-			      const struct seq_range *idx_range,
-			      unsigned int count)
-{
-	const ARRAY_TYPE(keywords) *idx_keywords;
-	string_t *keyword_str;
-	const char *const *idx_keyword_names;
-	unsigned int i, idx_keyword_count, new_pos;
-	int ret;
-
-	t_push();
-	keyword_str = t_str_new(2048);
-	idx_keywords = mail_index_get_keywords(mbox->ibox.index);
-	idx_keyword_names = array_get(idx_keywords, &idx_keyword_count);
-
-	/* make sure we've read the existing keywords */
-	if (!array_is_created(&file->idx_file_keywords)) {
-		ret = dbox_file_read_keywords(mbox, file);
-		if (ret < 0)
-			return -1;
-
-		if (ret == 0) {
-			/* broken keywords list. */
-			file->keyword_list_size_used = 0;
-		}
-	}
-
-	/* append existing keywords */
-	if (array_count(&file->idx_file_keywords) > 0) {
-		const unsigned int *file_idx;
-		unsigned int file_count;
-
-		file_idx = array_get(&file->file_idx_keywords, &file_count);
-		for (i = 0; i < file_count; i++) {
-			i_assert(file_idx[i] < idx_keyword_count);
-
-			str_append(keyword_str, idx_keyword_names[file_idx[i]]);
-			str_append_c(keyword_str, '\n');
-		}
-	}
-
-	/* append new keywords */
-	if (file->keyword_list_size_used == 0)
-		new_pos = 0;
-	else {
-		new_pos = str_len(keyword_str);
-		i_assert(new_pos == file->keyword_list_size_used);
-	}
-	for (i = 0; i < count; i++) {
-		unsigned int idx;
-
-		for (idx = idx_range[i].seq1; idx <= idx_range[i].seq2; idx++) {
-			size_t prev_len;
-
-			i_assert(idx < idx_keyword_count);
-			i_assert(!dbox_file_lookup_keyword(mbox, file,
-							   idx, NULL));
-
-			prev_len = str_len(keyword_str);
-			str_append(keyword_str, idx_keyword_names[idx]);
-			str_append_c(keyword_str, '\n');
-
-			if (str_len(keyword_str) >=
-			    file->keyword_list_size_alloc) {
-				/* FIXME: keyword list doesn't fit to the
-				   space allocated for it. create a new file
-				   where there's more space for keywords and
-				   move the mails there.
-
-				   for now we'll just ignore the problem. */
-				str_truncate(keyword_str, prev_len);
-				break;
-			}
-		}
-	}
-
-	str_append_c(keyword_str, '\n');
-	i_assert(str_len(keyword_str) <= file->keyword_list_size_alloc);
-	i_assert(new_pos < str_len(keyword_str));
-
-	/* we can reuse the existing keyword list position */
-	if (pwrite_full(file->fd, str_data(keyword_str) + new_pos,
-			str_len(keyword_str) - new_pos,
-			file->keyword_list_offset + new_pos) < 0) {
-		mail_storage_set_critical(&mbox->storage->storage,
-			"pwrite_full(%s) failed: %m", file->path);
-	}
-
-	/* FIXME: we could do this faster than by reading them.. */
-	ret = 0;
-	if (dbox_file_read_keywords(mbox, file) <= 0)
-		ret = -1;
-
-	t_pop();
-	return ret;
-}

--- a/src/lib-storage/index/dbox/dbox-keywords.h	Sat Sep 01 01:36:10 2007 +0300
+++ /dev/null	Thu Jan 01 00:00:00 1970 +0000
@@ -1,17 +0,0 @@
-#ifndef __DBOX_KEYWORDS_H
-#define __DBOX_KEYWORDS_H
-
-struct seq_range;
-
-/* Read keywords from file into memory. Returns 1 if ok, 0 if the list is
-   broken or -1 if I/O error. */
-int dbox_file_read_keywords(struct dbox_mailbox *mbox, struct dbox_file *file);
-/* Index file -> dbox file keyword index lookup. Returns TRUE if found. */
-bool dbox_file_lookup_keyword(struct dbox_mailbox *mbox, struct dbox_file *file,
-			      unsigned int index_idx, unsigned int *idx_r);
-/* Save keywords to dbox file. Returns -1 if error, 0 if ok. */
-int dbox_file_append_keywords(struct dbox_mailbox *mbox, struct dbox_file *file,
-			      const struct seq_range *idx_range,
-			      unsigned int count);
-
-#endif

--- a/src/lib-storage/index/dbox/dbox-mail.c	Sat Sep 01 01:36:10 2007 +0300
+++ b/src/lib-storage/index/dbox/dbox-mail.c	Sat Sep 01 03:04:02 2007 +0300
@@ -1,195 +1,162 @@
-/* Copyright (C) 2005 Timo Sirainen */
+/* Copyright (C) 2007 Timo Sirainen */
 
 #include "lib.h"
-#include "ioloop.h"
-#include "hex-dec.h"
-#include "read-full.h"
 #include "istream.h"
 #include "index-mail.h"
+#include "dbox-storage.h"
 #include "dbox-file.h"
-#include "dbox-sync.h"
-#include "dbox-storage.h"
+
+#include <stdlib.h>
+
+struct dbox_mail {
+	struct index_mail imail;
 
-#include <fcntl.h>
-#include <unistd.h>
-#include <sys/stat.h>
+	struct dbox_file *open_file;
+	uoff_t offset;
+};
 
-static int dbox_mail_parse_mail_header(struct index_mail *mail,
-				       struct dbox_file *file)
+struct mail *
+dbox_mail_alloc(struct mailbox_transaction_context *t,
+		enum mail_fetch_field wanted_fields,
+		struct mailbox_header_lookup_ctx *wanted_headers)
 {
-	struct dbox_mailbox *mbox =
-		(struct dbox_mailbox *)mail->mail.mail.box;
-	const struct dbox_mail_header *hdr = &file->seeked_mail_header;
-	uint32_t hdr_uid = hex2dec(hdr->uid_hex, sizeof(hdr->uid_hex));
+	struct dbox_mail *mail;
+	pool_t pool;
+
+	pool = pool_alloconly_create("mail", 1024);
+	mail = p_new(pool, struct dbox_mail, 1);
+	mail->imail.mail.pool = pool;
+
+	index_mail_init(&mail->imail, t, wanted_fields, wanted_headers);
+	return &mail->imail.mail.mail;
+}
+
+static void dbox_mail_close(struct mail *_mail)
+{
+	struct dbox_mail *mail = (struct dbox_mail *)_mail;
 
-	if (hdr_uid != mail->mail.mail.uid) {
-		mail_storage_set_critical(&mbox->storage->storage,
-			"dbox %s: Cached file offset broken",
-			mbox->file->path);
+	if (mail->open_file != NULL)
+		dbox_file_unref(&mail->open_file);
+	index_mail_close(_mail);
+}
 
-		/* make sure we get it fixed */
-		(void)dbox_sync(mbox, TRUE);
-		return -1;
+static int dbox_mail_lookup(struct dbox_mail *mail,
+			    uoff_t *offset_r, struct dbox_file **file_r)
+{
+	struct dbox_mailbox *mbox = (struct dbox_mailbox *)mail->imail.ibox;
+	unsigned int file_id;
+
+	if (mail->open_file == NULL) {
+		if (!dbox_file_lookup(mbox, mbox->ibox.view,
+				      mail->imail.mail.mail.seq,
+				      &file_id, &mail->offset)) {
+			mail_set_expunged(&mail->imail.mail.mail);
+			return -1;
+		}
+		mail->open_file = dbox_file_init(mbox, file_id);
 	}
 
-	/* Note that the mail may already have an expunge flag, but we don't
-	   care since we can still read it */
-	mail->data.physical_size = mail->data.virtual_size =
-		hex2dec(hdr->mail_size_hex, sizeof(hdr->mail_size_hex));
-	mail->data.received_date =
-		hex2dec(hdr->received_time_hex, sizeof(hdr->received_time_hex));
-	return 1;
+	*file_r = mail->open_file;
+	*offset_r = mail->offset;
+	return 0;
 }
 
-int dbox_mail_lookup_offset(struct index_transaction_context *trans,
-			    uint32_t seq, uint32_t *file_seq_r,
-			    uoff_t *offset_r)
+static int
+dbox_mail_metadata_seek(struct dbox_mail *mail, struct dbox_file **file_r)
 {
-	struct dbox_mailbox *mbox =
-		(struct dbox_mailbox *)trans->ibox;
-	uint32_t uid;
-	bool synced = FALSE;
+	struct mail *_mail = &mail->imail.mail.mail;
+	uoff_t offset, metadata_offset, physical_size;
+	bool expunged;
 	int ret;
 
-	for (;;) {
-		ret = dbox_file_lookup_offset(mbox, trans->trans_view, seq,
-					      file_seq_r, offset_r);
-		if (ret <= 0)
-			return ret;
-		if (*file_seq_r != 0)
-			return 1;
-
-		/* lost file sequence/offset */
-		if (synced)
-			return -1;
-
-		mail_index_lookup_uid(trans->trans_view, seq, &uid);
-		mail_storage_set_critical(&mbox->storage->storage,
-			"Cached message offset lost for uid %u in "
-			"dbox %s", uid, mbox->path);
+	if (dbox_mail_lookup(mail, &offset, file_r) < 0)
+		return -1;
 
-		/* resync and try again */
-		if (dbox_sync(mbox, TRUE) < 0)
-			return -1;
-		synced = TRUE;
-	}
-}
-
-static bool dbox_mail_try_open(struct index_mail *mail,
-			       uint32_t *file_seq_r, uoff_t *offset_r,
-			       int *ret_r)
-{
-	struct dbox_mailbox *mbox = (struct dbox_mailbox *)mail->ibox;
-	struct dbox_transaction_context *t =
-		(struct dbox_transaction_context *)mail->trans;
-	uint32_t seq = mail->mail.mail.seq;
-
-	*ret_r = dbox_mail_lookup_offset(mail->trans, seq,
-					 file_seq_r, offset_r);
-	if (*ret_r <= 0) {
-		if (*ret_r == 0)
-			mail_set_expunged(&mail->mail.mail);
-		return TRUE;
-	}
+	if (mail_get_physical_size(_mail, &physical_size) < 0)
+		return -1;
 
-	if ((*ret_r = dbox_file_seek(mbox, *file_seq_r, *offset_r,
-				     seq >= t->first_saved_mail_seq)) < 0)
-		return TRUE;
-	if (*ret_r > 0) {
-		/* ok */
-		*ret_r = dbox_mail_parse_mail_header(mail, mbox->file);
-		return TRUE;
+	metadata_offset =
+		dbox_file_get_metadata_offset(*file_r, offset, physical_size);
+	ret = dbox_file_metadata_seek(*file_r, metadata_offset, &expunged);
+	if (ret <= 0) {
+		if (ret < 0)
+			return -1;
+		/* FIXME */
+		return -1;
 	}
-	return FALSE;
-}
-
-static int dbox_mail_open(struct index_mail *mail, uoff_t *offset_r)
-{
-	struct dbox_mailbox *mbox = (struct dbox_mailbox *)mail->ibox;
-	uint32_t file_seq, prev_file_seq = 0;
-	uoff_t prev_offset = 0;
-	int i, ret;
-
-	if (mail->mail.mail.expunged || mbox->syncing)
-		return 0;
-
-	for (i = 0; i < 3; i++) {
-		if (dbox_mail_try_open(mail, &file_seq, offset_r, &ret))
-			return ret;
-
-		if (prev_file_seq == file_seq && prev_offset == *offset_r) {
-			/* broken offset */
-			break;
-		} else {
-			/* mail was moved. resync dbox to find out the new
-			   offset and try again. */
-			if (dbox_sync(mbox, FALSE) < 0)
-				return -1;
-		}
-
-		prev_file_seq = file_seq;
-		prev_offset = *offset_r;
+	if (expunged) {
+		mail_set_expunged(&mail->imail.mail.mail);
+		return -1;
 	}
-
-	mail_storage_set_critical(&mbox->storage->storage,
-				  "Cached message offset (%u, %"PRIuUOFF_T") "
-				  "broken for uid %u in dbox %s",
-				  file_seq, *offset_r, mail->mail.mail.uid,
-				  mbox->path);
-
-	if (dbox_sync(mbox, TRUE) < 0)
-		return -1;
-	if (dbox_mail_try_open(mail, &file_seq, offset_r, &ret))
-		return ret;
-	return -1;
+	return 0;
 }
 
 static int dbox_mail_get_received_date(struct mail *_mail, time_t *date_r)
 {
-	struct index_mail *mail = (struct index_mail *)_mail;
-	struct index_mail_data *data = &mail->data;
-	uoff_t offset;
+	struct dbox_mail *mail = (struct dbox_mail *)_mail;
+	struct index_mail_data *data = &mail->imail.data;
+	struct dbox_file *file;
+	const char *value;
 	uint32_t t;
 
 	(void)index_mail_get_received_date(_mail, date_r);
 	if (*date_r != (time_t)-1)
 		return 0;
 
-	if (dbox_mail_open(mail, &offset) <= 0)
-		return (time_t)-1;
-	if (data->received_date == (time_t)-1) {
-		/* it's broken and conflicts with our "not found"
-		   return value. change it. */
-		data->received_date = 0;
-	}
+	if (dbox_mail_metadata_seek(mail, &file) < 0)
+		return -1;
 
-	t = data->received_date;
-	index_mail_cache_add(mail, MAIL_CACHE_RECEIVED_DATE, &t, sizeof(t));
+	value = dbox_file_metadata_get(file, DBOX_METADATA_RECEIVED_TIME);
+	data->received_date = t = value == NULL ? 0 : strtoul(value, NULL, 16);
+	index_mail_cache_add(&mail->imail, MAIL_CACHE_RECEIVED_DATE,
+			     &t, sizeof(t));
 	*date_r = data->received_date;
 	return 0;
 }
 
 static int dbox_mail_get_save_date(struct mail *_mail, time_t *date_r)
 {
-	struct index_mail *mail = (struct index_mail *)_mail;
-	struct index_mail_data *data = &mail->data;
-	uoff_t offset;
+	struct dbox_mail *mail = (struct dbox_mail *)_mail;
+	struct index_mail_data *data = &mail->imail.data;
+	struct dbox_file *file;
+	const char *value;
+	uint32_t t;
 
 	(void)index_mail_get_save_date(_mail, date_r);
 	if (*date_r != (time_t)-1)
 		return 0;
 
-	if (dbox_mail_open(mail, &offset) <= 0)
+	if (dbox_mail_metadata_seek(mail, &file) < 0)
 		return -1;
-	if (data->save_date == (time_t)-1) {
-		/* it's broken and conflicts with our "not found"
-		   return value. change it. */
-		data->save_date = ioloop_time;
-	}
+
+	value = dbox_file_metadata_get(file, DBOX_METADATA_SAVE_TIME);
+	data->save_date = t = value == NULL ? 0 : strtoul(value, NULL, 16);
+	index_mail_cache_add(&mail->imail, MAIL_CACHE_SAVE_DATE, &t, sizeof(t));
+	*date_r = data->save_date;
+	return 0;
+}
+
+static int dbox_mail_get_virtual_size(struct mail *_mail, uoff_t *size_r)
+{
+	struct dbox_mail *mail = (struct dbox_mail *)_mail;
+	struct index_mail_data *data = &mail->imail.data;
+	struct dbox_file *file;
+	const char *value;
 
-	index_mail_cache_add(mail, MAIL_CACHE_SAVE_DATE,
-			     &data->save_date, sizeof(data->save_date));
-	*date_r = data->save_date;
+	if (index_mail_get_cached_virtual_size(&mail->imail, size_r))
+		return 0;
+
+	if (dbox_mail_metadata_seek(mail, &file) < 0)
+		return -1;
+
+	value = dbox_file_metadata_get(file, DBOX_METADATA_VIRTUAL_SIZE);
+	if (value == NULL)
+		return index_mail_get_virtual_size(_mail, size_r);
+
+	data->virtual_size = strtoul(value, NULL, 16);
+	index_mail_cache_add(&mail->imail, MAIL_CACHE_VIRTUAL_FULL_SIZE,
+			     &data->virtual_size, sizeof(data->virtual_size));
+	*size_r = data->virtual_size;
 	return 0;
 }
 
@@ -197,46 +164,62 @@
 {
 	struct index_mail *mail = (struct index_mail *)_mail;
 	struct index_mail_data *data = &mail->data;
-	uoff_t offset;
+	struct istream *input;
 
 	(void)index_mail_get_physical_size(_mail, size_r);
 	if (*size_r != (uoff_t)-1)
 		return 0;
 
-	if (dbox_mail_open(mail, &offset) <= 0)
+	if (mail_get_stream(_mail, NULL, NULL, &input) < 0)
 		return -1;
 
+	i_assert(data->physical_size != (uoff_t)-1);
 	index_mail_cache_add(mail, MAIL_CACHE_PHYSICAL_FULL_SIZE,
 			     &data->physical_size, sizeof(data->physical_size));
 	*size_r = data->physical_size;
 	return 0;
-
 }
 
-static int dbox_mail_get_stream(struct mail *_mail,
-				struct message_size *hdr_size,
-				struct message_size *body_size,
-				struct istream **stream_r)
+static int
+dbox_mail_get_stream(struct mail *_mail, struct message_size *hdr_size,
+		     struct message_size *body_size, struct istream **stream_r)
 {
-	struct index_mail *mail = (struct index_mail *)_mail;
-	struct dbox_mailbox *mbox = (struct dbox_mailbox *)mail->ibox;
+	struct dbox_mail *mail = (struct dbox_mail *)_mail;
+	struct index_mail_data *data = &mail->imail.data;
+	struct istream *input;
 	uoff_t offset;
+	uint32_t uid;
+	bool expunged;
+	int ret;
 
-	if (mail->data.stream == NULL) {
-		if (dbox_mail_open(mail, &offset) <= 0)
+	if (data->stream == NULL) {
+		if (dbox_mail_lookup(mail, &offset, &mail->open_file) < 0)
 			return -1;
 
-		offset += mbox->file->mail_header_size;
-		mail->data.stream =
-			i_stream_create_limit(mbox->file->input, offset,
-					      mbox->file->seeked_mail_size);
+		ret = dbox_file_get_mail_stream(mail->open_file, offset, &uid,
+						&data->physical_size, &input,
+						&expunged);
+		if (ret < 0)
+			return -1;
+		if (ret > 0 && expunged) {
+			mail_set_expunged(_mail);
+			return -1;
+		}
+		if (ret == 0 || uid != _mail->uid) {
+			/* FIXME: broken file/offset */
+			if (ret > 0)
+				i_stream_unref(&input);
+			return -1;
+		}
+		data->stream = input;
 	}
 
-	return index_mail_init_stream(mail, hdr_size, body_size, stream_r);
+	return index_mail_init_stream(&mail->imail, hdr_size, body_size,
+				      stream_r);
 }
 
 struct mail_vfuncs dbox_mail_vfuncs = {
-	index_mail_close,
+	dbox_mail_close,
 	index_mail_free,
 	index_mail_set_seq,
 	index_mail_set_uid,
@@ -247,7 +230,7 @@
 	index_mail_get_date,
 	dbox_mail_get_received_date,
 	dbox_mail_get_save_date,
-	dbox_mail_get_physical_size, /* physical = virtual in our case */
+	dbox_mail_get_virtual_size,
 	dbox_mail_get_physical_size,
 	index_mail_get_first_header,
 	index_mail_get_headers,

--- a/src/lib-storage/index/dbox/dbox-save.c	Sat Sep 01 01:36:10 2007 +0300
+++ b/src/lib-storage/index/dbox/dbox-save.c	Sat Sep 01 03:04:02 2007 +0300
@@ -1,98 +1,63 @@
-/* Copyright (C) 2005 Timo Sirainen */
+/* Copyright (C) 2007 Timo Sirainen */
 
 #include "lib.h"
 #include "array.h"
-#include "ioloop.h"
+#include "hex-dec.h"
+#include "str.h"
 #include "istream.h"
-#include "hex-dec.h"
+#include "ostream.h"
+#include "ostream-crlf.h"
 #include "write-full.h"
-#include "ostream.h"
-#include "seq-range-array.h"
 #include "index-mail.h"
-#include "dbox-uidlist.h"
-#include "dbox-keywords.h"
+#include "dbox-storage.h"
+#include "dbox-index.h"
+#include "dbox-file.h"
 #include "dbox-sync.h"
-#include "dbox-storage.h"
+
+#include <stdlib.h>
 
-#include <stddef.h>
+struct dbox_save_mail {
+	struct dbox_file *file;
+	uint32_t seq;
+	uint32_t append_offset;
+	uoff_t message_size;
+};
 
 struct dbox_save_context {
 	struct mail_save_context ctx;
 
 	struct dbox_mailbox *mbox;
 	struct mail_index_transaction *trans;
-	struct dbox_uidlist_append_ctx *append_ctx;
 
-	struct mail_index_sync_ctx *index_sync_ctx;
+	struct dbox_index_append_context *append_ctx;
+	struct dbox_sync_context *sync_ctx;
 
 	/* updated for each appended mail: */
 	uint32_t seq;
 	struct istream *input;
-	struct ostream *output;
-	struct dbox_file *file;
-	struct mail *mail;
-	uint64_t hdr_offset;
-	uint64_t mail_offset;
+	struct mail *mail, *cur_dest_mail;
+	time_t cur_received_date;
+	enum mail_flags cur_flags;
+	string_t *cur_keywords;
+
+	struct dbox_file *cur_file;
+	struct ostream *cur_output;
+
+	ARRAY_DEFINE(mails, struct dbox_save_mail);
 
 	unsigned int failed:1;
 	unsigned int finished:1;
 };
 
-static int
-dbox_save_add_keywords(struct dbox_save_context *ctx,
-		       const struct mail_keywords *keywords,
-		       buffer_t *file_keywords)
+static void dbox_save_keywords(struct dbox_save_context *ctx,
+			       struct mail_keywords *keywords)
 {
-	ARRAY_TYPE(seq_range) new_keywords;
-	const struct seq_range *range;
-	unsigned int i, count, file_idx;
-	int ret = 0;
-
-	/* Get a list of all new keywords. Using seq_range is the easiest
-	   way to do this and should be pretty fast too. */
-	t_push();
-	t_array_init(&new_keywords, 16);
-	for (i = 0; i < keywords->count; i++) {
-		/* check if it's already in the file */
-		if (dbox_file_lookup_keyword(ctx->mbox, ctx->file,
-					     keywords->idx[i], &file_idx)) {
-			buffer_write(file_keywords, file_idx, "1", 1);
-			continue;
-		}
-
-		/* add it. if it already exists, it's handled internally. */
-		seq_range_array_add(&new_keywords, 0, keywords->idx[i]);
-	}
-
-	/* now, write them to file */
-	range = array_get(&new_keywords, &count);
-	if (count > 0) {
-		if (dbox_file_append_keywords(ctx->mbox, ctx->file,
-					      range, count) < 0) {
-			ret = -1;
-			count = 0;
-		}
-
-		/* write the new keywords to file_keywords */
-		for (i = 0; i < count; i++) {
-			unsigned int kw;
-
-			for (kw = range[i].seq1; kw <= range[i].seq2; kw++) {
-				if (!dbox_file_lookup_keyword(ctx->mbox,
-							      ctx->file, kw,
-							      &file_idx)) {
-					/* it should have been found */
-					i_unreached();
-					continue;
-				}
-
-				buffer_write(file_keywords, file_idx, "1", 1);
-			}
-		}
-	}
-
-	t_pop();
-	return ret;
+	if (ctx->cur_keywords == NULL)
+		ctx->cur_keywords = str_new(default_pool, 128);
+	else
+		str_truncate(ctx->cur_keywords, 0);
+	dbox_mail_metadata_keywords_append(ctx->mbox, ctx->cur_keywords,
+					   keywords);
 }
 
 int dbox_save_init(struct mailbox_transaction_context *_t,
@@ -106,117 +71,33 @@
 		(struct dbox_transaction_context *)_t;
 	struct dbox_mailbox *mbox = (struct dbox_mailbox *)t->ictx.ibox;
 	struct dbox_save_context *ctx = t->save_ctx;
-	struct dbox_mail_header hdr;
-	const struct stat *st;
-	buffer_t *file_keywords = NULL;
+	struct ostream *output;
+	struct dbox_message_header dbox_msg_hdr;
+	struct dbox_save_mail *save_mail;
 	enum mail_flags save_flags;
-	unsigned int i, pos, left;
-	char buf[128];
-	int ret;
+	const struct stat *st;
+	uoff_t mail_size;
 
 	i_assert((t->ictx.flags & MAILBOX_TRANSACTION_FLAG_EXTERNAL) != 0);
 
-	if (received_date == (time_t)-1)
-		received_date = ioloop_time;
-
 	if (ctx == NULL) {
 		ctx = t->save_ctx = i_new(struct dbox_save_context, 1);
 		ctx->ctx.transaction = &t->ictx.mailbox_ctx;
 		ctx->mbox = mbox;
 		ctx->trans = t->ictx.trans;
-		ctx->append_ctx = dbox_uidlist_append_init(mbox->uidlist);
-
-		if ((ret = dbox_sync_is_changed(mbox)) < 0) {
-			ctx->failed = TRUE;
-			return -1;
-		}
-		if (ret > 0) {
-			if (dbox_sync(mbox, FALSE) < 0) {
-				ctx->failed = TRUE;
-				return -1;
-			}
-		}
+		ctx->append_ctx = dbox_index_append_begin(mbox->dbox_index);
+		i_array_init(&ctx->mails, 32);
 	}
-	ctx->input = input;
 
 	/* get the size of the mail to be saved, if possible */
 	st = i_stream_stat(input, TRUE);
-	if (st != NULL && st->st_size == -1)
-		st = NULL;
+	mail_size = st == NULL || st->st_size == -1 ? 0 : st->st_size;
 
-	if (dbox_uidlist_append_locked(ctx->append_ctx, &ctx->file,
-				       st != NULL ? st->st_size : 0) < 0) {
+	if (dbox_index_append_next(ctx->append_ctx, mail_size,
+				   &ctx->cur_file, &ctx->cur_output) < 0) {
 		ctx->failed = TRUE;
 		return -1;
 	}
-	ctx->hdr_offset = ctx->file->output->offset;
-
-	t_push();
-	if (keywords != NULL && keywords->count > 0) {
-		uint32_t uid;
-		time_t mtime;
-
-		/* uidlist must be locked while we're reading or modifying
-		   file's header */
-		if (dbox_uidlist_append_get_first_uid(ctx->append_ctx,
-						      &uid, &mtime) < 0) {
-			ctx->failed = TRUE;
-			t_pop();
-			return -1;
-		}
-
-		/* write keywords to the file */
-		file_keywords = buffer_create_dynamic(pool_datastack_create(),
-						      DBOX_KEYWORD_COUNT);
-		if (dbox_save_add_keywords(ctx, keywords, file_keywords) < 0) {
-			ctx->failed = TRUE;
-			t_pop();
-			return -1;
-		}
-		o_stream_seek(ctx->file->output, ctx->hdr_offset);
-	}
-
-	/* append mail header. UID and mail size are written later. */
-	memset(&hdr, '0', sizeof(hdr));
-	memcpy(hdr.magic, DBOX_MAIL_HEADER_MAGIC, sizeof(hdr.magic));
-	DEC2HEX(hdr.received_time_hex, received_date);
-	DEC2HEX(hdr.save_time_hex, ioloop_time);
-	hdr.answered = (flags & MAIL_ANSWERED) != 0 ? '1' : '0';
-	hdr.flagged = (flags & MAIL_FLAGGED) != 0 ? '1' : '0';
-	hdr.deleted = (flags & MAIL_DELETED) != 0 ? '1' : '0';
-	hdr.seen = (flags & MAIL_SEEN) != 0 ? '1' : '0';
-	hdr.draft = (flags & MAIL_DRAFT) != 0 ? '1' : '0';
-	hdr.expunged = '0';
-	o_stream_send(ctx->file->output, &hdr, sizeof(hdr));
-
-	/* write keywords */
-	if (file_keywords != NULL) {
-		unsigned char *keyword_string;
-		size_t size;
-
-		keyword_string =
-			buffer_get_modifiable_data(file_keywords, &size);
-
-		/* string should be filled with NULs and '1' now.
-		   Change NULs to '0'. */
-		for (i = 0; i < size; i++) {
-			if (keyword_string[i] == '\0')
-				keyword_string[i] = '0';
-		}
-		o_stream_send(ctx->file->output, keyword_string, size);
-	}
-
-	/* fill rest of the header with '0' characters */
-	pos = ctx->file->output->offset - ctx->hdr_offset;
-	i_assert(pos <= ctx->file->mail_header_size);
-	left = ctx->file->mail_header_size - pos;
-	memset(buf, '0', I_MIN(sizeof(buf), left));
-	while (left > sizeof(buf)) {
-		o_stream_send(ctx->file->output, buf, sizeof(buf));
-		left -= sizeof(buf);
-	}
-	o_stream_send(ctx->file->output, buf, left);
-	ctx->mail_offset = ctx->file->output->offset;
 
 	/* add to index */
 	save_flags = flags & ~MAIL_RECENT;
@@ -227,22 +108,36 @@
 		mail_index_update_keywords(ctx->trans, ctx->seq,
 					   MODIFY_REPLACE, keywords);
 	}
-	mail_index_update_ext(ctx->trans, ctx->seq, mbox->dbox_file_ext_idx,
-			      &ctx->file->file_seq, NULL);
-	mail_index_update_ext(ctx->trans, ctx->seq,
-			      mbox->dbox_offset_ext_idx, &ctx->hdr_offset,
-			      NULL);
 
 	if (dest_mail == NULL) {
 		if (ctx->mail == NULL)
-			ctx->mail = index_mail_alloc(_t, 0, NULL);
+			ctx->mail = mail_alloc(_t, 0, NULL);
 		dest_mail = ctx->mail;
 	}
 	mail_set_seq(dest_mail, ctx->seq);
 
-	if (t->first_saved_mail_seq == 0)
-		t->first_saved_mail_seq = ctx->seq;
-	t_pop();
+	ctx->cur_dest_mail = dest_mail;
+	ctx->input = index_mail_cache_parse_init(dest_mail, input);
+
+	save_mail = array_append_space(&ctx->mails);
+	save_mail->file = ctx->cur_file;
+	save_mail->seq = ctx->seq;
+	i_assert(ctx->cur_output->offset <= (uint32_t)-1);
+	save_mail->append_offset = ctx->cur_output->offset;
+
+	/* write a dummy header. it'll get rewritten when we're finished */
+	memset(&dbox_msg_hdr, 0, sizeof(dbox_msg_hdr));
+	o_stream_cork(ctx->cur_output);
+	o_stream_send(ctx->cur_output, &dbox_msg_hdr, sizeof(dbox_msg_hdr));
+
+	output = o_stream_create_lf(ctx->cur_output);
+	o_stream_unref(&ctx->cur_output);
+	ctx->cur_output = output;
+
+	ctx->cur_received_date = received_date != (time_t)-1 ?
+		received_date : ioloop_time;
+	ctx->cur_flags = flags;
+	dbox_save_keywords(ctx, keywords);
 
 	*ctx_r = &ctx->ctx;
 	return ctx->failed ? -1 : 0;
@@ -256,59 +151,109 @@
 	if (ctx->failed)
 		return -1;
 
-	if (o_stream_send_istream(ctx->file->output, ctx->input) < 0) {
-		if (!mail_storage_set_error_from_errno(storage)) {
-			mail_storage_set_critical(storage,
-				"o_stream_send_istream(%s) failed: %m",
-				ctx->file->path);
+	do {
+		if (o_stream_send_istream(ctx->cur_output, ctx->input) < 0) {
+			if (!mail_storage_set_error_from_errno(storage)) {
+				mail_storage_set_critical(storage,
+					"o_stream_send_istream(%s) failed: %m",
+					ctx->cur_file->path);
+			}
+			ctx->failed = TRUE;
+			return -1;
 		}
-		ctx->failed = TRUE;
-		return -1;
+		index_mail_cache_parse_continue(ctx->cur_dest_mail);
+
+		/* both tee input readers may consume data from our primary
+		   input stream. we'll have to make sure we don't return with
+		   one of the streams still having data in them. */
+	} while (i_stream_read(ctx->input) > 0);
+	return 0;
+}
+
+static void dbox_save_write_metadata(struct dbox_save_context *ctx)
+{
+	struct dbox_metadata_header metadata_hdr;
+	char space[DBOX_EXTRA_SPACE];
+	string_t *str;
+	uoff_t vsize;
+
+	memset(&metadata_hdr, 0, sizeof(metadata_hdr));
+	memcpy(metadata_hdr.magic_post, DBOX_MAGIC_POST,
+	       sizeof(metadata_hdr.magic_post));
+	o_stream_send(ctx->cur_output, &metadata_hdr, sizeof(metadata_hdr));
+
+	str = t_str_new(256);
+	/* write first fields that don't change */
+	str_printfa(str, "%c%lx\n", DBOX_METADATA_RECEIVED_TIME,
+		    (unsigned long)ctx->cur_received_date);
+	str_printfa(str, "%c%lx\n", DBOX_METADATA_SAVE_TIME,
+		    (unsigned long)ioloop_time);
+	if (mail_get_virtual_size(ctx->cur_dest_mail, &vsize) < 0)
+		i_unreached();
+	str_printfa(str, "%c%llx\n", DBOX_METADATA_VIRTUAL_SIZE,
+		    (unsigned long long)vsize);
+
+	/* flags */
+	str_append_c(str, DBOX_METADATA_FLAGS);
+	dbox_mail_metadata_flags_append(str, ctx->cur_flags);
+	str_append_c(str, '\n');
+
+	/* keywords */
+	if (ctx->cur_keywords != NULL && str_len(ctx->cur_keywords) > 0) {
+		str_append_c(str, DBOX_METADATA_KEYWORDS);
+		str_append_str(str, ctx->cur_keywords);
+		str_append_c(str, '\n');
 	}
-	return 0;
+
+	o_stream_send(ctx->cur_output, str_data(str), str_len(str));
+	memset(space, ' ', sizeof(space));
+	o_stream_send(ctx->cur_output, space, sizeof(space));
+	o_stream_send(ctx->cur_output, "\n", 1);
 }
 
 int dbox_save_finish(struct mail_save_context *_ctx)
 {
 	struct dbox_save_context *ctx = (struct dbox_save_context *)_ctx;
-	struct dbox_mail_header hdr;
+	struct mail_storage *storage = &ctx->mbox->storage->storage;
+	struct dbox_save_mail *save_mail;
+	uoff_t offset = 0;
+	unsigned int count;
 
 	ctx->finished = TRUE;
+	if (ctx->cur_output == NULL)
+		return -1;
 
-	if (ctx->file != NULL) {
-		/* Make sure the file ends here (we could have been overwriting
-		   some existing aborted mail). In case we failed, truncate the
-		   file to the size before writing. */
-		if (ftruncate(ctx->file->fd, ctx->failed ? ctx->hdr_offset :
-			      ctx->file->output->offset) < 0) {
-			mail_storage_set_critical(&ctx->mbox->storage->storage,
-						  "ftruncate(%s) failed: %m",
-						  ctx->file->path);
+	index_mail_cache_parse_deinit(ctx->cur_dest_mail,
+				      ctx->cur_received_date);
+
+	if (!ctx->failed) {
+		offset = ctx->cur_output->offset;
+		dbox_save_write_metadata(ctx);
+		if (o_stream_flush(ctx->cur_output) < 0) {
+			mail_storage_set_critical(storage,
+				"o_stream_flush(%s) failed: %m",
+				ctx->cur_file->path);
 			ctx->failed = TRUE;
 		}
 	}
 
-	if (!ctx->failed) {
-		/* write mail size to header */
-		DEC2HEX(hdr.mail_size_hex,
-			ctx->file->output->offset - ctx->mail_offset);
+	o_stream_destroy(&ctx->cur_output);
+	i_stream_unref(&ctx->input);
 
-		if (pwrite_full(ctx->file->fd, hdr.mail_size_hex,
-				sizeof(hdr.mail_size_hex), ctx->hdr_offset +
-				offsetof(struct dbox_mail_header,
-					 mail_size_hex)) < 0) {
-			mail_storage_set_critical(&ctx->mbox->storage->storage,
-						  "pwrite_full(%s) failed: %m",
-						  ctx->file->path);
-			ctx->failed = TRUE;
-		}
+	count = array_count(&ctx->mails);
+	save_mail = array_idx_modifiable(&ctx->mails, count - 1);
+	if (ctx->failed) {
+		dbox_file_cancel_append(save_mail->file,
+					save_mail->append_offset);
+		dbox_file_unref(&save_mail->file);
+		array_delete(&ctx->mails, count - 1, 1);
+		return -1;
+	} else {
+		dbox_file_finish_append(save_mail->file);
+		save_mail->message_size = offset - save_mail->append_offset -
+			save_mail->file->msg_header_size;
+		return 0;
 	}
-
-	if (ctx->failed)
-		return -1;
-
-	dbox_uidlist_append_finish_mail(ctx->append_ctx, ctx->file);
-	return 0;
 }
 
 void dbox_save_cancel(struct mail_save_context *_ctx)
@@ -319,102 +264,190 @@
 	(void)dbox_save_finish(_ctx);
 }
 
+static int
+dbox_save_mail_write_header(struct dbox_save_mail *mail, uint32_t uid)
+{
+	struct dbox_message_header dbox_msg_hdr;
+	struct ostream *output = mail->file->output;
+	uoff_t orig_offset;
+	int ret = 0;
+
+	i_assert(mail->file->msg_header_size == sizeof(dbox_msg_hdr));
+
+	mail->file->last_append_uid = uid;
+
+	memset(&dbox_msg_hdr, ' ', sizeof(dbox_msg_hdr));
+	memcpy(dbox_msg_hdr.magic_pre, DBOX_MAGIC_PRE,
+	       sizeof(dbox_msg_hdr.magic_pre));
+	dbox_msg_hdr.type = DBOX_MESSAGE_TYPE_NORMAL;
+	dec2hex(dbox_msg_hdr.uid_hex, uid, sizeof(dbox_msg_hdr.uid_hex));
+	dec2hex(dbox_msg_hdr.message_size_hex, mail->message_size,
+		sizeof(dbox_msg_hdr.message_size_hex));
+	dbox_msg_hdr.save_lf = '\n';
+
+	orig_offset = output->offset;
+	o_stream_seek(output, mail->append_offset);
+	if (o_stream_send(output, &dbox_msg_hdr, sizeof(dbox_msg_hdr)) < 0 ||
+	    o_stream_flush(output) < 0) {
+		dbox_file_set_syscall_error(mail->file, "write");
+		ret = -1;
+	}
+	o_stream_seek(output, orig_offset);
+	return ret;
+}
+
+static int
+dbox_save_file_write_append_offset(struct dbox_file *file, uoff_t append_offset)
+{
+	char buf[8+1];
+
+	i_assert(append_offset <= (uint32_t)-1);
+
+	i_snprintf(buf, sizeof(buf), "%08x", (unsigned int)append_offset);
+	if (pwrite_full(file->fd, buf, sizeof(buf)-1,
+			file->append_offset_header_pos) < 0) {
+		dbox_file_set_syscall_error(file, "pwrite");
+		return -1;
+	}
+	return 0;
+}
+
+static int dbox_save_file_commit_header(struct dbox_save_mail *mail)
+{
+	uoff_t append_offset;
+
+	append_offset = dbox_file_get_next_append_offset(mail->file);
+	return dbox_save_file_write_append_offset(mail->file, append_offset);
+}
+
+static void dbox_save_file_uncommit_header(struct dbox_save_mail *mail)
+{
+	if (mail->file->file_id == 0) {
+		/* temporary file, we'll just unlink it later */
+		return;
+	}
+	(void)dbox_save_file_write_append_offset(mail->file,
+						 mail->append_offset);
+}
+
+static int dbox_save_mail_file_cmp(const void *p1, const void *p2)
+{
+	const struct dbox_save_mail *m1 = p1, *m2 = p2;
+	int ret;
+
+	ret = strcmp(m1->file->path, m2->file->path);
+	if (ret == 0) {
+		/* the oldest sequence is first. this is needed for uncommit
+		   to work right. */
+		ret = (int)m1->seq - (int)m2->seq;
+	}
+	return ret;
+}
+
+static int dbox_save_commit(struct dbox_save_context *ctx, uint32_t first_uid)
+{
+	struct dbox_mail_index_record rec;
+	struct dbox_save_mail *mails;
+	unsigned int i, count;
+
+	/* first write updated mail headers and collect all files we wrote to */
+	mails = array_get_modifiable(&ctx->mails, &count);
+	for (i = 0; i < count; i++) {
+		if (dbox_save_mail_write_header(&mails[i], first_uid++) < 0)
+			return -1;
+	}
+
+	/* update append offsets in file headers */
+	qsort(mails, count, sizeof(*mails), dbox_save_mail_file_cmp);
+	for (i = 0; i < count; i++) {
+		if (i > 0 && mails[i].file == mails[i-1].file) {
+			/* already written */
+			continue;
+		}
+
+		if (dbox_save_file_commit_header(&mails[i]) < 0) {
+			/* have to uncommit all changes so far */
+			for (; i > 0; i--) {
+				if (i > 1 &&
+				    mails[i-2].file == mails[i-1].file)
+					continue;
+				dbox_save_file_uncommit_header(&mails[i-1]);
+			}
+			return -1;
+		}
+	}
+
+	/* set file_id / offsets to records */
+	if (dbox_index_append_assign_file_ids(ctx->append_ctx) < 0)
+		return -1;
+
+	memset(&rec, 0, sizeof(rec));
+	for (i = 0; i < count; i++) {
+		rec.file_id = mails[i].file->file_id;
+		rec.offset = mails[i].append_offset;
+
+		if ((rec.file_id & DBOX_FILE_ID_FLAG_UID) == 0) {
+			mail_index_update_ext(ctx->trans, mails[i].seq,
+					      ctx->mbox->dbox_ext_id,
+					      &rec, NULL);
+		}
+	}
+	return 0;
+}
+
 int dbox_transaction_save_commit_pre(struct dbox_save_context *ctx)
 {
 	struct dbox_transaction_context *t =
 		(struct dbox_transaction_context *)ctx->ctx.transaction;
-	struct dbox_mail_header hdr;
-	struct dbox_file *file;
-	struct mail_index_view *view;
-	const struct mail_index_header *idx_hdr;
-	uint32_t seq, uid, next_uid, file_seq;
-	time_t old_mtime, new_mtime;
-	uoff_t offset;
-	int ret;
+	const struct mail_index_header *hdr;
+	uint32_t uid, next_uid;
 
 	i_assert(ctx->finished);
 
-	/* uidlist locking is done before index locking. */
-	if (dbox_uidlist_append_get_first_uid(ctx->append_ctx,
-					      &uid, &old_mtime) < 0) {
-		ctx->failed = TRUE;
-		dbox_transaction_save_rollback(ctx);
-		return -1;
-	}
-	mail_index_append_assign_uids(ctx->trans, uid, &next_uid);
-
-	*t->ictx.first_saved_uid = uid;
-	*t->ictx.last_saved_uid = next_uid - 1;
-
-	/* update UIDs */
-	for (seq = t->first_saved_mail_seq; seq <= ctx->seq; seq++, uid++) {
-		ret = dbox_mail_lookup_offset(&t->ictx, seq,
-					      &file_seq, &offset);
-		i_assert(ret > 0); /* it's in memory, shouldn't fail! */
-
-		DEC2HEX(hdr.uid_hex, uid);
-
-		file = dbox_uidlist_append_lookup_file(ctx->append_ctx,
-						       file_seq);
-		if (pwrite_full(file->fd, hdr.uid_hex,
-				sizeof(hdr.uid_hex), offset +
-				offsetof(struct dbox_mail_header,
-					 uid_hex)) < 0) {
-			mail_storage_set_critical(&ctx->mbox->storage->storage,
-						  "pwrite_full(%s) failed: %m",
-						  file->path);
-			ctx->failed = TRUE;
-                        dbox_transaction_save_rollback(ctx);
-			return -1;
-		}
-	}
-
-	/* lock index lock before dropping uidlist lock in _append_commit() */
-	if (mail_index_sync_begin(ctx->mbox->ibox.index, &ctx->index_sync_ctx,
-				  &view, &ctx->trans, 0) < 0) {
+	if (dbox_sync_begin(ctx->mbox, &ctx->sync_ctx, FALSE) < 0) {
 		ctx->failed = TRUE;
 		dbox_transaction_save_rollback(ctx);
 		return -1;
 	}
 
-	if (dbox_uidlist_append_commit(ctx->append_ctx, &new_mtime) < 0) {
-		mail_index_sync_rollback(&ctx->index_sync_ctx);
-		i_free(ctx);
+	hdr = mail_index_get_header(ctx->sync_ctx->sync_view);
+	uid = hdr->next_uid;
+	mail_index_append_assign_uids(ctx->trans, uid, &next_uid);
+
+	if (dbox_save_commit(ctx, uid) < 0) {
+		ctx->failed = TRUE;
+		dbox_transaction_save_rollback(ctx);
 		return -1;
 	}
 
-	idx_hdr = mail_index_get_header(view);
-	if ((uint32_t)old_mtime == idx_hdr->sync_stamp &&
-	    old_mtime != new_mtime) {
-		/* index was fully synced. keep it that way. */
-		uint32_t sync_stamp = new_mtime;
+	*t->ictx.saved_uid_validity = hdr->uid_validity;
+	*t->ictx.first_saved_uid = uid;
+	*t->ictx.last_saved_uid = next_uid - 1;
 
-		mail_index_update_header(ctx->trans,
-			offsetof(struct mail_index_header, sync_stamp),
-			&sync_stamp, sizeof(sync_stamp), TRUE);
-	}
-
+	dbox_index_append_commit(&ctx->append_ctx);
 	return 0;
 }
 
 void dbox_transaction_save_commit_post(struct dbox_save_context *ctx)
 {
-	mail_index_sync_rollback(&ctx->index_sync_ctx);
-	if (ctx->mail != NULL)
-		index_mail_free(ctx->mail);
-	i_free(ctx);
+	(void)dbox_sync_finish(&ctx->sync_ctx, TRUE);
+	dbox_transaction_save_rollback(ctx);
 }
 
 void dbox_transaction_save_rollback(struct dbox_save_context *ctx)
 {
 	if (!ctx->finished)
 		dbox_save_cancel(&ctx->ctx);
+	if (ctx->append_ctx != NULL)
+		dbox_index_append_rollback(&ctx->append_ctx);
 
-	if (ctx->index_sync_ctx != NULL)
-		mail_index_sync_rollback(&ctx->index_sync_ctx);
+	if (ctx->sync_ctx != NULL)
+		(void)dbox_sync_finish(&ctx->sync_ctx, FALSE);
 
-        dbox_uidlist_append_rollback(ctx->append_ctx);
 	if (ctx->mail != NULL)
-		index_mail_free(ctx->mail);
+		mail_free(&ctx->mail);
+	if (ctx->cur_keywords != NULL)
+		str_free(&ctx->cur_keywords);
+	array_free(&ctx->mails);
 	i_free(ctx);
 }