--- a/src/lib-storage/index/dbox/dbox-file.c	Wed Mar 04 17:40:24 2009 -0500
+++ b/src/lib-storage/index/dbox/dbox-file.c	Wed Mar 04 17:40:46 2009 -0500
@@ -489,8 +489,8 @@
 
 void dbox_file_close(struct dbox_file *file)
 {
-	i_assert(file->lock == NULL);
-
+	if (file->lock != NULL)
+		file_lock_free(&file->lock);
 	if (file->input != NULL)
 		i_stream_unref(&file->input);
 	if (file->output != NULL)

--- a/src/lib-storage/index/dbox/dbox-map.c	Wed Mar 04 17:40:24 2009 -0500
+++ b/src/lib-storage/index/dbox/dbox-map.c	Wed Mar 04 17:40:46 2009 -0500
@@ -25,6 +25,7 @@
 	struct mail_index_view *view;
 
 	uint32_t map_ext_id, ref_ext_id;
+	ARRAY_TYPE(seq_range) ref0_file_ids;
 };
 
 struct dbox_map_append {
@@ -36,6 +37,10 @@
 	struct dbox_mailbox *mbox;
 	struct dbox_map *map;
 
+	struct mail_index_sync_ctx *sync_ctx;
+	struct mail_index_view *sync_view;
+	struct mail_index_transaction *trans;
+
 	ARRAY_DEFINE(files, struct dbox_file *);
 	ARRAY_DEFINE(appends, struct dbox_map_append);
 
@@ -71,6 +76,8 @@
 
 	*_map = NULL;
 
+	if (array_is_created(&map->ref0_file_ids))
+		array_free(&map->ref0_file_ids);
 	if (map->view != NULL)
 		mail_index_view_close(&map->view);
 	mail_index_free(&map->index);
@@ -108,6 +115,11 @@
 	struct mail_index_view_sync_ctx *ctx;
 	bool delayed_expunges;
 
+	if (mail_index_refresh(map->view->index) < 0) {
+		mail_storage_set_internal_error(&map->storage->storage);
+		mail_index_reset_error(map->index);
+		return -1;
+	}
 	ctx = mail_index_view_sync_begin(map->view,
 				MAIL_INDEX_VIEW_SYNC_FLAG_FIX_INCONSISTENT);
 	if (mail_index_view_sync_commit(&ctx, &delayed_expunges) < 0) {
@@ -144,11 +156,9 @@
 	return 0;
 }
 
-int dbox_map_lookup(struct dbox_map *map, uint32_t map_uid,
-		    uint32_t *file_id_r, uoff_t *offset_r)
+static int
+dbox_map_get_seq(struct dbox_map *map, uint32_t map_uid, uint32_t *seq_r)
 {
-	uint32_t seq;
-	uoff_t size;
 	int ret;
 
 	if ((ret = dbox_map_open(map, FALSE)) <= 0) {
@@ -156,19 +166,141 @@
 		return ret;
 	}
 
-	if (!mail_index_lookup_seq(map->view, map_uid, &seq)) {
+	if (!mail_index_lookup_seq(map->view, map_uid, seq_r)) {
 		/* not found - try again after a refresh */
 		if (dbox_map_refresh(map) < 0)
 			return -1;
-		if (!mail_index_lookup_seq(map->view, map_uid, &seq))
+		if (!mail_index_lookup_seq(map->view, map_uid, seq_r))
 			return 0;
 	}
+	return 1;
+}
+
+int dbox_map_lookup(struct dbox_map *map, uint32_t map_uid,
+		    uint32_t *file_id_r, uoff_t *offset_r)
+{
+	uint32_t seq;
+	uoff_t size;
+	int ret;
+
+	if ((ret = dbox_map_get_seq(map, map_uid, &seq)) <= 0)
+		return ret;
 
 	if (dbox_map_lookup_seq(map, seq, file_id_r, offset_r, &size) < 0)
 		return 0;
 	return 1;
 }
 
+int dbox_map_get_file_msgs(struct dbox_map *map, uint32_t file_id,
+			   ARRAY_TYPE(dbox_map_file_msg) *recs)
+{
+	const struct mail_index_header *hdr;
+	struct dbox_map_file_msg msg;
+	const struct dbox_mail_index_map_record *rec;
+	const uint16_t *ref16_p;
+	unsigned int seq;
+	const void *data;
+	bool expunged;
+
+	(void)dbox_map_refresh(map);
+	hdr = mail_index_get_header(map->view);
+
+	memset(&msg, 0, sizeof(msg));
+	for (seq = 1; seq <= hdr->messages_count; seq++) {
+		mail_index_lookup_uid(map->view, seq, &msg.map_uid);
+
+		mail_index_lookup_ext(map->view, seq, map->map_ext_id,
+				      &data, &expunged);
+		if (data == NULL) {
+			// FIXME
+			break;
+		}
+		rec = data;
+		if (rec->file_id != file_id)
+			continue;
+
+		msg.offset = rec->offset;
+		mail_index_lookup_ext(map->view, seq, map->ref_ext_id,
+				      &data, &expunged);
+		if (data == NULL) {
+			// FIXME
+			break;
+		}
+		ref16_p = data;
+		msg.refcount = *ref16_p;
+
+		array_append(recs, &msg, 1);
+	}
+	return 0;
+}
+
+const ARRAY_TYPE(seq_range) *dbox_map_get_zero_ref_files(struct dbox_map *map)
+{
+	const struct mail_index_header *hdr;
+	const struct dbox_mail_index_map_record *rec;
+	const uint16_t *ref16_p;
+	const void *data;
+	uint32_t seq;
+	bool expunged;
+
+	(void)dbox_map_refresh(map);
+
+	if (array_is_created(&map->ref0_file_ids))
+		array_clear(&map->ref0_file_ids);
+	else
+		i_array_init(&map->ref0_file_ids, 64);
+	hdr = mail_index_get_header(map->view);
+	for (seq = 1; seq <= hdr->messages_count; seq++) {
+		mail_index_lookup_ext(map->view, seq, map->ref_ext_id,
+				      &data, &expunged);
+		if (data != NULL && !expunged) {
+			ref16_p = data;
+			if (*ref16_p != 0)
+				continue;
+		}
+
+		mail_index_lookup_ext(map->view, seq, map->map_ext_id,
+				      &data, &expunged);
+		if (data != NULL && !expunged) {
+			rec = data;
+			seq_range_array_add(&map->ref0_file_ids, 0, seq);
+		}
+	}
+	return &map->ref0_file_ids;
+}
+
+int dbox_map_update_refcounts(struct dbox_map *map,
+			      const ARRAY_TYPE(seq_range) *map_uids, int diff)
+{
+	struct mail_index_transaction *trans;
+	struct seq_range_iter iter;
+	unsigned int i;
+	uint32_t map_uid, seq;
+	int ret = 0;
+
+	trans = mail_index_transaction_begin(map->view, 0);
+	seq_range_array_iter_init(&iter, map_uids); i = 0;
+	while (seq_range_array_iter_nth(&iter, i++, &map_uid)) {
+		if ((ret = dbox_map_get_seq(map, map_uid, &seq)) <= 0) {
+			if (ret < 0)
+				break;
+		} else {
+			mail_index_atomic_inc_ext(trans, seq,
+						  map->ref_ext_id, diff);
+		}
+	}
+	if (ret < 0) {
+		mail_index_transaction_rollback(&trans);
+		return -1;
+	} else {
+		uint32_t log_seq;
+		uoff_t log_offset;
+
+		return mail_index_transaction_commit(&trans, &log_seq,
+						     &log_offset);
+	}
+}
+
 struct dbox_map_append_context *
 dbox_map_append_begin(struct dbox_mailbox *mbox)
 {
@@ -235,6 +367,12 @@
 		return TRUE;
 	if (deleted)
 		return TRUE;
+	if (file->lock != NULL) {
+		/* already locked, we're possibly in the middle of cleaning
+		   it up in which case we really don't want to write there. */
+		dbox_file_unref(&file);
+		return TRUE;
+	}
 
 	if (file->create_time < stamp)
 		file_too_old = TRUE;
@@ -475,19 +613,75 @@
 	return 0;
 }
 
+static int dbox_map_assign_file_ids(struct dbox_map_append_context *ctx)
+{
+	struct dbox_file *const *files;
+	unsigned int i, count;
+	uint32_t first_file_id, file_id;
+	int ret;
+
+	/* start the syncing. we'll need it even if there are no file ids to
+	   be assigned. */
+	ret = mail_index_sync_begin(ctx->map->index, &ctx->sync_ctx,
+				    &ctx->sync_view, &ctx->trans, 0);
+	if (ret <= 0) {
+		i_assert(ret != 0);
+		mail_storage_set_internal_error(&ctx->map->storage->storage);
+		mail_index_reset_error(ctx->map->index);
+		return -1;
+	}
+
+	if (dbox_map_get_next_file_id(ctx->map, ctx->sync_view, &file_id) < 0) {
+		mail_index_sync_rollback(&ctx->sync_ctx);
+		return -1;
+	}
+
+	/* assign file_ids for newly created multi-files */
+	first_file_id = file_id;
+	files = array_get(&ctx->files, &count);
+	for (i = 0; i < count; i++) {
+		if (files[i]->single_mbox != NULL)
+			continue;
+
+		if (files[i]->output != NULL) {
+			if (dbox_file_flush_append(files[i]) < 0) {
+				ret = -1;
+				break;
+			}
+		}
+
+		if (files[i]->file_id == 0) {
+			if (dbox_file_assign_id(files[i], file_id++) < 0) {
+				ret = -1;
+				break;
+			}
+		}
+	}
+
+	if (ret < 0) {
+		/* FIXME: we have to rollback the changes we made */
+		mail_index_sync_rollback(&ctx->sync_ctx);
+		return -1;
+	}
+
+	/* update the highest used file_id */
+	if (first_file_id != file_id) {
+		file_id--;
+		mail_index_update_header_ext(ctx->trans, ctx->map->map_ext_id,
+					     0, &file_id, sizeof(file_id));
+	}
+	return 0;
+}
+
 int dbox_map_append_assign_map_uids(struct dbox_map_append_context *ctx,
 				    uint32_t *first_map_uid_r,
 				    uint32_t *last_map_uid_r)
 {
-	struct dbox_file *const *files;
 	const struct dbox_map_append *appends;
-	struct mail_index_sync_ctx *sync_ctx;
-	struct mail_index_view *sync_view;
-	struct mail_index_transaction *trans;
 	const struct mail_index_header *hdr;
 	struct dbox_mail_index_map_record rec;
 	unsigned int i, count;
-	uint32_t seq, first_uid, next_uid, first_file_id, file_id;
+	uint32_t seq, first_uid, next_uid;
 	uint16_t ref16;
 	int ret = 0;
 
@@ -497,55 +691,8 @@
 		return 0;
 	}
 
-	ret = mail_index_sync_begin(ctx->map->index, &sync_ctx, &sync_view,
-				    &trans, 0);
-	if (ret <= 0) {
-		i_assert(ret != 0);
-		mail_storage_set_internal_error(&ctx->map->storage->storage);
-		mail_index_reset_error(ctx->map->index);
-		return -1;
-	}
-
-	if (dbox_map_get_next_file_id(ctx->map, sync_view, &file_id) < 0) {
-		mail_index_sync_rollback(&sync_ctx);
+	if (dbox_map_assign_file_ids(ctx) < 0)
 		return -1;
-	}
-
-	/* assign file_ids for newly created multi-files */
-	files = array_get(&ctx->files, &count);
-	first_file_id = file_id;
-	for (i = 0; i < count; i++) {
-		if (files[i]->single_mbox != NULL)
-			continue;
-
-		if (files[i]->single_mbox == NULL && files[i]->output != NULL) {
-			if (dbox_file_flush_append(files[i]) < 0) {
-				ret = -1;
-				break;
-			}
-		}
-
-		if (files[i]->file_id != 0)
-			continue;
-
-		if (dbox_file_assign_id(files[i], file_id++) < 0) {
-			ret = -1;
-			break;
-		}
-	}
-
-	if (ret < 0) {
-		/* FIXME: we have to rollback the changes we made */
-		mail_index_sync_rollback(&sync_ctx);
-		return -1;
-	}
-
-	/* update the highest used file_id */
-	if (first_file_id != file_id) {
-		file_id--;
-		mail_index_update_header_ext(trans, ctx->map->map_ext_id,
-					     0, &file_id, sizeof(file_id));
-	}
 
 	/* append map records to index */
 	memset(&rec, 0, sizeof(rec));
@@ -559,28 +706,28 @@
 		rec.offset = appends[i].offset;
 		rec.size = appends[i].size;
 
-		mail_index_append(trans, 0, &seq);
-		mail_index_update_ext(trans, seq, ctx->map->map_ext_id,
+		mail_index_append(ctx->trans, 0, &seq);
+		mail_index_update_ext(ctx->trans, seq, ctx->map->map_ext_id,
 				      &rec, NULL);
-		mail_index_update_ext(trans, seq, ctx->map->ref_ext_id,
+		mail_index_update_ext(ctx->trans, seq, ctx->map->ref_ext_id,
 				      &ref16, NULL);
 	}
 
 	/* assign map UIDs for appended records */
-	hdr = mail_index_get_header(sync_view);
+	hdr = mail_index_get_header(ctx->sync_view);
 	first_uid = hdr->next_uid;
-	mail_index_append_assign_uids(trans, first_uid, &next_uid);
+	mail_index_append_assign_uids(ctx->trans, first_uid, &next_uid);
 	i_assert(next_uid - first_uid == count);
 
 	if (hdr->uid_validity == 0) {
 		/* we don't really care about uidvalidity, but it can't be 0 */
 		uint32_t uid_validity = ioloop_time;
-		mail_index_update_header(trans,
+		mail_index_update_header(ctx->trans,
 			offsetof(struct mail_index_header, uid_validity),
 			&uid_validity, sizeof(uid_validity), TRUE);
 	}
 
-	if (mail_index_sync_commit(&sync_ctx) < 0) {
+	if (mail_index_sync_commit(&ctx->sync_ctx) < 0) {
 		mail_storage_set_internal_error(&ctx->map->storage->storage);
 		mail_index_reset_error(ctx->map->index);
 		return -1;
@@ -591,6 +738,51 @@
 	return ret;
 }
 
+int dbox_map_append_move(struct dbox_map_append_context *ctx,
+			 ARRAY_TYPE(seq_range) *map_uids,
+			 ARRAY_TYPE(seq_range) *expunge_map_uids)
+{
+	const struct dbox_map_append *appends;
+	struct dbox_mail_index_map_record rec;
+	struct seq_range_iter iter;
+	unsigned int i, j, appends_count;
+	uint32_t uid, seq;
+
+	if (dbox_map_assign_file_ids(ctx) < 0)
+		return -1;
+
+	memset(&rec, 0, sizeof(rec));
+	appends = array_get(&ctx->appends, &appends_count);
+
+	seq_range_array_iter_init(&iter, map_uids); i = j = 0;
+	while (seq_range_array_iter_nth(&iter, i++, &uid)) {
+		i_assert(j < appends_count);
+		rec.file_id = appends[j].file->file_id;
+		rec.offset = appends[j].offset;
+		rec.size = appends[j].size;
+		j++;
+
+		if (!mail_index_lookup_seq(ctx->sync_view, uid, &seq))
+			i_unreached();
+		mail_index_update_ext(ctx->trans, seq, ctx->map->map_ext_id,
+				      &rec, NULL);
+	}
+
+	seq_range_array_iter_init(&iter, expunge_map_uids); i = 0;
+	while (seq_range_array_iter_nth(&iter, i++, &uid)) {
+		if (!mail_index_lookup_seq(ctx->sync_view, uid, &seq))
+			i_unreached();
+		mail_index_expunge(ctx->trans, seq);
+	}
+
+	if (mail_index_sync_commit(&ctx->sync_ctx) < 0) {
+		mail_storage_set_internal_error(&ctx->map->storage->storage);
+		mail_index_reset_error(ctx->map->index);
+		return -1;
+	}
+	return 0;
+}
+
 int dbox_map_append_assign_uids(struct dbox_map_append_context *ctx,
 				uint32_t first_uid, uint32_t last_uid)
 {

--- a/src/lib-storage/index/dbox/dbox-map.h	Wed Mar 04 17:40:24 2009 -0500
+++ b/src/lib-storage/index/dbox/dbox-map.h	Wed Mar 04 17:40:46 2009 -0500
@@ -5,12 +5,29 @@
 struct dbox_file;
 struct dbox_map_append_context;
 
+struct dbox_map_file_msg {
+	uint32_t map_uid;
+	uint32_t offset;
+	uint32_t refcount;
+};
+ARRAY_DEFINE_TYPE(dbox_map_file_msg, struct dbox_map_file_msg);
+
 struct dbox_map *dbox_map_init(struct dbox_storage *storage);
 void dbox_map_deinit(struct dbox_map **map);
 
 int dbox_map_lookup(struct dbox_map *map, uint32_t map_uid,
 		    uint32_t *file_id_r, uoff_t *offset_r);
 
+/* Get all messages from file */
+int dbox_map_get_file_msgs(struct dbox_map *map, uint32_t file_id,
+			   ARRAY_TYPE(dbox_map_file_msg) *recs);
+
+int dbox_map_update_refcounts(struct dbox_map *map,
+			      const ARRAY_TYPE(seq_range) *map_uids, int diff);
+
+/* Return all files containing messages with zero refcount. */
+const ARRAY_TYPE(seq_range) *dbox_map_get_zero_ref_files(struct dbox_map *map);
+
 struct dbox_map_append_context *
 dbox_map_append_begin(struct dbox_mailbox *mbox);
 /* Request file for saving a new message with given size (if available). If an
@@ -27,6 +44,11 @@
 /* Assign UIDs to all created single-files. */
 int dbox_map_append_assign_uids(struct dbox_map_append_context *ctx,
 				uint32_t first_uid, uint32_t last_uid);
+/* The appends are existing messages that were simply moved to a new file.
+   map_uids contains the moved messages' map UIDs. */
+int dbox_map_append_move(struct dbox_map_append_context *ctx,
+			 ARRAY_TYPE(seq_range) *map_uids,
+			 ARRAY_TYPE(seq_range) *expunge_map_uids);
 /* Returns 0 if ok, -1 if error. */
 void dbox_map_append_commit(struct dbox_map_append_context **ctx);
 void dbox_map_append_rollback(struct dbox_map_append_context **ctx);

--- a/src/lib-storage/index/dbox/dbox-sync-file.c	Wed Mar 04 17:40:24 2009 -0500
+++ b/src/lib-storage/index/dbox/dbox-sync-file.c	Wed Mar 04 17:40:46 2009 -0500
@@ -7,8 +7,15 @@
 #include "str.h"
 #include "dbox-storage.h"
 #include "dbox-file.h"
+#include "dbox-map.h"
 #include "dbox-sync.h"
 
+struct dbox_mail_move {
+	struct dbox_file *file;
+	uint32_t offset;
+};
+ARRAY_DEFINE_TYPE(dbox_mail_move, struct dbox_mail_move);
+
 static int dbox_sync_file_unlink(struct dbox_file *file)
 {
 	const char *path, *primary_path;
@@ -36,210 +43,133 @@
 }
 
 static int
-dbox_sync_file_expunge(struct dbox_sync_context *ctx, struct dbox_file *file,
-		       const struct dbox_sync_file_entry *entry)
+dbox_sync_file_cleanup(struct dbox_sync_context *ctx, struct dbox_file *file)
 {
-#if 0 //FIXME
-	const struct seq_range *expunges;
-	struct dbox_file *out_file = NULL;
+	struct dbox_file *out_file;
 	struct istream *input;
-	struct ostream *output;
-	uint32_t file_id, seq, uid;
-	uoff_t first_offset, offset, physical_size;
-	const char *out_path;
+	struct ostream *output = NULL;
+	struct dbox_metadata_header meta_hdr;
+	struct dbox_map_append_context *append_ctx;
+	ARRAY_TYPE(dbox_map_file_msg) msgs_arr;
+	const struct dbox_map_file_msg *msgs;
+	ARRAY_TYPE(seq_range) copied_map_uids, expunged_map_uids;
 	unsigned int i, count;
+	uoff_t physical_size, msg_size;
+	const unsigned char *data;
+	size_t size;
+	const char *line;
 	bool expunged;
 	int ret;
 
-	/* FIXME: lock the file first */
+	if ((ret = dbox_file_try_lock(file)) <= 0)
+		return ret;
+
+	append_ctx = dbox_map_append_begin(ctx->mbox);
 
-	expunges = array_get(&entry->expunges, &count);
-	if (!dbox_file_lookup(ctx->mbox, ctx->sync_view, expunges[0].seq1,
-			      &file_id, &first_offset))
-		return 0;
-	i_assert(file_id == file->file_id);
-	mail_index_expunge(ctx->trans, expunges[0].seq1);
-
-	offset = first_offset;
-	for (i = 0;;) {
-		if ((ret = dbox_file_seek_next(file, &offset,
-					       &physical_size)) <= 0)
+	t_array_init(&msgs_arr, 128);
+	if (dbox_map_get_file_msgs(ctx->mbox->storage->map, file->file_id,
+				   &msgs_arr) < 0) {
+		// FIXME
+		return -1;
+	}
+	msgs = array_get(&msgs_arr, &count);
+	t_array_init(&copied_map_uids, I_MIN(count, 1));
+	t_array_init(&expunged_map_uids, I_MIN(count, 1));
+	for (i = 0; i < count; i++) {
+		if (msgs[i].refcount == 0) {
+			seq_range_array_add(&expunged_map_uids, 0,
+					    msgs[i].map_uid);
+			continue;
+		}
+		ret = dbox_file_get_mail_stream(file, msgs[i].offset,
+						&physical_size,
+						NULL, &expunged);
+		if (ret <= 0) {
+			/* FIXME: handle corruption? */
+			ret = -1;
 			break;
-		if (physical_size == 0) {
-			/* EOF */
-			break;
-		}
-
-		if (i < count) {
-			mail_index_lookup_seq(ctx->sync_view, uid, &seq);
-			while (seq > expunges[i].seq2) {
-				if (++i == count)
-					break;
-			}
-		}
-		if (seq == 0 || (i < count && seq >= expunges[i].seq1 &&
-				 seq <= expunges[i].seq2)) {
-			/* this message gets expunged */
-			if (seq != 0)
-				mail_index_expunge(ctx->trans, seq);
-			continue;
 		}
 
 		/* non-expunged message. write it to output file. */
-		if (out_file == NULL) {
-			out_file = dbox_file_init(ctx->mbox, 0);
-			ret = dbox_file_get_append_stream(out_file,
-							  physical_size,
-							  &output);
-			if (ret <= 0)
-				break;
+		if (dbox_map_append_next(append_ctx, physical_size,
+					 &out_file, &output) < 0) {
+			// FIXME
+			ret = -1;
+			break;
+		}
+		i_assert(file->file_id != out_file->file_id);
+
+		i_stream_seek(file->input, msgs[i].offset);
+		msg_size = file->msg_header_size + physical_size;
+		input = i_stream_create_limit(file->input, msg_size);
+		ret = o_stream_send_istream(output, input);
+		i_stream_unref(&input);
+		if (ret != (off_t)(file->msg_header_size + physical_size)) {
+			// FIXME
+			ret = -1;
+			break;
 		}
 
-		i_stream_seek(file->input, offset);
-		input = i_stream_create_limit(file->input,
-					      file->msg_header_size +
-					      physical_size);
-		ret = o_stream_send_istream(output, input) < 0 ? -1 : 0;
-		i_stream_unref(&input);
-		if (ret < 0)
+		/* copy metadata */
+		i_stream_seek(file->input, msgs[i].offset + msg_size);
+		ret = i_stream_read_data(file->input, &data, &size,
+					 sizeof(meta_hdr));
+		if (ret <= 0) {
+			// FIXME
+			i_assert(ret != -2);
+			ret = -1;
 			break;
-
-		/* write metadata */
-		(void)dbox_file_metadata_seek_mail_offset(file, offset,
-							  &expunged);
-		if ((ret = dbox_file_metadata_write_to(file, output)) < 0)
+		}
+		memcpy(&meta_hdr, data, sizeof(meta_hdr));
+		if (memcmp(meta_hdr.magic_post, DBOX_MAGIC_POST,
+			   sizeof(meta_hdr.magic_post)) != 0) {
+			// FIXME
+			ret = -1;
 			break;
-
-		mail_index_update_flags(ctx->trans, seq, MODIFY_REMOVE,
-			(enum mail_flags)MAIL_INDEX_MAIL_FLAG_DIRTY);
-	}
-
-	out_path = out_file == NULL ? NULL :
-		dbox_file_get_path(out_file);
-	if (ret <= 0) {
-		if (out_file != NULL) {
-			if (unlink(out_path) < 0)
-				i_error("unlink(%s) failed: %m", out_path);
-			o_stream_unref(&output);
 		}
-	} else if (out_file != NULL) {
-		/* FIXME: rename out_file and add to index */
-		o_stream_unref(&output);
-	}
-
-	if (ret <= 0)
-		;
-	else if (first_offset == file->file_header_size) {
-		/* nothing exists in this file anymore */
-		ret = dbox_sync_file_unlink(file);
-	} else {
-		if (ftruncate(file->fd, first_offset) < 0) {
-			dbox_file_set_syscall_error(file, "ftruncate()");
+		i_stream_skip(file->input, sizeof(meta_hdr));
+		o_stream_send(output, &meta_hdr, sizeof(meta_hdr));
+		while ((line = i_stream_read_next_line(file->input)) != NULL) {
+			if (*line == DBOX_METADATA_OLDV1_SPACE || *line == '\0') {
+				/* end of metadata */
+				break;
+			}
+			o_stream_send_str(output, line);
+			o_stream_send(output, "\n", 1);
+		}
+		if (line == NULL) {
+			// FIXME
 			ret = -1;
+			break;
 		}
+		o_stream_send(output, "\n", 1);
+		dbox_map_append_finish_multi_mail(append_ctx);
+		seq_range_array_add(&copied_map_uids, 0, msgs[i].map_uid);
 	}
 
-	if (out_file != NULL)
-		dbox_file_unref(&out_file);
-	return ret;
-#endif
-	return -1;
-}
-
-#if 0
-static int
-dbox_sync_file_split(struct dbox_sync_context *ctx, struct dbox_file *in_file,
-		     uoff_t offset, uint32_t seq)
-{
-	static enum dbox_metadata_key maildir_metadata_keys[] = {
-		DBOX_METADATA_VIRTUAL_SIZE,
-		DBOX_METADATA_RECEIVED_TIME,
-		DBOX_METADATA_SAVE_TIME,
-		DBOX_METADATA_POP3_UIDL
-	};
-	struct dbox_index_append_context *append_ctx;
-	struct dbox_file *out_file;
-	struct istream *input;
-	struct ostream *output;
-	struct dbox_message_header dbox_msg_hdr;
-	struct dbox_mail_index_record rec;
-	const char *out_path, *value;
-	uoff_t size, append_offset;
-	unsigned int i;
-	int ret;
-	bool expunged;
-
-	/* FIXME: for now we handle only maildir file conversion */
-	i_assert(in_file->maildir_file);
-
-	ret = dbox_file_get_mail_stream(in_file, offset,
-					&size, &input, &expunged);
-	if (ret <= 0)
-		return ret;
-	if (expunged) {
-		mail_index_expunge(ctx->trans, seq);
-		return 1;
-	}
-
-	append_ctx = dbox_index_append_begin(ctx->mbox->dbox_index);
-	if (dbox_index_append_next(append_ctx, size, &out_file, &output) < 0 ||
-	    dbox_file_metadata_seek(out_file, 0, &expunged) < 0) {
-		dbox_index_append_rollback(&append_ctx);
-		i_stream_unref(&input);
+	if (ret < 0) {
+		dbox_map_append_rollback(&append_ctx);
 		return -1;
 	}
-	append_offset = output->offset;
-	dbox_msg_header_fill(&dbox_msg_hdr, size);
 
-	/* set static metadata */
-	for (i = 0; i < N_ELEMENTS(maildir_metadata_keys); i++) {
-		value = dbox_file_metadata_get(in_file,
-					       maildir_metadata_keys[i]);
-		if (value != NULL) {
-			dbox_file_metadata_set(out_file,
-					       maildir_metadata_keys[i], value);
-		}
+	if (output == NULL) {
+		/* everything expunged in this file, unlink it */
+		dbox_map_append_rollback(&append_ctx);
+		if (dbox_sync_file_unlink(file) < 0)
+			return -1;
+		return 0;
 	}
 
-	/* copy the message */
-	out_path = dbox_file_get_path(out_file);
-	o_stream_cork(output);
-	if (o_stream_send(output, &dbox_msg_hdr, sizeof(dbox_msg_hdr)) < 0 ||
-	    o_stream_send_istream(output, input) < 0 ||
-	    dbox_file_metadata_write_to(out_file, output) < 0 ||
-	    o_stream_flush(output) < 0) {
-		mail_storage_set_critical(&ctx->mbox->storage->storage,
-			"write(%s) failed: %m", out_path);
-		ret = -1;
-	} else {
-		dbox_file_finish_append(out_file);
-		out_file->last_append_uid = uid;
-
-		ret = dbox_index_append_assign_file_ids(append_ctx);
+	/* assign new file_id + offset to moved messages */
+	if (dbox_map_append_move(append_ctx, &copied_map_uids,
+				 &expunged_map_uids) < 0) {
+		// FIXME
+		return -1;
 	}
-
-	if (ret < 0)
-		dbox_index_append_rollback(&append_ctx);
-	else
-		ret = dbox_index_append_commit(&append_ctx);
-	i_stream_unref(&input);
-
-	if (ret == 0) {
-		/* update message position in index file */
-		memset(&rec, 0, sizeof(rec));
-		if ((rec.file_id & DBOX_FILE_ID_FLAG_UID) == 0) {
-			rec.file_id = out_file->file_id;
-			rec.offset = append_offset;
-		}
-		mail_index_update_ext(ctx->trans, seq, ctx->mbox->dbox_ext_id,
-				      &rec, NULL);
-
-		/* when everything is done, unlink the old file */
-		ret = dbox_sync_file_unlink(in_file);
-	}
-	return ret < 0 ? -1 : 1;
+	dbox_map_append_commit(&append_ctx);
+	(void)dbox_sync_file_unlink(file);
+	return 1;
 }
-#endif
 
 static void
 dbox_sync_file_move_if_needed(struct dbox_file *file,
@@ -256,21 +186,21 @@
 }
 
 static void
-dbox_sync_mark_single_file_expunged(struct dbox_sync_context *ctx,
-				    const struct dbox_sync_file_entry *entry)
+dbox_sync_mark_expunges(struct dbox_sync_context *ctx,
+			const ARRAY_TYPE(seq_range) *seqs)
 {
 	struct mailbox *box = &ctx->mbox->ibox.box;
-	const struct seq_range *expunges;
-	unsigned int count;
-	uint32_t uid;
+	struct seq_range_iter iter;
+	unsigned int i;
+	uint32_t seq, uid;
 
-	expunges = array_get(&entry->expunges, &count);
-	i_assert(count == 1 && expunges[0].seq1 == expunges[0].seq2);
-	mail_index_expunge(ctx->trans, expunges[0].seq1);
-
-	if (box->v.sync_notify != NULL) {
-		mail_index_lookup_uid(ctx->sync_view, expunges[0].seq1, &uid);
-		box->v.sync_notify(box, uid, MAILBOX_SYNC_TYPE_EXPUNGE);
+	seq_range_array_iter_init(&iter, seqs); i = 0;
+	while (seq_range_array_iter_nth(&iter, i++, &seq)) {
+		mail_index_expunge(ctx->trans, seq);
+		if (box->v.sync_notify != NULL) {
+			mail_index_lookup_uid(ctx->sync_view, seq, &uid);
+			box->v.sync_notify(box, uid, MAILBOX_SYNC_TYPE_EXPUNGE);
+		}
 	}
 }
 
@@ -284,20 +214,27 @@
 	file = entry->file_id != 0 ?
 		dbox_file_init_multi(ctx->mbox->storage, entry->file_id) :
 		dbox_file_init_single(ctx->mbox, entry->uid);
-	if (entry->uid != 0 && array_is_created(&entry->expunges)) {
+	if (!array_is_created(&entry->expunge_map_uids)) {
+		/* no expunges - we want to move it */
+		dbox_sync_file_move_if_needed(file, entry);
+	} else if (entry->uid != 0) {
 		/* fast path to expunging the whole file */
 		if ((ret = dbox_sync_file_unlink(file)) == 0) {
 			/* file was lost, delete it */
-			dbox_sync_mark_single_file_expunged(ctx, entry);
+			dbox_sync_mark_expunges(ctx, &entry->expunge_seqs);
 			ret = 1;
 		}
 	} else {
+		if (dbox_map_update_refcounts(ctx->mbox->storage->map,
+					      &entry->expunge_map_uids, -1) < 0)
+			return -1;
+
+		dbox_sync_mark_expunges(ctx, &entry->expunge_seqs);
+
+		/* FIXME: do this cleanup later */
 		ret = dbox_file_open_or_create(file, &deleted);
-		if (ret > 0 && !deleted) {
-			dbox_sync_file_move_if_needed(file, entry);
-			if (array_is_created(&entry->expunges))
-				ret = dbox_sync_file_expunge(ctx, file, entry);
-		}
+		if (ret > 0 && !deleted)
+			ret = dbox_sync_file_cleanup(ctx, file);
 	}
 	dbox_file_unref(&file);
 	return ret;

--- a/src/lib-storage/index/dbox/dbox-sync.c	Wed Mar 04 17:40:24 2009 -0500
+++ b/src/lib-storage/index/dbox/dbox-sync.c	Wed Mar 04 17:40:46 2009 -0500
@@ -67,11 +67,14 @@
 	}
 
 	if (sync_rec->type == MAIL_INDEX_SYNC_TYPE_EXPUNGE) {
-		if (!array_is_created(&entry->expunges)) {
-			p_array_init(&entry->expunges, ctx->pool,
+		if (!array_is_created(&entry->expunge_map_uids)) {
+			p_array_init(&entry->expunge_map_uids, ctx->pool,
+				     lookup_entry.uid != 0 ? 1 : 3);
+			p_array_init(&entry->expunge_seqs, ctx->pool,
 				     lookup_entry.uid != 0 ? 1 : 3);
 		}
-		seq_range_array_add(&entry->expunges, 0, seq);
+		seq_range_array_add(&entry->expunge_seqs, 0, seq);
+		seq_range_array_add(&entry->expunge_map_uids, 0, map_uid);
 	} else {
 		if ((sync_rec->add_flags & DBOX_INDEX_FLAG_ALT) != 0)
 			entry->move_to_alt = TRUE;
@@ -311,3 +314,11 @@
 
 	return index_mailbox_sync_init(box, flags, ret < 0);
 }
+
+void dbox_sync_cleanup(struct dbox_storage *storage)
+{
+	const ARRAY_TYPE(seq_range) *ref0_file_ids;
+	unsigned int i = 0;
+
+	ref0_file_ids = dbox_map_get_zero_ref_files(storage->map);
+}

--- a/src/lib-storage/index/dbox/dbox-sync.h	Wed Mar 04 17:40:24 2009 -0500
+++ b/src/lib-storage/index/dbox/dbox-sync.h	Wed Mar 04 17:40:46 2009 -0500
@@ -8,7 +8,8 @@
 
 	unsigned int move_from_alt:1;
 	unsigned int move_to_alt:1;
-	ARRAY_TYPE(seq_range) expunges;
+	ARRAY_TYPE(seq_range) expunge_seqs;
+	ARRAY_TYPE(seq_range) expunge_map_uids;
 };
 
 struct dbox_sync_context {