--- a/src/lib-index/Makefile.am	Sat May 09 12:32:04 2009 -0400
+++ b/src/lib-index/Makefile.am	Sat May 09 14:53:06 2009 -0400
@@ -19,6 +19,9 @@
         mail-index-map.c \
         mail-index-modseq.c \
         mail-index-transaction.c \
+        mail-index-transaction-export.c \
+        mail-index-transaction-finish.c \
+        mail-index-transaction-sort-appends.c \
         mail-index-transaction-view.c \
         mail-index-strmap.c \
         mail-index-sync.c \

--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/src/lib-index/mail-index-transaction-export.c	Sat May 09 14:53:06 2009 -0400
@@ -0,0 +1,429 @@
+/* Copyright (c) 2003-2009 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "array.h"
+#include "mail-index-private.h"
+#include "mail-index-modseq.h"
+#include "mail-transaction-log-private.h"
+#include "mail-index-transaction-private.h"
+
+struct mail_index_export_context {
+	struct mail_index_transaction *trans;
+	struct mail_transaction_log_append_ctx *append_ctx;
+};
+
+static void log_append_buffer(struct mail_index_export_context *ctx,
+			      const buffer_t *buf, const buffer_t *hdr_buf,
+			      enum mail_transaction_type type)
+{
+	buffer_t *output = ctx->append_ctx->output;
+	struct mail_transaction_header hdr;
+
+	i_assert((type & MAIL_TRANSACTION_TYPE_MASK) != 0);
+	i_assert((buf->used % 4) == 0);
+	i_assert(hdr_buf == NULL || (hdr_buf->used % 4) == 0);
+
+	if (buf->used == 0)
+		return;
+
+	memset(&hdr, 0, sizeof(hdr));
+	hdr.type = type;
+	if (type == MAIL_TRANSACTION_EXPUNGE)
+		hdr.type |= MAIL_TRANSACTION_EXPUNGE_PROT;
+	if ((ctx->trans->flags & MAIL_INDEX_TRANSACTION_FLAG_EXTERNAL) != 0)
+		hdr.type |= MAIL_TRANSACTION_EXTERNAL;
+	hdr.size = sizeof(hdr) + buf->used +
+		(hdr_buf == NULL ? 0 : hdr_buf->used);
+	hdr.size = mail_index_uint32_to_offset(hdr.size);
+
+	buffer_append(output, &hdr, sizeof(hdr));
+	if (hdr_buf != NULL)
+		buffer_append(output, hdr_buf->data, hdr_buf->used);
+	buffer_append(output, buf->data, buf->used);
+
+	if (mail_transaction_header_has_modseq(buf->data,
+				CONST_PTR_OFFSET(buf->data, sizeof(hdr)),
+				ctx->append_ctx->new_highest_modseq))
+		ctx->append_ctx->new_highest_modseq++;
+}
+
+static const buffer_t *
+log_get_hdr_update_buffer(struct mail_index_transaction *t, bool prepend)
+{
+	buffer_t *buf;
+	const unsigned char *data, *mask;
+	struct mail_transaction_header_update u;
+	uint16_t offset;
+	int state = 0;
+
+	memset(&u, 0, sizeof(u));
+
+	data = prepend ? t->pre_hdr_change : t->post_hdr_change;
+	mask = prepend ? t->pre_hdr_mask : t->post_hdr_mask;
+
+	buf = buffer_create_dynamic(pool_datastack_create(), 256);
+	for (offset = 0; offset <= sizeof(t->pre_hdr_change); offset++) {
+		if (offset < sizeof(t->pre_hdr_change) && mask[offset]) {
+			if (state == 0) {
+				u.offset = offset;
+				state++;
+			}
+		} else {
+			if (state > 0) {
+				u.size = offset - u.offset;
+				buffer_append(buf, &u, sizeof(u));
+				buffer_append(buf, data + u.offset, u.size);
+				state = 0;
+			}
+		}
+	}
+	return buf;
+}
+
+static void log_append_ext_intro(struct mail_index_export_context *ctx,
+				 uint32_t ext_id, uint32_t reset_id)
+{
+	struct mail_index_transaction *t = ctx->trans;
+	const struct mail_index_registered_ext *rext;
+        struct mail_transaction_ext_intro *intro, *resizes;
+	buffer_t *buf;
+	uint32_t idx;
+	unsigned int count;
+
+	i_assert(ext_id != (uint32_t)-1);
+
+	if (t->reset ||
+	    !mail_index_map_get_ext_idx(t->view->index->map, ext_id, &idx)) {
+		/* new extension */
+		idx = (uint32_t)-1;
+	}
+
+	rext = array_idx(&t->view->index->extensions, ext_id);
+	if (!array_is_created(&t->ext_resizes)) {
+		resizes = NULL;
+		count = 0;
+	} else {
+		resizes = array_get_modifiable(&t->ext_resizes, &count);
+	}
+
+	buf = buffer_create_dynamic(pool_datastack_create(), 128);
+	if (ext_id < count && resizes[ext_id].name_size != 0) {
+		/* we're resizing the extension. use the resize struct. */
+		intro = &resizes[ext_id];
+
+		i_assert(intro->ext_id == idx);
+		intro->name_size = idx != (uint32_t)-1 ? 0 :
+			strlen(rext->name);
+		buffer_append(buf, intro, sizeof(*intro));
+	} else {
+		/* generate a new intro structure */
+		intro = buffer_append_space_unsafe(buf, sizeof(*intro));
+		intro->ext_id = idx;
+		intro->hdr_size = rext->hdr_size;
+		intro->record_size = rext->record_size;
+		intro->record_align = rext->record_align;
+		intro->flags = MAIL_TRANSACTION_EXT_INTRO_FLAG_NO_SHRINK;
+		intro->name_size = idx != (uint32_t)-1 ? 0 :
+			strlen(rext->name);
+	}
+	if (reset_id != 0) {
+		/* we're going to reset this extension in this transaction */
+		intro->reset_id = reset_id;
+	} else if (idx != (uint32_t)-1) {
+		/* use the existing reset_id */
+		const struct mail_index_ext *map_ext =
+			array_idx(&t->view->index->map->extensions, idx);
+		intro->reset_id = map_ext->reset_id;
+	} else {
+		/* new extension, reset_id defaults to 0 */
+	}
+	buffer_append(buf, rext->name, intro->name_size);
+	if ((buf->used % 4) != 0)
+		buffer_append_zero(buf, 4 - (buf->used % 4));
+
+	if (ctx->append_ctx->new_highest_modseq == 0 &&
+	    strcmp(rext->name, MAIL_INDEX_MODSEQ_EXT_NAME) == 0) {
+		/* modseq tracking started */
+		ctx->append_ctx->new_highest_modseq = 1;
+	}
+
+	log_append_buffer(ctx, buf, NULL, MAIL_TRANSACTION_EXT_INTRO);
+}
+
+static void
+log_append_ext_hdr_update(struct mail_index_export_context *ctx,
+			const struct mail_index_transaction_ext_hdr_update *hdr)
+{
+	buffer_t *buf;
+	const unsigned char *data, *mask;
+	struct mail_transaction_ext_hdr_update u;
+	uint16_t offset;
+	bool started = FALSE;
+
+	memset(&u, 0, sizeof(u));
+
+	data = hdr->data;
+	mask = hdr->mask;
+
+	buf = buffer_create_dynamic(pool_datastack_create(), 256);
+	for (offset = 0; offset <= hdr->alloc_size; offset++) {
+		if (offset < hdr->alloc_size && mask[offset] != 0) {
+			if (!started) {
+				u.offset = offset;
+				started = TRUE;
+			}
+		} else {
+			if (started) {
+				u.size = offset - u.offset;
+				buffer_append(buf, &u, sizeof(u));
+				buffer_append(buf, data + u.offset, u.size);
+				started = FALSE;
+			}
+		}
+	}
+	if (buf->used % 4 != 0)
+		buffer_append_zero(buf, 4 - buf->used % 4);
+	log_append_buffer(ctx, buf, NULL, MAIL_TRANSACTION_EXT_HDR_UPDATE);
+}
+
+static void
+mail_transaction_log_append_ext_intros(struct mail_index_export_context *ctx)
+{
+	struct mail_index_transaction *t = ctx->trans;
+        const struct mail_transaction_ext_intro *resize;
+	const struct mail_index_transaction_ext_hdr_update *hdrs;
+	struct mail_transaction_ext_reset ext_reset;
+	unsigned int resize_count, ext_count = 0;
+	unsigned int hdrs_count, reset_id_count, reset_count;
+	uint32_t ext_id, reset_id;
+	const struct mail_transaction_ext_reset *reset;
+	const uint32_t *reset_ids;
+	buffer_t *reset_buf;
+
+	if (!array_is_created(&t->ext_resizes)) {
+		resize = NULL;
+		resize_count = 0;
+	} else {
+		resize = array_get(&t->ext_resizes, &resize_count);
+		if (ext_count < resize_count)
+			ext_count = resize_count;
+	}
+
+	if (!array_is_created(&t->ext_reset_ids)) {
+		reset_ids = NULL;
+		reset_id_count = 0;
+	} else {
+		reset_ids = array_get(&t->ext_reset_ids, &reset_id_count);
+	}
+
+	if (!array_is_created(&t->ext_resets)) {
+		reset = NULL;
+		reset_count = 0;
+	} else {
+		reset = array_get(&t->ext_resets, &reset_count);
+		if (ext_count < reset_count)
+			ext_count = reset_count;
+	}
+
+	if (!array_is_created(&t->ext_hdr_updates)) {
+		hdrs = NULL;
+		hdrs_count = 0;
+	} else {
+		hdrs = array_get(&t->ext_hdr_updates, &hdrs_count);
+		if (ext_count < hdrs_count)
+			ext_count = hdrs_count;
+	}
+
+	memset(&ext_reset, 0, sizeof(ext_reset));
+	reset_buf = buffer_create_data(pool_datastack_create(),
+				       &ext_reset, sizeof(ext_reset));
+	buffer_set_used_size(reset_buf, sizeof(ext_reset));
+
+	for (ext_id = 0; ext_id < ext_count; ext_id++) {
+		if (ext_id < reset_count)
+			ext_reset = reset[ext_id];
+		else
+			ext_reset.new_reset_id = 0;
+		if ((ext_id < resize_count && resize[ext_id].name_size) ||
+		    ext_reset.new_reset_id != 0 ||
+		    (ext_id < hdrs_count && hdrs[ext_id].alloc_size > 0)) {
+			if (ext_reset.new_reset_id != 0) {
+				/* we're going to reset this extension
+				   immediately after the intro */
+				reset_id = 0;
+			} else {
+				reset_id = ext_id < reset_id_count ?
+					reset_ids[ext_id] : 0;
+			}
+			log_append_ext_intro(ctx, ext_id, reset_id);
+		}
+		if (ext_reset.new_reset_id != 0) {
+			i_assert(ext_id < reset_id_count &&
+				 ext_reset.new_reset_id == reset_ids[ext_id]);
+			log_append_buffer(ctx, reset_buf, NULL,
+					  MAIL_TRANSACTION_EXT_RESET);
+		}
+		if (ext_id < hdrs_count && hdrs[ext_id].alloc_size > 0) {
+			T_BEGIN {
+				log_append_ext_hdr_update(ctx, &hdrs[ext_id]);
+			} T_END;
+		}
+	}
+}
+
+static void log_append_ext_recs(struct mail_index_export_context *ctx,
+				const ARRAY_TYPE(seq_array_array) *arr,
+				enum mail_transaction_type type)
+{
+	struct mail_index_transaction *t = ctx->trans;
+	const ARRAY_TYPE(seq_array) *updates;
+	const uint32_t *reset_ids;
+	unsigned int ext_id, count, reset_id_count;
+	uint32_t reset_id;
+
+	if (!array_is_created(&t->ext_reset_ids)) {
+		reset_ids = NULL;
+		reset_id_count = 0;
+	} else {
+		reset_ids = array_get_modifiable(&t->ext_reset_ids,
+						 &reset_id_count);
+	}
+
+	updates = array_get(arr, &count);
+	for (ext_id = 0; ext_id < count; ext_id++) {
+		if (!array_is_created(&updates[ext_id]))
+			continue;
+
+		reset_id = ext_id < reset_id_count ? reset_ids[ext_id] : 0;
+		log_append_ext_intro(ctx, ext_id, reset_id);
+
+		log_append_buffer(ctx, updates[ext_id].arr.buffer, NULL, type);
+	}
+}
+
+static void
+log_append_keyword_update(struct mail_index_export_context *ctx,
+			  buffer_t *hdr_buf, enum modify_type modify_type,
+			  const char *keyword, const buffer_t *buffer)
+{
+	struct mail_transaction_keyword_update kt_hdr;
+
+	memset(&kt_hdr, 0, sizeof(kt_hdr));
+	kt_hdr.modify_type = modify_type;
+	kt_hdr.name_size = strlen(keyword);
+
+	buffer_set_used_size(hdr_buf, 0);
+	buffer_append(hdr_buf, &kt_hdr, sizeof(kt_hdr));
+	buffer_append(hdr_buf, keyword, kt_hdr.name_size);
+	if ((hdr_buf->used % 4) != 0)
+		buffer_append_zero(hdr_buf, 4 - (hdr_buf->used % 4));
+
+	log_append_buffer(ctx, buffer, hdr_buf,
+			  MAIL_TRANSACTION_KEYWORD_UPDATE);
+}
+
+static enum mail_index_sync_type
+log_append_keyword_updates(struct mail_index_export_context *ctx)
+{
+        const struct mail_index_transaction_keyword_update *updates;
+	const char *const *keywords;
+	buffer_t *hdr_buf;
+	enum mail_index_sync_type change_mask = 0;
+	unsigned int i, count, keywords_count;
+
+	hdr_buf = buffer_create_dynamic(pool_datastack_create(), 64);
+
+	keywords = array_get_modifiable(&ctx->trans->view->index->keywords,
+					&keywords_count);
+	updates = array_get_modifiable(&ctx->trans->keyword_updates, &count);
+	i_assert(count <= keywords_count);
+
+	for (i = 0; i < count; i++) {
+		if (array_is_created(&updates[i].add_seq)) {
+			change_mask |= MAIL_INDEX_SYNC_TYPE_KEYWORD_ADD;
+			log_append_keyword_update(ctx, hdr_buf,
+					MODIFY_ADD, keywords[i],
+					updates[i].add_seq.arr.buffer);
+		}
+		if (array_is_created(&updates[i].remove_seq)) {
+			change_mask |= MAIL_INDEX_SYNC_TYPE_KEYWORD_REMOVE;
+			log_append_keyword_update(ctx, hdr_buf,
+					MODIFY_REMOVE, keywords[i],
+					updates[i].remove_seq.arr.buffer);
+		}
+	}
+	return change_mask;
+}
+
+void mail_index_transaction_export(struct mail_index_transaction *t,
+				   struct mail_transaction_log_append_ctx *append_ctx)
+{
+	enum mail_index_sync_type change_mask = 0;
+	struct mail_index_export_context ctx;
+
+	memset(&ctx, 0, sizeof(ctx));
+	ctx.trans = t;
+	ctx.append_ctx = append_ctx;
+
+	/* send all extension introductions and resizes before appends
+	   to avoid resize overhead as much as possible */
+        mail_transaction_log_append_ext_intros(&ctx);
+
+	if (t->pre_hdr_changed) {
+		log_append_buffer(&ctx,
+				  log_get_hdr_update_buffer(t, TRUE),
+				  NULL, MAIL_TRANSACTION_HEADER_UPDATE);
+	}
+	if (array_is_created(&t->appends)) {
+		change_mask |= MAIL_INDEX_SYNC_TYPE_APPEND;
+		log_append_buffer(&ctx, t->appends.arr.buffer, NULL,
+				  MAIL_TRANSACTION_APPEND);
+	}
+	if (array_is_created(&t->updates)) {
+		change_mask |= MAIL_INDEX_SYNC_TYPE_FLAGS;
+		log_append_buffer(&ctx, t->updates.arr.buffer, NULL,
+				  MAIL_TRANSACTION_FLAG_UPDATE);
+	}
+
+	if (array_is_created(&t->ext_rec_updates)) {
+		log_append_ext_recs(&ctx, &t->ext_rec_updates,
+				    MAIL_TRANSACTION_EXT_REC_UPDATE);
+	}
+	if (array_is_created(&t->ext_rec_atomics)) {
+		log_append_ext_recs(&ctx, &t->ext_rec_atomics,
+				    MAIL_TRANSACTION_EXT_ATOMIC_INC);
+	}
+
+	/* keyword resets before updates */
+	if (array_is_created(&t->keyword_resets)) {
+		change_mask |= MAIL_INDEX_SYNC_TYPE_KEYWORD_RESET;
+		log_append_buffer(&ctx, t->keyword_resets.arr.buffer,
+				  NULL, MAIL_TRANSACTION_KEYWORD_RESET);
+	}
+	if (array_is_created(&t->keyword_updates))
+		change_mask |= log_append_keyword_updates(&ctx);
+
+	if (array_is_created(&t->expunges)) {
+		/* non-external expunges are only requests, ignore them when
+		   checking fsync_mask */
+		if ((t->flags & MAIL_INDEX_TRANSACTION_FLAG_EXTERNAL) != 0)
+			change_mask |= MAIL_INDEX_SYNC_TYPE_EXPUNGE;
+		log_append_buffer(&ctx, t->expunges.arr.buffer, NULL,
+				  MAIL_TRANSACTION_EXPUNGE);
+	}
+
+	if (t->post_hdr_changed) {
+		log_append_buffer(&ctx, log_get_hdr_update_buffer(t, FALSE),
+				  NULL, MAIL_TRANSACTION_HEADER_UPDATE);
+	}
+
+	/* Update the tail offsets only when committing the sync transaction.
+	   Other transactions may not know the latest tail offset and might
+	   end up shrinking it. (Alternatively the shrinking tail offsets could
+	   just be ignored, which would probably work fine too.) */
+	append_ctx->append_sync_offset = t->sync_transaction;
+
+	append_ctx->want_fsync =
+		(t->view->index->fsync_mask & change_mask) != 0 ||
+		(t->flags & MAIL_INDEX_TRANSACTION_FLAG_FSYNC) != 0;
+}

--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/src/lib-index/mail-index-transaction-finish.c	Sat May 09 14:53:06 2009 -0400
@@ -0,0 +1,294 @@
+/* Copyright (c) 2003-2009 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "array.h"
+#include "mail-index-private.h"
+#include "mail-index-modseq.h"
+#include "mail-index-transaction-private.h"
+
+static void
+ext_reset_update_atomic(struct mail_index_transaction *t,
+			uint32_t ext_id, uint32_t expected_reset_id)
+{
+	const struct mail_index_ext *map_ext;
+	struct mail_transaction_ext_reset *reset;
+	uint32_t idx, reset_id;
+
+	if (!mail_index_map_get_ext_idx(t->view->index->map, ext_id, &idx)) {
+		/* new extension */
+		reset_id = 1;
+	} else {
+		map_ext = array_idx(&t->view->index->map->extensions, idx);
+		reset_id = map_ext->reset_id + 1;
+	}
+	if (reset_id != expected_reset_id) {
+		/* ignore this extension update */
+		mail_index_ext_set_reset_id(t, ext_id, 0);
+		return;
+	}
+
+	if (reset_id == 0)
+		reset_id++;
+
+	array_idx_set(&t->ext_reset_ids, ext_id, &reset_id);
+
+	/* reseting existing data is optional */
+	if (array_is_created(&t->ext_resets)) {
+		reset = array_idx_modifiable(&t->ext_resets, ext_id);
+		if (reset->new_reset_id == (uint32_t)-1)
+			reset->new_reset_id = reset_id;
+	}
+}
+
+static void
+transaction_update_atomic_reset_ids(struct mail_index_transaction *t)
+{
+	const uint32_t *expected_reset_ids;
+	unsigned int ext_id, count;
+
+	if (!array_is_created(&t->ext_reset_atomic))
+		return;
+
+	expected_reset_ids = array_get(&t->ext_reset_atomic, &count);
+	for (ext_id = 0; ext_id < count; ext_id++) {
+		if (expected_reset_ids[ext_id] != 0) {
+			ext_reset_update_atomic(t, ext_id,
+						expected_reset_ids[ext_id]);
+		}
+	}
+}
+
+static bool
+mail_index_update_cancel_array(ARRAY_TYPE(seq_range) *array, uint32_t seq)
+{
+	if (array_is_created(array)) {
+		if (seq_range_array_remove(array, seq)) {
+			if (array_count(array) == 0)
+				array_free(array);
+			return TRUE;
+		}
+	}
+	return FALSE;
+}
+
+static bool
+mail_index_update_cancel(struct mail_index_transaction *t, uint32_t seq)
+{
+	struct mail_index_transaction_keyword_update *kw;
+	struct mail_transaction_flag_update *updates, tmp_update;
+	unsigned int i, count;
+	bool ret, have_kw_changes = FALSE;
+
+	ret = mail_index_update_cancel_array(&t->keyword_resets, seq);
+	if (array_is_created(&t->keyword_updates)) {
+		kw = array_get_modifiable(&t->keyword_updates, &count);
+		for (i = 0; i < count; i++) {
+			if (mail_index_update_cancel_array(&kw[i].add_seq, seq))
+				ret = TRUE;
+			if (mail_index_update_cancel_array(&kw[i].remove_seq,
+							   seq))
+				ret = TRUE;
+			if (array_is_created(&kw[i].add_seq) ||
+			    array_is_created(&kw[i].remove_seq))
+				have_kw_changes = TRUE;
+		}
+		if (!have_kw_changes)
+			array_free(&t->keyword_updates);
+	}
+
+	if (!array_is_created(&t->updates))
+		return ret;
+
+	updates = array_get_modifiable(&t->updates, &count);
+	i = mail_index_transaction_get_flag_update_pos(t, 0, count, seq);
+	if (i < count && updates[i].uid1 <= seq && updates[i].uid2 >= seq) {
+		/* exists */
+		ret = TRUE;
+		if (updates[i].uid1 == seq && updates[i].uid2 == seq) {
+			if (count > 1)
+				array_delete(&t->updates, i, 1);
+			else
+				array_free(&t->updates);
+		} else if (updates[i].uid1 == seq)
+			updates[i].uid1++;
+		else if (updates[i].uid2 == seq)
+			updates[i].uid2--;
+		else {
+			/* need to split it in two */
+			tmp_update = updates[i];
+			tmp_update.uid1 = seq+1;
+			updates[i].uid2 = seq-1;
+			array_insert(&t->updates, i + 1, &tmp_update, 1);
+		}
+	}
+	return ret;
+}
+
+static void
+mail_index_transaction_check_conflicts(struct mail_index_transaction *t)
+{
+	uint32_t seq;
+
+	i_assert(t->max_modseq != 0);
+	i_assert(t->conflict_seqs != NULL);
+
+	if (t->max_modseq == mail_index_modseq_get_highest(t->view)) {
+		/* no conflicts possible */
+		return;
+	}
+	if (t->min_flagupdate_seq == 0) {
+		/* no flag updates */
+		return;
+	}
+
+	for (seq = t->min_flagupdate_seq; seq <= t->max_flagupdate_seq; seq++) {
+		if (mail_index_modseq_lookup(t->view, seq) > t->max_modseq) {
+			if (mail_index_update_cancel(t, seq))
+				seq_range_array_add(t->conflict_seqs, 0, seq);
+		}
+	}
+	mail_index_transaction_set_log_updates(t);
+}
+
+static uint32_t
+mail_index_transaction_get_uid(struct mail_index_transaction *t, uint32_t seq)
+{
+	const struct mail_index_record *rec;
+
+	i_assert(seq > 0);
+
+	if (seq >= t->first_new_seq)
+		rec = mail_index_transaction_lookup(t, seq);
+	else {
+		i_assert(seq <= t->view->map->hdr.messages_count);
+		rec = MAIL_INDEX_MAP_IDX(t->view->map, seq - 1);
+	}
+	i_assert(rec->uid != 0);
+	return rec->uid;
+}
+
+static void
+mail_index_convert_to_uids(struct mail_index_transaction *t,
+			   ARRAY_TYPE(seq_array) *array)
+{
+	uint32_t *seq;
+	unsigned int i, count;
+
+	if (!array_is_created(array))
+		return;
+
+	count = array_count(array);
+	for (i = 0; i < count; i++) {
+		seq = array_idx_modifiable(array, i);
+		*seq = mail_index_transaction_get_uid(t, *seq);
+	}
+}
+
+static uint32_t
+get_nonexpunged_uid2(struct mail_index_transaction *t,
+		     uint32_t uid1, uint32_t seq1)
+{
+	seq1++;
+
+	while (mail_index_transaction_get_uid(t, seq1) == uid1 + 1) {
+		seq1++;
+		uid1++;
+	}
+	return uid1;
+}
+
+static void
+mail_index_convert_to_uid_ranges(struct mail_index_transaction *t,
+				 ARRAY_TYPE(seq_range) *array)
+{
+	struct seq_range *range, *new_range;
+	unsigned int i, count;
+	uint32_t uid1, uid2;
+
+	if (!array_is_created(array))
+		return;
+
+	count = array_count(array);
+	for (i = 0; i < count; i++) {
+		range = array_idx_modifiable(array, i);
+
+		uid1 = mail_index_transaction_get_uid(t, range->seq1);
+		uid2 = mail_index_transaction_get_uid(t, range->seq2);
+		if (uid2 - uid1 == range->seq2 - range->seq1) {
+			/* simple conversion */
+			range->seq1 = uid1;
+			range->seq2 = uid2;
+		} else {
+			/* remove expunged UIDs */
+			new_range = array_insert_space(array, i);
+			range = array_idx_modifiable(array, i + 1);
+			count++;
+
+			memcpy(new_range, range, array->arr.element_size);
+			new_range->seq1 = uid1;
+			new_range->seq2 = get_nonexpunged_uid2(t, uid1,
+							       range->seq1);
+
+			/* continue the range without the inserted seqs */
+			range->seq1 += new_range->seq2 - new_range->seq1 + 1;
+		}
+	}
+}
+
+static void keyword_updates_convert_to_uids(struct mail_index_transaction *t)
+{
+        struct mail_index_transaction_keyword_update *updates;
+	unsigned int i, count;
+
+	if (!array_is_created(&t->keyword_updates))
+		return;
+
+	updates = array_get_modifiable(&t->keyword_updates, &count);
+	for (i = 0; i < count; i++) {
+		mail_index_convert_to_uid_ranges(t, &updates[i].add_seq);
+		mail_index_convert_to_uid_ranges(t, &updates[i].remove_seq);
+	}
+}
+
+static void
+mail_index_transaction_convert_to_uids(struct mail_index_transaction *t)
+{
+	ARRAY_TYPE(seq_array) *updates;
+	unsigned int i, count;
+
+	if (array_is_created(&t->ext_rec_updates)) {
+		updates = array_get_modifiable(&t->ext_rec_updates, &count);
+		for (i = 0; i < count; i++)
+			mail_index_convert_to_uids(t, (void *)&updates[i]);
+	}
+	if (array_is_created(&t->ext_rec_atomics)) {
+		updates = array_get_modifiable(&t->ext_rec_atomics, &count);
+		for (i = 0; i < count; i++)
+			mail_index_convert_to_uids(t, (void *)&updates[i]);
+	}
+
+        keyword_updates_convert_to_uids(t);
+
+	mail_index_convert_to_uid_ranges(t, &t->expunges);
+	mail_index_convert_to_uid_ranges(t, (void *)&t->updates);
+	mail_index_convert_to_uid_ranges(t, &t->keyword_resets);
+}
+
+int mail_index_transaction_finish(struct mail_index_transaction *t)
+{
+	mail_index_transaction_sort_appends(t);
+
+	if (array_is_created(&t->ext_reset_atomic) || t->max_modseq != 0) {
+		if (mail_index_map(t->view->index,
+				   MAIL_INDEX_SYNC_HANDLER_HEAD) <= 0)
+			return -1;
+	}
+	if (array_is_created(&t->ext_reset_atomic))
+		transaction_update_atomic_reset_ids(t);
+	if (t->max_modseq != 0)
+		mail_index_transaction_check_conflicts(t);
+	/* finally convert all sequences to UIDs before we write them,
+	   but after we've checked and removed conflicts */
+	mail_index_transaction_convert_to_uids(t);
+	return 0;
+}

--- a/src/lib-index/mail-index-transaction-private.h	Sat May 09 12:32:04 2009 -0400
+++ b/src/lib-index/mail-index-transaction-private.h	Sat May 09 14:53:06 2009 -0400
@@ -89,6 +89,10 @@
 	unsigned int log_ext_updates:1;
 };
 
+#define MAIL_INDEX_TRANSACTION_HAS_CHANGES(t) \
+	((t)->log_updates || (t)->log_ext_updates || \
+	 (array_is_created(&(t)->updates) && array_count(&(t)->updates) > 0))
+
 extern void (*hook_mail_index_transaction_created)
 		(struct mail_index_transaction *t);
 
@@ -100,8 +104,7 @@
 
 void mail_index_transaction_sort_appends(struct mail_index_transaction *t);
 uint32_t mail_index_transaction_get_next_uid(struct mail_index_transaction *t);
-void mail_index_transaction_convert_to_uids(struct mail_index_transaction *t);
-void mail_index_transaction_check_conflicts(struct mail_index_transaction *t);
+void mail_index_transaction_set_log_updates(struct mail_index_transaction *t);
 
 unsigned int
 mail_index_transaction_get_flag_update_pos(struct mail_index_transaction *t,
@@ -111,5 +114,12 @@
 
 bool mail_index_seq_array_lookup(const ARRAY_TYPE(seq_array) *array,
 				 uint32_t seq, unsigned int *idx_r);
+bool mail_index_seq_array_add(ARRAY_TYPE(seq_array) *array, uint32_t seq,
+			      const void *record, size_t record_size,
+			      void *old_record);
+
+int mail_index_transaction_finish(struct mail_index_transaction *t);
+void mail_index_transaction_export(struct mail_index_transaction *t,
+				   struct mail_transaction_log_append_ctx *append_ctx);
 
 #endif

--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/src/lib-index/mail-index-transaction-sort-appends.c	Sat May 09 14:53:06 2009 -0400
@@ -0,0 +1,182 @@
+/* Copyright (c) 2003-2009 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "array.h"
+#include "seq-range-array.h"
+#include "mail-index-private.h"
+#include "mail-index-transaction-private.h"
+
+#include <stdlib.h>
+
+struct uid_map {
+	uint32_t idx;
+	uint32_t uid;
+};
+
+static int uid_map_cmp(const void *p1, const void *p2)
+{
+	const struct uid_map *m1 = p1, *m2 = p2;
+
+	return m1->uid < m2->uid ? -1 :
+		(m1->uid > m2->uid ? 1 : 0);
+}
+
+static void
+mail_index_transaction_sort_appends_ext(ARRAY_TYPE(seq_array_array) *updates,
+					uint32_t first_new_seq,
+					const uint32_t *old_to_newseq_map)
+{
+	ARRAY_TYPE(seq_array) *ext_rec_arrays;
+	ARRAY_TYPE(seq_array) *old_array;
+	ARRAY_TYPE(seq_array) new_array;
+	unsigned int ext_count;
+	const uint32_t *ext_rec;
+	uint32_t seq;
+	unsigned int i, j, count;
+
+	if (!array_is_created(updates))
+		return;
+
+	ext_rec_arrays = array_get_modifiable(updates, &count);
+	for (j = 0; j < count; j++) {
+		old_array = &ext_rec_arrays[j];
+		if (!array_is_created(old_array))
+			continue;
+
+		ext_count = array_count(old_array);
+		array_create(&new_array, default_pool,
+			     old_array->arr.element_size, ext_count);
+		for (i = 0; i < ext_count; i++) {
+			ext_rec = array_idx(old_array, i);
+
+			seq = *ext_rec < first_new_seq ? *ext_rec :
+				old_to_newseq_map[*ext_rec - first_new_seq];
+			mail_index_seq_array_add(&new_array, seq, ext_rec+1,
+						 old_array->arr.element_size -
+						 sizeof(*ext_rec), NULL);
+		}
+		array_free(old_array);
+		ext_rec_arrays[j] = new_array;
+	}
+}
+
+static void
+sort_appends_seq_range(ARRAY_TYPE(seq_range) *array, uint32_t first_new_seq,
+		       const uint32_t *old_to_newseq_map)
+{
+	struct seq_range *range, temp_range;
+	ARRAY_TYPE(seq_range) old_seqs;
+	uint32_t idx, idx1, idx2;
+	unsigned int i, count;
+
+	range = array_get_modifiable(array, &count);
+	for (i = 0; i < count; i++) {
+		if (range[i].seq2 >= first_new_seq)
+			break;
+	}
+	if (i == count) {
+		/* nothing to do */
+		return;
+	}
+
+	i_array_init(&old_seqs, count - i);
+	if (range[i].seq1 < first_new_seq) {
+		temp_range.seq1 = first_new_seq;
+		temp_range.seq2 = range[i].seq2;
+		array_append(&old_seqs, &temp_range, 1);
+		range[i].seq2 = first_new_seq - 1;
+		i++;
+	}
+	array_append(&old_seqs, &range[i], count - i);
+	array_delete(array, i, count - i);
+
+	range = array_get_modifiable(&old_seqs, &count);
+	for (i = 0; i < count; i++) {
+		idx1 = range[i].seq1 - first_new_seq;
+		idx2 = range[i].seq2 - first_new_seq;
+		for (idx = idx1; idx <= idx2; idx++)
+			seq_range_array_add(array, 0, old_to_newseq_map[idx]);
+	}
+	array_free(&old_seqs);
+}
+
+static void
+mail_index_transaction_sort_appends_keywords(struct mail_index_transaction *t,
+					     const uint32_t *old_to_newseq_map)
+{
+	struct mail_index_transaction_keyword_update *updates;
+	unsigned int i, count;
+
+	if (array_is_created(&t->keyword_updates)) {
+		updates = array_get_modifiable(&t->keyword_updates, &count);
+		for (i = 0; i < count; i++) {
+			if (array_is_created(&updates[i].add_seq)) {
+				sort_appends_seq_range(&updates[i].add_seq,
+						       t->first_new_seq,
+						       old_to_newseq_map);
+			}
+			if (array_is_created(&updates[i].remove_seq)) {
+				sort_appends_seq_range(&updates[i].remove_seq,
+						       t->first_new_seq,
+						       old_to_newseq_map);
+			}
+		}
+	}
+
+	if (array_is_created(&t->keyword_resets)) {
+		sort_appends_seq_range(&t->keyword_resets, t->first_new_seq,
+				       old_to_newseq_map);
+	}
+}
+
+void mail_index_transaction_sort_appends(struct mail_index_transaction *t)
+{
+	struct mail_index_record *recs, *sorted_recs;
+	struct uid_map *new_uid_map;
+	uint32_t *old_to_newseq_map;
+	unsigned int i, count;
+
+	if (!t->appends_nonsorted || !array_is_created(&t->appends))
+		return;
+
+	/* first make a copy of the UIDs and map them to sequences */
+	recs = array_get_modifiable(&t->appends, &count);
+	i_assert(count > 0);
+
+	new_uid_map = i_new(struct uid_map, count);
+	for (i = 0; i < count; i++) {
+		new_uid_map[i].idx = i;
+		new_uid_map[i].uid = recs[i].uid;
+	}
+
+	/* now sort the UID map */
+	qsort(new_uid_map, count, sizeof(*new_uid_map), uid_map_cmp);
+
+	/* sort mail records */
+	sorted_recs = i_new(struct mail_index_record, count);
+	sorted_recs[0] = recs[new_uid_map[0].idx];
+	for (i = 1; i < count; i++) {
+		sorted_recs[i] = recs[new_uid_map[i].idx];
+		if (sorted_recs[i].uid == sorted_recs[i-1].uid)
+			i_panic("Duplicate UIDs added in transaction");
+	}
+	buffer_write(t->appends.arr.buffer, 0, sorted_recs,
+		     sizeof(*sorted_recs) * count);
+	i_free(sorted_recs);
+
+	old_to_newseq_map = i_new(uint32_t, count);
+	for (i = 0; i < count; i++)
+		old_to_newseq_map[new_uid_map[i].idx] = i + t->first_new_seq;
+	i_free(new_uid_map);
+
+	mail_index_transaction_sort_appends_ext(&t->ext_rec_updates,
+						t->first_new_seq,
+						old_to_newseq_map);
+	mail_index_transaction_sort_appends_ext(&t->ext_rec_atomics,
+						t->first_new_seq,
+						old_to_newseq_map);
+	mail_index_transaction_sort_appends_keywords(t, old_to_newseq_map);
+	i_free(old_to_newseq_map);
+
+	t->appends_nonsorted = FALSE;
+}

--- a/src/lib-index/mail-index-transaction.c	Sat May 09 12:32:04 2009 -0400
+++ b/src/lib-index/mail-index-transaction.c	Sat May 09 14:53:06 2009 -0400
@@ -8,17 +8,11 @@
 #include "ioloop.h"
 #include "array.h"
 #include "bsearch-insert-pos.h"
-#include "seq-range-array.h"
-#include "mail-index-view-private.h"
-#include "mail-index-modseq.h"
-#include "mail-transaction-log.h"
-#include "mail-cache-private.h"
+#include "mail-index-private.h"
+#include "mail-transaction-log-private.h"
+#include "mail-cache.h"
 #include "mail-index-transaction-private.h"
 
-#include <stddef.h>
-#include <stdlib.h>
-#include <time.h>
-
 void (*hook_mail_index_transaction_created)
 		(struct mail_index_transaction *t) = NULL;
 
@@ -106,10 +100,10 @@
 	t->log_ext_updates = FALSE;
 }
 
-static bool mail_index_transaction_has_changes(struct mail_index_transaction *t)
+void mail_index_transaction_set_log_updates(struct mail_index_transaction *t)
 {
 	/* flag updates aren't included in log_updates */
-	return array_is_created(&t->appends) ||
+	t->log_updates = array_is_created(&t->appends) ||
 		array_is_created(&t->expunges) ||
 		array_is_created(&t->keyword_resets) ||
 		array_is_created(&t->keyword_updates) ||
@@ -172,9 +166,9 @@
 				  mail_index_seq_record_cmp, idx_r);
 }
 
-static bool mail_index_seq_array_add(ARRAY_TYPE(seq_array) *array, uint32_t seq,
-				     const void *record, size_t record_size,
-				     void *old_record)
+bool mail_index_seq_array_add(ARRAY_TYPE(seq_array) *array, uint32_t seq,
+			      const void *record, size_t record_size,
+			      void *old_record)
 {
 	void *p;
 	unsigned int idx, aligned_record_size;
@@ -208,140 +202,98 @@
 	}
 }
 
-static uint32_t
-mail_index_transaction_get_uid(struct mail_index_transaction *t, uint32_t seq)
-{
-	const struct mail_index_record *rec;
-
-	i_assert(seq > 0);
-
-	if (seq >= t->first_new_seq)
-		rec = mail_index_transaction_lookup(t, seq);
-	else {
-		i_assert(seq <= t->view->map->hdr.messages_count);
-		rec = MAIL_INDEX_MAP_IDX(t->view->map, seq - 1);
-	}
-	i_assert(rec->uid != 0);
-	return rec->uid;
-}
-
-static void
-mail_index_convert_to_uids(struct mail_index_transaction *t,
-			   ARRAY_TYPE(seq_array) *array)
+uint32_t mail_index_transaction_get_next_uid(struct mail_index_transaction *t)
 {
-	uint32_t *seq;
-	unsigned int i, count;
+	const struct mail_index_header *head_hdr, *hdr;
+	unsigned int offset;
+	uint32_t next_uid;
 
-	if (!array_is_created(array))
-		return;
-
-	count = array_count(array);
-	for (i = 0; i < count; i++) {
-		seq = array_idx_modifiable(array, i);
-		*seq = mail_index_transaction_get_uid(t, *seq);
+	head_hdr = &t->view->index->map->hdr;
+	hdr = &t->view->map->hdr;
+	next_uid = t->reset || head_hdr->uid_validity != hdr->uid_validity ?
+		1 : hdr->next_uid;
+	if (array_is_created(&t->appends) && t->highest_append_uid != 0) {
+		/* get next_uid from appends if they have UIDs */
+		i_assert(next_uid <= t->highest_append_uid);
+		next_uid = t->highest_append_uid + 1;
 	}
-}
 
-static uint32_t
-get_nonexpunged_uid2(struct mail_index_transaction *t,
-		     uint32_t uid1, uint32_t seq1)
-{
-	seq1++;
-
-	while (mail_index_transaction_get_uid(t, seq1) == uid1 + 1) {
-		seq1++;
-		uid1++;
+	/* see if it's been updated in pre/post header changes */
+	offset = offsetof(struct mail_index_header, next_uid);
+	if (t->post_hdr_mask[offset] != 0) {
+		hdr = (const void *)t->post_hdr_change;
+		if (hdr->next_uid > next_uid)
+			next_uid = hdr->next_uid;
 	}
-	return uid1;
+	if (t->pre_hdr_mask[offset] != 0) {
+		hdr = (const void *)t->pre_hdr_change;
+		if (hdr->next_uid > next_uid)
+			next_uid = hdr->next_uid;
+	}
+	return next_uid;
 }
 
-static void
-mail_index_convert_to_uid_ranges(struct mail_index_transaction *t,
-				 ARRAY_TYPE(seq_range) *array)
+static int
+mail_transaction_log_file_refresh(struct mail_index_transaction *t,
+				  struct mail_transaction_log_append_ctx *ctx)
 {
-	struct seq_range *range, *new_range;
-	unsigned int i, count;
-	uint32_t uid1, uid2;
-
-	if (!array_is_created(array))
-		return;
-
-	count = array_count(array);
-	for (i = 0; i < count; i++) {
-		range = array_idx_modifiable(array, i);
+	struct mail_transaction_log_file *file;
 
-		uid1 = mail_index_transaction_get_uid(t, range->seq1);
-		uid2 = mail_index_transaction_get_uid(t, range->seq2);
-		if (uid2 - uid1 == range->seq2 - range->seq1) {
-			/* simple conversion */
-			range->seq1 = uid1;
-			range->seq2 = uid2;
-		} else {
-			/* remove expunged UIDs */
-			new_range = array_insert_space(array, i);
-			range = array_idx_modifiable(array, i + 1);
-			count++;
+	if (t->reset) {
+		/* Reset the whole index, preserving only indexid. Begin by
+		   rotating the log. We don't care if we skip some non-synced
+		   transactions. */
+		if (mail_transaction_log_rotate(t->view->index->log, TRUE) < 0)
+			return -1;
 
-			memcpy(new_range, range, array->arr.element_size);
-			new_range->seq1 = uid1;
-			new_range->seq2 = get_nonexpunged_uid2(t, uid1,
-							       range->seq1);
-
-			/* continue the range without the inserted seqs */
-			range->seq1 += new_range->seq2 - new_range->seq1 + 1;
+		if (!MAIL_INDEX_TRANSACTION_HAS_CHANGES(t)) {
+			/* we only wanted to reset */
+			return 0;
 		}
 	}
+	file = t->view->index->log->head;
+
+	if (!t->view->index->log_locked) {
+		/* update sync_offset */
+		if (mail_transaction_log_file_map(file, file->sync_offset,
+						  (uoff_t)-1) <= 0)
+			return -1;
+	}
+	i_assert(file->sync_offset >= file->buffer_offset);
+	ctx->new_highest_modseq = file->sync_highest_modseq;
+	return 1;
 }
 
-static void keyword_updates_convert_to_uids(struct mail_index_transaction *t)
+static int mail_index_transaction_commit_real(struct mail_index_transaction *t)
 {
-        struct mail_index_transaction_keyword_update *updates;
-	unsigned int i, count;
-
-	if (!array_is_created(&t->keyword_updates))
-		return;
-
-	updates = array_get_modifiable(&t->keyword_updates, &count);
-	for (i = 0; i < count; i++) {
-		mail_index_convert_to_uid_ranges(t, &updates[i].add_seq);
-		mail_index_convert_to_uid_ranges(t, &updates[i].remove_seq);
-	}
-}
+	struct mail_transaction_log *log = t->view->index->log;
+	struct mail_transaction_log_append_ctx *ctx;
+	uint32_t log_seq1, log_seq2;
+	uoff_t log_offset1, log_offset2;
+	int ret;
 
-void mail_index_transaction_convert_to_uids(struct mail_index_transaction *t)
-{
-	ARRAY_TYPE(seq_array) *updates;
-	unsigned int i, count;
-
-	if (array_is_created(&t->ext_rec_updates)) {
-		updates = array_get_modifiable(&t->ext_rec_updates, &count);
-		for (i = 0; i < count; i++)
-			mail_index_convert_to_uids(t, (void *)&updates[i]);
-	}
-	if (array_is_created(&t->ext_rec_atomics)) {
-		updates = array_get_modifiable(&t->ext_rec_atomics, &count);
-		for (i = 0; i < count; i++)
-			mail_index_convert_to_uids(t, (void *)&updates[i]);
+	if (mail_transaction_log_append_begin(t, &ctx) < 0)
+		return -1;
+	ret = mail_transaction_log_file_refresh(t, ctx);
+	if (ret > 0) {
+		ret = mail_index_transaction_finish(t);
+		if (ret == 0)
+			mail_index_transaction_export(t, ctx);
 	}
 
-        keyword_updates_convert_to_uids(t);
-
-	mail_index_convert_to_uid_ranges(t, &t->expunges);
-	mail_index_convert_to_uid_ranges(t, (void *)&t->updates);
-	mail_index_convert_to_uid_ranges(t, &t->keyword_resets);
-}
+	mail_transaction_log_get_head(log, &log_seq1, &log_offset1);
+	if (mail_transaction_log_append_commit(&ctx) < 0 || ret < 0)
+		return -1;
+	mail_transaction_log_get_head(log, &log_seq2, &log_offset2);
+	i_assert(log_seq1 == log_seq2);
 
-struct uid_map {
-	uint32_t idx;
-	uint32_t uid;
-};
-
-static int uid_map_cmp(const void *p1, const void *p2)
-{
-	const struct uid_map *m1 = p1, *m2 = p2;
-
-	return m1->uid < m2->uid ? -1 :
-		(m1->uid > m2->uid ? 1 : 0);
+	if ((t->flags & MAIL_INDEX_TRANSACTION_FLAG_HIDE) != 0 &&
+	    log_offset1 != log_offset2) {
+		/* mark the area covered by this transaction hidden */
+		mail_index_view_add_hidden_transaction(t->view, log_seq1,
+			log_offset1, log_offset2 - log_offset1);
+	}
+	return 0;
 }
 
 static void
@@ -390,199 +342,11 @@
 		hdr.day_first_uid, sizeof(hdr.day_first_uid), FALSE);
 }
 
-static void
-mail_index_transaction_sort_appends_ext(struct mail_index_transaction *t,
-					ARRAY_TYPE(seq_array_array) *updates,
-					const uint32_t *old_to_newseq_map)
-{
-	ARRAY_TYPE(seq_array) *ext_rec_arrays;
-	ARRAY_TYPE(seq_array) *old_array;
-	ARRAY_TYPE(seq_array) new_array;
-	unsigned int ext_count;
-	const uint32_t *ext_rec;
-	uint32_t seq;
-	unsigned int i, j, count;
-
-	if (!array_is_created(updates))
-		return;
-
-	ext_rec_arrays = array_get_modifiable(updates, &count);
-	for (j = 0; j < count; j++) {
-		old_array = &ext_rec_arrays[j];
-		if (!array_is_created(old_array))
-			continue;
-
-		ext_count = array_count(old_array);
-		array_create(&new_array, default_pool,
-			     old_array->arr.element_size, ext_count);
-		for (i = 0; i < ext_count; i++) {
-			ext_rec = array_idx(old_array, i);
-
-			seq = *ext_rec < t->first_new_seq ? *ext_rec :
-				old_to_newseq_map[*ext_rec - t->first_new_seq];
-			mail_index_seq_array_add(&new_array, seq, ext_rec+1,
-						 old_array->arr.element_size -
-						 sizeof(*ext_rec), NULL);
-		}
-		array_free(old_array);
-		ext_rec_arrays[j] = new_array;
-	}
-}
-
-static void
-sort_appends_seq_range(struct mail_index_transaction *t,
-		       ARRAY_TYPE(seq_range) *array,
-		       const uint32_t *old_to_newseq_map)
-{
-	struct seq_range *range, temp_range;
-	ARRAY_TYPE(seq_range) old_seqs;
-	uint32_t idx, idx1, idx2;
-	unsigned int i, count;
-
-	range = array_get_modifiable(array, &count);
-	for (i = 0; i < count; i++) {
-		if (range[i].seq2 >= t->first_new_seq)
-			break;
-	}
-	if (i == count) {
-		/* nothing to do */
-		return;
-	}
-
-	i_array_init(&old_seqs, count - i);
-	if (range[i].seq1 < t->first_new_seq) {
-		temp_range.seq1 = t->first_new_seq;
-		temp_range.seq2 = range[i].seq2;
-		array_append(&old_seqs, &temp_range, 1);
-		range[i].seq2 = t->first_new_seq - 1;
-		i++;
-	}
-	array_append(&old_seqs, &range[i], count - i);
-	array_delete(array, i, count - i);
-
-	range = array_get_modifiable(&old_seqs, &count);
-	for (i = 0; i < count; i++) {
-		idx1 = range[i].seq1 - t->first_new_seq;
-		idx2 = range[i].seq2 - t->first_new_seq;
-		for (idx = idx1; idx <= idx2; idx++)
-			seq_range_array_add(array, 0, old_to_newseq_map[idx]);
-	}
-	array_free(&old_seqs);
-}
-
-static void
-mail_index_transaction_sort_appends_keywords(struct mail_index_transaction *t,
-					     const uint32_t *old_to_newseq_map)
-{
-	struct mail_index_transaction_keyword_update *updates;
-	unsigned int i, count;
-
-	if (array_is_created(&t->keyword_updates)) {
-		updates = array_get_modifiable(&t->keyword_updates, &count);
-		for (i = 0; i < count; i++) {
-			if (array_is_created(&updates[i].add_seq)) {
-				sort_appends_seq_range(t, &updates[i].add_seq,
-						       old_to_newseq_map);
-			}
-			if (array_is_created(&updates[i].remove_seq)) {
-				sort_appends_seq_range(t,
-						       &updates[i].remove_seq,
-						       old_to_newseq_map);
-			}
-		}
-	}
-
-	if (array_is_created(&t->keyword_resets)) {
-		sort_appends_seq_range(t, &t->keyword_resets,
-				       old_to_newseq_map);
-	}
-}
-
-void mail_index_transaction_sort_appends(struct mail_index_transaction *t)
-{
-	struct mail_index_record *recs, *sorted_recs;
-	struct uid_map *new_uid_map;
-	uint32_t *old_to_newseq_map;
-	unsigned int i, count;
-
-	if (!t->appends_nonsorted)
-		return;
-
-	/* first make a copy of the UIDs and map them to sequences */
-	recs = array_get_modifiable(&t->appends, &count);
-	i_assert(count > 0);
-
-	new_uid_map = i_new(struct uid_map, count);
-	for (i = 0; i < count; i++) {
-		new_uid_map[i].idx = i;
-		new_uid_map[i].uid = recs[i].uid;
-	}
-
-	/* now sort the UID map */
-	qsort(new_uid_map, count, sizeof(*new_uid_map), uid_map_cmp);
-
-	/* sort mail records */
-	sorted_recs = i_new(struct mail_index_record, count);
-	sorted_recs[0] = recs[new_uid_map[0].idx];
-	for (i = 1; i < count; i++) {
-		sorted_recs[i] = recs[new_uid_map[i].idx];
-		if (sorted_recs[i].uid == sorted_recs[i-1].uid)
-			i_panic("Duplicate UIDs added in transaction");
-	}
-	buffer_write(t->appends.arr.buffer, 0, sorted_recs,
-		     sizeof(*sorted_recs) * count);
-	i_free(sorted_recs);
-
-	old_to_newseq_map = i_new(uint32_t, count);
-	for (i = 0; i < count; i++)
-		old_to_newseq_map[new_uid_map[i].idx] = i + t->first_new_seq;
-	i_free(new_uid_map);
-
-	mail_index_transaction_sort_appends_ext(t, &t->ext_rec_updates,
-						old_to_newseq_map);
-	mail_index_transaction_sort_appends_ext(t, &t->ext_rec_atomics,
-						old_to_newseq_map);
-	mail_index_transaction_sort_appends_keywords(t, old_to_newseq_map);
-	i_free(old_to_newseq_map);
-
-	t->appends_nonsorted = FALSE;
-}
-
-uint32_t mail_index_transaction_get_next_uid(struct mail_index_transaction *t)
-{
-	const struct mail_index_header *head_hdr, *hdr;
-	unsigned int offset;
-	uint32_t next_uid;
-
-	head_hdr = &t->view->index->map->hdr;
-	hdr = &t->view->map->hdr;
-	next_uid = t->reset || head_hdr->uid_validity != hdr->uid_validity ?
-		1 : hdr->next_uid;
-	if (array_is_created(&t->appends) && t->highest_append_uid != 0) {
-		/* get next_uid from appends if they have UIDs */
-		i_assert(next_uid <= t->highest_append_uid);
-		next_uid = t->highest_append_uid + 1;
-	}
-
-	/* see if it's been updated in pre/post header changes */
-	offset = offsetof(struct mail_index_header, next_uid);
-	if (t->post_hdr_mask[offset] != 0) {
-		hdr = (const void *)t->post_hdr_change;
-		if (hdr->next_uid > next_uid)
-			next_uid = hdr->next_uid;
-	}
-	if (t->pre_hdr_mask[offset] != 0) {
-		hdr = (const void *)t->pre_hdr_change;
-		if (hdr->next_uid > next_uid)
-			next_uid = hdr->next_uid;
-	}
-	return next_uid;
-}
-
 static int mail_index_transaction_commit_v(struct mail_index_transaction *t,
 					   uint32_t *log_file_seq_r,
 					   uoff_t *log_file_offset_r)
 {
+	struct mail_index *index = t->view->index;
 	int ret;
 
 	i_assert(t->first_new_seq >
@@ -591,14 +355,19 @@
 	if (t->cache_trans_ctx != NULL)
 		mail_cache_transaction_commit(&t->cache_trans_ctx);
 
-	if (array_is_created(&t->appends)) {
-		mail_index_transaction_sort_appends(t);
+	if (array_is_created(&t->appends))
 		mail_index_update_day_headers(t);
+
+	if (!MAIL_INDEX_TRANSACTION_HAS_CHANGES(t) && !t->reset) {
+		/* nothing to append */
+		ret = 0;
+	} else {
+		ret = mail_index_transaction_commit_real(t);
 	}
+	mail_transaction_log_get_head(index->log, log_file_seq_r,
+				      log_file_offset_r);
 
-	ret = mail_transaction_log_append(t, log_file_seq_r, log_file_offset_r);
-
-	if (ret == 0 && !t->view->index->syncing) {
+	if (ret == 0 && !index->syncing) {
 		/* if we're committing a normal transaction, we want to
 		   have those changes in the index mapping immediately. this
 		   is especially important when committing cache offset
@@ -608,7 +377,7 @@
 		   be done later as MAIL_INDEX_SYNC_HANDLER_FILE so that
 		   expunge handlers get run for the newly expunged messages
 		   (and sync handlers that require HANDLER_FILE as well). */
-		(void)mail_index_refresh(t->view->index);
+		(void)mail_index_refresh(index);
 	}
 
 	mail_index_transaction_unref(&t);
@@ -782,7 +551,7 @@
 		t->appends_nonsorted = FALSE;
 		array_free(&t->appends);
 	}
-	t->log_updates = mail_index_transaction_has_changes(t);
+	mail_index_transaction_set_log_updates(t);
 }
 
 void mail_index_expunge(struct mail_index_transaction *t, uint32_t seq)
@@ -1209,7 +978,7 @@
 	if (array_is_created(&t->ext_hdr_updates)) {
 		const struct mail_index_transaction_ext_hdr_update *hdr;
 
-		hdr = array_get(&t->ext_hdr_updates, &count);
+		hdr = array_get(&t->ext_hdr_updates, &count);
 		for (i = 0; i < count; i++) {
 			if (hdr[i].alloc_size > 0)
 				return TRUE;
@@ -1603,97 +1372,6 @@
 	t->conflict_seqs = seqs;
 }
 
-static bool
-mail_index_update_cancel_array(ARRAY_TYPE(seq_range) *array, uint32_t seq)
-{
-	if (array_is_created(array)) {
-		if (seq_range_array_remove(array, seq)) {
-			if (array_count(array) == 0)
-				array_free(array);
-			return TRUE;
-		}
-	}
-	return FALSE;
-}
-
-static bool
-mail_index_update_cancel(struct mail_index_transaction *t, uint32_t seq)
-{
-	struct mail_index_transaction_keyword_update *kw;
-	struct mail_transaction_flag_update *updates, tmp_update;
-	unsigned int i, count;
-	bool ret, have_kw_changes = FALSE;
-
-	ret = mail_index_update_cancel_array(&t->keyword_resets, seq);
-	if (array_is_created(&t->keyword_updates)) {
-		kw = array_get_modifiable(&t->keyword_updates, &count);
-		for (i = 0; i < count; i++) {
-			if (mail_index_update_cancel_array(&kw[i].add_seq, seq))
-				ret = TRUE;
-			if (mail_index_update_cancel_array(&kw[i].remove_seq,
-							   seq))
-				ret = TRUE;
-			if (array_is_created(&kw[i].add_seq) ||
-			    array_is_created(&kw[i].remove_seq))
-				have_kw_changes = TRUE;
-		}
-		if (!have_kw_changes)
-			array_free(&t->keyword_updates);
-	}
-
-	if (!array_is_created(&t->updates))
-		return ret;
-
-	updates = array_get_modifiable(&t->updates, &count);
-	i = mail_index_transaction_get_flag_update_pos(t, 0, count, seq);
-	if (i < count && updates[i].uid1 <= seq && updates[i].uid2 >= seq) {
-		/* exists */
-		ret = TRUE;
-		if (updates[i].uid1 == seq && updates[i].uid2 == seq) {
-			if (count > 1)
-				array_delete(&t->updates, i, 1);
-			else
-				array_free(&t->updates);
-		} else if (updates[i].uid1 == seq)
-			updates[i].uid1++;
-		else if (updates[i].uid2 == seq)
-			updates[i].uid2--;
-		else {
-			/* need to split it in two */
-			tmp_update = updates[i];
-			tmp_update.uid1 = seq+1;
-			updates[i].uid2 = seq-1;
-			array_insert(&t->updates, i + 1, &tmp_update, 1);
-		}
-	}
-	return ret;
-}
-
-void mail_index_transaction_check_conflicts(struct mail_index_transaction *t)
-{
-	uint32_t seq;
-
-	i_assert(t->max_modseq != 0);
-	i_assert(t->conflict_seqs != NULL);
-
-	if (t->max_modseq == mail_index_modseq_get_highest(t->view)) {
-		/* no conflicts possible */
-		return;
-	}
-	if (t->min_flagupdate_seq == 0) {
-		/* no flag updates */
-		return;
-	}
-
-	for (seq = t->min_flagupdate_seq; seq <= t->max_flagupdate_seq; seq++) {
-		if (mail_index_modseq_lookup(t->view, seq) > t->max_modseq) {
-			if (mail_index_update_cancel(t, seq))
-				seq_range_array_add(t->conflict_seqs, 0, seq);
-		}
-	}
-	t->log_updates = mail_index_transaction_has_changes(t);
-}
-
 static struct mail_index_transaction_vfuncs trans_vfuncs = {
 	mail_index_transaction_commit_v,
 	mail_index_transaction_rollback_v

--- a/src/lib-index/mail-transaction-log-append.c	Sat May 09 12:32:04 2009 -0400
+++ b/src/lib-index/mail-transaction-log-append.c	Sat May 09 14:53:06 2009 -0400
@@ -2,75 +2,14 @@
 
 #include "lib.h"
 #include "array.h"
-#include "buffer.h"
 #include "write-full.h"
 #include "mail-index-private.h"
-#include "mail-index-view-private.h"
-#include "mail-index-modseq.h"
-#include "mail-index-transaction-private.h"
 #include "mail-transaction-log-private.h"
 
-struct log_append_context {
-	struct mail_transaction_log_file *file;
-	struct mail_index_transaction *trans;
-	buffer_t *output;
-
-	uint64_t modseq;
-	uint32_t first_append_size;
-	bool sync_includes_this;
-};
-
-static void log_append_buffer(struct log_append_context *ctx,
-			      const buffer_t *buf, const buffer_t *hdr_buf,
-			      enum mail_transaction_type type)
+static int
+log_buffer_move_to_memory(struct mail_transaction_log_append_ctx *ctx)
 {
-	struct mail_transaction_header hdr;
-	uint32_t hdr_size;
-	size_t hdr_pos;
-
-	i_assert((type & MAIL_TRANSACTION_TYPE_MASK) != 0);
-	i_assert((buf->used % 4) == 0);
-	i_assert(hdr_buf == NULL || (hdr_buf->used % 4) == 0);
-
-	if (buf->used == 0)
-		return;
-
-	memset(&hdr, 0, sizeof(hdr));
-	hdr.type = type;
-	if (type == MAIL_TRANSACTION_EXPUNGE)
-		hdr.type |= MAIL_TRANSACTION_EXPUNGE_PROT;
-	if ((ctx->trans->flags & MAIL_INDEX_TRANSACTION_FLAG_EXTERNAL) != 0)
-		hdr.type |= MAIL_TRANSACTION_EXTERNAL;
-	hdr.size = sizeof(hdr) + buf->used +
-		(hdr_buf == NULL ? 0 : hdr_buf->used);
-
-	hdr_pos = ctx->output->used;
-	buffer_append(ctx->output, &hdr, sizeof(hdr));
-	if (hdr_buf != NULL)
-		buffer_append(ctx->output, hdr_buf->data, hdr_buf->used);
-	buffer_append(ctx->output, buf->data, buf->used);
-
-	if (mail_transaction_header_has_modseq(buf->data,
-			CONST_PTR_OFFSET(buf->data, sizeof(hdr)), ctx->modseq))
-		ctx->modseq++;
-
-	/* update the size */
-	hdr_size = mail_index_uint32_to_offset(hdr.size);
-	if (!MAIL_TRANSACTION_LOG_FILE_IN_MEMORY(ctx->file) &&
-	    ctx->first_append_size == 0) {
-		/* size will be written later once everything
-		   is in disk */
-		ctx->first_append_size = hdr_size;
-		hdr.size = 0;
-	} else {
-		hdr.size = hdr_size;
-	}
-	buffer_write(ctx->output, hdr_pos, &hdr, sizeof(hdr));
-}
-
-static int log_buffer_move_to_memory(struct log_append_context *ctx)
-{
-	struct mail_transaction_log_file *file = ctx->file;
+	struct mail_transaction_log_file *file = ctx->log->head;
 
 	/* first we need to truncate this latest write so that log syncing
 	   doesn't break */
@@ -87,15 +26,15 @@
 	i_assert(file->buffer_offset + file->buffer->used ==
 		 file->sync_offset);
 	buffer_append_buf(file->buffer, ctx->output, 0, (size_t)-1);
-	buffer_write(file->buffer, file->sync_offset - file->buffer_offset,
-		     &ctx->first_append_size, sizeof(uint32_t));
 	file->sync_offset = file->buffer_offset + file->buffer->used;
 	return 0;
 }
 
-static int log_buffer_write(struct log_append_context *ctx, bool want_fsync)
+static int log_buffer_write(struct mail_transaction_log_append_ctx *ctx)
 {
-	struct mail_transaction_log_file *file = ctx->file;
+	struct mail_transaction_log_file *file = ctx->log->head;
+	struct mail_transaction_header *hdr;
+	uint32_t first_size;
 
 	if (MAIL_TRANSACTION_LOG_FILE_IN_MEMORY(file)) {
 		if (file->buffer == NULL) {
@@ -107,10 +46,16 @@
 		return 0;
 	}
 
-	i_assert(ctx->first_append_size != 0);
+	/* size will be written later once everything is in disk */
+	hdr = buffer_get_space_unsafe(ctx->output, 0, sizeof(*hdr));
+	first_size = hdr->size;
+	i_assert(first_size != 0);
+	hdr->size = 0;
+
 	if (pwrite_full(file->fd, ctx->output->data, ctx->output->used,
 			file->sync_offset) < 0) {
 		/* write failure, fallback to in-memory indexes. */
+		hdr->size = first_size;
 		mail_index_file_set_syscall_error(file->log->index,
 						  file->filepath,
 						  "pwrite_full()");
@@ -123,15 +68,16 @@
 
 	/* now that the whole transaction has been written, rewrite the first
 	   record's size so the transaction becomes visible */
-	if (pwrite_full(file->fd, &ctx->first_append_size,
-			sizeof(uint32_t), file->sync_offset) < 0) {
+	if (pwrite_full(file->fd, &first_size, sizeof(uint32_t),
+			file->sync_offset +
+			offsetof(struct mail_transaction_header, size)) < 0) {
 		mail_index_file_set_syscall_error(file->log->index,
 						  file->filepath,
 						  "pwrite_full()");
 		return log_buffer_move_to_memory(ctx);
 	}
 
-	if ((want_fsync && !file->log->index->fsync_disable) ||
+	if ((ctx->want_fsync && !file->log->index->fsync_disable) ||
 	    file->log->index->nfs_flush) {
 		if (fdatasync(file->fd) < 0) {
 			mail_index_file_set_syscall_error(file->log->index,
@@ -150,531 +96,44 @@
 	return 0;
 }
 
-static const buffer_t *
-log_get_hdr_update_buffer(struct mail_index_transaction *t, bool prepend)
-{
-	buffer_t *buf;
-	const unsigned char *data, *mask;
-	struct mail_transaction_header_update u;
-	uint16_t offset;
-	int state = 0;
-
-	memset(&u, 0, sizeof(u));
-
-	data = prepend ? t->pre_hdr_change : t->post_hdr_change;
-	mask = prepend ? t->pre_hdr_mask : t->post_hdr_mask;
-
-	buf = buffer_create_dynamic(pool_datastack_create(), 256);
-	for (offset = 0; offset <= sizeof(t->pre_hdr_change); offset++) {
-		if (offset < sizeof(t->pre_hdr_change) && mask[offset]) {
-			if (state == 0) {
-				u.offset = offset;
-				state++;
-			}
-		} else {
-			if (state > 0) {
-				u.size = offset - u.offset;
-				buffer_append(buf, &u, sizeof(u));
-				buffer_append(buf, data + u.offset, u.size);
-				state = 0;
-			}
-		}
-	}
-	return buf;
-}
-
 static void
-ext_reset_update_atomic(struct mail_index_transaction *t,
-			uint32_t ext_id, uint32_t expected_reset_id)
-{
-	const struct mail_index_ext *map_ext;
-	struct mail_transaction_ext_reset *reset;
-	uint32_t idx, reset_id;
-
-	if (!mail_index_map_get_ext_idx(t->view->index->map, ext_id, &idx)) {
-		/* new extension */
-		reset_id = 1;
-	} else {
-		map_ext = array_idx(&t->view->index->map->extensions, idx);
-		reset_id = map_ext->reset_id + 1;
-	}
-	if (reset_id != expected_reset_id) {
-		/* ignore this extension update */
-		mail_index_ext_set_reset_id(t, ext_id, 0);
-		return;
-	}
-
-	if (reset_id == 0)
-		reset_id++;
-
-	array_idx_set(&t->ext_reset_ids, ext_id, &reset_id);
-
-	/* reseting existing data is optional */
-	if (array_is_created(&t->ext_resets)) {
-		reset = array_idx_modifiable(&t->ext_resets, ext_id);
-		if (reset->new_reset_id == (uint32_t)-1)
-			reset->new_reset_id = reset_id;
-	}
-}
-
-static void
-transaction_update_atomic_reset_ids(struct mail_index_transaction *t)
-{
-	const uint32_t *expected_reset_ids;
-	unsigned int ext_id, count;
-
-	if (!array_is_created(&t->ext_reset_atomic))
-		return;
-
-	expected_reset_ids = array_get(&t->ext_reset_atomic, &count);
-	for (ext_id = 0; ext_id < count; ext_id++) {
-		if (expected_reset_ids[ext_id] != 0) {
-			ext_reset_update_atomic(t, ext_id,
-						expected_reset_ids[ext_id]);
-		}
-	}
-}
-
-static void log_append_ext_intro(struct log_append_context *ctx,
-				 uint32_t ext_id, uint32_t reset_id)
-{
-	struct mail_index_transaction *t = ctx->trans;
-	const struct mail_index_registered_ext *rext;
-        struct mail_transaction_ext_intro *intro, *resizes;
-	buffer_t *buf;
-	uint32_t idx;
-	unsigned int count;
-
-	i_assert(ext_id != (uint32_t)-1);
-
-	if (t->reset ||
-	    !mail_index_map_get_ext_idx(t->view->index->map, ext_id, &idx)) {
-		/* new extension */
-		idx = (uint32_t)-1;
-	}
-
-	rext = array_idx(&t->view->index->extensions, ext_id);
-	if (!array_is_created(&t->ext_resizes)) {
-		resizes = NULL;
-		count = 0;
-	} else {
-		resizes = array_get_modifiable(&t->ext_resizes, &count);
-	}
-
-	buf = buffer_create_dynamic(pool_datastack_create(), 128);
-	if (ext_id < count && resizes[ext_id].name_size != 0) {
-		/* we're resizing the extension. use the resize struct. */
-		intro = &resizes[ext_id];
-
-		i_assert(intro->ext_id == idx);
-		intro->name_size = idx != (uint32_t)-1 ? 0 :
-			strlen(rext->name);
-		buffer_append(buf, intro, sizeof(*intro));
-	} else {
-		/* generate a new intro structure */
-		intro = buffer_append_space_unsafe(buf, sizeof(*intro));
-		intro->ext_id = idx;
-		intro->hdr_size = rext->hdr_size;
-		intro->record_size = rext->record_size;
-		intro->record_align = rext->record_align;
-		intro->flags = MAIL_TRANSACTION_EXT_INTRO_FLAG_NO_SHRINK;
-		intro->name_size = idx != (uint32_t)-1 ? 0 :
-			strlen(rext->name);
-	}
-	if (reset_id != 0) {
-		/* we're going to reset this extension in this transaction */
-		intro->reset_id = reset_id;
-	} else if (idx != (uint32_t)-1) {
-		/* use the existing reset_id */
-		const struct mail_index_ext *map_ext =
-			array_idx(&t->view->index->map->extensions, idx);
-		intro->reset_id = map_ext->reset_id;
-	} else {
-		/* new extension, reset_id defaults to 0 */
-	}
-	buffer_append(buf, rext->name, intro->name_size);
-	if ((buf->used % 4) != 0)
-		buffer_append_zero(buf, 4 - (buf->used % 4));
-
-	if (ctx->file->sync_highest_modseq == 0 &&
-	    strcmp(rext->name, MAIL_INDEX_MODSEQ_EXT_NAME) == 0) {
-		/* modseq tracking started */
-		ctx->file->sync_highest_modseq = 1;
-	}
-
-	log_append_buffer(ctx, buf, NULL, MAIL_TRANSACTION_EXT_INTRO);
-}
-
-static void
-log_append_ext_hdr_update(struct log_append_context *ctx,
-			const struct mail_index_transaction_ext_hdr_update *hdr)
+log_append_sync_offset_if_needed(struct mail_transaction_log_append_ctx *ctx)
 {
-	buffer_t *buf;
-	const unsigned char *data, *mask;
-	struct mail_transaction_ext_hdr_update u;
-	uint16_t offset;
-	bool started = FALSE;
-
-	memset(&u, 0, sizeof(u));
-
-	data = hdr->data;
-	mask = hdr->mask;
-
-	buf = buffer_create_dynamic(pool_datastack_create(), 256);
-	for (offset = 0; offset <= hdr->alloc_size; offset++) {
-		if (offset < hdr->alloc_size && mask[offset] != 0) {
-			if (!started) {
-				u.offset = offset;
-				started = TRUE;
-			}
-		} else {
-			if (started) {
-				u.size = offset - u.offset;
-				buffer_append(buf, &u, sizeof(u));
-				buffer_append(buf, data + u.offset, u.size);
-				started = FALSE;
-			}
-		}
-	}
-	if (buf->used % 4 != 0)
-		buffer_append_zero(buf, 4 - buf->used % 4);
-	log_append_buffer(ctx, buf, NULL, MAIL_TRANSACTION_EXT_HDR_UPDATE);
-}
-
-static void
-mail_transaction_log_append_ext_intros(struct log_append_context *ctx)
-{
-	struct mail_index_transaction *t = ctx->trans;
-        const struct mail_transaction_ext_intro *resize;
-	const struct mail_index_transaction_ext_hdr_update *hdrs;
-	struct mail_transaction_ext_reset ext_reset;
-	unsigned int resize_count, ext_count = 0;
-	unsigned int hdrs_count, reset_id_count, reset_count;
-	uint32_t ext_id, reset_id;
-	const struct mail_transaction_ext_reset *reset;
-	const uint32_t *reset_ids;
-	buffer_t *reset_buf;
-
-	if (!array_is_created(&t->ext_resizes)) {
-		resize = NULL;
-		resize_count = 0;
-	} else {
-		resize = array_get(&t->ext_resizes, &resize_count);
-		if (ext_count < resize_count)
-			ext_count = resize_count;
-	}
-
-	if (!array_is_created(&t->ext_reset_ids)) {
-		reset_ids = NULL;
-		reset_id_count = 0;
-	} else {
-		reset_ids = array_get(&t->ext_reset_ids, &reset_id_count);
-	}
-
-	if (!array_is_created(&t->ext_resets)) {
-		reset = NULL;
-		reset_count = 0;
-	} else {
-		reset = array_get(&t->ext_resets, &reset_count);
-		if (ext_count < reset_count)
-			ext_count = reset_count;
-	}
-
-	if (!array_is_created(&t->ext_hdr_updates)) {
-		hdrs = NULL;
-		hdrs_count = 0;
-	} else {
-		hdrs = array_get(&t->ext_hdr_updates, &hdrs_count);
-		if (ext_count < hdrs_count)
-			ext_count = hdrs_count;
-	}
-
-	memset(&ext_reset, 0, sizeof(ext_reset));
-	reset_buf = buffer_create_data(pool_datastack_create(),
-				       &ext_reset, sizeof(ext_reset));
-	buffer_set_used_size(reset_buf, sizeof(ext_reset));
-
-	for (ext_id = 0; ext_id < ext_count; ext_id++) {
-		if (ext_id < reset_count)
-			ext_reset = reset[ext_id];
-		else
-			ext_reset.new_reset_id = 0;
-		if ((ext_id < resize_count && resize[ext_id].name_size) ||
-		    ext_reset.new_reset_id != 0 ||
-		    (ext_id < hdrs_count && hdrs[ext_id].alloc_size > 0)) {
-			if (ext_reset.new_reset_id != 0) {
-				/* we're going to reset this extension
-				   immediately after the intro */
-				reset_id = 0;
-			} else {
-				reset_id = ext_id < reset_id_count ?
-					reset_ids[ext_id] : 0;
-			}
-			log_append_ext_intro(ctx, ext_id, reset_id);
-		}
-		if (ext_reset.new_reset_id != 0) {
-			i_assert(ext_id < reset_id_count &&
-				 ext_reset.new_reset_id == reset_ids[ext_id]);
-			log_append_buffer(ctx, reset_buf, NULL,
-					  MAIL_TRANSACTION_EXT_RESET);
-		}
-		if (ext_id < hdrs_count && hdrs[ext_id].alloc_size > 0) {
-			T_BEGIN {
-				log_append_ext_hdr_update(ctx, &hdrs[ext_id]);
-			} T_END;
-		}
-	}
-}
-
-static void log_append_ext_recs(struct log_append_context *ctx,
-				const ARRAY_TYPE(seq_array_array) *arr,
-				enum mail_transaction_type type)
-{
-	struct mail_index_transaction *t = ctx->trans;
-	const ARRAY_TYPE(seq_array) *updates;
-	const uint32_t *reset_ids;
-	unsigned int ext_id, count, reset_id_count;
-	uint32_t reset_id;
-
-	if (!array_is_created(&t->ext_reset_ids)) {
-		reset_ids = NULL;
-		reset_id_count = 0;
-	} else {
-		reset_ids = array_get_modifiable(&t->ext_reset_ids,
-						 &reset_id_count);
-	}
-
-	updates = array_get(arr, &count);
-	for (ext_id = 0; ext_id < count; ext_id++) {
-		if (!array_is_created(&updates[ext_id]))
-			continue;
-
-		reset_id = ext_id < reset_id_count ? reset_ids[ext_id] : 0;
-		log_append_ext_intro(ctx, ext_id, reset_id);
-
-		log_append_buffer(ctx, updates[ext_id].arr.buffer, NULL, type);
-	}
-}
-
-static void
-log_append_keyword_update(struct log_append_context *ctx,
-			  buffer_t *hdr_buf, enum modify_type modify_type,
-			  const char *keyword, const buffer_t *buffer)
-{
-	struct mail_transaction_keyword_update kt_hdr;
-
-	memset(&kt_hdr, 0, sizeof(kt_hdr));
-	kt_hdr.modify_type = modify_type;
-	kt_hdr.name_size = strlen(keyword);
-
-	buffer_set_used_size(hdr_buf, 0);
-	buffer_append(hdr_buf, &kt_hdr, sizeof(kt_hdr));
-	buffer_append(hdr_buf, keyword, kt_hdr.name_size);
-	if ((hdr_buf->used % 4) != 0)
-		buffer_append_zero(hdr_buf, 4 - (hdr_buf->used % 4));
-
-	log_append_buffer(ctx, buffer, hdr_buf,
-			  MAIL_TRANSACTION_KEYWORD_UPDATE);
-}
-
-static enum mail_index_sync_type
-log_append_keyword_updates(struct log_append_context *ctx)
-{
-        const struct mail_index_transaction_keyword_update *updates;
-	const char *const *keywords;
-	buffer_t *hdr_buf;
-	enum mail_index_sync_type change_mask = 0;
-	unsigned int i, count, keywords_count;
-
-	hdr_buf = buffer_create_dynamic(pool_datastack_create(), 64);
-
-	keywords = array_get_modifiable(&ctx->trans->view->index->keywords,
-					&keywords_count);
-	updates = array_get_modifiable(&ctx->trans->keyword_updates, &count);
-	i_assert(count <= keywords_count);
-
-	for (i = 0; i < count; i++) {
-		if (array_is_created(&updates[i].add_seq)) {
-			change_mask |= MAIL_INDEX_SYNC_TYPE_KEYWORD_ADD;
-			log_append_keyword_update(ctx, hdr_buf,
-					MODIFY_ADD, keywords[i],
-					updates[i].add_seq.arr.buffer);
-		}
-		if (array_is_created(&updates[i].remove_seq)) {
-			change_mask |= MAIL_INDEX_SYNC_TYPE_KEYWORD_REMOVE;
-			log_append_keyword_update(ctx, hdr_buf,
-					MODIFY_REMOVE, keywords[i],
-					updates[i].remove_seq.arr.buffer);
-		}
-	}
-	return change_mask;
-}
-
-static void log_append_sync_offset_if_needed(struct log_append_context *ctx)
-{
+	struct mail_transaction_log_file *file = ctx->log->head;
 	struct mail_transaction_header_update *u;
-	buffer_t *buf;
+	struct mail_transaction_header *hdr;
 	uint32_t offset;
 
-	/* Update the tail offsets only when committing the sync transaction.
-	   Other transactions may not know the latest tail offset and might
-	   end up shrinking it. (Alternatively the shrinking tail offsets could
-	   just be ignored, which would probably work fine too.) */
-	if (!ctx->trans->sync_transaction)
-		return;
-
-	if (ctx->file->max_tail_offset == ctx->file->sync_offset) {
+	if (file->max_tail_offset == file->sync_offset) {
 		/* FIXME: when we remove exclusive log locking, we
 		   can't rely on this. then write non-changed offset + check
 		   real offset + rewrite the new offset if other transactions
 		   weren't written in the middle */
-		ctx->file->max_tail_offset += ctx->output->used +
-			sizeof(struct mail_transaction_header) +
-			sizeof(*u) + sizeof(offset);
+		file->max_tail_offset += ctx->output->used +
+			sizeof(*hdr) + sizeof(*u) + sizeof(offset);
 		ctx->sync_includes_this = TRUE;
 	}
-	offset = ctx->file->max_tail_offset;
+	offset = file->max_tail_offset;
 
-	if (ctx->file->saved_tail_offset == offset)
+	if (file->saved_tail_offset == offset)
 		return;
-	i_assert(offset > ctx->file->saved_tail_offset);
+	i_assert(offset > file->saved_tail_offset);
 
-	buf = buffer_create_static_hard(pool_datastack_create(),
-					sizeof(*u) + sizeof(offset));
-	u = buffer_append_space_unsafe(buf, sizeof(*u));
+	hdr = buffer_append_space_unsafe(ctx->output, sizeof(*hdr));
+	hdr->type = MAIL_TRANSACTION_HEADER_UPDATE | MAIL_TRANSACTION_EXTERNAL;
+	hdr->size = sizeof(*hdr) + sizeof(*u) + sizeof(uint32_t);
+	hdr->size = mail_index_uint32_to_offset(hdr->size);
+
+	u = buffer_append_space_unsafe(ctx->output, sizeof(*u));
 	u->offset = offsetof(struct mail_index_header, log_file_tail_offset);
 	u->size = sizeof(offset);
-	buffer_append(buf, &offset, sizeof(offset));
-
-	log_append_buffer(ctx, buf, NULL, MAIL_TRANSACTION_HEADER_UPDATE);
+	buffer_append(ctx->output, &offset, sizeof(offset));
 }
 
-#define TRANSACTION_HAS_CHANGES(t) \
-	((t)->log_updates || (t)->log_ext_updates || \
-	 (array_is_created(&(t)->updates) && array_count(&(t)->updates) > 0))
-
 static int
-mail_transaction_log_append_locked(struct mail_index_transaction *t,
-				   uint32_t *log_file_seq_r,
-				   uoff_t *log_file_offset_r)
+mail_transaction_log_append_locked(struct mail_transaction_log_append_ctx *ctx)
 {
-	enum mail_index_sync_type change_mask = 0;
-	struct mail_index_view *view = t->view;
-	struct mail_index *index;
-	struct mail_transaction_log *log;
-	struct mail_transaction_log_file *file;
-	struct log_append_context ctx;
-	uoff_t append_offset;
-	bool want_fsync;
-
-	index = mail_index_view_get_index(view);
-	log = index->log;
-
-	if (t->reset) {
-		/* Reset the whole index, preserving only indexid. Begin by
-		   rotating the log. We don't care if we skip some non-synced
-		   transactions. */
-		if (mail_transaction_log_rotate(log, TRUE) < 0)
-			return -1;
-
-		if (!TRANSACTION_HAS_CHANGES(t)) {
-			/* we only wanted to reset */
-			return 0;
-		}
-	}
-
-	if (!index->log_locked) {
-		/* update sync_offset */
-		if (mail_transaction_log_file_map(log->head,
-						  log->head->sync_offset,
-						  (uoff_t)-1) <= 0)
-			return -1;
-	}
-
-	if (array_is_created(&t->ext_reset_atomic) || t->max_modseq != 0) {
-		if (mail_index_map(t->view->index,
-				   MAIL_INDEX_SYNC_HANDLER_HEAD) <= 0)
-			return -1;
-	}
-	if (array_is_created(&t->ext_reset_atomic))
-		transaction_update_atomic_reset_ids(t);
-	if (t->max_modseq != 0)
-		mail_index_transaction_check_conflicts(t);
-	if (!TRANSACTION_HAS_CHANGES(t)) {
-		/* we aborted all changes, nothing else to do */
-		return 0;
-	}
-	/* finally convert all sequences to UIDs before we write them,
-	   but after we've checked and removed conflicts */
-	mail_index_transaction_convert_to_uids(t);
-
-	file = log->head;
-
-	i_assert(file->sync_offset >= file->buffer_offset);
-
-	memset(&ctx, 0, sizeof(ctx));
-	ctx.file = file;
-	ctx.trans = t;
-	ctx.output = buffer_create_dynamic(default_pool, 1024);
-	ctx.modseq = file->sync_highest_modseq;
-
-	/* send all extension introductions and resizes before appends
-	   to avoid resize overhead as much as possible */
-        mail_transaction_log_append_ext_intros(&ctx);
-
-	if (t->pre_hdr_changed) {
-		log_append_buffer(&ctx,
-				  log_get_hdr_update_buffer(t, TRUE),
-				  NULL, MAIL_TRANSACTION_HEADER_UPDATE);
-	}
-	if (array_is_created(&t->appends)) {
-		change_mask |= MAIL_INDEX_SYNC_TYPE_APPEND;
-		log_append_buffer(&ctx, t->appends.arr.buffer, NULL,
-				  MAIL_TRANSACTION_APPEND);
-	}
-	if (array_is_created(&t->updates)) {
-		change_mask |= MAIL_INDEX_SYNC_TYPE_FLAGS;
-		log_append_buffer(&ctx, t->updates.arr.buffer, NULL,
-				  MAIL_TRANSACTION_FLAG_UPDATE);
-	}
-
-	if (array_is_created(&t->ext_rec_updates)) {
-		log_append_ext_recs(&ctx, &t->ext_rec_updates,
-				    MAIL_TRANSACTION_EXT_REC_UPDATE);
-	}
-	if (array_is_created(&t->ext_rec_atomics)) {
-		log_append_ext_recs(&ctx, &t->ext_rec_atomics,
-				    MAIL_TRANSACTION_EXT_ATOMIC_INC);
-	}
-
-	/* keyword resets before updates */
-	if (array_is_created(&t->keyword_resets)) {
-		change_mask |= MAIL_INDEX_SYNC_TYPE_KEYWORD_RESET;
-		log_append_buffer(&ctx, t->keyword_resets.arr.buffer,
-				  NULL, MAIL_TRANSACTION_KEYWORD_RESET);
-	}
-	if (array_is_created(&t->keyword_updates))
-		change_mask |= log_append_keyword_updates(&ctx);
-
-	if (array_is_created(&t->expunges)) {
-		/* non-external expunges are only requests, ignore them when
-		   checking fsync_mask */
-		if ((t->flags & MAIL_INDEX_TRANSACTION_FLAG_EXTERNAL) != 0)
-			change_mask |= MAIL_INDEX_SYNC_TYPE_EXPUNGE;
-		log_append_buffer(&ctx, t->expunges.arr.buffer, NULL,
-				  MAIL_TRANSACTION_EXPUNGE);
-	}
-
-	if (t->post_hdr_changed) {
-		log_append_buffer(&ctx, log_get_hdr_update_buffer(t, FALSE),
-				  NULL, MAIL_TRANSACTION_HEADER_UPDATE);
-	}
-
-	/* NOTE: mailbox sync offset update must be the last change.
-	   it may update the sync offset to include this transaction, so it
-	   needs to know this transaction's size */
-	if ((t->flags & MAIL_INDEX_TRANSACTION_FLAG_EXTERNAL) != 0)
-		log_append_sync_offset_if_needed(&ctx);
+	struct mail_transaction_log_file *file = ctx->log->head;
 
 	if (file->sync_offset < file->last_size) {
 		/* there is some garbage at the end of the transaction log
@@ -684,58 +143,55 @@
 				     file->sync_offset - file->buffer_offset);
 		if (!MAIL_TRANSACTION_LOG_FILE_IN_MEMORY(file)) {
 			if (ftruncate(file->fd, file->sync_offset) < 0) {
-				mail_index_file_set_syscall_error(index,
+				mail_index_file_set_syscall_error(ctx->log->index,
 					file->filepath, "ftruncate()");
 			}
 		}
 	}
 
-	want_fsync = (view->index->fsync_mask & change_mask) != 0 ||
-		(t->flags & MAIL_INDEX_TRANSACTION_FLAG_FSYNC) != 0;
-	append_offset = file->sync_offset;
-	if (log_buffer_write(&ctx, want_fsync) < 0) {
-		buffer_free(&ctx.output);
+	if (ctx->append_sync_offset)
+		log_append_sync_offset_if_needed(ctx);
+
+	if (log_buffer_write(ctx) < 0)
 		return -1;
-	}
-	file->sync_highest_modseq = ctx.modseq;
-	buffer_free(&ctx.output);
-
-	if ((t->flags & MAIL_INDEX_TRANSACTION_FLAG_HIDE) != 0) {
-		/* mark the area covered by this transaction hidden */
-		mail_index_view_add_hidden_transaction(view, file->hdr.file_seq,
-			append_offset, file->sync_offset - append_offset);
-	}
-
-	*log_file_seq_r = file->hdr.file_seq;
-	*log_file_offset_r = file->sync_offset;
+	file->sync_highest_modseq = ctx->new_highest_modseq;
 	return 0;
 }
 
-int mail_transaction_log_append(struct mail_index_transaction *t,
-				uint32_t *log_file_seq_r,
-				uoff_t *log_file_offset_r)
+int mail_transaction_log_append_begin(struct mail_index_transaction *t,
+				      struct mail_transaction_log_append_ctx **ctx_r)
 {
+	struct mail_transaction_log_append_ctx *ctx;
 	struct mail_index *index;
-	int ret;
-
-	*log_file_seq_r = 0;
-	*log_file_offset_r = 0;
-
-	if (!TRANSACTION_HAS_CHANGES(t) && !t->reset) {
-		/* nothing to append */
-		return 0;
-	}
 
 	index = mail_index_view_get_index(t->view);
 	if (!index->log_locked) {
 		if (mail_transaction_log_lock_head(index->log) < 0)
 			return -1;
 	}
+	ctx = i_new(struct mail_transaction_log_append_ctx, 1);
+	ctx->log = index->log;
+	ctx->output = buffer_create_dynamic(default_pool, 1024);
 
-	ret = mail_transaction_log_append_locked(t, log_file_seq_r,
-						 log_file_offset_r);
+	*ctx_r = ctx;
+	return 0;
+}
+
+int mail_transaction_log_append_commit(struct mail_transaction_log_append_ctx **_ctx)
+{
+	struct mail_transaction_log_append_ctx *ctx = *_ctx;
+	struct mail_index *index = ctx->log->index;
+	int ret = 0;
+
+	*_ctx = NULL;
+
+	if (ctx->output->used > 0)
+		ret = mail_transaction_log_append_locked(ctx);
 
 	if (!index->log_locked)
 		mail_transaction_log_file_unlock(index->log->head);
+
+	buffer_free(&ctx->output);
+	i_free(ctx);
 	return ret;
 }

--- a/src/lib-index/mail-transaction-log.h	Sat May 09 12:32:04 2009 -0400
+++ b/src/lib-index/mail-transaction-log.h	Sat May 09 14:53:06 2009 -0400
@@ -131,6 +131,16 @@
 	int32_t diff;
 };
 
+struct mail_transaction_log_append_ctx {
+	struct mail_transaction_log *log;
+	buffer_t *output;
+
+	uint64_t new_highest_modseq;
+	unsigned int append_sync_offset:1;
+	unsigned int sync_includes_this:1;
+	unsigned int want_fsync:1;
+};
+
 #define LOG_IS_BEFORE(seq1, offset1, seq2, offset2) \
 	(((offset1) < (offset2) && (seq1) == (seq2)) || (seq1) < (seq2))
 
@@ -213,13 +223,9 @@
 
 void mail_transaction_log_views_close(struct mail_transaction_log *log);
 
-/* Write data to transaction log. This is atomic operation. Sequences in
-   updates[] and expunges[] are relative to given view, they're modified
-   to real ones. If nothing is written, log_file_seq_r and log_file_offset_r
-   will be set to 0. */
-int mail_transaction_log_append(struct mail_index_transaction *t,
-				uint32_t *log_file_seq_r,
-				uoff_t *log_file_offset_r);
+int mail_transaction_log_append_begin(struct mail_index_transaction *t,
+				      struct mail_transaction_log_append_ctx **ctx_r);
+int mail_transaction_log_append_commit(struct mail_transaction_log_append_ctx **ctx);
 
 /* Lock transaction log for index synchronization. Log cannot be read or
    written to while it's locked. Returns end offset. */