--- a/src/lib-index/mail-index-map.c	Fri Jun 15 23:49:52 2007 +0300
+++ b/src/lib-index/mail-index-map.c	Fri Jun 15 23:52:47 2007 +0300
@@ -697,8 +697,8 @@
 		return ret;
 	}
 
-	index->last_read_log_file_index_offset =
-		new_map->hdr.log_file_index_int_offset;
+	index->last_read_log_file_tail_offset =
+		new_map->hdr.log_file_tail_offset;
 	mail_index_unmap(index, map);
 	*map = new_map;
 	return 1;

--- a/src/lib-index/mail-index-private.h	Fri Jun 15 23:49:52 2007 +0300
+++ b/src/lib-index/mail-index-private.h	Fri Jun 15 23:52:47 2007 +0300
@@ -159,7 +159,9 @@
 	const struct mail_index_header *hdr;
 	struct mail_index_map *map;
 	uint32_t indexid;
-	uint32_t last_read_log_file_index_offset;
+	/* last known log_file_tail_offset in main index file. used for
+	   optimizing main index updates. */
+	uint32_t last_read_log_file_tail_offset;
 
 	int lock_type, shared_lock_count, excl_lock_count;
 	unsigned int lock_id;

--- a/src/lib-index/mail-index-sync-update.c	Fri Jun 15 23:49:52 2007 +0300
+++ b/src/lib-index/mail-index-sync-update.c	Fri Jun 15 23:52:47 2007 +0300
@@ -9,6 +9,7 @@
 #include "mail-transaction-log.h"
 #include "mail-transaction-log-private.h"
 
+#if 0 // FIXME: can we / do we want to support this?
 static void
 mail_index_sync_update_log_offset(struct mail_index_sync_map_ctx *ctx,
 				  struct mail_index_map *map, bool eol)
@@ -32,22 +33,11 @@
 		prev_offset = ctx->ext_intro_offset;
 	}
 
-	i_assert(prev_offset >= map->hdr.log_file_index_int_offset ||
-		 prev_seq > map->hdr.log_file_seq);
-	map->hdr.log_file_index_int_offset = prev_offset;
-
-	/* we might be in the middle of syncing internal transactions, with
-	   some of the following external transactions already synced. */
-	i_assert(prev_seq > map->hdr.log_file_seq ||
-		 prev_offset >= map->hdr.log_file_index_ext_offset || !eol);
-	if (map->hdr.log_file_seq != prev_seq ||
-	    prev_offset > map->hdr.log_file_index_ext_offset) {
-		map->hdr.log_file_seq = prev_seq;
-		map->hdr.log_file_index_ext_offset = prev_offset;
-	}
+	i_assert(prev_seq == map->hdr.log_file_seq);
+	i_assert(prev_offset >= map->hdr.log_file_head_offset);
+	map->hdr.log_file_head_offset = prev_offset;
 }
 
-#if 0 // FIXME: can we / do we want to support this?
 static int
 mail_index_map_msync(struct mail_index *index, struct mail_index_map *map)
 {
@@ -730,7 +720,7 @@
 	const struct mail_transaction_header *thdr;
 	const void *tdata;
 	uint32_t prev_seq, mailbox_sync_seq;
-	uoff_t prev_offset, mailbox_sync_offset;
+	uoff_t start_offset, prev_offset, mailbox_sync_offset;
 	int ret;
 	bool had_dirty;
 
@@ -748,15 +738,16 @@
 		/* this isn't necessary correct currently, but it should be
 		   close enough */
 		log_size = index->log->head->last_size;
-		if (log_size > map->hdr.log_file_index_int_offset &&
-		    log_size - map->hdr.log_file_index_int_offset > index_size)
+		if (log_size > map->hdr.log_file_tail_offset &&
+		    log_size - map->hdr.log_file_tail_offset > index_size)
 			return 0;
 	}
 
+	start_offset = type == MAIL_INDEX_SYNC_HANDLER_FILE ?
+		map->hdr.log_file_tail_offset : map->hdr.log_file_head_offset;
 	view = mail_index_view_open_with_map(index, map);
 	if (mail_transaction_log_view_set(view->log_view,
-					  map->hdr.log_file_seq,
-					  map->hdr.log_file_index_int_offset,
+					  map->hdr.log_file_seq, start_offset,
 					  (uint32_t)-1, (uoff_t)-1) <= 0) {
 		/* can't use it. sync by re-reading index. */
 		mail_index_view_close(&view);
@@ -806,12 +797,16 @@
 		mail_transaction_log_view_get_prev_pos(view->log_view,
 						       &prev_seq, &prev_offset);
 
-		if ((thdr->type & MAIL_TRANSACTION_EXTERNAL) != 0) {
-			/* see if this transaction is already synced */
-			if (prev_seq < view->map->hdr.log_file_seq ||
-			    (prev_seq == view->map->hdr.log_file_seq &&
-			     prev_offset <
-			     view->map->hdr.log_file_index_ext_offset))
+		if (LOG_IS_BEFORE(prev_seq, prev_offset,
+				  view->map->hdr.log_file_seq,
+				  view->map->hdr.log_file_head_offset)) {
+			/* this has been synced already. we're here only to call
+			   expunge handlers and extension update handlers. */
+			i_assert(type == MAIL_INDEX_SYNC_HANDLER_FILE);
+
+			if ((thdr->type & MAIL_TRANSACTION_EXTERNAL) != 0)
+				continue;
+			if ((thdr->type & MAIL_TRANSACTION_EXT_MASK) == 0)
 				continue;
 		}
 
@@ -827,15 +822,17 @@
 		mail_index_sync_update_hdr_dirty_flag(map);
 
 	/* update sync position */
-	// FIXME: eol=TRUE gives intro errors
-	mail_index_sync_update_log_offset(&sync_map_ctx, map, FALSE);
+	mail_transaction_log_view_get_prev_pos(view->log_view,
+					       &prev_seq, &prev_offset);
+	i_assert(index->log->head->hdr.file_seq == prev_seq);
+	map->hdr.log_file_seq = prev_seq;
+	map->hdr.log_file_head_offset = prev_offset;
 
-	/* although mailbox_sync_update gets updated by the header update
-	   records, transaction log syncing can internally also update
-	   mailbox_sync_max_offset to skip over following external
-	   transactions. use it to avoid extra unneeded log reading. */
-	map->hdr.log_file_mailbox_offset =
-		index->log->head->mailbox_sync_max_offset;
+	/* transaction log tracks internally the current tail offset.
+	   besides using header updates, it also updates the offset to skip
+	   over following external transactions to avoid extra unneeded log
+	   reading. */
+	map->hdr.log_file_tail_offset = index->log->head->max_tail_offset;
 
 	if (map->write_base_header) {
 		i_assert(MAIL_INDEX_MAP_IS_IN_MEMORY(map));

--- a/src/lib-index/mail-index-sync.c	Fri Jun 15 23:49:52 2007 +0300
+++ b/src/lib-index/mail-index-sync.c	Fri Jun 15 23:52:47 2007 +0300
@@ -22,7 +22,7 @@
 
 	ARRAY_DEFINE(sync_list, struct mail_index_sync_list);
 	uint32_t next_uid;
-	uint32_t last_mailbox_seq, last_mailbox_offset;
+	uint32_t last_tail_seq, last_tail_offset;
 
 	uint32_t append_uid_first, append_uid_last;
 
@@ -211,8 +211,8 @@
 	mail_transaction_log_view_get_prev_pos(ctx->view->log_view,
 					       &seq, &offset);
 
-	ctx->last_mailbox_seq = seq;
-	ctx->last_mailbox_offset = offset + ctx->hdr->size + sizeof(*ctx->hdr);
+	ctx->last_tail_seq = seq;
+	ctx->last_tail_offset = offset + ctx->hdr->size + sizeof(*ctx->hdr);
 }
 
 static int
@@ -302,7 +302,7 @@
 
 	if (hdr->log_file_seq < log_file_seq ||
 	     (hdr->log_file_seq == log_file_seq &&
-	      hdr->log_file_mailbox_offset < log_file_offset))
+	      hdr->log_file_tail_offset < log_file_offset))
 		return TRUE;
 
 	/* already synced */
@@ -373,11 +373,9 @@
 		return 0;
 	}
 
-	if (hdr->log_file_index_int_offset > hdr->log_file_index_ext_offset ||
+	if (hdr->log_file_tail_offset > hdr->log_file_head_offset ||
 	    hdr->log_file_seq > seq ||
-	    (hdr->log_file_seq == seq &&
-	     (hdr->log_file_index_ext_offset > offset ||
-	      hdr->log_file_mailbox_offset > offset))) {
+	    (hdr->log_file_seq == seq && hdr->log_file_tail_offset > offset)) {
 		/* broken sync positions. fix them. */
 		mail_index_set_error(index,
 			"broken sync positions in index file %s",
@@ -394,8 +392,8 @@
 	ctx->lock_id = lock_id;
 	ctx->sync_recent = sync_recent;
 	ctx->sync_dirty = sync_dirty;
-	ctx->last_mailbox_seq = hdr->log_file_seq;
-	ctx->last_mailbox_offset = hdr->log_file_mailbox_offset;
+	ctx->last_tail_seq = hdr->log_file_seq;
+	ctx->last_tail_offset = hdr->log_file_tail_offset;
 
 	ctx->view = mail_index_view_open(index);
 
@@ -406,7 +404,7 @@
 	/* we wish to see all the changes from last mailbox sync position to
 	   the end of the transaction log */
 	if (mail_index_sync_set_log_view(ctx->view, hdr->log_file_seq,
-					 hdr->log_file_mailbox_offset) < 0) {
+					 hdr->log_file_tail_offset) < 0) {
                 mail_index_sync_rollback(&ctx);
 		return -1;
 	}
@@ -721,7 +719,7 @@
 		mail_index_unlock(index, lock_id);
 	}
 
-	index->last_read_log_file_index_offset = hdr->log_file_index_int_offset;
+	index->last_read_log_file_tail_offset = hdr->log_file_tail_offset;
 
 	map->write_atomic = FALSE;
 	map->write_seq_first = map->write_seq_last = 0;
@@ -729,8 +727,7 @@
 	map->write_ext_header = FALSE;
 
 	if (want_rotate &&
-	    hdr->log_file_index_int_offset == hdr->log_file_index_ext_offset &&
-	    hdr->log_file_index_int_offset == hdr->log_file_mailbox_offset)
+	    hdr->log_file_tail_offset == hdr->log_file_head_offset)
 		(void)mail_transaction_log_rotate(index->log);
 }
 
@@ -745,11 +742,9 @@
 					       &seq, &offset);
 	mail_transaction_log_set_mailbox_sync_pos(ctx->index->log, seq, offset);
 
-	/* This sync may have seen only external transactions, in which case
-	   it's not required to write the mailbox sync offset. Otherwise we
-	   must update the offset even if nothing else is going to be
-	   written. */
-	if (hdr->log_file_mailbox_offset != ctx->last_mailbox_offset)
+	/* If tail offset has changed, make sure it gets written to
+	   transaction log. */
+	if (hdr->log_file_tail_offset != ctx->last_tail_offset)
 		ctx->ext_trans->log_updates = TRUE;
 }
 
@@ -788,8 +783,8 @@
 
 	/* FIXME: create a better rule? */
 	want_rotate = mail_transaction_log_want_rotate(index->log);
-	diff = index->map->hdr.log_file_index_int_offset -
-		index->last_read_log_file_index_offset;
+	diff = index->map->hdr.log_file_tail_offset -
+		index->last_read_log_file_tail_offset;
 	if (ret == 0 && (diff > 1024 || want_rotate))
 		mail_index_write(index, want_rotate);
 	mail_index_sync_end(_ctx);

--- a/src/lib-index/mail-index-view-sync.c	Fri Jun 15 23:49:52 2007 +0300
+++ b/src/lib-index/mail-index-view-sync.c	Fri Jun 15 23:52:47 2007 +0300
@@ -119,7 +119,7 @@
 	ret = mail_transaction_log_view_set(view->log_view,
 					    tail_seq, tail_offset,
 					    hdr->log_file_seq,
-					    hdr->log_file_index_int_offset);
+					    hdr->log_file_head_offset);
 	if (ret <= 0) {
 		if (ret == 0) {
 			/* FIXME: use the new index to get needed changes */
@@ -256,8 +256,7 @@
 
 #define VIEW_IS_SYNCED_TO_SAME(hdr, tail_seq, tail_offset) \
 	((hdr)->log_file_seq == (tail_seq) && \
-	 (hdr)->log_file_index_int_offset == (tail_offset) && \
-	 (hdr)->log_file_index_ext_offset == (tail_offset))
+	 (hdr)->log_file_head_offset == (tail_offset))
 
 int mail_index_view_sync_begin(struct mail_index_view *view,
                                enum mail_index_view_sync_type sync_type,
@@ -525,14 +524,14 @@
 
 		mail_transaction_log_view_get_prev_pos(log_view, &seq, &offset);
 
-		/* if we started from a map that we didn't create ourself,
-		   some of the external transactions may already be synced.
-		   at the end of view sync we'll update the file_seq=0 in the
-		   header so that this check always becomes FALSE for
-		   subsequent syncs. */
-		synced_to_map = offset < view->hdr.log_file_index_ext_offset &&
-			seq == view->hdr.log_file_seq &&
-			(hdr->type & MAIL_TRANSACTION_EXTERNAL) != 0;
+		/* If we started from a map that we didn't create ourself,
+		   some of the transactions may already be synced. at the end
+		   of this view sync we'll update file_seq=0 so that this check
+		   always becomes FALSE for subsequent syncs. */
+		synced_to_map = view->map->hdr.log_file_seq != 0 &&
+			!LOG_IS_BEFORE(seq, offset,
+				       view->map->hdr.log_file_seq,
+				       view->map->hdr.log_file_head_offset);
 
 		/* Apply transaction to view's mapping if needed (meaning we
 		   didn't just re-map the view to head mapping). */
@@ -740,8 +739,8 @@
 		/* log offsets have no meaning in views. make sure they're not
 		   tried to be used wrong by setting them to zero. */
 		view->map->hdr.log_file_seq = 0;
-		view->map->hdr.log_file_index_int_offset = 0;
-		view->map->hdr.log_file_index_ext_offset = 0;
+		view->map->hdr.log_file_head_offset = 0;
+		view->map->hdr.log_file_tail_offset = 0;
 	}
 	view->hdr = view->map->hdr;
 

--- a/src/lib-index/mail-index-view.c	Fri Jun 15 23:49:52 2007 +0300
+++ b/src/lib-index/mail-index-view.c	Fri Jun 15 23:52:47 2007 +0300
@@ -698,7 +698,7 @@
 	view->log_file_append_offset =
 		view->log_file_expunge_offset =
 		view->log_file_head_offset =
-		view->map->hdr.log_file_index_int_offset;
+		view->map->hdr.log_file_head_offset;
 
 	i_array_init(&view->module_contexts,
 		     I_MIN(5, mail_index_module_register.id));

--- a/src/lib-index/mail-index.h	Fri Jun 15 23:49:52 2007 +0300
+++ b/src/lib-index/mail-index.h	Fri Jun 15 23:52:47 2007 +0300
@@ -72,8 +72,10 @@
 	uint32_t first_deleted_uid_lowwater;
 
 	uint32_t log_file_seq;
-	uint32_t log_file_index_int_offset;
-	uint32_t log_file_index_ext_offset;
+	/* non-external records between tail..head haven't been committed to
+	   mailbox yet. */
+	uint32_t log_file_tail_offset;
+	uint32_t log_file_head_offset;
 
 	uint64_t sync_size;
 	uint32_t sync_stamp;
@@ -81,9 +83,6 @@
 	/* daily first UIDs that have been added to index. */
 	uint32_t day_stamp;
 	uint32_t day_first_uid[8];
-
-	uint32_t log_file_mailbox_offset;
-	uint32_t padding;
 };
 
 struct mail_index_record {

--- a/src/lib-index/mail-transaction-log-append.c	Fri Jun 15 23:49:52 2007 +0300
+++ b/src/lib-index/mail-transaction-log-append.c	Fri Jun 15 23:52:47 2007 +0300
@@ -101,7 +101,7 @@
 
 	i_assert(!ctx->sync_includes_this ||
 		 file->sync_offset + ctx->output->used ==
-		 file->mailbox_sync_max_offset);
+		 file->max_tail_offset);
 
 	if (!file->log->index->fsync_disable && fdatasync(file->fd) < 0) {
 		mail_index_file_set_syscall_error(file->log->index,
@@ -369,25 +369,25 @@
 	buffer_t *buf;
 	uint32_t offset;
 
-	if (ctx->file->mailbox_sync_max_offset == ctx->file->sync_offset) {
+	if (ctx->file->max_tail_offset == ctx->file->sync_offset) {
 		/* FIXME: when we remove exclusive log locking, we
 		   can't rely on this. then write non-changed offset + check
 		   real offset + rewrite the new offset if other transactions
 		   weren't written in the middle */
-		ctx->file->mailbox_sync_max_offset += ctx->output->used +
+		ctx->file->max_tail_offset += ctx->output->used +
 			sizeof(struct mail_transaction_header) +
 			sizeof(*u) + sizeof(offset);
 		ctx->sync_includes_this = TRUE;
 	}
-	offset = ctx->file->mailbox_sync_max_offset;
+	offset = ctx->file->max_tail_offset;
 
-	if (ctx->file->mailbox_sync_saved_offset == offset)
+	if (ctx->file->saved_tail_offset == offset)
 		return;
 
 	buf = buffer_create_static_hard(pool_datastack_create(),
 					sizeof(*u) + sizeof(offset));
 	u = buffer_append_space_unsafe(buf, sizeof(*u));
-	u->offset = offsetof(struct mail_index_header, log_file_mailbox_offset);
+	u->offset = offsetof(struct mail_index_header, log_file_tail_offset);
 	u->size = sizeof(offset);
 	buffer_append(buf, &offset, sizeof(offset));
 

--- a/src/lib-index/mail-transaction-log-file.c	Fri Jun 15 23:49:52 2007 +0300
+++ b/src/lib-index/mail-transaction-log-file.c	Fri Jun 15 23:52:47 2007 +0300
@@ -107,20 +107,19 @@
 	struct mail_index_map *map = log->index->map;
 
 	if (map != NULL && file->hdr.file_seq == map->hdr.log_file_seq &&
-	    map->hdr.log_file_index_int_offset != 0) {
+	    map->hdr.log_file_head_offset != 0) {
 		/* we can get a valid log offset from index file. initialize
 		   sync_offset from it so we don't have to read the whole log
 		   file from beginning. */
-		if (map->hdr.log_file_index_int_offset >= file->hdr.hdr_size)
-			file->sync_offset = map->hdr.log_file_index_int_offset;
+		if (map->hdr.log_file_head_offset >= file->hdr.hdr_size)
+			file->sync_offset = map->hdr.log_file_head_offset;
 		else {
 			mail_index_set_error(log->index,
-				"%s: log_file_index_int_offset too small",
+				"%s: log_file_head_offset too small",
 				log->index->filepath);
 			file->sync_offset = file->hdr.hdr_size;
 		}
-		file->mailbox_sync_saved_offset =
-			map->hdr.log_file_mailbox_offset;
+		file->saved_tail_offset = map->hdr.log_file_tail_offset;
 	} else {
 		file->sync_offset = file->hdr.hdr_size;
 	}
@@ -161,7 +160,7 @@
 	}
 	if (index->hdr != NULL) {
 		hdr->prev_file_seq = index->hdr->log_file_seq;
-		hdr->prev_file_offset = index->hdr->log_file_index_int_offset;
+		hdr->prev_file_offset = index->hdr->log_file_head_offset;
 		hdr->file_seq = index->hdr->log_file_seq + 1;
 	} else {
 		hdr->file_seq = 1;
@@ -628,8 +627,8 @@
 	const struct mail_transaction_header_update *u = data;
 	const struct mail_index_header *ihdr;
 	const unsigned int offset_pos =
-		offsetof(struct mail_index_header, log_file_mailbox_offset);
-	const unsigned int offset_size = sizeof(ihdr->log_file_mailbox_offset);
+		offsetof(struct mail_index_header, log_file_tail_offset);
+	const unsigned int offset_size = sizeof(ihdr->log_file_tail_offset);
 	uint32_t sync_offset;
 
 	i_assert(offset_size == sizeof(sync_offset));
@@ -646,14 +645,14 @@
 		       CONST_PTR_OFFSET(u + 1, offset_pos - u->offset),
 		       sizeof(sync_offset));
 
-		if (sync_offset < file->mailbox_sync_saved_offset) {
+		if (sync_offset < file->saved_tail_offset) {
 			mail_transaction_log_file_set_corrupted(file,
-				"mailbox_sync_offset shrinked");
+				"log_file_tail_offset shrinked");
 			return -1;
 		}
-		file->mailbox_sync_saved_offset = sync_offset;
-		if (sync_offset > file->mailbox_sync_max_offset)
-			file->mailbox_sync_max_offset = sync_offset;
+		file->saved_tail_offset = sync_offset;
+		if (sync_offset > file->max_tail_offset)
+			file->max_tail_offset = sync_offset;
 		return 1;
 	}
 	return 0;
@@ -678,11 +677,11 @@
 			return ret < 0 ? -1 : 0;
 	}
 
-	if (file->mailbox_sync_max_offset == file->sync_offset) {
+	if (file->max_tail_offset == file->sync_offset) {
 		/* external transactions aren't synced to mailbox. we can
 		   update mailbox sync offset to skip this transaction to
 		   avoid re-reading it at the next sync. */
-		file->mailbox_sync_max_offset += trans_size;
+		file->max_tail_offset += trans_size;
 	}
 	return 0;
 }

--- a/src/lib-index/mail-transaction-log-private.h	Fri Jun 15 23:49:52 2007 +0300
+++ b/src/lib-index/mail-transaction-log-private.h	Fri Jun 15 23:52:47 2007 +0300
@@ -46,12 +46,12 @@
 
 	/* points to the next uncommitted transaction. usually same as EOF. */
 	uoff_t sync_offset;
-	/* saved_offset is the offset that was last written to transaction log.
-	   max_offset is what should be written to the log the next time a
-	   transaction is written. transaction log handling may update
-	   max_offset automatically by making it skip external transactions
+	/* saved_tail_offset is the offset that was last written to transaction
+	   log. max_tail_offset is what should be written to the log the next
+	   time a transaction is written. transaction log handling may update
+	   max_tail_offset automatically by making it skip external transactions
 	   after the last saved offset (to avoid re-reading them unneededly). */
-	uoff_t mailbox_sync_saved_offset, mailbox_sync_max_offset;
+	uoff_t saved_tail_offset, max_tail_offset;
 
 	struct file_lock *file_lock;
 

--- a/src/lib-index/mail-transaction-log.c	Fri Jun 15 23:49:52 2007 +0300
+++ b/src/lib-index/mail-transaction-log.c	Fri Jun 15 23:52:47 2007 +0300
@@ -248,7 +248,7 @@
 					       uoff_t *file_offset_r)
 {
 	*file_seq_r = log->head->hdr.file_seq;
-	*file_offset_r = log->head->mailbox_sync_max_offset;
+	*file_offset_r = log->head->max_tail_offset;
 }
 
 void mail_transaction_log_set_mailbox_sync_pos(struct mail_transaction_log *log,
@@ -256,10 +256,10 @@
 					       uoff_t file_offset)
 {
 	i_assert(file_seq == log->head->hdr.file_seq);
-	i_assert(file_offset >= log->head->mailbox_sync_saved_offset);
+	i_assert(file_offset >= log->head->saved_tail_offset);
 
-	if (file_offset >= log->head->mailbox_sync_max_offset)
-		log->head->mailbox_sync_max_offset = file_offset;
+	if (file_offset >= log->head->max_tail_offset)
+		log->head->max_tail_offset = file_offset;
 }
 
 int mail_transaction_log_find_file(struct mail_transaction_log *log,

--- a/src/lib-index/mail-transaction-log.h	Fri Jun 15 23:49:52 2007 +0300
+++ b/src/lib-index/mail-transaction-log.h	Fri Jun 15 23:52:47 2007 +0300
@@ -31,6 +31,10 @@
 
 	MAIL_TRANSACTION_TYPE_MASK		= 0x0000ffff,
 
+#define MAIL_TRANSACTION_EXT_MASK \
+	(MAIL_TRANSACTION_EXT_INTRO | MAIL_TRANSACTION_EXT_RESET | \
+	MAIL_TRANSACTION_EXT_HDR_UPDATE | MAIL_TRANSACTION_EXT_REC_UPDATE)
+
 	/* since we'll expunge mails based on data read from transaction log,
 	   try to avoid the possibility of corrupted transaction log expunging
 	   messages. this value is ORed to the actual MAIL_TRANSACTION_EXPUNGE