# HG changeset patch
# User Timo Sirainen <tss@iki.fi>
# Date 1186404226 -10800
# Node ID 606f3d9f276905959c57cf9b27f30e7a82a8f5cb
# Parent  00453421620282425ec117971294774ed3d7916c
Prevent copying record data when appending new messages.

diff -r 004534216202 -r 606f3d9f2769 src/lib-index/mail-index-map.c
--- a/src/lib-index/mail-index-map.c	Fri Aug 03 03:25:13 2007 +0300
+++ b/src/lib-index/mail-index-map.c	Mon Aug 06 15:43:46 2007 +0300
@@ -845,11 +845,6 @@
 {
 	size_t size;
 
-	if (src == NULL) {
-		dest->buffer = buffer_create_dynamic(default_pool, 1024);
-		return;
-	}
-
 	size = src->records_count * record_size;
 	dest->buffer = buffer_create_dynamic(default_pool, I_MIN(size, 1024));
 	buffer_append(dest->buffer, src->records, size);
@@ -878,9 +873,12 @@
 	}
 	buffer_append(dest->hdr_copy_buf, &dest->hdr,
 		      I_MIN(sizeof(dest->hdr), src->hdr.base_header_size));
-	buffer_write(dest->hdr_copy_buf, src->hdr.base_header_size,
-		     CONST_PTR_OFFSET(src->hdr_base, src->hdr.base_header_size),
-		     src->hdr.header_size - src->hdr.base_header_size);
+	if (src != dest) {
+		buffer_write(dest->hdr_copy_buf, src->hdr.base_header_size,
+			     CONST_PTR_OFFSET(src->hdr_base,
+					      src->hdr.base_header_size),
+			     src->hdr.header_size - src->hdr.base_header_size);
+	}
 	dest->hdr_base = buffer_get_modifiable_data(dest->hdr_copy_buf, NULL);
 }
 
@@ -904,10 +902,15 @@
 	mem_map = i_new(struct mail_index_map, 1);
 	mem_map->index = map->index;
 	mem_map->refcount = 1;
-	mem_map->rec_map = mail_index_record_map_alloc(mem_map);
+	if (map->rec_map == NULL) {
+		mem_map->rec_map = mail_index_record_map_alloc(mem_map);
+		mem_map->rec_map->buffer =
+			buffer_create_dynamic(default_pool, 1024);
+	} else {
+		mem_map->rec_map = map->rec_map;
+		array_append(&mem_map->rec_map->maps, &mem_map, 1);
+	}
 
-	mail_index_map_copy_records(mem_map->rec_map, map->rec_map,
-				    map->hdr.record_size);
 	mail_index_map_copy_header(mem_map, map);
 
 	mem_map->write_atomic = map->write_atomic;
@@ -936,6 +939,22 @@
 	return mem_map;
 }
 
+void mail_index_record_map_move_to_private(struct mail_index_map *map)
+{
+	struct mail_index_record_map *new_map;
+
+	if (array_count(&map->rec_map->maps) == 1)
+		return;
+
+	new_map = mail_index_record_map_alloc(map);
+	mail_index_map_copy_records(new_map, map->rec_map,
+				    map->hdr.record_size);
+
+	mail_index_map_unlock(map);
+	mail_index_record_map_unlink(map);
+	map->rec_map = new_map;
+}
+
 void mail_index_map_move_to_memory(struct mail_index_map *map)
 {
 	struct mail_index_record_map *new_map;
diff -r 004534216202 -r 606f3d9f2769 src/lib-index/mail-index-private.h
--- a/src/lib-index/mail-index-private.h	Fri Aug 03 03:25:13 2007 +0300
+++ b/src/lib-index/mail-index-private.h	Mon Aug 06 15:43:46 2007 +0300
@@ -116,6 +116,8 @@
 	void *records; /* struct mail_index_record[] */
 	unsigned int records_count;
 
+	uint32_t last_appended_uid;
+
 	/* If this mapping is written to disk and write_atomic=FALSE,
 	   write_seq_* specify the message sequence range that needs to be
 	   written. */
@@ -277,6 +279,7 @@
 
 /* Clone a map. The returned map is always in memory. */
 struct mail_index_map *mail_index_map_clone(const struct mail_index_map *map);
+void mail_index_record_map_move_to_private(struct mail_index_map *map);
 /* Move a mmaped map to memory. */
 void mail_index_map_move_to_memory(struct mail_index_map *map);
 
diff -r 004534216202 -r 606f3d9f2769 src/lib-index/mail-index-sync-private.h
--- a/src/lib-index/mail-index-sync-private.h	Fri Aug 03 03:25:13 2007 +0300
+++ b/src/lib-index/mail-index-sync-private.h	Mon Aug 06 15:43:46 2007 +0300
@@ -54,7 +54,6 @@
 			   const struct mail_transaction_header *hdr,
 			   const void *data);
 
-void mail_index_sync_move_to_private(struct mail_index_sync_map_ctx *ctx);
 struct mail_index_map *
 mail_index_sync_get_atomic_map(struct mail_index_sync_map_ctx *ctx);
 void mail_index_sync_write_seq_update(struct mail_index_sync_map_ctx *ctx,
diff -r 004534216202 -r 606f3d9f2769 src/lib-index/mail-index-sync-update.c
--- a/src/lib-index/mail-index-sync-update.c	Fri Aug 03 03:25:13 2007 +0300
+++ b/src/lib-index/mail-index-sync-update.c	Mon Aug 06 15:43:46 2007 +0300
@@ -59,7 +59,8 @@
 		view->index->map = map;
 }
 
-void mail_index_sync_move_to_private(struct mail_index_sync_map_ctx *ctx)
+static void
+mail_index_sync_move_to_private_memory(struct mail_index_sync_map_ctx *ctx)
 {
 	struct mail_index_map *map = ctx->view->map;
 
@@ -70,12 +71,6 @@
 		map = mail_index_map_clone(map);
 		mail_index_sync_replace_map(ctx, map);
 	}
-}
-
-static void
-mail_index_sync_move_to_private_memory(struct mail_index_sync_map_ctx *ctx)
-{
-	mail_index_sync_move_to_private(ctx);
 
 	if (!MAIL_INDEX_MAP_IS_IN_MEMORY(ctx->view->map))
 		mail_index_map_move_to_memory(ctx->view->map);
@@ -85,6 +80,7 @@
 mail_index_sync_get_atomic_map(struct mail_index_sync_map_ctx *ctx)
 {
 	mail_index_sync_move_to_private_memory(ctx);
+	mail_index_record_map_move_to_private(ctx->view->map);
 	ctx->view->map->write_atomic = TRUE;
 	return ctx->view->map;
 }
@@ -137,6 +133,7 @@
 
 static void
 mail_index_sync_header_update_counts_all(struct mail_index_sync_map_ctx *ctx,
+					 uint32_t uid,
 					 uint8_t old_flags, uint8_t new_flags)
 {
 	struct mail_index_map *const *maps;
@@ -145,6 +142,9 @@
 
 	maps = array_get(&ctx->view->map->rec_map->maps, &count);
 	for (i = 0; i < count; i++) {
+		if (uid >= maps[i]->hdr.next_uid)
+			continue;
+
 		if (mail_index_header_update_counts(&maps[i]->hdr,
 						    old_flags, new_flags,
 						    &error) < 0)
@@ -154,15 +154,16 @@
 
 static void
 mail_index_sync_header_update_counts(struct mail_index_sync_map_ctx *ctx,
-				     uint8_t old_flags, uint8_t new_flags,
-				     bool all)
+				     uint32_t uid, uint8_t old_flags,
+				     uint8_t new_flags, bool all)
 {
 	const char *error;
 
 	if (all) {
-		mail_index_sync_header_update_counts_all(ctx, old_flags,
+		mail_index_sync_header_update_counts_all(ctx, uid, old_flags,
 							 new_flags);
 	} else {
+		i_assert(uid < ctx->view->map->hdr.next_uid);
 		if (mail_index_header_update_counts(&ctx->view->map->hdr,
 						    old_flags, new_flags,
 						    &error) < 0)
@@ -240,7 +241,7 @@
 		map = mail_index_sync_get_atomic_map(ctx);
 		for (seq = seq1; seq <= seq2; seq++) {
 			rec = MAIL_INDEX_MAP_IDX(map, seq-1);
-			mail_index_sync_header_update_counts(ctx,
+			mail_index_sync_header_update_counts(ctx, rec->uid,
 							     rec->flags, 0,
 							     FALSE);
 		}
@@ -294,30 +295,36 @@
 	mail_index_sync_move_to_private_memory(ctx);
 	map = view->map;
 
-	/* don't rely on buffer->used being at the correct position.
-	   at least expunges can move it */
-	append_pos = map->rec_map->records_count * map->hdr.record_size;
-	dest = buffer_get_space_unsafe(map->rec_map->buffer, append_pos,
-				       map->hdr.record_size);
-	map->rec_map->records =
-		buffer_get_modifiable_data(map->rec_map->buffer, NULL);
+	if (rec->uid <= map->rec_map->last_appended_uid) {
+		i_assert(map->hdr.messages_count < map->rec_map->records_count);
+	} else {
+		/* don't rely on buffer->used being at the correct position.
+		   at least expunges can move it */
+		append_pos = map->rec_map->records_count * map->hdr.record_size;
+		dest = buffer_get_space_unsafe(map->rec_map->buffer, append_pos,
+					       map->hdr.record_size);
+		map->rec_map->records =
+			buffer_get_modifiable_data(map->rec_map->buffer, NULL);
 
-	memcpy(dest, rec, sizeof(*rec));
-	memset(PTR_OFFSET(dest, sizeof(*rec)), 0,
-	       map->hdr.record_size - sizeof(*rec));
+		memcpy(dest, rec, sizeof(*rec));
+		memset(PTR_OFFSET(dest, sizeof(*rec)), 0,
+		       map->hdr.record_size - sizeof(*rec));
+		map->rec_map->records_count++;
+		map->rec_map->last_appended_uid = rec->uid;
+
+		mail_index_sync_write_seq_update(ctx, map->hdr.messages_count,
+						 map->hdr.messages_count);
+	}
 
 	map->hdr.messages_count++;
 	map->hdr.next_uid = rec->uid+1;
-	map->rec_map->records_count++;
-
-	mail_index_sync_write_seq_update(ctx, map->hdr.messages_count,
-					 map->hdr.messages_count);
 
 	if ((rec->flags & MAIL_INDEX_MAIL_FLAG_DIRTY) != 0)
 		map->hdr.flags |= MAIL_INDEX_HDR_FLAG_HAVE_DIRTY;
 
 	mail_index_header_update_lowwaters(ctx, rec);
-	mail_index_sync_header_update_counts(ctx, 0, rec->flags, FALSE);
+	mail_index_sync_header_update_counts(ctx, rec->uid,
+					     0, rec->flags, FALSE);
 	return 1;
 }
 
@@ -358,7 +365,8 @@
 			rec->flags = (rec->flags & flag_mask) | u->add_flags;
 
 			mail_index_header_update_lowwaters(ctx, rec);
-			mail_index_sync_header_update_counts(ctx, old_flags,
+			mail_index_sync_header_update_counts(ctx, rec->uid,
+							     old_flags,
 							     rec->flags, TRUE);
 		}
 	}
@@ -585,9 +593,6 @@
 		i_unreached();
 	}
 	t_pop();
-
-	i_assert(ctx->view->map->rec_map->records_count ==
-		 ctx->view->map->hdr.messages_count);
 	return ret;
 }
 
diff -r 004534216202 -r 606f3d9f2769 src/lib-index/mail-index-view-sync.c
--- a/src/lib-index/mail-index-view-sync.c	Fri Aug 03 03:25:13 2007 +0300
+++ b/src/lib-index/mail-index-view-sync.c	Mon Aug 06 15:43:46 2007 +0300
@@ -324,8 +324,6 @@
 		} else {
 			map = view->map;
 		}
-		i_assert(map->rec_map->records_count ==
-			 map->hdr.messages_count);
 	}
 
 #ifdef DEBUG