Mercurial > dovecot > original-hg > dovecot-1.2

changeset 6165:6418139447fa HEAD

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Created a new struct mail_index_record_map which can be shared by multiple mail_index_maps. This avoids having to copy records when appending new messages.
author Timo Sirainen <tss@iki.fi>
date Fri, 03 Aug 2007 03:22:04 +0300
parents 22398d619cac
children 004534216202
files src/lib-index/mail-cache.c src/lib-index/mail-index-fsck.c src/lib-index/mail-index-map.c src/lib-index/mail-index-private.h src/lib-index/mail-index-sync-ext.c src/lib-index/mail-index-sync-update.c src/lib-index/mail-index-transaction.c src/lib-index/mail-index-view-sync.c src/lib-index/mail-index-view.c src/lib-index/mail-index-write.c src/lib-index/mail-index.c
diffstat 11 files changed, 278 insertions(+), 166 deletions(-) [+]
line wrap: on
line diff
--- a/src/lib-index/mail-cache.c	Thu Aug 02 19:06:34 2007 +0300
+++ b/src/lib-index/mail-cache.c	Fri Aug 03 03:22:04 2007 +0300
@@ -527,8 +527,8 @@
 	uoff_t max_del_space;
 
         cont_percentage = hdr->continued_record_count * 100 /
-		(cache->index->map->records_count == 0 ? 1 :
-		 cache->index->map->records_count);
+		(cache->index->map->rec_map->records_count == 0 ? 1 :
+		 cache->index->map->rec_map->records_count);
 	if (cont_percentage >= MAIL_CACHE_COMPRESS_CONTINUED_PERCENTAGE &&
 	    hdr->used_file_size >= MAIL_CACHE_COMPRESS_MIN_SIZE) {
 		/* too many continued rows, compress */
--- a/src/lib-index/mail-index-fsck.c	Thu Aug 02 19:06:34 2007 +0300
+++ b/src/lib-index/mail-index-fsck.c	Fri Aug 03 03:22:04 2007 +0300
@@ -78,8 +78,8 @@
 	hdr.first_unseen_uid_lowwater = 0;
 	hdr.first_deleted_uid_lowwater = 0;
 
-	rec = map->records; last_uid = 0;
-	for (i = 0; i < map->records_count; i++) {
+	rec = map->rec_map->records; last_uid = 0;
+	for (i = 0; i < map->rec_map->records_count; i++) {
 		if (rec->uid <= last_uid) {
 			*error_r = "Record UIDs are not ordered";
 			return 0;
--- a/src/lib-index/mail-index-map.c	Thu Aug 02 19:06:34 2007 +0300
+++ b/src/lib-index/mail-index-map.c	Fri Aug 03 03:22:04 2007 +0300
@@ -310,12 +310,12 @@
 	    hdr->first_deleted_uid_lowwater > hdr->next_uid)
 		return 0;
 
-	if (map->records_count > 0) {
+	if (hdr->messages_count > 0) {
 		/* last message's UID must be smaller than next_uid.
 		   also make sure it's not zero. */
 		const struct mail_index_record *rec;
 
-		rec = MAIL_INDEX_MAP_IDX(map, map->records_count-1);
+		rec = MAIL_INDEX_MAP_IDX(map, hdr->messages_count-1);
 		if (rec->uid == 0 || rec->uid >= hdr->next_uid)
 			return 0;
 	}
@@ -323,28 +323,6 @@
 	return 1;
 }
 
-static void mail_index_map_clear(struct mail_index_map *map)
-{
-	if (map->buffer != NULL) {
-		i_assert(map->mmap_base == NULL);
-		buffer_free(map->buffer);
-		map->buffer = NULL;
-	} else if (map->mmap_base != NULL) {
-		i_assert(map->buffer == NULL);
-		if (munmap(map->mmap_base, map->mmap_size) < 0)
-			mail_index_set_syscall_error(map->index, "munmap()");
-		map->mmap_base = NULL;
-	}
-
-	if (map->refcount > 0) {
-		memset(&map->hdr, 0, sizeof(map->hdr));
-		map->mmap_size = 0;
-		map->mmap_used_size = 0;
-		map->records = NULL;
-		map->records_count = 0;
-	}
-}
-
 static void mail_index_map_copy_hdr(struct mail_index_map *map,
 				    const struct mail_index_header *hdr)
 {
@@ -361,14 +339,14 @@
 static int mail_index_mmap(struct mail_index_map *map, uoff_t file_size)
 {
 	struct mail_index *index = map->index;
+	struct mail_index_record_map *rec_map = map->rec_map;
 	const struct mail_index_header *hdr;
 	unsigned int records_count;
 
-	if (map->buffer != NULL) {
-		/* we had temporarily used a buffer, eg. for updating index */
-		buffer_free(map->buffer);
-		map->buffer = NULL;
-	}
+	i_assert(rec_map->mmap_base == NULL);
+
+	buffer_free(rec_map->buffer);
+	rec_map->buffer = NULL;
 
 	if (file_size > SSIZE_T_MAX) {
 		/* too large file to map into memory */
@@ -377,40 +355,40 @@
 		return -1;
 	}
 
-	map->mmap_base = mmap(NULL, file_size, PROT_READ | PROT_WRITE,
-			      MAP_PRIVATE, index->fd, 0);
-	if (map->mmap_base == MAP_FAILED) {
-		map->mmap_base = NULL;
+	rec_map->mmap_base = mmap(NULL, file_size, PROT_READ | PROT_WRITE,
+				  MAP_PRIVATE, index->fd, 0);
+	if (rec_map->mmap_base == MAP_FAILED) {
+		rec_map->mmap_base = NULL;
 		mail_index_set_syscall_error(index, "mmap()");
 		return -1;
 	}
-	map->mmap_size = file_size;
+	rec_map->mmap_size = file_size;
 
-	hdr = map->mmap_base;
-	if (map->mmap_size >
+	hdr = rec_map->mmap_base;
+	if (rec_map->mmap_size >
 	    offsetof(struct mail_index_header, major_version) &&
 	    hdr->major_version != MAIL_INDEX_MAJOR_VERSION) {
 		/* major version change - handle silently */
 		return 0;
 	}
 
-	if (map->mmap_size < MAIL_INDEX_HEADER_MIN_SIZE) {
+	if (rec_map->mmap_size < MAIL_INDEX_HEADER_MIN_SIZE) {
 		mail_index_set_error(index, "Corrupted index file %s: "
 				     "File too small (%"PRIuSIZE_T")",
-				     index->filepath, map->mmap_size);
+				     index->filepath, rec_map->mmap_size);
 		return 0;
 	}
 
-	if (!mail_index_check_header_compat(index, hdr, map->mmap_size)) {
+	if (!mail_index_check_header_compat(index, hdr, rec_map->mmap_size)) {
 		/* Can't use this file */
 		return 0;
 	}
 
-	map->mmap_used_size = hdr->header_size +
+	rec_map->mmap_used_size = hdr->header_size +
 		hdr->messages_count * hdr->record_size;
 
-	if (map->mmap_used_size > map->mmap_size) {
-		records_count = (map->mmap_size - hdr->header_size) /
+	if (rec_map->mmap_used_size > rec_map->mmap_size) {
+		records_count = (rec_map->mmap_size - hdr->header_size) /
 			hdr->record_size;
 		mail_index_set_error(index, "Corrupted index file %s: "
 				     "messages_count too large (%u > %u)",
@@ -421,9 +399,9 @@
 
 	mail_index_map_copy_hdr(map, hdr);
 
-	map->hdr_base = map->mmap_base;
-	map->records = PTR_OFFSET(map->mmap_base, map->hdr.header_size);
-	map->records_count = map->hdr.messages_count;
+	map->hdr_base = rec_map->mmap_base;
+	rec_map->records = PTR_OFFSET(rec_map->mmap_base, map->hdr.header_size);
+	rec_map->records_count = map->hdr.messages_count;
 	return 1;
 }
 
@@ -464,7 +442,7 @@
 	size_t pos, records_size, initial_buf_pos = 0;
 	unsigned int records_count, extra;
 
-	i_assert(map->mmap_base == NULL);
+	i_assert(map->rec_map->mmap_base == NULL);
 
 	*retry_r = FALSE;
 	ret = mail_index_read_header(index, read_buf, sizeof(read_buf), &pos);
@@ -517,23 +495,24 @@
 			return 0;
 		}
 
-		if (map->buffer == NULL) {
-			map->buffer = buffer_create_dynamic(default_pool,
-							    records_size);
+		if (map->rec_map->buffer == NULL) {
+			map->rec_map->buffer =
+				buffer_create_dynamic(default_pool,
+						      records_size);
 		}
 
 		/* @UNSAFE */
-		buffer_set_used_size(map->buffer, 0);
+		buffer_set_used_size(map->rec_map->buffer, 0);
 		if (initial_buf_pos <= hdr->header_size)
 			extra = 0;
 		else {
 			extra = initial_buf_pos - hdr->header_size;
-			buffer_append(map->buffer,
+			buffer_append(map->rec_map->buffer,
 				      CONST_PTR_OFFSET(buf, hdr->header_size),
 				      extra);
 		}
 		if (records_size > extra) {
-			data = buffer_append_space_unsafe(map->buffer,
+			data = buffer_append_space_unsafe(map->rec_map->buffer,
 							  records_size - extra);
 			ret = pread_full(index->fd, data, records_size - extra,
 					 hdr->header_size + extra);
@@ -556,8 +535,9 @@
 		return 0;
 	}
 
-	map->records = buffer_get_modifiable_data(map->buffer, NULL);
-	map->records_count = hdr->messages_count;
+	map->rec_map->records =
+		buffer_get_modifiable_data(map->rec_map->buffer, NULL);
+	map->rec_map->records_count = hdr->messages_count;
 
 	mail_index_map_copy_hdr(map, hdr);
 	map->hdr_base = map->hdr_copy_buf->data;
@@ -700,7 +680,9 @@
 
 	new_map = mail_index_map_alloc(index);
 	if (use_mmap) {
-		new_map->lock_id = lock_id;
+		new_map->rec_map->lock_id = lock_id;
+		new_map->rec_map->lock_count++;
+		new_map->locked = TRUE;
 		ret = mail_index_mmap(new_map, file_size);
 	} else {
 		ret = mail_index_read_map(new_map, file_size);
@@ -718,6 +700,7 @@
 		mail_index_unmap(&new_map);
 		return ret;
 	}
+	i_assert(new_map->rec_map->records != NULL);
 
 	index->last_read_log_file_seq = new_map->hdr.log_file_seq;
 	index->last_read_log_file_head_offset =
@@ -773,6 +756,40 @@
 	return ret;
 }
 
+static void mail_index_record_map_free(struct mail_index_map *map,
+				       struct mail_index_record_map *rec_map)
+{
+	if (rec_map->buffer != NULL) {
+		i_assert(rec_map->mmap_base == NULL);
+		buffer_free(rec_map->buffer);
+		rec_map->buffer = NULL;
+	} else if (rec_map->mmap_base != NULL) {
+		i_assert(rec_map->buffer == NULL);
+		if (munmap(rec_map->mmap_base, rec_map->mmap_size) < 0)
+			mail_index_set_syscall_error(map->index, "munmap()");
+		rec_map->mmap_base = NULL;
+	}
+	array_free(&rec_map->maps);
+	i_free(rec_map);
+}
+
+static void mail_index_record_map_unlink(struct mail_index_map *map)
+{
+	struct mail_index_map *const *maps;
+	unsigned int i, count;
+
+	maps = array_get(&map->rec_map->maps, &count);
+	for (i = 0; i < count; i++) {
+		if (maps[i] == map) {
+			array_delete(&map->rec_map->maps, i, 1);
+			if (i == 0 && count == 1)
+				mail_index_record_map_free(map, map->rec_map);
+			return;
+		}
+	}
+	i_unreached();
+}
+
 void mail_index_unmap(struct mail_index_map **_map)
 {
 	struct mail_index_map *map = *_map;
@@ -782,8 +799,8 @@
 		return;
 
 	i_assert(map->refcount == 0);
-	mail_index_map_clear(map);
 	mail_index_map_unlock(map);
+	mail_index_record_map_unlink(map);
 
 	if (map->extension_pool != NULL)
 		pool_unref(map->extension_pool);
@@ -795,35 +812,61 @@
 
 int mail_index_map_lock(struct mail_index_map *map)
 {
-	if (map->lock_id != 0 || MAIL_INDEX_MAP_IS_IN_MEMORY(map))
+	if (map->locked)
 		return 0;
 
-	if (mail_index_lock_shared(map->index, &map->lock_id) < 0)
+	if (map->rec_map->lock_id != 0 || MAIL_INDEX_MAP_IS_IN_MEMORY(map)) {
+		map->rec_map->lock_count++;
+		map->locked = TRUE;
+		return 0;
+	}
+
+	if (mail_index_lock_shared(map->index, &map->rec_map->lock_id) < 0)
 		return -1;
 
-	mail_index_map_copy_hdr(map, map->mmap_base);
+	mail_index_map_copy_hdr(map, map->rec_map->mmap_base);
+	map->rec_map->lock_count++;
+	map->locked = TRUE;
 	return 0;
 }
 
 void mail_index_map_unlock(struct mail_index_map *map)
 {
-	mail_index_unlock(map->index, &map->lock_id);
+	if (!map->locked)
+		return;
+
+	if (--map->rec_map->lock_count == 0)
+		mail_index_unlock(map->index, &map->rec_map->lock_id);
 }
 
-static void mail_index_map_copy(struct mail_index_map *dest,
-				const struct mail_index_map *src)
+static void mail_index_map_copy_records(struct mail_index_record_map *dest,
+					const struct mail_index_record_map *src,
+					unsigned int record_size)
 {
 	size_t size;
 
-	/* copy records */
-	size = src->records_count * src->hdr.record_size;
-	dest->buffer = buffer_create_dynamic(default_pool, size);
+	if (src == NULL) {
+		dest->buffer = buffer_create_dynamic(default_pool, 1024);
+		return;
+	}
+
+	size = src->records_count * record_size;
+	dest->buffer = buffer_create_dynamic(default_pool, I_MIN(size, 1024));
 	buffer_append(dest->buffer, src->records, size);
 
 	dest->records = buffer_get_modifiable_data(dest->buffer, NULL);
 	dest->records_count = src->records_count;
 
-	/* copy header. use src->hdr copy directly, because if we got here
+	/* if the map is ever written back to disk, we need to keep track of
+	   what has changed. */
+	dest->write_seq_first = src->write_seq_first;
+	dest->write_seq_last = src->write_seq_last;
+}
+
+static void mail_index_map_copy_header(struct mail_index_map *dest,
+				       const struct mail_index_map *src)
+{
+	/* use src->hdr copy directly, because if we got here
 	   from syncing it has the latest changes. */
 	dest->hdr = src->hdr;
 	if (dest->hdr_copy_buf != NULL)
@@ -841,6 +884,17 @@
 	dest->hdr_base = buffer_get_modifiable_data(dest->hdr_copy_buf, NULL);
 }
 
+static struct mail_index_record_map *
+mail_index_record_map_alloc(struct mail_index_map *map)
+{
+	struct mail_index_record_map *rec_map;
+
+	rec_map = i_new(struct mail_index_record_map, 1);
+	i_array_init(&rec_map->maps, 4);
+	array_append(&rec_map->maps, &map, 1);
+	return rec_map;
+}
+
 struct mail_index_map *mail_index_map_clone(const struct mail_index_map *map)
 {
 	struct mail_index_map *mem_map;
@@ -850,19 +904,15 @@
 	mem_map = i_new(struct mail_index_map, 1);
 	mem_map->index = map->index;
 	mem_map->refcount = 1;
-
-	mail_index_map_copy(mem_map, map);
+	mem_map->rec_map = mail_index_record_map_alloc(mem_map);
 
-	/* if the map is ever written back to disk, we need to keep track of
-	   what has changed. */
-	if (map->write_atomic)
-		mem_map->write_atomic = TRUE;
-	else {
-		mem_map->write_seq_first = map->write_seq_first;
-		mem_map->write_seq_last = map->write_seq_last;
-		mem_map->write_base_header = map->write_base_header;
-		mem_map->write_ext_header = map->write_ext_header;
-	}
+	mail_index_map_copy_records(mem_map->rec_map, map->rec_map,
+				    map->hdr.record_size);
+	mail_index_map_copy_header(mem_map, map);
+
+	mem_map->write_atomic = map->write_atomic;
+	mem_map->write_base_header = map->write_base_header;
+	mem_map->write_ext_header = map->write_ext_header;
 
 	/* copy extensions */
 	if (array_is_created(&map->ext_id_map)) {
@@ -888,17 +938,29 @@
 
 void mail_index_map_move_to_memory(struct mail_index_map *map)
 {
-	if (map->mmap_base == NULL)
+	struct mail_index_record_map *new_map;
+
+	if (map->rec_map->mmap_base == NULL)
 		return;
 
-	i_assert(map->lock_id != 0);
+	i_assert(map->rec_map->lock_id != 0);
+
+	new_map = array_count(&map->rec_map->maps) == 1 ? map->rec_map :
+		mail_index_record_map_alloc(map);
 
-	mail_index_map_copy(map, map);
+	mail_index_map_copy_records(new_map, map->rec_map,
+				    map->hdr.record_size);
+	mail_index_map_copy_header(map, map);
+
 	mail_index_map_unlock(map);
-
-	if (munmap(map->mmap_base, map->mmap_size) < 0)
-		i_error("munmap(index map) failed: %m");
-	map->mmap_base = NULL;
+	if (new_map != map->rec_map) {
+		mail_index_record_map_unlink(map);
+		map->rec_map = new_map;
+	} else if (new_map->mmap_base != NULL) {
+		if (munmap(new_map->mmap_base, new_map->mmap_size) < 0)
+			mail_index_set_syscall_error(map->index, "munmap()");
+		new_map->mmap_base = NULL;
+	}
 }
 
 int mail_index_map_get_ext_idx(struct mail_index_map *map,
--- a/src/lib-index/mail-index-private.h	Thu Aug 02 19:06:34 2007 +0300
+++ b/src/lib-index/mail-index-private.h	Fri Aug 03 03:22:04 2007 +0300
@@ -25,11 +25,11 @@
 	((index)->dir == NULL)
 
 #define MAIL_INDEX_MAP_IS_IN_MEMORY(map) \
-	((map)->buffer != NULL)
+	((map)->rec_map->mmap_base == NULL)
 
 #define MAIL_INDEX_MAP_IDX(map, idx) \
 	((struct mail_index_record *) \
-		PTR_OFFSET((map)->records, (idx) * (map)->hdr.record_size))
+	 PTR_OFFSET((map)->rec_map->records, (idx) * (map)->hdr.record_size))
 
 typedef int mail_index_expunge_handler_t(struct mail_index_sync_map_ctx *ctx,
 					 uint32_t seq, const void *data,
@@ -102,33 +102,43 @@
 	unsigned int expunge_handler_call_always:1;
 };
 
+struct mail_index_record_map {
+	ARRAY_DEFINE(maps, struct mail_index_map *);
+
+	void *mmap_base;
+	size_t mmap_size, mmap_used_size;
+
+	unsigned int lock_count;
+	unsigned int lock_id;
+
+	buffer_t *buffer;
+
+	void *records; /* struct mail_index_record[] */
+	unsigned int records_count;
+
+	/* If this mapping is written to disk and write_atomic=FALSE,
+	   write_seq_* specify the message sequence range that needs to be
+	   written. */
+	uint32_t write_seq_first, write_seq_last;
+};
+
 struct mail_index_map {
 	struct mail_index *index;
 	int refcount;
 
 	struct mail_index_header hdr;
 	const void *hdr_base;
-	void *records; /* struct mail_index_record[] */
-	unsigned int records_count;
+	buffer_t *hdr_copy_buf;
 
 	pool_t extension_pool;
 	ARRAY_DEFINE(extensions, struct mail_index_ext);
 	ARRAY_DEFINE(ext_id_map, uint32_t); /* index -> file */
 
-	void *mmap_base;
-	size_t mmap_size, mmap_used_size;
-	unsigned int lock_id;
-
-	buffer_t *buffer;
-	buffer_t *hdr_copy_buf;
-
 	ARRAY_DEFINE(keyword_idx_map, unsigned int); /* file -> index */
 
-	/* If this mapping is written to disk and write_atomic=FALSE,
-	   write_seq_* specify the message sequence range that needs to be
-	   written. */
-	uint32_t write_seq_first, write_seq_last;
+	struct mail_index_record_map *rec_map;
 
+	unsigned int locked:1;
 	unsigned int keywords_read:1;
 	unsigned int write_base_header:1;
 	unsigned int write_ext_header:1;
--- a/src/lib-index/mail-index-sync-ext.c	Thu Aug 02 19:06:34 2007 +0300
+++ b/src/lib-index/mail-index-sync-ext.c	Fri Aug 03 03:22:04 2007 +0300
@@ -223,11 +223,11 @@
 	new_record_size = offset;
 
 	/* copy the records to new buffer */
-	new_buffer_size = map->records_count * new_record_size;
+	new_buffer_size = map->rec_map->records_count * new_record_size;
 	new_buffer = buffer_create_dynamic(default_pool, new_buffer_size);
-	src = map->records;
+	src = map->rec_map->records;
 	offset = 0;
-	for (rec_idx = 0; rec_idx < map->records_count; rec_idx++) {
+	for (rec_idx = 0; rec_idx < map->rec_map->records_count; rec_idx++) {
 		/* write the base record */
 		buffer_write(new_buffer, offset, src,
 			     sizeof(struct mail_index_record));
@@ -249,9 +249,10 @@
 		buffer_append_zero(new_buffer, space);
 	}
 
-	buffer_free(map->buffer);
-	map->buffer = new_buffer;
-	map->records = buffer_get_modifiable_data(map->buffer, NULL);
+	buffer_free(map->rec_map->buffer);
+	map->rec_map->buffer = new_buffer;
+	map->rec_map->records =
+		buffer_get_modifiable_data(map->rec_map->buffer, NULL);
 	map->hdr.record_size = new_record_size;
 
 	/* update record offsets in headers */
@@ -468,13 +469,13 @@
 				       ext->hdr_size), 0, ext->hdr_size);
 	map->hdr_base = map->hdr_copy_buf->data;
 
-	for (i = 0; i < view->map->records_count; i++) {
+	for (i = 0; i < view->map->rec_map->records_count; i++) {
 		rec = MAIL_INDEX_MAP_IDX(view->map, i);
 		memset(PTR_OFFSET(rec, ext->record_offset), 0,
 		       ext->record_size);
 	}
-	map->write_seq_first = 1;
-	map->write_seq_last = view->map->records_count;
+	map->rec_map->write_seq_first = 1;
+	map->rec_map->write_seq_last = view->map->rec_map->records_count;
 
 	ext_hdr = get_ext_header(map, ext);
 	ext_hdr->reset_id = u->new_reset_id;
@@ -526,7 +527,6 @@
 
 	if (seq == 0)
 		return 1;
-	mail_index_sync_move_to_private(ctx);
 
 	ext = array_idx(&view->map->extensions, ctx->cur_ext_id);
 
--- a/src/lib-index/mail-index-sync-update.c	Thu Aug 02 19:06:34 2007 +0300
+++ b/src/lib-index/mail-index-sync-update.c	Fri Aug 03 03:22:04 2007 +0300
@@ -63,7 +63,8 @@
 {
 	struct mail_index_map *map = ctx->view->map;
 
-	i_assert(MAIL_INDEX_MAP_IS_IN_MEMORY(map) || map->lock_id != 0);
+	i_assert(MAIL_INDEX_MAP_IS_IN_MEMORY(map) ||
+		 map->rec_map->lock_id != 0);
 
 	if (map->refcount > 1) {
 		map = mail_index_map_clone(map);
@@ -135,26 +136,56 @@
 }
 
 static void
+mail_index_sync_header_update_counts_all(struct mail_index_sync_map_ctx *ctx,
+					 uint8_t old_flags, uint8_t new_flags)
+{
+	struct mail_index_map *const *maps;
+	const char *error;
+	unsigned int i, count;
+
+	maps = array_get(&ctx->view->map->rec_map->maps, &count);
+	for (i = 0; i < count; i++) {
+		if (mail_index_header_update_counts(&maps[i]->hdr,
+						    old_flags, new_flags,
+						    &error) < 0)
+			mail_index_sync_set_corrupted(ctx, "%s", error);
+	}
+}
+
+static void
 mail_index_sync_header_update_counts(struct mail_index_sync_map_ctx *ctx,
-				     uint8_t old_flags, uint8_t new_flags)
+				     uint8_t old_flags, uint8_t new_flags,
+				     bool all)
 {
 	const char *error;
 
-	if (mail_index_header_update_counts(&ctx->view->map->hdr,
-					    old_flags, new_flags, &error) < 0)
-		mail_index_sync_set_corrupted(ctx, "%s", error);
+	if (all) {
+		mail_index_sync_header_update_counts_all(ctx, old_flags,
+							 new_flags);
+	} else {
+		if (mail_index_header_update_counts(&ctx->view->map->hdr,
+						    old_flags, new_flags,
+						    &error) < 0)
+			mail_index_sync_set_corrupted(ctx, "%s", error);
+	}
 }
 
 static void
-mail_index_header_update_lowwaters(struct mail_index_header *hdr,
+mail_index_header_update_lowwaters(struct mail_index_sync_map_ctx *ctx,
 				   const struct mail_index_record *rec)
 {
-	if ((rec->flags & MAIL_SEEN) == 0 &&
-	    rec->uid < hdr->first_unseen_uid_lowwater)
-		hdr->first_unseen_uid_lowwater = rec->uid;
-	if ((rec->flags & MAIL_DELETED) != 0 &&
-	    rec->uid < hdr->first_deleted_uid_lowwater)
-		hdr->first_deleted_uid_lowwater = rec->uid;
+	struct mail_index_map *const *maps;
+	unsigned int i, count;
+
+	maps = array_get(&ctx->view->map->rec_map->maps, &count);
+	for (i = 0; i < count; i++) {
+		if ((rec->flags & MAIL_SEEN) == 0 &&
+		    rec->uid < maps[i]->hdr.first_unseen_uid_lowwater)
+			maps[i]->hdr.first_unseen_uid_lowwater = rec->uid;
+		if ((rec->flags & MAIL_DELETED) != 0 &&
+		    rec->uid < maps[i]->hdr.first_deleted_uid_lowwater)
+			maps[i]->hdr.first_deleted_uid_lowwater = rec->uid;
+	}
 }
 
 static int
@@ -210,7 +241,8 @@
 		for (seq = seq1; seq <= seq2; seq++) {
 			rec = MAIL_INDEX_MAP_IDX(map, seq-1);
 			mail_index_sync_header_update_counts(ctx,
-							     rec->flags, 0);
+							     rec->flags, 0,
+							     FALSE);
 		}
 
 		if (sync_expunge_call_handlers(ctx, seq1, seq2) < 0)
@@ -219,10 +251,11 @@
 		/* @UNSAFE */
 		memmove(MAIL_INDEX_MAP_IDX(map, seq1-1),
 			MAIL_INDEX_MAP_IDX(map, seq2),
-			(map->records_count - seq2) * map->hdr.record_size);
+			(map->rec_map->records_count - seq2) *
+			map->hdr.record_size);
 
 		seq_count = seq2 - seq1 + 1;
-		map->records_count -= seq_count;
+		map->rec_map->records_count -= seq_count;
 		map->hdr.messages_count -= seq_count;
 	}
 	return 1;
@@ -233,11 +266,11 @@
 {
 	struct mail_index_map *map = ctx->view->map;
 
-	if (map->write_seq_first == 0 ||
-	    map->write_seq_first > seq1)
-		map->write_seq_first = seq1;
-	if (map->write_seq_last < seq2)
-		map->write_seq_last = seq2;
+	if (map->rec_map->write_seq_first == 0 ||
+	    map->rec_map->write_seq_first > seq1)
+		map->rec_map->write_seq_first = seq1;
+	if (map->rec_map->write_seq_last < seq2)
+		map->rec_map->write_seq_last = seq2;
 }
 
 static int sync_append(const struct mail_index_record *rec,
@@ -263,10 +296,11 @@
 
 	/* don't rely on buffer->used being at the correct position.
 	   at least expunges can move it */
-	append_pos = map->records_count * map->hdr.record_size;
-	dest = buffer_get_space_unsafe(map->buffer, append_pos,
+	append_pos = map->rec_map->records_count * map->hdr.record_size;
+	dest = buffer_get_space_unsafe(map->rec_map->buffer, append_pos,
 				       map->hdr.record_size);
-	map->records = buffer_get_modifiable_data(map->buffer, NULL);
+	map->rec_map->records =
+		buffer_get_modifiable_data(map->rec_map->buffer, NULL);
 
 	memcpy(dest, rec, sizeof(*rec));
 	memset(PTR_OFFSET(dest, sizeof(*rec)), 0,
@@ -274,7 +308,7 @@
 
 	map->hdr.messages_count++;
 	map->hdr.next_uid = rec->uid+1;
-	map->records_count++;
+	map->rec_map->records_count++;
 
 	mail_index_sync_write_seq_update(ctx, map->hdr.messages_count,
 					 map->hdr.messages_count);
@@ -282,8 +316,8 @@
 	if ((rec->flags & MAIL_INDEX_MAIL_FLAG_DIRTY) != 0)
 		map->hdr.flags |= MAIL_INDEX_HDR_FLAG_HAVE_DIRTY;
 
-	mail_index_header_update_lowwaters(&map->hdr, rec);
-	mail_index_sync_header_update_counts(ctx, 0, rec->flags);
+	mail_index_header_update_lowwaters(ctx, rec);
+	mail_index_sync_header_update_counts(ctx, 0, rec->flags, FALSE);
 	return 1;
 }
 
@@ -291,7 +325,6 @@
 			    struct mail_index_sync_map_ctx *ctx)
 {
 	struct mail_index_view *view = ctx->view;
-	struct mail_index_header *hdr;
 	struct mail_index_record *rec;
 	uint8_t flag_mask, old_flags;
 	uint32_t idx, seq1, seq2;
@@ -305,9 +338,8 @@
 
 	mail_index_sync_write_seq_update(ctx, seq1, seq2);
 
-	hdr = &view->map->hdr;
 	if ((u->add_flags & MAIL_INDEX_MAIL_FLAG_DIRTY) != 0)
-		hdr->flags |= MAIL_INDEX_HDR_FLAG_HAVE_DIRTY;
+		view->map->hdr.flags |= MAIL_INDEX_HDR_FLAG_HAVE_DIRTY;
 
         flag_mask = ~u->remove_flags;
 
@@ -325,9 +357,9 @@
 			old_flags = rec->flags;
 			rec->flags = (rec->flags & flag_mask) | u->add_flags;
 
-			mail_index_header_update_lowwaters(hdr, rec);
+			mail_index_header_update_lowwaters(ctx, rec);
 			mail_index_sync_header_update_counts(ctx, old_flags,
-							     rec->flags);
+							     rec->flags, TRUE);
 		}
 	}
 	return 1;
@@ -554,7 +586,7 @@
 	}
 	t_pop();
 
-	i_assert(ctx->view->map->records_count ==
+	i_assert(ctx->view->map->rec_map->records_count ==
 		 ctx->view->map->hdr.messages_count);
 	return ret;
 }
@@ -590,7 +622,7 @@
 		return;
 
 	/* do we have dirty flags anymore? */
-	for (i = 0; i < map->records_count; i++) {
+	for (i = 0; i < map->rec_map->records_count; i++) {
 		rec = MAIL_INDEX_MAP_IDX(map, i);
 		if ((rec->flags & MAIL_INDEX_MAIL_FLAG_DIRTY) != 0) {
 			map->hdr.flags |= MAIL_INDEX_HDR_FLAG_HAVE_DIRTY;
@@ -659,7 +691,8 @@
 			index_size = MAIL_INDEX_SYNC_MIN_READ_INDEX_SIZE;
 		} else {
 			index_size = map->hdr.header_size +
-				map->records_count * map->hdr.record_size;
+				map->rec_map->records_count *
+				map->hdr.record_size;
 		}
 
 		/* this isn't necessary correct currently, but it should be
@@ -765,7 +798,7 @@
 
 	buffer_write(map->hdr_copy_buf, 0, &map->hdr, sizeof(map->hdr));
 	if (!MAIL_INDEX_MAP_IS_IN_MEMORY(map)) {
-		memcpy(map->mmap_base, map->hdr_copy_buf->data,
+		memcpy(map->rec_map->mmap_base, map->hdr_copy_buf->data,
 		       map->hdr_copy_buf->used);
 	}
 
--- a/src/lib-index/mail-index-transaction.c	Thu Aug 02 19:06:34 2007 +0300
+++ b/src/lib-index/mail-index-transaction.c	Fri Aug 03 03:22:04 2007 +0300
@@ -223,7 +223,7 @@
 			if (*seq >= t->first_new_seq)
 				rec = mail_index_transaction_lookup(t, *seq);
 			else {
-				i_assert(*seq <= view->map->records_count);
+				i_assert(*seq <= view->map->hdr.messages_count);
 				rec = MAIL_INDEX_MAP_IDX(view->map, *seq - 1);
 			}
 
@@ -233,9 +233,9 @@
 				/* FIXME: replace with simple assert once we
 				   figure out why this happens.. */
 				i_panic("seq = %u, rec->uid = %u, "
-					"first_new_seq = %u, records = %u",
+					"first_new_seq = %u, messages = %u",
 					*seq, rec->uid, t->first_new_seq,
-					view->map->records_count);
+					view->map->hdr.messages_count);
 			}
 			*seq = rec->uid;
 		}
--- a/src/lib-index/mail-index-view-sync.c	Thu Aug 02 19:06:34 2007 +0300
+++ b/src/lib-index/mail-index-view-sync.c	Fri Aug 03 03:22:04 2007 +0300
@@ -324,7 +324,8 @@
 		} else {
 			map = view->map;
 		}
-		i_assert(map->records_count == map->hdr.messages_count);
+		i_assert(map->rec_map->records_count ==
+			 map->hdr.messages_count);
 	}
 
 #ifdef DEBUG
--- a/src/lib-index/mail-index-view.c	Thu Aug 02 19:06:34 2007 +0300
+++ b/src/lib-index/mail-index-view.c	Fri Aug 03 03:22:04 2007 +0300
@@ -244,13 +244,14 @@
 	const struct mail_index_record *rec_base, *rec;
 	uint32_t idx, right_idx, record_size;
 
-	i_assert(view->map->hdr.messages_count <= view->map->records_count);
+	i_assert(view->map->hdr.messages_count <=
+		 view->map->rec_map->records_count);
 
 	if (uid == 1) {
 		/* optimization: the message can be only the first one */
 		return 1;
 	}
-	rec_base = view->map->records;
+	rec_base = view->map->rec_map->records;
 	record_size = view->map->hdr.record_size;
 
 	idx = left_idx;
@@ -355,7 +356,8 @@
 			return 0;
 	}
 
-	i_assert(view->map->hdr.messages_count <= view->map->records_count);
+	i_assert(view->map->hdr.messages_count <=
+		 view->map->rec_map->records_count);
 
 	/* we can delay locking until we're looking at the flags */
 	if (mail_index_map_lock(view->map) < 0)
--- a/src/lib-index/mail-index-write.c	Thu Aug 02 19:06:34 2007 +0300
+++ b/src/lib-index/mail-index-write.c	Fri Aug 03 03:22:04 2007 +0300
@@ -35,7 +35,8 @@
 				 map->hdr.header_size - base_size);
 	}
 	if (ret == 0) {
-		ret = write_full(fd, map->records, map->records_count *
+		ret = write_full(fd, map->rec_map->records,
+				 map->rec_map->records_count *
 				 map->hdr.record_size);
 	}
 	if (ret < 0)
@@ -69,6 +70,7 @@
 static int mail_index_write_map_over(struct mail_index *index)
 {
 	struct mail_index_map *map = index->map;
+	struct mail_index_record_map *rec_map = map->rec_map;
 	unsigned int base_size;
 
 	if (MAIL_INDEX_IS_IN_MEMORY(index))
@@ -85,14 +87,15 @@
 	}
 
 	/* write records. */
-	if (map->write_seq_first != 0) {
+	if (rec_map->write_seq_first != 0) {
 		size_t rec_offset =
-			(map->write_seq_first-1) * map->hdr.record_size;
+			(rec_map->write_seq_first-1) * map->hdr.record_size;
 		size_t recs_size = map->hdr.record_size *
-			(map->write_seq_last - map->write_seq_first + 1);
+			(rec_map->write_seq_last -
+			 rec_map->write_seq_first + 1);
 
 		if (pwrite_full(index->fd,
-				CONST_PTR_OFFSET(map->records, rec_offset),
+				CONST_PTR_OFFSET(rec_map->records, rec_offset),
 				recs_size,
 				map->hdr.header_size + rec_offset) < 0)
 			return -1;
@@ -130,7 +133,7 @@
 
 #define mail_index_map_has_changed(map) \
 	((map)->write_base_header || (map)->write_ext_header || \
-	 (map)->write_seq_first != 0)
+	 (map)->rec_map->write_seq_first != 0)
 
 void mail_index_write(struct mail_index *index, bool want_rotate)
 {
@@ -154,8 +157,9 @@
 	}
 
 	if (index->last_read_stat.st_size < MAIL_INDEX_MIN_UPDATE_SIZE ||
-	    (map->write_seq_last - map->write_seq_first + 1) +
-	    MAIL_INDEX_MAX_OVERWRITE_NEG_SEQ_COUNT >= map->records_count) {
+	    (map->rec_map->write_seq_last - map->rec_map->write_seq_first + 1) +
+	    MAIL_INDEX_MAX_OVERWRITE_NEG_SEQ_COUNT >=
+	    map->rec_map->records_count) {
 		/* the file is so small that we don't even bother trying to
 		   update it / changes are so large we might as well recreate */
 		map->write_atomic = TRUE;
@@ -216,8 +220,8 @@
 	index->last_read_log_file_head_offset = hdr->log_file_head_offset;
 	index->last_read_log_file_tail_offset = hdr->log_file_tail_offset;
 
+	map->rec_map->write_seq_first = map->rec_map->write_seq_last = 0;
 	map->write_atomic = FALSE;
-	map->write_seq_first = map->write_seq_last = 0;
 	map->write_base_header = FALSE;
 	map->write_ext_header = FALSE;
 
--- a/src/lib-index/mail-index.c	Thu Aug 02 19:06:34 2007 +0300
+++ b/src/lib-index/mail-index.c	Fri Aug 03 03:22:04 2007 +0300
@@ -358,7 +358,7 @@
 	if (MAIL_INDEX_IS_IN_MEMORY(index))
 		return 0;
 
-	i_assert(index->map == NULL || index->map->lock_id == 0);
+	i_assert(index->map == NULL || index->map->rec_map->lock_id == 0);
 	ret = mail_index_map(index, MAIL_INDEX_SYNC_HANDLER_HEAD);
 	if (ret == 0) {
 		/* it's corrupted - recreate it */