Mercurial > dovecot > original-hg > dovecot-1.2

changeset 7931:502cfdcc5650 HEAD

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Keep modseqs as 1 until the first modseq ext intro record enables them. Performance should be better again when modseqs are disabled.
author Timo Sirainen <tss@iki.fi>
date Sat, 21 Jun 2008 15:06:46 +0300
parents b4286cde68d0
children 91758686277a
files src/lib-index/mail-index-modseq.c src/lib-index/mail-index-modseq.h src/lib-index/mail-transaction-log-append.c src/lib-index/mail-transaction-log-file.c src/lib-index/mail-transaction-log-private.h src/lib-index/mail-transaction-log-view.c src/util/logview.c
diffstat 7 files changed, 134 insertions(+), 75 deletions(-) [+]
line wrap: on
line diff
--- a/src/lib-index/mail-index-modseq.c	Sat Jun 21 14:03:59 2008 +0300
+++ b/src/lib-index/mail-index-modseq.c	Sat Jun 21 15:06:46 2008 +0300
@@ -7,8 +7,6 @@
 #include "mail-index-sync-private.h"
 #include "mail-index-modseq.h"
 
-#define MAIL_INDEX_MODSEQ_EXT_NAME "modseq"
-
 ARRAY_DEFINE_TYPE(modseqs, uint64_t);
 
 enum modseq_metadata_idx {
@@ -51,7 +49,7 @@
 static uint64_t mail_index_modseq_get_head(struct mail_index *index)
 {
 	return index->log->head == NULL ? 1 :
-		index->log->head->sync_highest_modseq;
+		I_MAX(index->log->head->sync_highest_modseq, 1);
 }
 
 void mail_index_modseq_enable(struct mail_index *index)
--- a/src/lib-index/mail-index-modseq.h	Sat Jun 21 14:03:59 2008 +0300
+++ b/src/lib-index/mail-index-modseq.h	Sat Jun 21 15:06:46 2008 +0300
@@ -1,6 +1,8 @@
 #ifndef MAIL_INDEX_MODSEQ_H
 #define MAIL_INDEX_MODSEQ_H
 
+#define MAIL_INDEX_MODSEQ_EXT_NAME "modseq"
+
 enum mail_flags;
 struct mail_keywords;
 struct mail_index;
--- a/src/lib-index/mail-transaction-log-append.c	Sat Jun 21 14:03:59 2008 +0300
+++ b/src/lib-index/mail-transaction-log-append.c	Sat Jun 21 15:06:46 2008 +0300
@@ -6,6 +6,7 @@
 #include "write-full.h"
 #include "mail-index-private.h"
 #include "mail-index-view-private.h"
+#include "mail-index-modseq.h"
 #include "mail-index-transaction-private.h"
 #include "mail-transaction-log-private.h"
 
@@ -14,7 +15,7 @@
 	struct mail_index_transaction *trans;
 	buffer_t *output;
 
-	unsigned int modseq_change_count;
+	uint64_t modseq;
 	uint32_t first_append_size;
 	bool sync_includes_this;
 };
@@ -25,6 +26,7 @@
 {
 	struct mail_transaction_header hdr;
 	uint32_t hdr_size;
+	size_t hdr_pos;
 
 	i_assert((type & MAIL_TRANSACTION_TYPE_MASK) != 0);
 	i_assert((buf->used % 4) == 0);
@@ -39,26 +41,31 @@
 		hdr.type |= MAIL_TRANSACTION_EXPUNGE_PROT;
 	if ((ctx->trans->flags & MAIL_INDEX_TRANSACTION_FLAG_EXTERNAL) != 0)
 		hdr.type |= MAIL_TRANSACTION_EXTERNAL;
-
-	if (mail_transaction_header_has_modseq(&hdr))
-		ctx->modseq_change_count++;
+	hdr.size = sizeof(hdr) + buf->used +
+		(hdr_buf == NULL ? 0 : hdr_buf->used);
 
-	hdr_size = mail_index_uint32_to_offset(sizeof(hdr) + buf->used +
-					       (hdr_buf == NULL ? 0 :
-						hdr_buf->used));
+	hdr_pos = ctx->output->used;
+	buffer_append(ctx->output, &hdr, sizeof(hdr));
+	if (hdr_buf != NULL)
+		buffer_append(ctx->output, hdr_buf->data, hdr_buf->used);
+	buffer_append(ctx->output, buf->data, buf->used);
+
+	if (mail_transaction_header_has_modseq(buf->data,
+			CONST_PTR_OFFSET(buf->data, sizeof(hdr)), ctx->modseq))
+		ctx->modseq++;
+
+	/* update the size */
+	hdr_size = mail_index_uint32_to_offset(hdr.size);
 	if (!MAIL_TRANSACTION_LOG_FILE_IN_MEMORY(ctx->file) &&
 	    ctx->first_append_size == 0) {
 		/* size will be written later once everything
 		   is in disk */
 		ctx->first_append_size = hdr_size;
+		hdr.size = 0;
 	} else {
 		hdr.size = hdr_size;
 	}
-
-	buffer_append(ctx->output, &hdr, sizeof(hdr));
-	if (hdr_buf != NULL)
-		buffer_append(ctx->output, hdr_buf->data, hdr_buf->used);
-	buffer_append(ctx->output, buf->data, buf->used);
+	buffer_write(ctx->output, hdr_pos, &hdr, sizeof(hdr));
 }
 
 static int log_buffer_move_to_memory(struct log_append_context *ctx)
@@ -280,10 +287,15 @@
 		/* new extension, reset_id defaults to 0 */
 	}
 	buffer_append(buf, rext->name, intro->name_size);
-
 	if ((buf->used % 4) != 0)
 		buffer_append_zero(buf, 4 - (buf->used % 4));
 
+	if (ctx->file->sync_highest_modseq == 0 &&
+	    strcmp(rext->name, MAIL_INDEX_MODSEQ_EXT_NAME) == 0) {
+		/* modseq tracking started */
+		ctx->file->sync_highest_modseq = 1;
+	}
+
 	log_append_buffer(ctx, buf, NULL, MAIL_TRANSACTION_EXT_INTRO);
 }
 
@@ -601,6 +613,7 @@
 	ctx.file = file;
 	ctx.trans = t;
 	ctx.output = buffer_create_dynamic(default_pool, 1024);
+	ctx.modseq = file->sync_highest_modseq;
 
 	/* send all extension introductions and resizes before appends
 	   to avoid resize overhead as much as possible */
@@ -674,7 +687,7 @@
 		buffer_free(&ctx.output);
 		return -1;
 	}
-	file->sync_highest_modseq += ctx.modseq_change_count;
+	file->sync_highest_modseq = ctx.modseq;
 	buffer_free(&ctx.output);
 
 	if ((t->flags & MAIL_INDEX_TRANSACTION_FLAG_HIDE) != 0) {
--- a/src/lib-index/mail-transaction-log-file.c	Sat Jun 21 14:03:59 2008 +0300
+++ b/src/lib-index/mail-transaction-log-file.c	Sat Jun 21 15:06:46 2008 +0300
@@ -102,45 +102,62 @@
 }
 
 static void
-mail_transaction_log_file_add_to_list(struct mail_transaction_log_file *file)
+mail_transaction_log_file_skip_to_head(struct mail_transaction_log_file *file)
 {
 	struct mail_transaction_log *log = file->log;
-	struct mail_transaction_log_file **p;
 	struct mail_index_map *map = log->index->map;
 	const struct mail_index_modseq_header *modseq_hdr;
+	uoff_t head_offset;
 
-	if (map != NULL && file->hdr.file_seq == map->hdr.log_file_seq &&
-	    map->hdr.log_file_head_offset != 0) {
-		/* we can get a valid log offset from index file. initialize
-		   sync_offset from it so we don't have to read the whole log
-		   file from beginning. */
-		uoff_t head_offset = map->hdr.log_file_head_offset;
+	if (map == NULL || file->hdr.file_seq != map->hdr.log_file_seq ||
+	    map->hdr.log_file_head_offset == 0)
+		return;
+
+	/* we can get a valid log offset from index file. initialize
+	   sync_offset from it so we don't have to read the whole log
+	   file from beginning. */
+	head_offset = map->hdr.log_file_head_offset;
 
-		modseq_hdr = mail_index_map_get_modseq_header(map);
-		if (head_offset < file->hdr.hdr_size) {
-			mail_index_set_error(log->index,
-				"%s: log_file_head_offset too small",
-				log->index->filepath);
-			file->sync_offset = file->hdr.hdr_size;
-			file->sync_highest_modseq = file->hdr.initial_modseq;
-		} else if (modseq_hdr == NULL ||
-			   modseq_hdr->log_seq != file->hdr.file_seq ||
-			   modseq_hdr->log_offset != head_offset) {
-			/* highest_modseq not synced, start from beginning */
-			file->sync_offset = file->hdr.hdr_size;
-			file->sync_highest_modseq = file->hdr.initial_modseq;
-		} else {
-			file->sync_offset = head_offset;
-			file->sync_highest_modseq = modseq_hdr->highest_modseq;
-		}
-		file->saved_tail_offset = map->hdr.log_file_tail_offset;
-	} else {
+	modseq_hdr = mail_index_map_get_modseq_header(map);
+	if (head_offset < file->hdr.hdr_size) {
+		mail_index_set_error(log->index,
+				     "%s: log_file_head_offset too small",
+				     log->index->filepath);
+		file->sync_offset = file->hdr.hdr_size;
+		file->sync_highest_modseq = file->hdr.initial_modseq;
+	} else if (modseq_hdr == NULL && file->hdr.initial_modseq == 0) {
+		/* modseqs not used yet */
+		file->sync_offset = head_offset;
+		file->sync_highest_modseq = 0;
+	} else if (modseq_hdr->log_seq != file->hdr.file_seq) {
+		/* highest_modseq not synced, start from beginning */
 		file->sync_offset = file->hdr.hdr_size;
 		file->sync_highest_modseq = file->hdr.initial_modseq;
+	} else if (modseq_hdr->log_offset > head_offset) {
+		mail_index_set_error(log->index,
+				     "%s: modseq_hdr.log_offset too large",
+				     log->index->filepath);
+		file->sync_offset = file->hdr.hdr_size;
+		file->sync_highest_modseq = file->hdr.initial_modseq;
+	} else {
+		/* start from where we last stopped tracking modseqs */
+		file->sync_offset = modseq_hdr->log_offset;
+		file->sync_highest_modseq = modseq_hdr->highest_modseq;
 	}
+	file->saved_tail_offset = log->index->map->hdr.log_file_tail_offset;
+}
+
+static void
+mail_transaction_log_file_add_to_list(struct mail_transaction_log_file *file)
+{
+	struct mail_transaction_log_file **p;
+
+	file->sync_offset = file->hdr.hdr_size;
+	file->sync_highest_modseq = file->hdr.initial_modseq;
+	mail_transaction_log_file_skip_to_head(file);
 
 	/* insert it to correct position */
-	for (p = &log->files; *p != NULL; p = &(*p)->next) {
+	for (p = &file->log->files; *p != NULL; p = &(*p)->next) {
 		if ((*p)->hdr.file_seq > file->hdr.file_seq)
 			break;
 		i_assert((*p)->hdr.file_seq < file->hdr.file_seq);
@@ -179,7 +196,8 @@
 		hdr->prev_file_seq = index->map->hdr.log_file_seq;
 		hdr->prev_file_offset = index->map->hdr.log_file_head_offset;
 		hdr->file_seq = index->map->hdr.log_file_seq + 1;
-		hdr->initial_modseq =
+		hdr->initial_modseq = log->head == NULL ||
+			log->head->sync_highest_modseq == 0 ? 0 :
 			mail_index_map_modseq_get_highest(index->map);
 	} else {
 		hdr->file_seq = 1;
@@ -368,8 +386,6 @@
 		   shouldn't have filled */
 		memset(PTR_OFFSET(&file->hdr, file->hdr.hdr_size), 0,
 		       sizeof(file->hdr) - file->hdr.hdr_size);
-		if (file->hdr.minor_version == 0)
-			file->hdr.initial_modseq = 1;
 	}
 
 	if (file->hdr.indexid == 0) {
@@ -524,7 +540,7 @@
 	if (reset) {
 		file->hdr.prev_file_seq = 0;
 		file->hdr.prev_file_offset = 0;
-		file->hdr.initial_modseq = 1;
+		file->hdr.initial_modseq = 0;
 	}
 
 	if (write_full(new_fd, &file->hdr, sizeof(file->hdr)) < 0) {
@@ -724,8 +740,31 @@
 }
 
 bool
-mail_transaction_header_has_modseq(const struct mail_transaction_header *hdr)
+mail_transaction_header_has_modseq(const struct mail_transaction_header *hdr,
+				   const void *data,
+				   uint64_t cur_modseq)
 {
+	if (cur_modseq != 0) {
+		/* tracking modseqs */
+	} else if ((hdr->type & MAIL_TRANSACTION_TYPE_MASK) ==
+		   MAIL_TRANSACTION_EXT_INTRO) {
+		/* modseqs not tracked yet. see if this is a modseq
+		   extension introduction. */
+		const struct mail_transaction_ext_intro *intro = data;
+		const unsigned int modseq_ext_len =
+			strlen(MAIL_INDEX_MODSEQ_EXT_NAME);
+
+		if (intro->name_size == modseq_ext_len &&
+		    memcmp(intro + 1, MAIL_INDEX_MODSEQ_EXT_NAME,
+			   modseq_ext_len) == 0) {
+			/* modseq tracking started */
+			return TRUE;
+		}
+	} else {
+		/* not tracking modseqs */
+		return FALSE;
+	}
+
 	switch (hdr->type & MAIL_TRANSACTION_TYPE_MASK) {
 	case MAIL_TRANSACTION_EXPUNGE | MAIL_TRANSACTION_EXPUNGE_PROT:
 		if ((hdr->type & MAIL_TRANSACTION_EXTERNAL) == 0) {
@@ -882,7 +921,8 @@
 	while (cur_offset < offset) {
 		if (log_get_synced_record(file, &cur_offset, &hdr) < 0)
 			return- 1;
-		if (mail_transaction_header_has_modseq(hdr))
+		if (mail_transaction_header_has_modseq(hdr, hdr + 1,
+						       cur_modseq))
 			cur_modseq++;
 	}
 
@@ -943,7 +983,8 @@
 		prev_offset = cur_offset;
 		if (log_get_synced_record(file, &cur_offset, &hdr) < 0)
 			return -1;
-		if (mail_transaction_header_has_modseq(hdr)) {
+		if (mail_transaction_header_has_modseq(hdr, hdr + 1,
+						       cur_modseq)) {
 			if (++cur_modseq == modseq)
 				break;
 		}
@@ -973,11 +1014,12 @@
 		    const struct mail_transaction_header *hdr,
 		    unsigned int trans_size)
 {
+	const void *data = hdr + 1;
 	int ret;
 
-	if (mail_transaction_header_has_modseq(hdr))
+	if (mail_transaction_header_has_modseq(hdr, hdr + 1,
+					       file->sync_highest_modseq))
 		file->sync_highest_modseq++;
-
 	if ((hdr->type & MAIL_TRANSACTION_EXTERNAL) == 0)
 		return 0;
 
@@ -985,7 +1027,7 @@
 	if ((hdr->type & MAIL_TRANSACTION_TYPE_MASK) ==
 	    MAIL_TRANSACTION_HEADER_UPDATE) {
 		/* see if this updates mailbox_sync_offset */
-		ret = log_file_track_mailbox_sync_offset_hdr(file, hdr + 1,
+		ret = log_file_track_mailbox_sync_offset_hdr(file, data,
 							     trans_size -
 							     sizeof(*hdr));
 		if (ret != 0)
@@ -1209,13 +1251,6 @@
 		}
 	}
 
-	if (start_offset > file->sync_offset) {
-		/* although we could just skip over the unwanted data, we have
-		   to sync everything so that modseqs are calculated
-		   correctly */
-		start_offset = file->sync_offset;
-	}
-
 	if (file->buffer != NULL && file->buffer_offset > start_offset) {
 		/* we have to insert missing data to beginning of buffer */
 		ret = mail_transaction_log_file_insert_read(file, start_offset);
@@ -1419,6 +1454,15 @@
 						  end_offset);
 	}
 
+	if (start_offset > file->sync_offset)
+		mail_transaction_log_file_skip_to_head(file);
+	if (start_offset > file->sync_offset) {
+		/* although we could just skip over the unwanted data, we have
+		   to sync everything so that modseqs are calculated
+		   correctly */
+		start_offset = file->sync_offset;
+	}
+
 	if (!index->mmap_disable)
 		ret = mail_transaction_log_file_map_mmap(file, start_offset);
 	else {
--- a/src/lib-index/mail-transaction-log-private.h	Sat Jun 21 14:03:59 2008 +0300
+++ b/src/lib-index/mail-transaction-log-private.h	Sat Jun 21 15:06:46 2008 +0300
@@ -122,7 +122,9 @@
 void mail_transaction_log_file_unlock(struct mail_transaction_log_file *file);
 
 bool
-mail_transaction_header_has_modseq(const struct mail_transaction_header *hdr);
+mail_transaction_header_has_modseq(const struct mail_transaction_header *hdr,
+				   const void *data,
+				   uint64_t cur_modseq);
 int mail_transaction_log_file_get_highest_modseq_at(
 		struct mail_transaction_log_file *file,
 		uoff_t offset, uint64_t *highest_modseq_r);
--- a/src/lib-index/mail-transaction-log-view.c	Sat Jun 21 14:03:59 2008 +0300
+++ b/src/lib-index/mail-transaction-log-view.c	Sat Jun 21 15:06:46 2008 +0300
@@ -21,12 +21,7 @@
         struct mail_transaction_log_file *cur, *head, *tail;
 	uoff_t cur_offset;
 
-	/* prev_modseq doesn't contain correct values until we know that
-	   caller is really interested in modseqs. so the prev_modseq begins
-	   from 0 and it's relative to prev_modseq_start_offset. when
-	   prev_modseq_initialized=TRUE prev_modseq contains a correct value */
 	uint64_t prev_modseq;
-
 	uint32_t prev_file_seq;
 	uoff_t prev_file_offset;
 
@@ -614,7 +609,8 @@
 		ret = log_view_is_record_valid(file, hdr, data) ? 1 : -1;
 	} T_END;
 	if (ret > 0) {
-		if (mail_transaction_header_has_modseq(hdr))
+		if (mail_transaction_header_has_modseq(hdr, data,
+						       view->prev_modseq))
 			view->prev_modseq++;
 		*hdr_r = hdr;
 		*data_r = data;
--- a/src/util/logview.c	Sat Jun 21 14:03:59 2008 +0300
+++ b/src/util/logview.c	Sat Jun 21 15:06:46 2008 +0300
@@ -45,7 +45,7 @@
 	printf("create stamp = %u\n", hdr.create_stamp);
 	printf("initial modseq = %llu\n",
 	       (unsigned long long)hdr.initial_modseq);
-	*modseq_r = I_MAX(hdr.initial_modseq, 1);
+	*modseq_r = hdr.initial_modseq;
 }
 
 static bool
@@ -220,7 +220,7 @@
 }
 
 static void log_record_print(const struct mail_transaction_header *hdr,
-			     const void *data)
+			     const void *data, uint64_t *modseq)
 {
 	unsigned int size = hdr->size - sizeof(*hdr);
 
@@ -275,8 +275,12 @@
 		printf(" - flags = %u\n", intro->flags);
 		printf(" - name_size = %u\n", intro->name_size);
 		if (intro->name_size > 0) {
-			printf(" - name = '%.*s'\n",
-			       intro->name_size, (const char *)(intro+1));
+			const char *name = (const char *)(intro+1);
+
+			printf(" - name = '%.*s'\n", intro->name_size, name);
+			if (*modseq == 0 && intro->name_size == 6 &&
+			    memcmp(name, "modseq", 6) == 0)
+				*modseq = 1;
 		}
 		break;
 	}
@@ -372,7 +376,7 @@
 
 	printf("record: offset=%"PRIuUOFF_T", type=%s, size=%u",
 	       offset, log_record_type(hdr.type), hdr.size);
-	if (mail_transaction_header_has_modseq(&hdr)) {
+	if (*modseq > 0 && mail_transaction_header_has_modseq(&hdr)) {
 		*modseq += 1;
 		printf(", modseq=%llu", (unsigned long long)*modseq);
 	}
@@ -386,7 +390,7 @@
 			i_fatal("rec data read() %"PRIuSIZE_T" != %"PRIuSIZE_T,
 				ret, hdr.size - sizeof(hdr));
 		}
-		log_record_print(&hdr, buf);
+		log_record_print(&hdr, buf, modseq);
 	} else {
 		lseek(fd, hdr.size - sizeof(hdr), SEEK_CUR);
 	}