Mercurial > dovecot > core-2.2

changeset 21876:ccbdafa83dbc

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
lib-index: Don't increase modseq for backend/dirty flag changes These flags are used only for internal changes and they shouldn't be triggering any modseq changes. To avoid modseqs from unexpectedly shrinking, the new modseq counting behavior is enabled only for newly rotated transaction log files that have a new minor_version.
author Timo Sirainen <timo.sirainen@dovecot.fi>
date Mon, 27 Mar 2017 18:05:29 +0300
parents 7c51ff56653a
children d703b0fde204
files src/doveadm/doveadm-dump-log.c src/lib-index/mail-index-transaction-export.c src/lib-index/mail-transaction-log-append.c src/lib-index/mail-transaction-log-file.c src/lib-index/mail-transaction-log-private.h src/lib-index/mail-transaction-log-view.c src/lib-index/mail-transaction-log.c src/lib-index/mail-transaction-log.h src/lib-index/test-mail-transaction-log-append.c src/lib-index/test-mail-transaction-log-view.c
diffstat 10 files changed, 96 insertions(+), 18 deletions(-) [+]
line wrap: on
line diff
--- a/src/doveadm/doveadm-dump-log.c	Thu Mar 30 20:38:44 2017 +0300
+++ b/src/doveadm/doveadm-dump-log.c	Mon Mar 27 18:05:29 2017 +0300
@@ -11,7 +11,8 @@
 
 static struct mail_transaction_ext_intro prev_intro;
 
-static void dump_hdr(struct istream *input, uint64_t *modseq_r)
+static void dump_hdr(struct istream *input, uint64_t *modseq_r,
+		     unsigned int *version_r)
 {
 	struct mail_transaction_log_header hdr;
 	const unsigned char *data;
@@ -42,6 +43,7 @@
 	       (unsigned long long)hdr.initial_modseq);
 	printf("compat flags = %x\n", hdr.compat_flags);
 	*modseq_r = hdr.initial_modseq;
+	*version_r = MAIL_TRANSACTION_LOG_HDR_VERSION(&hdr);
 }
 
 static const char *log_record_type(unsigned int type)
@@ -464,7 +466,8 @@
 	}
 }
 
-static int dump_record(struct istream *input, uint64_t *modseq)
+static int dump_record(struct istream *input, uint64_t *modseq,
+		       unsigned int version)
 {
 	struct mail_transaction_header hdr;
 	unsigned int hdr_size;
@@ -505,7 +508,7 @@
 	}
 
 	uint64_t prev_modseq = *modseq;
-	mail_transaction_update_modseq(&hdr, data, modseq);
+	mail_transaction_update_modseq(&hdr, data, modseq, version);
 	if (*modseq > prev_modseq)
 		printf(", modseq=%llu", (unsigned long long)*modseq);
 	printf("\n");
@@ -519,13 +522,14 @@
 {
 	struct istream *input;
 	uint64_t modseq;
+	unsigned int version;
 	int ret;
 
 	input = i_stream_create_file(argv[1], (size_t)-1);
-	dump_hdr(input, &modseq);
+	dump_hdr(input, &modseq, &version);
 	do {
 		T_BEGIN {
-			ret = dump_record(input, &modseq);
+			ret = dump_record(input, &modseq, version);
 		} T_END;
 	} while (ret > 0);
 	i_stream_unref(&input);
--- a/src/lib-index/mail-index-transaction-export.c	Thu Mar 30 20:38:44 2017 +0300
+++ b/src/lib-index/mail-index-transaction-export.c	Mon Mar 27 18:05:29 2017 +0300
@@ -536,6 +536,32 @@
 	return count;
 }
 
+static bool
+transaction_flag_updates_have_non_internal(struct mail_index_transaction *t)
+{
+	struct mail_transaction_log_file *file = t->view->index->log->head;
+	const uint8_t internal_flags =
+		MAIL_INDEX_MAIL_FLAG_BACKEND | MAIL_INDEX_MAIL_FLAG_DIRTY;
+	const struct mail_index_flag_update *u;
+	const unsigned int hdr_version =
+		MAIL_TRANSACTION_LOG_HDR_VERSION(&file->hdr);
+
+	if (!MAIL_TRANSACTION_LOG_VERSION_HAVE(hdr_version, HIDE_INTERNAL_MODSEQS)) {
+		/* this check can be a bit racy if the call isn't done while
+		   transaction log is locked. practically it won't matter
+		   now though. */
+		return array_count(&t->updates) > 0;
+	}
+
+	array_foreach(&t->updates, u) {
+		uint8_t changed_flags = u->add_flags | u->remove_flags;
+
+		if ((changed_flags & ~internal_flags) != 0)
+			return TRUE;
+	}
+	return FALSE;
+}
+
 uint64_t mail_index_transaction_get_highest_modseq(struct mail_index_transaction *t)
 {
 	struct mail_transaction_log_file *file = t->view->index->log->head;
@@ -565,7 +591,8 @@
 		/* sorting may change the order of keyword_updates,  */
 		new_highest_modseq++;
 	}
-	if (array_is_created(&t->updates) && array_count(&t->updates) > 0)
+	if (array_is_created(&t->updates) &&
+	    transaction_flag_updates_have_non_internal(t) > 0)
 		new_highest_modseq++;
 	if (array_is_created(&t->keyword_updates)) {
 		new_highest_modseq +=
--- a/src/lib-index/mail-transaction-log-append.c	Thu Mar 30 20:38:44 2017 +0300
+++ b/src/lib-index/mail-transaction-log-append.c	Mon Mar 27 18:05:29 2017 +0300
@@ -31,7 +31,8 @@
 	buffer_append(ctx->output, &hdr, sizeof(hdr));
 	buffer_append(ctx->output, data, size);
 
-	mail_transaction_update_modseq(&hdr, data, &ctx->new_highest_modseq);
+	mail_transaction_update_modseq(&hdr, data, &ctx->new_highest_modseq,
+		MAIL_TRANSACTION_LOG_HDR_VERSION(&ctx->log->head->hdr));
 	ctx->transaction_count++;
 }
 
--- a/src/lib-index/mail-transaction-log-file.c	Thu Mar 30 20:38:44 2017 +0300
+++ b/src/lib-index/mail-transaction-log-file.c	Mon Mar 27 18:05:29 2017 +0300
@@ -1007,8 +1007,31 @@
 	return 0;
 }
 
+static bool
+flag_updates_have_non_internal(const struct mail_transaction_flag_update *u,
+			       unsigned int count, unsigned int version)
+{
+	const uint8_t internal_flags =
+		MAIL_INDEX_MAIL_FLAG_BACKEND | MAIL_INDEX_MAIL_FLAG_DIRTY;
+
+	/* Hide internal flags from modseqs if the log file's version
+	   is new enough. This allows upgrading without the modseqs suddenly
+	   shrinking. */
+	if (!MAIL_TRANSACTION_LOG_VERSION_HAVE(version, HIDE_INTERNAL_MODSEQS))
+		return TRUE;
+
+	for (unsigned int i = 0; i < count; i++) {
+		uint8_t changed_flags = u->add_flags | u->remove_flags;
+
+		if ((changed_flags & ~internal_flags) != 0)
+			return TRUE;
+	}
+	return FALSE;
+}
+
 void mail_transaction_update_modseq(const struct mail_transaction_header *hdr,
-				    const void *data, uint64_t *cur_modseq)
+				    const void *data, uint64_t *cur_modseq,
+				    unsigned int version)
 {
 	uint32_t trans_size;
 
@@ -1046,13 +1069,21 @@
 			break;
 		}
 	case MAIL_TRANSACTION_APPEND:
-	case MAIL_TRANSACTION_FLAG_UPDATE:
 	case MAIL_TRANSACTION_KEYWORD_UPDATE:
 	case MAIL_TRANSACTION_KEYWORD_RESET:
 	case MAIL_TRANSACTION_ATTRIBUTE_UPDATE:
 		/* these changes increase modseq */
 		*cur_modseq += 1;
 		break;
+	case MAIL_TRANSACTION_FLAG_UPDATE: {
+		const struct mail_transaction_flag_update *rec = data;
+		unsigned int count;
+
+		count = (trans_size - sizeof(*hdr)) / sizeof(*rec);
+		if (flag_updates_have_non_internal(rec, count, version))
+			*cur_modseq += 1;
+		break;
+	}
 	case MAIL_TRANSACTION_MODSEQ_UPDATE: {
 		const struct mail_transaction_modseq_update *rec, *end;
 
@@ -1209,7 +1240,8 @@
 	while (cur_offset < offset) {
 		if (log_get_synced_record(file, &cur_offset, &hdr) < 0)
 			return- 1;
-		mail_transaction_update_modseq(hdr, hdr + 1, &cur_modseq);
+		mail_transaction_update_modseq(hdr, hdr + 1, &cur_modseq,
+			MAIL_TRANSACTION_LOG_HDR_VERSION(&file->hdr));
 	}
 
 	/* @UNSAFE: cache the value */
@@ -1280,7 +1312,8 @@
 	while (cur_offset < file->sync_offset) {
 		if (log_get_synced_record(file, &cur_offset, &hdr) < 0)
 			return -1;
-		mail_transaction_update_modseq(hdr, hdr + 1, &cur_modseq);
+		mail_transaction_update_modseq(hdr, hdr + 1, &cur_modseq,
+			MAIL_TRANSACTION_LOG_HDR_VERSION(&file->hdr));
 		if (cur_modseq >= modseq)
 			break;
 	}
@@ -1312,8 +1345,8 @@
 	const void *data = hdr + 1;
 	int ret;
 
-	mail_transaction_update_modseq(hdr, hdr + 1,
-				       &file->sync_highest_modseq);
+	mail_transaction_update_modseq(hdr, hdr + 1, &file->sync_highest_modseq,
+		MAIL_TRANSACTION_LOG_HDR_VERSION(&file->hdr));
 	if ((hdr->type & MAIL_TRANSACTION_EXTERNAL) == 0)
 		return 1;
 
--- a/src/lib-index/mail-transaction-log-private.h	Thu Mar 30 20:38:44 2017 +0300
+++ b/src/lib-index/mail-transaction-log-private.h	Mon Mar 27 18:05:29 2017 +0300
@@ -148,7 +148,8 @@
 				      const char *lock_reason);
 
 void mail_transaction_update_modseq(const struct mail_transaction_header *hdr,
-				    const void *data, uint64_t *cur_modseq);
+				    const void *data, uint64_t *cur_modseq,
+				    unsigned int version);
 int mail_transaction_log_file_get_highest_modseq_at(
 		struct mail_transaction_log_file *file,
 		uoff_t offset, uint64_t *highest_modseq_r);
--- a/src/lib-index/mail-transaction-log-view.c	Thu Mar 30 20:38:44 2017 +0300
+++ b/src/lib-index/mail-transaction-log-view.c	Mon Mar 27 18:05:29 2017 +0300
@@ -792,7 +792,8 @@
 		ret = log_view_is_record_valid(file, hdr, data) ? 1 : -1;
 	} T_END;
 	if (ret > 0) {
-		mail_transaction_update_modseq(hdr, data, &view->prev_modseq);
+		mail_transaction_update_modseq(hdr, data, &view->prev_modseq,
+			MAIL_TRANSACTION_LOG_HDR_VERSION(&file->hdr));
 		*hdr_r = hdr;
 		*data_r = data;
 		view->cur_offset += full_size;
--- a/src/lib-index/mail-transaction-log.c	Thu Mar 30 20:38:44 2017 +0300
+++ b/src/lib-index/mail-transaction-log.c	Mon Mar 27 18:05:29 2017 +0300
@@ -218,6 +218,13 @@
 {
 	struct mail_transaction_log_file *file = log->head;
 
+	if (file->hdr.major_version < MAIL_TRANSACTION_LOG_MAJOR_VERSION ||
+	    (file->hdr.major_version == MAIL_TRANSACTION_LOG_MAJOR_VERSION &&
+	     file->hdr.minor_version < MAIL_TRANSACTION_LOG_MINOR_VERSION)) {
+		/* upgrade immediately to a new log file format */
+		return TRUE;
+	}
+
 	if (file->sync_offset > log->index->log_rotate_max_size) {
 		/* file is too large, definitely rotate */
 		return TRUE;
--- a/src/lib-index/mail-transaction-log.h	Thu Mar 30 20:38:44 2017 +0300
+++ b/src/lib-index/mail-transaction-log.h	Mon Mar 27 18:05:29 2017 +0300
@@ -6,7 +6,7 @@
 #define MAIL_TRANSACTION_LOG_SUFFIX ".log"
 
 #define MAIL_TRANSACTION_LOG_MAJOR_VERSION 1
-#define MAIL_TRANSACTION_LOG_MINOR_VERSION 2
+#define MAIL_TRANSACTION_LOG_MINOR_VERSION 3
 #define MAIL_TRANSACTION_LOG_HEADER_MIN_SIZE 24
 
 #define MAIL_TRANSACTION_LOG_VERSION_FULL(major, minor) \
@@ -18,6 +18,8 @@
 
 #define MAIL_TRANSACTION_LOG_VERSION_COMPAT_FLAGS \
 	MAIL_TRANSACTION_LOG_VERSION_FULL(1, 2)
+#define MAIL_TRANSACTION_LOG_VERSION_HIDE_INTERNAL_MODSEQS \
+	MAIL_TRANSACTION_LOG_VERSION_FULL(1, 3)
 
 struct mail_transaction_log_header {
 	uint8_t major_version;
--- a/src/lib-index/test-mail-transaction-log-append.c	Thu Mar 30 20:38:44 2017 +0300
+++ b/src/lib-index/test-mail-transaction-log-append.c	Mon Mar 27 18:05:29 2017 +0300
@@ -27,7 +27,8 @@
 
 void mail_transaction_update_modseq(const struct mail_transaction_header *hdr,
 				    const void *data ATTR_UNUSED,
-				    uint64_t *cur_modseq)
+				    uint64_t *cur_modseq,
+				    unsigned int version ATTR_UNUSED)
 {
 	if ((hdr->type & MAIL_TRANSACTION_EXPUNGE) != 0)
 		*cur_modseq += 1;
--- a/src/lib-index/test-mail-transaction-log-view.c	Thu Mar 30 20:38:44 2017 +0300
+++ b/src/lib-index/test-mail-transaction-log-view.c	Mon Mar 27 18:05:29 2017 +0300
@@ -51,7 +51,8 @@
 
 void mail_transaction_update_modseq(const struct mail_transaction_header *hdr ATTR_UNUSED,
 				    const void *data ATTR_UNUSED,
-				    uint64_t *cur_modseq)
+				    uint64_t *cur_modseq,
+				    unsigned int version ATTR_UNUSED)
 {
 	*cur_modseq += 1;
 }