changeset 22611:65ada9976a7f

lib-index: Add mail_index_cache_optimization_settings
author Timo Sirainen <timo.sirainen@dovecot.fi>
date Mon, 09 Oct 2017 15:37:51 +0300
parents bf758ce6ee6d
children 9cc4ce6f3920
files src/doveadm/dsync/dsync-brain-mailbox.c src/lib-index/mail-cache-compress.c src/lib-index/mail-cache-fields.c src/lib-index/mail-cache-private.h src/lib-index/mail-cache-transaction.c src/lib-index/mail-cache.c src/lib-index/mail-index.c src/lib-index/mail-index.h
diffstat 8 files changed, 62 insertions(+), 30 deletions(-) [+]
line wrap: on
line diff
--- a/src/doveadm/dsync/dsync-brain-mailbox.c	Mon Oct 09 15:24:45 2017 +0300
+++ b/src/doveadm/dsync/dsync-brain-mailbox.c	Mon Oct 09 15:37:51 2017 +0300
@@ -599,6 +599,7 @@
 static void
 dsync_cache_fields_update(const struct dsync_mailbox *local_box,
 			  const struct dsync_mailbox *remote_box,
+			  struct mailbox *box,
 			  struct mailbox_update *update)
 {
 	ARRAY_TYPE(mailbox_cache_field) local_sorted, remote_sorted, changes;
@@ -630,7 +631,8 @@
 	local_fields = array_get(&local_sorted, &local_count);
 	remote_fields = array_get(&remote_sorted, &remote_count);
 	t_array_init(&changes, local_count + remote_count);
-	drop_older_timestamp = ioloop_time - MAIL_CACHE_FIELD_DROP_SECS;
+	drop_older_timestamp = ioloop_time -
+		box->index->optimization_set.cache.unaccessed_field_drop_secs;
 
 	for (li = ri = 0; li < local_count || ri < remote_count; ) {
 		ret = li == local_count ? 1 :
@@ -702,7 +704,7 @@
 		}
 	}
 
-	dsync_cache_fields_update(local_box, remote_box, &update);
+	dsync_cache_fields_update(local_box, remote_box, box, &update);
 
 	if (update.uid_validity == 0 &&
 	    update.cache_updates == NULL) {
--- a/src/lib-index/mail-cache-compress.c	Mon Oct 09 15:24:45 2017 +0300
+++ b/src/lib-index/mail-cache-compress.c	Mon Oct 09 15:37:51 2017 +0300
@@ -207,7 +207,8 @@
 	   used fields */
 	idx_hdr = mail_index_get_header(view);
 	max_drop_time = idx_hdr->day_stamp == 0 ? 0 :
-		idx_hdr->day_stamp - MAIL_CACHE_FIELD_DROP_SECS;
+		idx_hdr->day_stamp -
+		cache->index->optimization_set.cache.unaccessed_field_drop_secs;
 
 	orig_fields_count = cache->fields_count;
 	if (cache->file_fields_count == 0) {
@@ -272,7 +273,7 @@
 			mail_cache_compress_field(&ctx, &field);
 
 		if (ctx.buffer->used == sizeof(cache_rec) ||
-		    ctx.buffer->used > MAIL_CACHE_RECORD_MAX_SIZE) {
+		    ctx.buffer->used > cache->index->optimization_set.cache.record_max_size) {
 			/* nothing cached */
 			ext_offset = 0;
 		} else {
--- a/src/lib-index/mail-cache-fields.c	Mon Oct 09 15:24:45 2017 +0300
+++ b/src/lib-index/mail-cache-fields.c	Mon Oct 09 15:37:51 2017 +0300
@@ -287,7 +287,7 @@
 	}
 	cache->last_field_header_offset = offset;
 
-	if (next_count > MAIL_CACHE_HEADER_FIELD_CONTINUE_COUNT)
+	if (next_count > cache->index->optimization_set.cache.compress_header_continue_count)
 		cache->need_compress_file_seq = cache->hdr->file_seq;
 
 	if (field_hdr_r != NULL) {
@@ -372,7 +372,8 @@
 		cache->field_file_map[i] = (uint32_t)-1;
 
 	max_drop_time = cache->index->map->hdr.day_stamp == 0 ? 0 :
-		cache->index->map->hdr.day_stamp - MAIL_CACHE_FIELD_DROP_SECS;
+		cache->index->map->hdr.day_stamp -
+		cache->index->optimization_set.cache.unaccessed_field_drop_secs;
 
 	i_zero(&field);
 	for (i = 0; i < field_hdr->fields_count; i++) {
--- a/src/lib-index/mail-cache-private.h	Mon Oct 09 15:24:45 2017 +0300
+++ b/src/lib-index/mail-cache-private.h	Mon Oct 09 15:37:51 2017 +0300
@@ -8,26 +8,6 @@
 #define MAIL_CACHE_MAJOR_VERSION 1
 #define MAIL_CACHE_MINOR_VERSION 1
 
-/* Drop fields that haven't been accessed for n seconds */
-#define MAIL_CACHE_FIELD_DROP_SECS (3600*24*30)
-
-/* Never compress the file if it's smaller than this */
-#define MAIL_CACHE_COMPRESS_MIN_SIZE (1024*32)
-
-/* Compress the file when n% of records are deleted */
-#define MAIL_CACHE_COMPRESS_DELETE_PERCENTAGE 20
-
-/* Compress the file when n% of rows contain continued rows.
-   200% means that there's 2 continued rows per record. */
-#define MAIL_CACHE_COMPRESS_CONTINUED_PERCENTAGE 200
-
-/* Compress the file when we need to follow more than n next_offsets to find
-   the latest cache header. */
-#define MAIL_CACHE_HEADER_FIELD_CONTINUE_COUNT 4
-
-/* If cache record becomes larger than this, don't add it. */
-#define MAIL_CACHE_RECORD_MAX_SIZE (64*1024)
-
 #define MAIL_CACHE_LOCK_TIMEOUT 10
 #define MAIL_CACHE_LOCK_CHANGE_TIMEOUT 300
 
--- a/src/lib-index/mail-cache-transaction.c	Mon Oct 09 15:24:45 2017 +0300
+++ b/src/lib-index/mail-cache-transaction.c	Mon Oct 09 15:37:51 2017 +0300
@@ -521,7 +521,7 @@
 	size_t size;
 
 	size = mail_cache_transaction_update_last_rec_size(ctx);
-	if (size > MAIL_CACHE_RECORD_MAX_SIZE) {
+	if (size > ctx->cache->index->optimization_set.cache.record_max_size) {
 		buffer_set_used_size(ctx->cache_data, ctx->last_rec_pos);
 		return;
 	}
--- a/src/lib-index/mail-cache.c	Mon Oct 09 15:24:45 2017 +0300
+++ b/src/lib-index/mail-cache.c	Mon Oct 09 15:37:51 2017 +0300
@@ -245,6 +245,8 @@
 
 static void mail_cache_update_need_compress(struct mail_cache *cache)
 {
+	const struct mail_index_cache_optimization_settings *set =
+		&cache->index->optimization_set.cache;
 	const struct mail_cache_header *hdr = cache->hdr;
 	struct stat st;
 	unsigned int msg_count;
@@ -274,14 +276,14 @@
 	}
 
 	cont_percentage = hdr->continued_record_count * 100 / records_count;
-	if (cont_percentage >= MAIL_CACHE_COMPRESS_CONTINUED_PERCENTAGE) {
+	if (cont_percentage >= set->compress_continued_percentage) {
 		/* too many continued rows, compress */
 		want_compress = TRUE;
 	}
 
 	delete_percentage = hdr->deleted_record_count * 100 /
 		(records_count + hdr->deleted_record_count);
-	if (delete_percentage >= MAIL_CACHE_COMPRESS_DELETE_PERCENTAGE) {
+	if (delete_percentage >= set->compress_delete_percentage) {
 		/* too many deleted records, compress */
 		want_compress = TRUE;
 	}
@@ -292,7 +294,7 @@
 				mail_cache_set_syscall_error(cache, "fstat()");
 			return;
 		}
-		if (st.st_size >= MAIL_CACHE_COMPRESS_MIN_SIZE)
+		if ((uoff_t)st.st_size >= set->compress_min_size)
 			cache->need_compress_file_seq = hdr->file_seq;
 	}
 
--- a/src/lib-index/mail-index.c	Mon Oct 09 15:24:45 2017 +0300
+++ b/src/lib-index/mail-index.c	Mon Oct 09 15:37:51 2017 +0300
@@ -40,6 +40,14 @@
 		.min_age_secs = 5 * 60,
 		.log2_max_age_secs = 3600 * 24 * 2,
 	},
+	.cache = {
+		.unaccessed_field_drop_secs = 3600 * 24 * 30,
+		.record_max_size = 64 * 1024,
+		.compress_min_size = 32 * 1024,
+		.compress_delete_percentage = 20,
+		.compress_continued_percentage = 200,
+		.compress_header_continue_count = 4,
+	},
 };
 
 struct mail_index *mail_index_alloc(const char *dir, const char *prefix)
@@ -182,6 +190,24 @@
 		dest->log.min_age_secs = set->log.min_age_secs;
 	if (set->log.log2_max_age_secs != 0)
 		dest->log.log2_max_age_secs = set->log.log2_max_age_secs;
+
+	/* cache */
+	if (set->cache.unaccessed_field_drop_secs != 0)
+		dest->cache.unaccessed_field_drop_secs =
+			set->cache.unaccessed_field_drop_secs;
+	if (set->cache.compress_min_size != 0)
+		dest->cache.compress_min_size = set->cache.compress_min_size;
+	if (set->cache.compress_delete_percentage != 0)
+		dest->cache.compress_delete_percentage =
+			set->cache.compress_delete_percentage;
+	if (set->cache.compress_continued_percentage != 0)
+		dest->cache.compress_continued_percentage =
+			set->cache.compress_continued_percentage;
+	if (set->cache.compress_header_continue_count != 0)
+		dest->cache.compress_header_continue_count =
+			set->cache.compress_header_continue_count;
+	if (set->cache.record_max_size != 0)
+		dest->cache.record_max_size = set->cache.record_max_size;
 }
 
 void mail_index_set_ext_init_data(struct mail_index *index, uint32_t ext_id,
--- a/src/lib-index/mail-index.h	Mon Oct 09 15:24:45 2017 +0300
+++ b/src/lib-index/mail-index.h	Mon Oct 09 15:37:51 2017 +0300
@@ -256,9 +256,29 @@
 	unsigned int log2_max_age_secs;
 };
 
+struct mail_index_cache_optimization_settings {
+	/* Drop fields that haven't been accessed for n seconds */
+	unsigned int unaccessed_field_drop_secs;
+	/* If cache record becomes larger than this, don't add it. */
+	unsigned int record_max_size;
+
+	/* Never compress the file if it's smaller than this */
+	uoff_t compress_min_size;
+	/* Compress the file when n% of records are deleted */
+	unsigned int compress_delete_percentage;
+	/* Compress the file when n% of rows contain continued rows.
+	   For example 200% means that the record has 2 continued rows, i.e.
+	   it exists in 3 separate segments in the cache file. */
+	unsigned int compress_continued_percentage;
+	/* Compress the file when we need to follow more than n next_offsets to
+	   find the latest cache header. */
+	unsigned int compress_header_continue_count;
+};
+
 struct mail_index_optimization_settings {
 	struct mail_index_base_optimization_settings index;
 	struct mail_index_log_optimization_settings log;
+	struct mail_index_cache_optimization_settings cache;
 };
 
 struct mail_index;