Mercurial > dovecot > core-2.2

changeset 22442:a6f1f04b70e4

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
lib-index: Add mail_cache_close_mail() to smartly drop cached data with INDEX=MEMORY Instead of reseting the entire transaction buffer when 256 kB is reached, just drop mails have have been marked closed with mail_cache_close_mail(). If that's not enough, continue deleting forcibly until the buffer is below 256 kB. This is especially important when mail_prefetch_count>0 and INDEX=MEMORY. In that case there can be multiple mails that are being added to cache and used later on. If they were dropped from cache too early, the work would have to be done all over again.
author Timo Sirainen <timo.sirainen@dovecot.fi>
date Fri, 18 Aug 2017 17:34:14 +0300
parents 31ab1147d15d
children 6ad4c4203dba
files src/lib-index/mail-cache-transaction.c src/lib-index/mail-cache.h
diffstat 2 files changed, 54 insertions(+), 1 deletions(-) [+]
line wrap: on
line diff
--- a/src/lib-index/mail-cache-transaction.c	Wed Aug 16 12:44:02 2017 +0300
+++ b/src/lib-index/mail-cache-transaction.c	Fri Aug 18 17:34:14 2017 +0300
@@ -39,6 +39,7 @@
 
 	buffer_t *cache_data;
 	ARRAY(struct mail_cache_transaction_rec) cache_data_seq;
+	ARRAY_TYPE(seq_range) cache_data_wanted_seqs;
 	uint32_t prev_seq, min_seq;
 	size_t last_rec_pos;
 
@@ -159,6 +160,8 @@
 		buffer_free(&ctx->cache_data);
 	if (array_is_created(&ctx->cache_data_seq))
 		array_free(&ctx->cache_data_seq);
+	if (array_is_created(&ctx->cache_data_wanted_seqs))
+		array_free(&ctx->cache_data_wanted_seqs);
 	i_free(ctx);
 }
 
@@ -463,9 +466,40 @@
 	ctx->min_seq = 0;
 
 	array_clear(&ctx->cache_data_seq);
+	array_clear(&ctx->cache_data_wanted_seqs);
 	return ret;
 }
 
+static void
+mail_cache_transaction_drop_unwanted(struct mail_cache_transaction_ctx *ctx,
+				     size_t space_needed)
+{
+	struct mail_cache_transaction_rec *recs;
+	unsigned int i, count;
+
+	recs = array_get_modifiable(&ctx->cache_data_seq, &count);
+	/* find out how many records to delete. delete all unwanted sequences,
+	   and if that's not enough delete some more. */
+	for (i = 0; i < count; i++) {
+		if (seq_range_exists(&ctx->cache_data_wanted_seqs, recs[i].seq)) {
+			if (recs[i].cache_data_pos >= space_needed)
+				break;
+			/* we're going to forcibly delete it - remove it also
+			   from the array since it's no longer useful there */
+			seq_range_array_remove(&ctx->cache_data_wanted_seqs,
+					       recs[i].seq);
+		}
+	}
+	unsigned int deleted_count = i;
+	size_t deleted_space = i < count ?
+		recs[i].cache_data_pos : ctx->last_rec_pos;
+	for (; i < count; i++)
+		recs[i].cache_data_pos -= deleted_space;
+	ctx->last_rec_pos -= deleted_space;
+	array_delete(&ctx->cache_data_seq, 0, deleted_count);
+	buffer_delete(ctx->cache_data, 0, deleted_space);
+}
+
 static size_t
 mail_cache_transaction_update_last_rec_size(struct mail_cache_transaction_ctx *ctx)
 {
@@ -513,6 +547,7 @@
 			buffer_create_dynamic(default_pool,
 					      MAIL_CACHE_INIT_WRITE_BUFFER);
 		i_array_init(&ctx->cache_data_seq, 64);
+		i_array_init(&ctx->cache_data_wanted_seqs, 32);
 	}
 
 	i_zero(&new_rec);
@@ -727,6 +762,7 @@
 	if (ctx->prev_seq != seq) {
 		mail_cache_transaction_switch_seq(ctx);
 		ctx->prev_seq = seq;
+		seq_range_array_add(&ctx->cache_data_wanted_seqs, seq);
 
 		/* remember roughly what we have modified, so cache lookups can
 		   look into transactions to see changes. */
@@ -750,7 +786,12 @@
 		   cache file had been compressed and was reopened, return
 		   without adding the cached data since cache_data buffer
 		   doesn't contain the cache_rec anymore. */
-		if (mail_cache_transaction_flush(ctx) < 0) {
+		if (MAIL_INDEX_IS_IN_MEMORY(ctx->cache->index)) {
+			/* just drop the old data to free up memory */
+			size_t space_needed = ctx->cache_data->used +
+				full_size - MAIL_CACHE_MAX_WRITE_BUFFER;
+			mail_cache_transaction_drop_unwanted(ctx, space_needed);
+		} else if (mail_cache_transaction_flush(ctx) < 0) {
 			/* make sure the transaction is reset, so we don't
 			   constantly try to flush for each call to this
 			   function */
@@ -812,3 +853,10 @@
 
 	return mail_cache_field_exists(ctx->view, seq, field_idx) == 0;
 }
+
+void mail_cache_close_mail(struct mail_cache_transaction_ctx *ctx,
+			   uint32_t seq)
+{
+	if (array_is_created(&ctx->cache_data_wanted_seqs))
+		seq_range_array_remove(&ctx->cache_data_wanted_seqs, seq);
+}
--- a/src/lib-index/mail-cache.h	Wed Aug 16 12:44:02 2017 +0300
+++ b/src/lib-index/mail-cache.h	Fri Aug 18 17:34:14 2017 +0300
@@ -115,6 +115,11 @@
    returned only if the decision is a forced no. */
 bool mail_cache_field_can_add(struct mail_cache_transaction_ctx *ctx,
 			      uint32_t seq, unsigned int field_idx);
+/* Notify cache that the mail is now closed. Any records added with
+   mail_cache_add() are unlikely to be required again. This mainly tells
+   INDEX=MEMORY that it can free up the memory used by the mail. */
+void mail_cache_close_mail(struct mail_cache_transaction_ctx *ctx,
+			   uint32_t seq);
 
 /* Returns 1 if field exists, 0 if not, -1 if error. */
 int mail_cache_field_exists(struct mail_cache_view *view, uint32_t seq,