changeset 22041:21fab4826117

imapc: Try to merge STOREs together as much as possible when syncing
author Timo Sirainen <timo.sirainen@dovecot.fi>
date Thu, 04 May 2017 19:59:41 +0300
parents 99752d3e656b
children ba158fa10ff8
files src/lib-storage/index/imapc/imapc-sync.c src/lib-storage/index/imapc/imapc-sync.h
diffstat 2 files changed, 127 insertions(+), 25 deletions(-) [+]
line wrap: on
line diff
--- a/src/lib-storage/index/imapc/imapc-sync.c	Fri May 05 15:20:05 2017 +0300
+++ b/src/lib-storage/index/imapc/imapc-sync.c	Thu May 04 19:59:41 2017 +0300
@@ -2,7 +2,9 @@
 
 #include "lib.h"
 #include "ioloop.h"
+#include "hash.h"
 #include "str.h"
+#include "sort.h"
 #include "imap-util.h"
 #include "mail-cache.h"
 #include "mail-index-modseq.h"
@@ -75,10 +77,103 @@
 	return imapc_sync_cmd_full(ctx, cmd_str, FALSE);
 }
 
-static struct imapc_command *
-imapc_sync_store_cmd(struct imapc_sync_context *ctx, const char *cmd_str)
+static unsigned int imapc_sync_store_hash(const struct imapc_sync_store *store)
+{
+	return str_hash(store->flags) ^ store->modify_type;
+}
+
+static int imapc_sync_store_cmp(const struct imapc_sync_store *store1,
+				const struct imapc_sync_store *store2)
+{
+	if (store1->modify_type != store2->modify_type)
+		return 1;
+	return strcmp(store1->flags, store2->flags);
+}
+
+static const char *imapc_sync_flags_sort(const char *flags)
+{
+	if (strchr(flags, ' ') == NULL)
+		return flags;
+
+	const char **str = t_strsplit(flags, " ");
+	i_qsort(str, str_array_length(str), sizeof(const char *),
+		i_strcasecmp_p);
+	return t_strarray_join(str, " ");
+}
+
+static void
+imapc_sync_store_flush(struct imapc_sync_context *ctx)
 {
-	return imapc_sync_cmd_full(ctx, cmd_str, TRUE);
+	struct imapc_sync_store *store;
+	const char *sorted_flags;
+
+	if (ctx->prev_uid1 == 0)
+		return;
+
+	sorted_flags = imapc_sync_flags_sort(str_c(ctx->prev_flags));
+	struct imapc_sync_store store_lookup = {
+		.modify_type = ctx->prev_modify_type,
+		.flags = sorted_flags,
+	};
+	store = hash_table_lookup(ctx->stores, &store_lookup);
+	if (store == NULL) {
+		store = p_new(ctx->pool, struct imapc_sync_store, 1);
+		store->modify_type = ctx->prev_modify_type;
+		store->flags = p_strdup(ctx->pool, sorted_flags);
+		p_array_init(&store->uids, ctx->pool, 4);
+		hash_table_insert(ctx->stores, store, store);
+	}
+	seq_range_array_add_range(&store->uids, ctx->prev_uid1, ctx->prev_uid2);
+}
+
+static void
+imapc_sync_store(struct imapc_sync_context *ctx,
+		 enum modify_type modify_type, uint32_t uid1, uint32_t uid2,
+		 const char *flags)
+{
+	if (ctx->prev_flags == NULL) {
+		ctx->prev_flags = str_new(ctx->pool, 128);
+		hash_table_create(&ctx->stores, ctx->pool, 0,
+				  imapc_sync_store_hash, imapc_sync_store_cmp);
+	}
+
+	if (ctx->prev_uid1 != uid1 || ctx->prev_uid2 != uid2 ||
+	    ctx->prev_modify_type != modify_type) {
+		imapc_sync_store_flush(ctx);
+		ctx->prev_uid1 = uid1;
+		ctx->prev_uid2 = uid2;
+		ctx->prev_modify_type = modify_type;
+		str_truncate(ctx->prev_flags, 0);
+	}
+	if (str_len(ctx->prev_flags) > 0)
+		str_append_c(ctx->prev_flags, ' ');
+	str_append(ctx->prev_flags, flags);
+}
+
+static void
+imapc_sync_finish_store(struct imapc_sync_context *ctx)
+{
+	struct hash_iterate_context *iter;
+	struct imapc_sync_store *store;
+	string_t *cmd = t_str_new(128);
+
+	imapc_sync_store_flush(ctx);
+
+	if (!hash_table_is_created(ctx->stores))
+		return;
+
+	iter = hash_table_iterate_init(ctx->stores);
+	while (hash_table_iterate(iter, ctx->stores, &store, &store)) {
+		str_truncate(cmd, 0);
+		str_append(cmd, "UID STORE ");
+		imap_write_seq_range(cmd, &store->uids);
+		str_printfa(cmd, " %cFLAGS (%s)",
+			    store->modify_type == MODIFY_ADD ? '+' : '-',
+			    store->flags);
+		imapc_sync_cmd_full(ctx, str_c(cmd), TRUE);
+	}
+	hash_table_iterate_deinit(&iter);
+	hash_table_destroy(&ctx->stores);
 }
 
 static void
@@ -87,7 +182,6 @@
 {
 	const struct mail_index_record *rec;
 	uint32_t seq, uid1, uid2;
-	const char *cmd;
 
 	/* if any of them has a missing \Deleted flag,
 	   just add it to all of them. */
@@ -100,9 +194,8 @@
 	if (seq <= seq2) {
 		mail_index_lookup_uid(ctx->sync_view, seq1, &uid1);
 		mail_index_lookup_uid(ctx->sync_view, seq2, &uid2);
-		cmd = t_strdup_printf("UID STORE %u:%u +FLAGS \\Deleted",
-				      uid1, uid2);
-		imapc_sync_store_cmd(ctx, cmd);
+
+		imapc_sync_store(ctx, MODIFY_ADD, uid1, uid2, "\\Deleted");
 	}
 }
 
@@ -115,21 +208,18 @@
 
 	if (sync_rec->add_flags != 0) {
 		i_assert((sync_rec->add_flags & MAIL_RECENT) == 0);
-		str_printfa(str, "UID STORE %u:%u +FLAGS (",
-			    sync_rec->uid1, sync_rec->uid2);
+
 		imap_write_flags(str, sync_rec->add_flags, NULL);
-		str_append_c(str, ')');
-		imapc_sync_store_cmd(ctx, str_c(str));
+		imapc_sync_store(ctx, MODIFY_ADD, sync_rec->uid1,
+				 sync_rec->uid2, str_c(str));
 	}
 
 	if (sync_rec->remove_flags != 0) {
 		i_assert((sync_rec->remove_flags & MAIL_RECENT) == 0);
 		str_truncate(str, 0);
-		str_printfa(str, "UID STORE %u:%u -FLAGS (",
-			    sync_rec->uid1, sync_rec->uid2);
 		imap_write_flags(str, sync_rec->remove_flags, NULL);
-		str_append_c(str, ')');
-		imapc_sync_store_cmd(ctx, str_c(str));
+		imapc_sync_store(ctx, MODIFY_REMOVE, sync_rec->uid1,
+				 sync_rec->uid2, str_c(str));
 	}
 }
 
@@ -137,28 +227,23 @@
 imapc_sync_index_keyword(struct imapc_sync_context *ctx,
 			 const struct mail_index_sync_rec *sync_rec)
 {
-	string_t *str = t_str_new(128);
 	const char *const *kw_p;
-	char change_char;
+	enum modify_type modify_type;
 
 	switch (sync_rec->type) {
 	case MAIL_INDEX_SYNC_TYPE_KEYWORD_ADD:
-		change_char = '+';
+		modify_type = MODIFY_ADD;
 		break;
 	case MAIL_INDEX_SYNC_TYPE_KEYWORD_REMOVE:
-		change_char = '-';
+		modify_type = MODIFY_REMOVE;
 		break;
 	default:
 		i_unreached();
 	}
 
-	str_printfa(str, "UID STORE %u:%u %cFLAGS (",
-		    sync_rec->uid1, sync_rec->uid2, change_char);
-
 	kw_p = array_idx(ctx->keywords, sync_rec->keyword_idx);
-	str_append(str, *kw_p);
-	str_append_c(str, ')');
-	imapc_sync_store_cmd(ctx, str_c(str));
+	imapc_sync_store(ctx, modify_type, sync_rec->uid1,
+			 sync_rec->uid2, *kw_p);
 }
 
 static void imapc_sync_expunge_finish(struct imapc_sync_context *ctx)
@@ -359,6 +444,7 @@
 
 	i_array_init(&ctx->expunged_uids, 64);
 	ctx->keywords = mail_index_get_keywords(mbox->box.index);
+	ctx->pool = pool_alloconly_create("imapc sync pool", 1024);
 
 	imapc_sync_uid_validity(ctx);
 	while (mail_index_sync_next(ctx->index_sync_ctx, &sync_rec)) T_BEGIN {
@@ -381,6 +467,8 @@
 			break;
 		}
 	} T_END;
+	imapc_sync_finish_store(ctx);
+	pool_unref(&ctx->pool);
 
 	if (!mbox->initial_sync_done) {
 		/* with initial syncing we're fetching all messages' flags and
--- a/src/lib-storage/index/imapc/imapc-sync.h	Fri May 05 15:20:05 2017 +0300
+++ b/src/lib-storage/index/imapc/imapc-sync.h	Thu May 04 19:59:41 2017 +0300
@@ -4,6 +4,13 @@
 struct mailbox;
 struct mailbox_sync_status;
 
+struct imapc_sync_store {
+	enum modify_type modify_type;
+	const char *flags;
+
+	ARRAY_TYPE(seq_range) uids;
+};
+
 struct imapc_sync_context {
 	struct imapc_mailbox *mbox;
         struct mail_index_sync_ctx *index_sync_ctx;
@@ -14,6 +21,13 @@
 	ARRAY_TYPE(seq_range) expunged_uids;
 	unsigned int sync_command_count;
 
+	pool_t pool;
+	HASH_TABLE(struct imapc_sync_store *, struct imapc_sync_store *) stores;
+
+	uint32_t prev_uid1, prev_uid2;
+	enum modify_type prev_modify_type;
+	string_t *prev_flags;
+
 	unsigned int failed:1;
 };