# HG changeset patch # User Timo Sirainen # Date 1493917181 -10800 # Node ID 21fab4826117e2245804f50a92e3eb2eb39936a4 # Parent 99752d3e656ba3a900dc2d1697c210a627f7ac51 imapc: Try to merge STOREs together as much as possible when syncing diff -r 99752d3e656b -r 21fab4826117 src/lib-storage/index/imapc/imapc-sync.c --- a/src/lib-storage/index/imapc/imapc-sync.c Fri May 05 15:20:05 2017 +0300 +++ b/src/lib-storage/index/imapc/imapc-sync.c Thu May 04 19:59:41 2017 +0300 @@ -2,7 +2,9 @@ #include "lib.h" #include "ioloop.h" +#include "hash.h" #include "str.h" +#include "sort.h" #include "imap-util.h" #include "mail-cache.h" #include "mail-index-modseq.h" @@ -75,10 +77,103 @@ return imapc_sync_cmd_full(ctx, cmd_str, FALSE); } -static struct imapc_command * -imapc_sync_store_cmd(struct imapc_sync_context *ctx, const char *cmd_str) +static unsigned int imapc_sync_store_hash(const struct imapc_sync_store *store) +{ + return str_hash(store->flags) ^ store->modify_type; +} + +static int imapc_sync_store_cmp(const struct imapc_sync_store *store1, + const struct imapc_sync_store *store2) +{ + if (store1->modify_type != store2->modify_type) + return 1; + return strcmp(store1->flags, store2->flags); +} + +static const char *imapc_sync_flags_sort(const char *flags) +{ + if (strchr(flags, ' ') == NULL) + return flags; + + const char **str = t_strsplit(flags, " "); + i_qsort(str, str_array_length(str), sizeof(const char *), + i_strcasecmp_p); + return t_strarray_join(str, " "); +} + +static void +imapc_sync_store_flush(struct imapc_sync_context *ctx) { - return imapc_sync_cmd_full(ctx, cmd_str, TRUE); + struct imapc_sync_store *store; + const char *sorted_flags; + + if (ctx->prev_uid1 == 0) + return; + + sorted_flags = imapc_sync_flags_sort(str_c(ctx->prev_flags)); + struct imapc_sync_store store_lookup = { + .modify_type = ctx->prev_modify_type, + .flags = sorted_flags, + }; + store = hash_table_lookup(ctx->stores, &store_lookup); + if (store == NULL) { + store = p_new(ctx->pool, struct imapc_sync_store, 1); + store->modify_type = ctx->prev_modify_type; + store->flags = p_strdup(ctx->pool, sorted_flags); + p_array_init(&store->uids, ctx->pool, 4); + hash_table_insert(ctx->stores, store, store); + } + seq_range_array_add_range(&store->uids, ctx->prev_uid1, ctx->prev_uid2); +} + +static void +imapc_sync_store(struct imapc_sync_context *ctx, + enum modify_type modify_type, uint32_t uid1, uint32_t uid2, + const char *flags) +{ + if (ctx->prev_flags == NULL) { + ctx->prev_flags = str_new(ctx->pool, 128); + hash_table_create(&ctx->stores, ctx->pool, 0, + imapc_sync_store_hash, imapc_sync_store_cmp); + } + + if (ctx->prev_uid1 != uid1 || ctx->prev_uid2 != uid2 || + ctx->prev_modify_type != modify_type) { + imapc_sync_store_flush(ctx); + ctx->prev_uid1 = uid1; + ctx->prev_uid2 = uid2; + ctx->prev_modify_type = modify_type; + str_truncate(ctx->prev_flags, 0); + } + if (str_len(ctx->prev_flags) > 0) + str_append_c(ctx->prev_flags, ' '); + str_append(ctx->prev_flags, flags); +} + +static void +imapc_sync_finish_store(struct imapc_sync_context *ctx) +{ + struct hash_iterate_context *iter; + struct imapc_sync_store *store; + string_t *cmd = t_str_new(128); + + imapc_sync_store_flush(ctx); + + if (!hash_table_is_created(ctx->stores)) + return; + + iter = hash_table_iterate_init(ctx->stores); + while (hash_table_iterate(iter, ctx->stores, &store, &store)) { + str_truncate(cmd, 0); + str_append(cmd, "UID STORE "); + imap_write_seq_range(cmd, &store->uids); + str_printfa(cmd, " %cFLAGS (%s)", + store->modify_type == MODIFY_ADD ? '+' : '-', + store->flags); + imapc_sync_cmd_full(ctx, str_c(cmd), TRUE); + } + hash_table_iterate_deinit(&iter); + hash_table_destroy(&ctx->stores); } static void @@ -87,7 +182,6 @@ { const struct mail_index_record *rec; uint32_t seq, uid1, uid2; - const char *cmd; /* if any of them has a missing \Deleted flag, just add it to all of them. */ @@ -100,9 +194,8 @@ if (seq <= seq2) { mail_index_lookup_uid(ctx->sync_view, seq1, &uid1); mail_index_lookup_uid(ctx->sync_view, seq2, &uid2); - cmd = t_strdup_printf("UID STORE %u:%u +FLAGS \\Deleted", - uid1, uid2); - imapc_sync_store_cmd(ctx, cmd); + + imapc_sync_store(ctx, MODIFY_ADD, uid1, uid2, "\\Deleted"); } } @@ -115,21 +208,18 @@ if (sync_rec->add_flags != 0) { i_assert((sync_rec->add_flags & MAIL_RECENT) == 0); - str_printfa(str, "UID STORE %u:%u +FLAGS (", - sync_rec->uid1, sync_rec->uid2); + imap_write_flags(str, sync_rec->add_flags, NULL); - str_append_c(str, ')'); - imapc_sync_store_cmd(ctx, str_c(str)); + imapc_sync_store(ctx, MODIFY_ADD, sync_rec->uid1, + sync_rec->uid2, str_c(str)); } if (sync_rec->remove_flags != 0) { i_assert((sync_rec->remove_flags & MAIL_RECENT) == 0); str_truncate(str, 0); - str_printfa(str, "UID STORE %u:%u -FLAGS (", - sync_rec->uid1, sync_rec->uid2); imap_write_flags(str, sync_rec->remove_flags, NULL); - str_append_c(str, ')'); - imapc_sync_store_cmd(ctx, str_c(str)); + imapc_sync_store(ctx, MODIFY_REMOVE, sync_rec->uid1, + sync_rec->uid2, str_c(str)); } } @@ -137,28 +227,23 @@ imapc_sync_index_keyword(struct imapc_sync_context *ctx, const struct mail_index_sync_rec *sync_rec) { - string_t *str = t_str_new(128); const char *const *kw_p; - char change_char; + enum modify_type modify_type; switch (sync_rec->type) { case MAIL_INDEX_SYNC_TYPE_KEYWORD_ADD: - change_char = '+'; + modify_type = MODIFY_ADD; break; case MAIL_INDEX_SYNC_TYPE_KEYWORD_REMOVE: - change_char = '-'; + modify_type = MODIFY_REMOVE; break; default: i_unreached(); } - str_printfa(str, "UID STORE %u:%u %cFLAGS (", - sync_rec->uid1, sync_rec->uid2, change_char); - kw_p = array_idx(ctx->keywords, sync_rec->keyword_idx); - str_append(str, *kw_p); - str_append_c(str, ')'); - imapc_sync_store_cmd(ctx, str_c(str)); + imapc_sync_store(ctx, modify_type, sync_rec->uid1, + sync_rec->uid2, *kw_p); } static void imapc_sync_expunge_finish(struct imapc_sync_context *ctx) @@ -359,6 +444,7 @@ i_array_init(&ctx->expunged_uids, 64); ctx->keywords = mail_index_get_keywords(mbox->box.index); + ctx->pool = pool_alloconly_create("imapc sync pool", 1024); imapc_sync_uid_validity(ctx); while (mail_index_sync_next(ctx->index_sync_ctx, &sync_rec)) T_BEGIN { @@ -381,6 +467,8 @@ break; } } T_END; + imapc_sync_finish_store(ctx); + pool_unref(&ctx->pool); if (!mbox->initial_sync_done) { /* with initial syncing we're fetching all messages' flags and diff -r 99752d3e656b -r 21fab4826117 src/lib-storage/index/imapc/imapc-sync.h --- a/src/lib-storage/index/imapc/imapc-sync.h Fri May 05 15:20:05 2017 +0300 +++ b/src/lib-storage/index/imapc/imapc-sync.h Thu May 04 19:59:41 2017 +0300 @@ -4,6 +4,13 @@ struct mailbox; struct mailbox_sync_status; +struct imapc_sync_store { + enum modify_type modify_type; + const char *flags; + + ARRAY_TYPE(seq_range) uids; +}; + struct imapc_sync_context { struct imapc_mailbox *mbox; struct mail_index_sync_ctx *index_sync_ctx; @@ -14,6 +21,13 @@ ARRAY_TYPE(seq_range) expunged_uids; unsigned int sync_command_count; + pool_t pool; + HASH_TABLE(struct imapc_sync_store *, struct imapc_sync_store *) stores; + + uint32_t prev_uid1, prev_uid2; + enum modify_type prev_modify_type; + string_t *prev_flags; + unsigned int failed:1; };