Mercurial > dovecot > core-2.2

comparison src/lib-storage/index/imapc/imapc-sync.c @ 22041:21fab4826117

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
imapc: Try to merge STOREs together as much as possible when syncing
author Timo Sirainen <timo.sirainen@dovecot.fi>
date Thu, 04 May 2017 19:59:41 +0300
parents 3acdf933fc65
children c24c32983eae
comparison
equal deleted inserted replaced
22040:99752d3e656b 22041:21fab4826117
1 /* Copyright (c) 2007-2017 Dovecot authors, see the included COPYING file */ 1 /* Copyright (c) 2007-2017 Dovecot authors, see the included COPYING file */
2 2
3 #include "lib.h" 3 #include "lib.h"
4 #include "ioloop.h" 4 #include "ioloop.h"
5 #include "hash.h"
5 #include "str.h" 6 #include "str.h"
7 #include "sort.h"
6 #include "imap-util.h" 8 #include "imap-util.h"
7 #include "mail-cache.h" 9 #include "mail-cache.h"
8 #include "mail-index-modseq.h" 10 #include "mail-index-modseq.h"
9 #include "index-sync-private.h" 11 #include "index-sync-private.h"
10 #include "imapc-msgmap.h" 12 #include "imapc-msgmap.h"
73 imapc_sync_cmd(struct imapc_sync_context *ctx, const char *cmd_str) 75 imapc_sync_cmd(struct imapc_sync_context *ctx, const char *cmd_str)
74 { 76 {
75 return imapc_sync_cmd_full(ctx, cmd_str, FALSE); 77 return imapc_sync_cmd_full(ctx, cmd_str, FALSE);
76 } 78 }
77 79
78 static struct imapc_command * 80 static unsigned int imapc_sync_store_hash(const struct imapc_sync_store *store)
79 imapc_sync_store_cmd(struct imapc_sync_context *ctx, const char *cmd_str) 81 {
80 { 82 return str_hash(store->flags) ^ store->modify_type;
81 return imapc_sync_cmd_full(ctx, cmd_str, TRUE); 83 }
84
85 static int imapc_sync_store_cmp(const struct imapc_sync_store *store1,
86 const struct imapc_sync_store *store2)
87 {
88 if (store1->modify_type != store2->modify_type)
89 return 1;
90 return strcmp(store1->flags, store2->flags);
91 }
92
93 static const char *imapc_sync_flags_sort(const char *flags)
94 {
95 if (strchr(flags, ' ') == NULL)
96 return flags;
97
98 const char **str = t_strsplit(flags, " ");
99 i_qsort(str, str_array_length(str), sizeof(const char *),
100 i_strcasecmp_p);
101 return t_strarray_join(str, " ");
102 }
103
104 static void
105 imapc_sync_store_flush(struct imapc_sync_context *ctx)
106 {
107 struct imapc_sync_store *store;
108 const char *sorted_flags;
109
110 if (ctx->prev_uid1 == 0)
111 return;
112
113 sorted_flags = imapc_sync_flags_sort(str_c(ctx->prev_flags));
114 struct imapc_sync_store store_lookup = {
115 .modify_type = ctx->prev_modify_type,
116 .flags = sorted_flags,
117 };
118 store = hash_table_lookup(ctx->stores, &store_lookup);
119 if (store == NULL) {
120 store = p_new(ctx->pool, struct imapc_sync_store, 1);
121 store->modify_type = ctx->prev_modify_type;
122 store->flags = p_strdup(ctx->pool, sorted_flags);
123 p_array_init(&store->uids, ctx->pool, 4);
124 hash_table_insert(ctx->stores, store, store);
125 }
126 seq_range_array_add_range(&store->uids, ctx->prev_uid1, ctx->prev_uid2);
127 }
128
129 static void
130 imapc_sync_store(struct imapc_sync_context *ctx,
131 enum modify_type modify_type, uint32_t uid1, uint32_t uid2,
132 const char *flags)
133 {
134 if (ctx->prev_flags == NULL) {
135 ctx->prev_flags = str_new(ctx->pool, 128);
136 hash_table_create(&ctx->stores, ctx->pool, 0,
137 imapc_sync_store_hash, imapc_sync_store_cmp);
138 }
139
140 if (ctx->prev_uid1 != uid1 || ctx->prev_uid2 != uid2 ||
141 ctx->prev_modify_type != modify_type) {
142 imapc_sync_store_flush(ctx);
143 ctx->prev_uid1 = uid1;
144 ctx->prev_uid2 = uid2;
145 ctx->prev_modify_type = modify_type;
146 str_truncate(ctx->prev_flags, 0);
147 }
148 if (str_len(ctx->prev_flags) > 0)
149 str_append_c(ctx->prev_flags, ' ');
150 str_append(ctx->prev_flags, flags);
151 }
152
153 static void
154 imapc_sync_finish_store(struct imapc_sync_context *ctx)
155 {
156 struct hash_iterate_context *iter;
157 struct imapc_sync_store *store;
158 string_t *cmd = t_str_new(128);
159
160 imapc_sync_store_flush(ctx);
161
162 if (!hash_table_is_created(ctx->stores))
163 return;
164
165 iter = hash_table_iterate_init(ctx->stores);
166 while (hash_table_iterate(iter, ctx->stores, &store, &store)) {
167 str_truncate(cmd, 0);
168 str_append(cmd, "UID STORE ");
169 imap_write_seq_range(cmd, &store->uids);
170 str_printfa(cmd, " %cFLAGS (%s)",
171 store->modify_type == MODIFY_ADD ? '+' : '-',
172 store->flags);
173 imapc_sync_cmd_full(ctx, str_c(cmd), TRUE);
174 }
175 hash_table_iterate_deinit(&iter);
176 hash_table_destroy(&ctx->stores);
82 } 177 }
83 178
84 static void 179 static void
85 imapc_sync_add_missing_deleted_flags(struct imapc_sync_context *ctx, 180 imapc_sync_add_missing_deleted_flags(struct imapc_sync_context *ctx,
86 uint32_t seq1, uint32_t seq2) 181 uint32_t seq1, uint32_t seq2)
87 { 182 {
88 const struct mail_index_record *rec; 183 const struct mail_index_record *rec;
89 uint32_t seq, uid1, uid2; 184 uint32_t seq, uid1, uid2;
90 const char *cmd;
91 185
92 /* if any of them has a missing \Deleted flag, 186 /* if any of them has a missing \Deleted flag,
93 just add it to all of them. */ 187 just add it to all of them. */
94 for (seq = seq1; seq <= seq2; seq++) { 188 for (seq = seq1; seq <= seq2; seq++) {
95 rec = mail_index_lookup(ctx->sync_view, seq); 189 rec = mail_index_lookup(ctx->sync_view, seq);
98 } 192 }
99 193
100 if (seq <= seq2) { 194 if (seq <= seq2) {
101 mail_index_lookup_uid(ctx->sync_view, seq1, &uid1); 195 mail_index_lookup_uid(ctx->sync_view, seq1, &uid1);
102 mail_index_lookup_uid(ctx->sync_view, seq2, &uid2); 196 mail_index_lookup_uid(ctx->sync_view, seq2, &uid2);
103 cmd = t_strdup_printf("UID STORE %u:%u +FLAGS \\Deleted", 197
104 uid1, uid2); 198 imapc_sync_store(ctx, MODIFY_ADD, uid1, uid2, "\\Deleted");
105 imapc_sync_store_cmd(ctx, cmd);
106 } 199 }
107 } 200 }
108 201
109 static void imapc_sync_index_flags(struct imapc_sync_context *ctx, 202 static void imapc_sync_index_flags(struct imapc_sync_context *ctx,
110 const struct mail_index_sync_rec *sync_rec) 203 const struct mail_index_sync_rec *sync_rec)
113 206
114 i_assert(sync_rec->type == MAIL_INDEX_SYNC_TYPE_FLAGS); 207 i_assert(sync_rec->type == MAIL_INDEX_SYNC_TYPE_FLAGS);
115 208
116 if (sync_rec->add_flags != 0) { 209 if (sync_rec->add_flags != 0) {
117 i_assert((sync_rec->add_flags & MAIL_RECENT) == 0); 210 i_assert((sync_rec->add_flags & MAIL_RECENT) == 0);
118 str_printfa(str, "UID STORE %u:%u +FLAGS (", 211
119 sync_rec->uid1, sync_rec->uid2);
120 imap_write_flags(str, sync_rec->add_flags, NULL); 212 imap_write_flags(str, sync_rec->add_flags, NULL);
121 str_append_c(str, ')'); 213 imapc_sync_store(ctx, MODIFY_ADD, sync_rec->uid1,
122 imapc_sync_store_cmd(ctx, str_c(str)); 214 sync_rec->uid2, str_c(str));
123 } 215 }
124 216
125 if (sync_rec->remove_flags != 0) { 217 if (sync_rec->remove_flags != 0) {
126 i_assert((sync_rec->remove_flags & MAIL_RECENT) == 0); 218 i_assert((sync_rec->remove_flags & MAIL_RECENT) == 0);
127 str_truncate(str, 0); 219 str_truncate(str, 0);
128 str_printfa(str, "UID STORE %u:%u -FLAGS (",
129 sync_rec->uid1, sync_rec->uid2);
130 imap_write_flags(str, sync_rec->remove_flags, NULL); 220 imap_write_flags(str, sync_rec->remove_flags, NULL);
131 str_append_c(str, ')'); 221 imapc_sync_store(ctx, MODIFY_REMOVE, sync_rec->uid1,
132 imapc_sync_store_cmd(ctx, str_c(str)); 222 sync_rec->uid2, str_c(str));
133 } 223 }
134 } 224 }
135 225
136 static void 226 static void
137 imapc_sync_index_keyword(struct imapc_sync_context *ctx, 227 imapc_sync_index_keyword(struct imapc_sync_context *ctx,
138 const struct mail_index_sync_rec *sync_rec) 228 const struct mail_index_sync_rec *sync_rec)
139 { 229 {
140 string_t *str = t_str_new(128);
141 const char *const *kw_p; 230 const char *const *kw_p;
142 char change_char; 231 enum modify_type modify_type;
143 232
144 switch (sync_rec->type) { 233 switch (sync_rec->type) {
145 case MAIL_INDEX_SYNC_TYPE_KEYWORD_ADD: 234 case MAIL_INDEX_SYNC_TYPE_KEYWORD_ADD:
146 change_char = '+'; 235 modify_type = MODIFY_ADD;
147 break; 236 break;
148 case MAIL_INDEX_SYNC_TYPE_KEYWORD_REMOVE: 237 case MAIL_INDEX_SYNC_TYPE_KEYWORD_REMOVE:
149 change_char = '-'; 238 modify_type = MODIFY_REMOVE;
150 break; 239 break;
151 default: 240 default:
152 i_unreached(); 241 i_unreached();
153 } 242 }
154 243
155 str_printfa(str, "UID STORE %u:%u %cFLAGS (",
156 sync_rec->uid1, sync_rec->uid2, change_char);
157
158 kw_p = array_idx(ctx->keywords, sync_rec->keyword_idx); 244 kw_p = array_idx(ctx->keywords, sync_rec->keyword_idx);
159 str_append(str, *kw_p); 245 imapc_sync_store(ctx, modify_type, sync_rec->uid1,
160 str_append_c(str, ')'); 246 sync_rec->uid2, *kw_p);
161 imapc_sync_store_cmd(ctx, str_c(str));
162 } 247 }
163 248
164 static void imapc_sync_expunge_finish(struct imapc_sync_context *ctx) 249 static void imapc_sync_expunge_finish(struct imapc_sync_context *ctx)
165 { 250 {
166 string_t *str; 251 string_t *str;
357 struct mail_index_sync_rec sync_rec; 442 struct mail_index_sync_rec sync_rec;
358 uint32_t seq1, seq2; 443 uint32_t seq1, seq2;
359 444
360 i_array_init(&ctx->expunged_uids, 64); 445 i_array_init(&ctx->expunged_uids, 64);
361 ctx->keywords = mail_index_get_keywords(mbox->box.index); 446 ctx->keywords = mail_index_get_keywords(mbox->box.index);
447 ctx->pool = pool_alloconly_create("imapc sync pool", 1024);
362 448
363 imapc_sync_uid_validity(ctx); 449 imapc_sync_uid_validity(ctx);
364 while (mail_index_sync_next(ctx->index_sync_ctx, &sync_rec)) T_BEGIN { 450 while (mail_index_sync_next(ctx->index_sync_ctx, &sync_rec)) T_BEGIN {
365 if (!mail_index_lookup_seq_range(ctx->sync_view, 451 if (!mail_index_lookup_seq_range(ctx->sync_view,
366 sync_rec.uid1, sync_rec.uid2, 452 sync_rec.uid1, sync_rec.uid2,
379 case MAIL_INDEX_SYNC_TYPE_KEYWORD_REMOVE: 465 case MAIL_INDEX_SYNC_TYPE_KEYWORD_REMOVE:
380 imapc_sync_index_keyword(ctx, &sync_rec); 466 imapc_sync_index_keyword(ctx, &sync_rec);
381 break; 467 break;
382 } 468 }
383 } T_END; 469 } T_END;
470 imapc_sync_finish_store(ctx);
471 pool_unref(&ctx->pool);
384 472
385 if (!mbox->initial_sync_done) { 473 if (!mbox->initial_sync_done) {
386 /* with initial syncing we're fetching all messages' flags and 474 /* with initial syncing we're fetching all messages' flags and
387 expunge mails from local index that no longer exist on 475 expunge mails from local index that no longer exist on
388 remote server */ 476 remote server */