Mercurial > dovecot > core-2.2
changeset 11170:c2c2c1dd252e HEAD
mdbox: Optimization: Don't update map UIDs' refcounts via array when it's not necessary.
author | Timo Sirainen <tss@iki.fi> |
---|---|
date | Mon, 19 Apr 2010 16:15:57 +0300 |
parents | 0eb423848dc1 |
children | 8e6cb40f577f |
files | src/lib-storage/index/dbox-multi/mdbox-map.c src/lib-storage/index/dbox-multi/mdbox-map.h src/lib-storage/index/dbox-multi/mdbox-storage.c src/lib-storage/index/dbox-multi/mdbox-sync.c src/lib-storage/index/dbox-multi/mdbox-sync.h |
diffstat | 5 files changed, 77 insertions(+), 85 deletions(-) [+] |
line wrap: on
line diff
--- a/src/lib-storage/index/dbox-multi/mdbox-map.c Mon Apr 19 16:14:37 2010 +0300 +++ b/src/lib-storage/index/dbox-multi/mdbox-map.c Mon Apr 19 16:15:57 2010 +0300 @@ -409,50 +409,61 @@ i_free(ctx); } -int dbox_map_update_refcounts(struct dbox_map_transaction_context *ctx, - const ARRAY_TYPE(uint32_t) *map_uids, int diff) +int dbox_map_update_refcount(struct dbox_map_transaction_context *ctx, + uint32_t map_uid, int diff) { struct dbox_map *map = ctx->map; - const uint32_t *uidp; - unsigned int i, count; const void *data; uint32_t seq; bool expunged; int cur_diff; - if (ctx->trans == NULL) + if (unlikely(ctx->trans == NULL)) + return -1; + + if (!mail_index_lookup_seq(map->view, map_uid, &seq)) { + /* we can't refresh map here since view has a + transaction open. */ + dbox_map_set_corrupted(map, "refcount update lost map_uid=%u", + map_uid); + return -1; + } + mail_index_lookup_ext(map->view, seq, map->ref_ext_id, + &data, &expunged); + cur_diff = data == NULL ? 0 : *((const uint16_t *)data); + ctx->changed = TRUE; + cur_diff += mail_index_atomic_inc_ext(ctx->trans, seq, + map->ref_ext_id, diff); + if (cur_diff < 0) { + dbox_map_set_corrupted(map, "map_uid=%u refcount too low", + map_uid); + return -1; + } + if (cur_diff >= 32768) { + /* we're getting close to the 64k limit. fail early + to make it less likely that two processes increase + the refcount enough times to cross the limit */ + mail_storage_set_error(MAP_STORAGE(map), MAIL_ERROR_NOTPOSSIBLE, + "Message has been copied too many times"); + return -1; + } + return 0; +} + +int dbox_map_update_refcounts(struct dbox_map_transaction_context *ctx, + const ARRAY_TYPE(uint32_t) *map_uids, int diff) +{ + const uint32_t *uidp; + unsigned int i, count; + + if (unlikely(ctx->trans == NULL)) return -1; count = array_count(map_uids); for (i = 0; i < count; i++) { uidp = array_idx(map_uids, i); - if (!mail_index_lookup_seq(map->view, *uidp, &seq)) { - /* we can't refresh map here since view has a - transaction open. */ - dbox_map_set_corrupted(map, - "refcount update lost map_uid=%u", *uidp); + if (dbox_map_update_refcount(ctx, *uidp, diff) < 0) return -1; - } - mail_index_lookup_ext(map->view, seq, map->ref_ext_id, - &data, &expunged); - cur_diff = data == NULL ? 0 : *((const uint16_t *)data); - ctx->changed = TRUE; - cur_diff += mail_index_atomic_inc_ext(ctx->trans, seq, - map->ref_ext_id, diff); - if (cur_diff < 0) { - dbox_map_set_corrupted(map, - "map_uid=%u refcount too low", *uidp); - return -1; - } - if (cur_diff >= 32768) { - /* we're getting close to the 64k limit. fail early - to make it less likely that two processes increase - the refcount enough times to cross the limit */ - mail_storage_set_error(MAP_STORAGE(map), - MAIL_ERROR_NOTPOSSIBLE, - "Message has been copied too many times"); - return -1; - } } return 0; }
--- a/src/lib-storage/index/dbox-multi/mdbox-map.h Mon Apr 19 16:14:37 2010 +0300 +++ b/src/lib-storage/index/dbox-multi/mdbox-map.h Mon Apr 19 16:15:57 2010 +0300 @@ -58,6 +58,8 @@ void dbox_map_transaction_set_failed(struct dbox_map_transaction_context *ctx); void dbox_map_transaction_free(struct dbox_map_transaction_context **ctx); +int dbox_map_update_refcount(struct dbox_map_transaction_context *ctx, + uint32_t map_uid, int diff); int dbox_map_update_refcounts(struct dbox_map_transaction_context *ctx, const ARRAY_TYPE(uint32_t) *map_uids, int diff); int dbox_map_remove_file_id(struct dbox_map *map, uint32_t file_id);
--- a/src/lib-storage/index/dbox-multi/mdbox-storage.c Mon Apr 19 16:14:37 2010 +0300 +++ b/src/lib-storage/index/dbox-multi/mdbox-storage.c Mon Apr 19 16:15:57 2010 +0300 @@ -282,35 +282,27 @@ { struct dbox_map_transaction_context *map_trans; const struct mail_index_header *hdr; - const struct mdbox_mail_index_record *dbox_rec; - ARRAY_TYPE(uint32_t) map_uids; - const void *data; - bool expunged; - uint32_t seq; - int ret; + uint32_t seq, map_uid; + int ret = 0; - /* get a list of all map_uids in this mailbox */ - i_array_init(&map_uids, 128); + map_trans = dbox_map_transaction_begin(mbox->storage->map, FALSE); hdr = mail_index_get_header(mbox->box.view); for (seq = 1; seq <= hdr->messages_count; seq++) { - mail_index_lookup_ext(mbox->box.view, seq, mbox->ext_id, - &data, &expunged); - dbox_rec = data; - if (dbox_rec == NULL) { - /* no multi-mails */ + if (mdbox_mail_lookup(mbox, mbox->box.view, seq, + &map_uid) < 0) { + ret = -1; break; } - if (dbox_rec->map_uid != 0) - array_append(&map_uids, &dbox_rec->map_uid, 1); + + if (dbox_map_update_refcount(map_trans, map_uid, -1) < 0) { + ret = -1; + break; + } } - /* unreference the map_uids */ - map_trans = dbox_map_transaction_begin(mbox->storage->map, FALSE); - ret = dbox_map_update_refcounts(map_trans, &map_uids, -1); if (ret == 0) ret = dbox_map_transaction_commit(map_trans); dbox_map_transaction_free(&map_trans); - array_free(&map_uids); return ret; }
--- a/src/lib-storage/index/dbox-multi/mdbox-sync.c Mon Apr 19 16:14:37 2010 +0300 +++ b/src/lib-storage/index/dbox-multi/mdbox-sync.c Mon Apr 19 16:15:57 2010 +0300 @@ -55,12 +55,13 @@ if (mdbox_mail_lookup(ctx->mbox, ctx->sync_view, seq, &map_uid) < 0) return -1; + if (dbox_map_update_refcount(ctx->map_trans, map_uid, -1) < 0) + return -1; seq_range_array_add(&ctx->expunged_seqs, 0, seq); - array_append(&ctx->expunged_map_uids, &map_uid, 1); return 0; } -static int mdbox_sync_add(struct mdbox_sync_context *ctx, +static int mdbox_sync_rec(struct mdbox_sync_context *ctx, const struct mail_index_sync_rec *sync_rec) { uint32_t seq, seq1, seq2; @@ -119,30 +120,6 @@ return 0; } -static int mdbox_sync_index_finish_expunges(struct mdbox_sync_context *ctx) -{ - struct dbox_map_transaction_context *map_trans; - int ret; - - map_trans = dbox_map_transaction_begin(ctx->mbox->storage->map, FALSE); - ret = dbox_map_update_refcounts(map_trans, &ctx->expunged_map_uids, -1); - if (ret == 0) { - /* write refcount changes to map index */ - ret = dbox_map_transaction_commit(map_trans); - if (ret == 0) { - /* write changes to mailbox index */ - ret = dbox_sync_mark_expunges(ctx); - } - } - - /* this finally finishes the map sync and makes it clear that the - map transaction was successfully finished. */ - if (ret < 0) - dbox_map_transaction_set_failed(map_trans); - dbox_map_transaction_free(&map_trans); - return ret; -} - static int mdbox_sync_index(struct mdbox_sync_context *ctx) { struct mailbox *box = &ctx->mbox->box; @@ -164,21 +141,33 @@ seq1, seq2); } - /* read all changes and group changes to same file_id together */ + ctx->map_trans = + dbox_map_transaction_begin(ctx->mbox->storage->map, FALSE); i_array_init(&ctx->expunged_seqs, 64); - i_array_init(&ctx->expunged_map_uids, 64); while (mail_index_sync_next(ctx->index_sync_ctx, &sync_rec)) { - if ((ret = mdbox_sync_add(ctx, &sync_rec)) < 0) + if ((ret = mdbox_sync_rec(ctx, &sync_rec)) < 0) break; } - if (ret == 0 && array_count(&ctx->expunged_seqs) > 0) - ret = mdbox_sync_index_finish_expunges(ctx); + + if (ret == 0) { + /* write refcount changes to map index */ + ret = dbox_map_transaction_commit(ctx->map_trans); + } + if (ret == 0) { + /* write changes to mailbox index */ + ret = dbox_sync_mark_expunges(ctx); + } + + /* this finally finishes the map sync and marks the map transaction + to be successfully finished. */ + if (ret < 0) + dbox_map_transaction_set_failed(ctx->map_trans); + dbox_map_transaction_free(&ctx->map_trans); if (box->v.sync_notify != NULL) box->v.sync_notify(box, 0, 0); array_free(&ctx->expunged_seqs); - array_free(&ctx->expunged_map_uids); return ret == 0 ? 1 : (ctx->mbox->storage->storage.files_corrupted ? 0 : -1); }
--- a/src/lib-storage/index/dbox-multi/mdbox-sync.h Mon Apr 19 16:14:37 2010 +0300 +++ b/src/lib-storage/index/dbox-multi/mdbox-sync.h Mon Apr 19 16:15:57 2010 +0300 @@ -16,12 +16,10 @@ struct mail_index_sync_ctx *index_sync_ctx; struct mail_index_view *sync_view; struct mail_index_transaction *trans; + struct dbox_map_transaction_context *map_trans; enum mdbox_sync_flags flags; ARRAY_TYPE(seq_range) expunged_seqs; - /* list of expunged map_uids. the same map_uid may be listed more than - once in case message has been copied multiple times to mailbox. */ - ARRAY_TYPE(uint32_t) expunged_map_uids; }; int mdbox_sync_begin(struct mdbox_mailbox *mbox, enum mdbox_sync_flags flags,