Mercurial > dovecot > core-2.2
changeset 20393:53ba3feb039a
dbox: Optimize POP3 MAIL_FETCH_UIDL_BACKEND.
We keep track of the highest UID known to have POP3 UIDL in index's header.
If saving adds a newer message, it'll also update the header. When fetching
UIDL_BACKEND, we can need to check only mails with lower UIDs. There are
some race conditions here, but normally UIDLs are set only once during
migration so it shouldn't matter.
| author | Timo Sirainen <timo.sirainen@dovecot.fi> |
|---|---|
| date | Mon, 20 Jun 2016 20:06:38 +0300 |
| parents | 596b61c6a86c |
| children | 0993d4b55bc8 |
| files | src/lib-storage/index/Makefile.am src/lib-storage/index/dbox-common/dbox-mail.c src/lib-storage/index/dbox-common/dbox-save.c src/lib-storage/index/dbox-common/dbox-save.h src/lib-storage/index/dbox-multi/mdbox-save.c src/lib-storage/index/dbox-single/sdbox-save.c src/lib-storage/index/index-pop3-uidl.c src/lib-storage/index/index-pop3-uidl.h src/lib-storage/index/index-storage.c src/lib-storage/index/index-transaction.c src/lib-storage/mail-storage-private.h |
| diffstat | 11 files changed, 173 insertions(+), 2 deletions(-) [+] |
line wrap: on
line diff
--- a/src/lib-storage/index/Makefile.am Mon Jun 20 11:39:55 2016 +0300 +++ b/src/lib-storage/index/Makefile.am Mon Jun 20 20:06:38 2016 +0300 @@ -20,6 +20,7 @@ index-mail-binary.c \ index-mail-headers.c \ index-mailbox-size.c \ + index-pop3-uidl.c \ index-rebuild.c \ index-search.c \ index-search-result.c \ @@ -41,6 +42,7 @@ index-attachment.h \ index-mail.h \ index-mailbox-size.h \ + index-pop3-uidl.h \ index-rebuild.h \ index-search-private.h \ index-search-result.h \
--- a/src/lib-storage/index/dbox-common/dbox-mail.c Mon Jun 20 11:39:55 2016 +0300 +++ b/src/lib-storage/index/dbox-common/dbox-mail.c Mon Jun 20 20:06:38 2016 +0300 @@ -5,6 +5,7 @@ #include "str.h" #include "index-storage.h" #include "index-mail.h" +#include "index-pop3-uidl.h" #include "dbox-attachment.h" #include "dbox-storage.h" #include "dbox-file.h" @@ -223,15 +224,31 @@ const char **value_r) { struct dbox_mail *mail = (struct dbox_mail *)_mail; + int ret; /* keep the UIDL in cache file, otherwise POP3 would open all mail files and read the metadata. same for GUIDs if they're used. */ switch (field) { case MAIL_FETCH_UIDL_BACKEND: - return dbox_get_cached_metadata(mail, DBOX_METADATA_POP3_UIDL, - MAIL_CACHE_POP3_UIDL, value_r); + if (!index_pop3_uidl_can_exist(_mail)) { + *value_r = ""; + return 0; + } + ret = dbox_get_cached_metadata(mail, DBOX_METADATA_POP3_UIDL, + MAIL_CACHE_POP3_UIDL, value_r); + if (ret == 0) { + index_pop3_uidl_update_exists(&mail->imail.mail.mail, + (*value_r)[0] != '\0'); + } + return ret; case MAIL_FETCH_POP3_ORDER: + if (!index_pop3_uidl_can_exist(_mail)) { + /* we're assuming that if there's a POP3 order, there's + also a UIDL */ + *value_r = ""; + return 0; + } return dbox_get_cached_metadata(mail, DBOX_METADATA_POP3_ORDER, MAIL_CACHE_POP3_ORDER, value_r); case MAIL_FETCH_GUID:
--- a/src/lib-storage/index/dbox-common/dbox-save.c Mon Jun 20 11:39:55 2016 +0300 +++ b/src/lib-storage/index/dbox-common/dbox-save.c Mon Jun 20 20:06:38 2016 +0300 @@ -166,11 +166,15 @@ str_printfa(str, "%c%s\n", DBOX_METADATA_POP3_UIDL, mdata->pop3_uidl); ctx->have_pop3_uidls = TRUE; + ctx->highest_pop3_uidl_seq = + I_MAX(ctx->highest_pop3_uidl_seq, ctx->seq); } if (mdata->pop3_order != 0) { str_printfa(str, "%c%u\n", DBOX_METADATA_POP3_ORDER, mdata->pop3_order); ctx->have_pop3_orders = TRUE; + ctx->highest_pop3_uidl_seq = + I_MAX(ctx->highest_pop3_uidl_seq, ctx->seq); } guid = mdata->guid;
--- a/src/lib-storage/index/dbox-common/dbox-save.h Mon Jun 20 11:39:55 2016 +0300 +++ b/src/lib-storage/index/dbox-common/dbox-save.h Mon Jun 20 20:06:38 2016 +0300 @@ -14,6 +14,7 @@ struct ostream *dbox_output; + uint32_t highest_pop3_uidl_seq; unsigned int failed:1; unsigned int finished:1; unsigned int have_pop3_uidls:1;
--- a/src/lib-storage/index/dbox-multi/mdbox-save.c Mon Jun 20 11:39:55 2016 +0300 +++ b/src/lib-storage/index/dbox-multi/mdbox-save.c Mon Jun 20 20:06:38 2016 +0300 @@ -11,6 +11,7 @@ #include "ostream.h" #include "write-full.h" #include "index-mail.h" +#include "index-pop3-uidl.h" #include "mail-copy.h" #include "dbox-save.h" #include "mdbox-storage.h" @@ -326,6 +327,17 @@ mail_index_append_finish_uids(ctx->ctx.trans, hdr->next_uid, &_t->changes->saved_uids); + if (ctx->ctx.highest_pop3_uidl_seq != 0) { + struct seq_range_iter iter; + uint32_t uid; + + seq_range_array_iter_init(&iter, &_t->changes->saved_uids); + if (!seq_range_array_iter_nth(&iter, + ctx->ctx.highest_pop3_uidl_seq-1, &uid)) + i_unreached(); + index_pop3_uidl_set_max_uid(&ctx->mbox->box, ctx->ctx.trans, uid); + } + /* save map UIDs to mailbox index */ if (first_map_uid != 0) mdbox_save_set_map_uids(ctx, first_map_uid, last_map_uid);
--- a/src/lib-storage/index/dbox-single/sdbox-save.c Mon Jun 20 11:39:55 2016 +0300 +++ b/src/lib-storage/index/dbox-single/sdbox-save.c Mon Jun 20 20:06:38 2016 +0300 @@ -12,6 +12,7 @@ #include "write-full.h" #include "index-mail.h" #include "mail-copy.h" +#include "index-pop3-uidl.h" #include "dbox-attachment.h" #include "dbox-save.h" #include "sdbox-storage.h" @@ -248,6 +249,10 @@ i_assert(ret); if (sdbox_file_assign_uid(sfile, uid) < 0) return -1; + if (ctx->ctx.highest_pop3_uidl_seq == i+1) { + index_pop3_uidl_set_max_uid(&ctx->mbox->box, + ctx->ctx.trans, uid); + } } i_assert(!seq_range_array_iter_nth(&iter, n, &uid)); return 0;
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-storage/index/index-pop3-uidl.c Mon Jun 20 20:06:38 2016 +0300 @@ -0,0 +1,101 @@ +/* Copyright (c) 2016 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "index-storage.h" +#include "index-mail.h" +#include "index-pop3-uidl.h" + +void index_pop3_uidl_set_max_uid(struct mailbox *box, + struct mail_index_transaction *trans, + uint32_t uid) +{ + struct mailbox_index_pop3_uidl uidl; + + memset(&uidl, 0, sizeof(uidl)); + uidl.max_uid_with_pop3_uidl = uid; + + mail_index_update_header_ext(trans, box->pop3_uidl_hdr_ext_id, + 0, &uidl, sizeof(uidl)); +} + +bool index_pop3_uidl_can_exist(struct mail *mail) +{ + struct mailbox_index_pop3_uidl uidl; + const void *data; + size_t size; + + /* We'll assume that if the header exists, it's up-to-date. normally + UIDLs are set only during migration, so this value never changes. + Also even if it does, it becomes out-of-date only when the mailbox + is modified with old Dovecot versions. To fix that we'd have to + add and keep updating "max tracked uid" in this header for every + saved mail, which isn't worth it. */ + mail_index_get_header_ext(mail->transaction->view, + mail->box->pop3_uidl_hdr_ext_id, + &data, &size); + if (size < sizeof(uidl)) { + /* this header isn't set yet */ + return TRUE; + } + memcpy(&uidl, data, size); + return mail->uid <= uidl.max_uid_with_pop3_uidl; +} + +void index_pop3_uidl_update_exists(struct mail *mail, bool exists) +{ + struct mailbox_transaction_context *trans = mail->transaction; + + if (exists) { + if (trans->highest_pop3_uidl_uid < mail->uid) { + trans->highest_pop3_uidl_uid = mail->uid; + trans->prev_pop3_uidl_tracking_seq = mail->seq; + } + } else if (mail->seq == trans->prev_pop3_uidl_tracking_seq+1) { + trans->prev_pop3_uidl_tracking_seq++; + } else { + /* skipping mails. we don't know the state. */ + } +} + +void index_pop3_uidl_update_exists_finish(struct mailbox_transaction_context *trans) +{ + struct mail_index_view *view; + struct mailbox_index_pop3_uidl uidl; + const void *data; + size_t size; + bool seen_all_msgs; + + if (trans->highest_pop3_uidl_uid == 0) + return; + + /* First check that we actually looked at UIDL for all messages. + Otherwise we can't say for sure if the newest messages had UIDLs. */ + if (trans->prev_pop3_uidl_tracking_seq != + mail_index_view_get_messages_count(trans->view)) + return; + + /* Just to be sure: Refresh the index and check again. POP3 keeps + transactions open for duration of the entire session. Maybe another + process already added new mails (and already updated this header). + This check is racy, but normally UIDLs aren't added after migration + so it's a bit questionable if it's even worth having this check in + there. */ + view = mail_index_view_open(trans->box->index); + seen_all_msgs = mail_index_refresh(trans->box->index) == 0 && + trans->prev_pop3_uidl_tracking_seq == + mail_index_view_get_messages_count(view); + mail_index_view_close(&view); + if (!seen_all_msgs) + return; + + /* check if we have already the same header */ + mail_index_get_header_ext(trans->view, trans->box->pop3_uidl_hdr_ext_id, + &data, &size); + if (size >= sizeof(uidl)) { + memcpy(&uidl, data, size); + if (trans->highest_pop3_uidl_uid == uidl.max_uid_with_pop3_uidl) + return; + } + index_pop3_uidl_set_max_uid(trans->box, trans->itrans, + trans->highest_pop3_uidl_uid); +}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-storage/index/index-pop3-uidl.h Mon Jun 20 20:06:38 2016 +0300 @@ -0,0 +1,16 @@ +#ifndef INDEX_POP3_H +#define INDEX_POP3_H + +struct mail_index_transaction; +struct mail; +struct mailbox; +struct mailbox_transaction_context; + +void index_pop3_uidl_set_max_uid(struct mailbox *box, + struct mail_index_transaction *trans, + uint32_t uid); +bool index_pop3_uidl_can_exist(struct mail *mail); +void index_pop3_uidl_update_exists(struct mail *mail, bool exists); +void index_pop3_uidl_update_exists_finish(struct mailbox_transaction_context *trans); + +#endif
--- a/src/lib-storage/index/index-storage.c Mon Jun 20 11:39:55 2016 +0300 +++ b/src/lib-storage/index/index-storage.c Mon Jun 20 20:06:38 2016 +0300 @@ -293,6 +293,9 @@ mail_index_ext_register(box->index, "hdr-vsize", sizeof(struct mailbox_index_vsize), 0, sizeof(uint64_t)); + box->pop3_uidl_hdr_ext_id = + mail_index_ext_register(box->index, "hdr-pop3-uidl", + sizeof(struct mailbox_index_pop3_uidl), 0, 0); box->opened = TRUE;
--- a/src/lib-storage/index/index-transaction.c Mon Jun 20 11:39:55 2016 +0300 +++ b/src/lib-storage/index/index-transaction.c Mon Jun 20 20:06:38 2016 +0300 @@ -5,6 +5,7 @@ #include "dict.h" #include "index-storage.h" #include "index-sync-private.h" +#include "index-pop3-uidl.h" #include "index-mail.h" static void index_transaction_free(struct mailbox_transaction_context *t) @@ -28,6 +29,7 @@ struct index_mailbox_sync_pvt_context *pvt_sync_ctx = NULL; int ret = 0; + index_pop3_uidl_update_exists_finish(t); if (t->nontransactional_changes) t->changes->changed = TRUE;
--- a/src/lib-storage/mail-storage-private.h Mon Jun 20 11:39:55 2016 +0300 +++ b/src/lib-storage/mail-storage-private.h Mon Jun 20 20:06:38 2016 +0300 @@ -292,6 +292,10 @@ uint32_t message_count; }; +struct mailbox_index_pop3_uidl { + uint32_t max_uid_with_pop3_uidl; +}; + struct mailbox_index_first_saved { uint32_t uid; uint32_t timestamp; @@ -346,6 +350,7 @@ enum mailbox_feature enabled_features; struct mail_msgpart_partial_cache partial_cache; uint32_t vsize_hdr_ext_id; + uint32_t pop3_uidl_hdr_ext_id; /* MAIL_RECENT flags handling */ ARRAY_TYPE(seq_range) recent_flags; @@ -551,6 +556,9 @@ struct mail_transaction_commit_changes *changes; ARRAY(union mailbox_transaction_module_context *) module_contexts; + uint32_t prev_pop3_uidl_tracking_seq; + uint32_t highest_pop3_uidl_uid; + struct mail_save_context *save_ctx; /* number of mails saved/copied within this transaction. */ unsigned int save_count;