Mercurial > dovecot > core-2.2
changeset 17845:d2e1b3f6d13b
dsync: Added DSYNC_BRAIN_FLAG_NO_MAIL_PREFETCH to avoid opening mails unnecessarily.
author | Timo Sirainen <tss@iki.fi> |
---|---|
date | Mon, 29 Sep 2014 14:16:06 +0300 |
parents | 35c3194900b7 |
children | 475ff24ac365 |
files | src/doveadm/dsync/dsync-brain-mailbox.c src/doveadm/dsync/dsync-brain-private.h src/doveadm/dsync/dsync-brain.c src/doveadm/dsync/dsync-brain.h src/doveadm/dsync/dsync-ibc-stream.c src/doveadm/dsync/dsync-mail.c src/doveadm/dsync/dsync-mail.h src/doveadm/dsync/dsync-mailbox-export.c src/doveadm/dsync/dsync-mailbox-export.h src/doveadm/dsync/dsync-mailbox-import.c |
diffstat | 10 files changed, 103 insertions(+), 40 deletions(-) [+] |
line wrap: on
line diff
--- a/src/doveadm/dsync/dsync-brain-mailbox.c Fri Sep 26 00:32:03 2014 +0300 +++ b/src/doveadm/dsync/dsync-brain-mailbox.c Mon Sep 29 14:16:06 2014 +0300 @@ -304,6 +304,8 @@ (brain->local_dsync_box.have_save_guids || (brain->backup_send && brain->local_dsync_box.have_guids))) exporter_flags |= DSYNC_MAILBOX_EXPORTER_FLAG_MAILS_HAVE_GUIDS; + if (brain->no_mail_prefetch) + exporter_flags |= DSYNC_MAILBOX_EXPORTER_FLAG_MINIMAL_DMAIL_FILL; brain->box_exporter = brain->backup_recv ? NULL : dsync_mailbox_export_init(brain->box, brain->log_scan,
--- a/src/doveadm/dsync/dsync-brain-private.h Fri Sep 26 00:32:03 2014 +0300 +++ b/src/doveadm/dsync/dsync-brain-private.h Mon Sep 29 14:16:06 2014 +0300 @@ -100,6 +100,7 @@ unsigned int sync_visible_namespaces:1; unsigned int no_mail_sync:1; unsigned int no_backup_overwrite:1; + unsigned int no_mail_prefetch:1; unsigned int changes_during_sync:1; unsigned int require_full_resync:1; unsigned int verbose_proctitle:1;
--- a/src/doveadm/dsync/dsync-brain.c Fri Sep 26 00:32:03 2014 +0300 +++ b/src/doveadm/dsync/dsync-brain.c Mon Sep 29 14:16:06 2014 +0300 @@ -137,6 +137,8 @@ brain->no_mail_sync = (flags & DSYNC_BRAIN_FLAG_NO_MAIL_SYNC) != 0; brain->no_backup_overwrite = (flags & DSYNC_BRAIN_FLAG_NO_BACKUP_OVERWRITE) != 0; + brain->no_mail_prefetch = + (flags & DSYNC_BRAIN_FLAG_NO_MAIL_PREFETCH) != 0; } struct dsync_brain *
--- a/src/doveadm/dsync/dsync-brain.h Fri Sep 26 00:32:03 2014 +0300 +++ b/src/doveadm/dsync/dsync-brain.h Mon Sep 29 14:16:06 2014 +0300 @@ -22,7 +22,11 @@ DSYNC_BRAIN_FLAG_NO_BACKUP_OVERWRITE = 0x40, /* Run storage purge on the remote after syncing. Useful with e.g. a nightly doveadm backup. */ - DSYNC_BRAIN_FLAG_PURGE_REMOTE = 0x80 + DSYNC_BRAIN_FLAG_PURGE_REMOTE = 0x80, + /* Don't prefetch mail bodies until they're actually needed. This works + only with pipe ibc. It's useful if most of the mails can be copied + directly within filesystem without having to read them. */ + DSYNC_BRAIN_FLAG_NO_MAIL_PREFETCH = 0x100 }; enum dsync_brain_sync_type {
--- a/src/doveadm/dsync/dsync-ibc-stream.c Fri Sep 26 00:32:03 2014 +0300 +++ b/src/doveadm/dsync/dsync-ibc-stream.c Mon Sep 29 14:16:06 2014 +0300 @@ -1681,6 +1681,7 @@ struct dsync_serializer_encoder *encoder; string_t *str = t_str_new(128); + i_assert(!mail->minimal_fields); i_assert(ibc->value_output == NULL); str_append_c(str, items[ITEM_MAIL].chr);
--- a/src/doveadm/dsync/dsync-mail.c Fri Sep 26 00:32:03 2014 +0300 +++ b/src/doveadm/dsync/dsync-mail.c Mon Sep 29 14:16:06 2014 +0300 @@ -60,10 +60,10 @@ return ret; } -int dsync_mail_fill(struct mail *mail, struct dsync_mail *dmail_r, - const char **error_field_r) +int dsync_mail_fill(struct mail *mail, bool minimal_fill, + struct dsync_mail *dmail_r, const char **error_field_r) { - const char *guid, *str; + const char *guid; memset(dmail_r, 0, sizeof(*dmail_r)); @@ -76,6 +76,22 @@ dmail_r->input_mail = mail; dmail_r->input_mail_uid = mail->uid; + + if (mail_get_save_date(mail, &dmail_r->saved_date) < 0) { + *error_field_r = "saved-date"; + return -1; + } + if (!minimal_fill) + return dsync_mail_fill_nonminimal(mail, dmail_r, error_field_r); + dmail_r->minimal_fields = TRUE; + return 0; +} + +int dsync_mail_fill_nonminimal(struct mail *mail, struct dsync_mail *dmail_r, + const char **error_field_r) +{ + const char *str; + if (mail_get_stream(mail, NULL, NULL, &dmail_r->input) < 0) { *error_field_r = "body"; return -1; @@ -97,10 +113,6 @@ *error_field_r = "received-date"; return -1; } - if (mail_get_save_date(mail, &dmail_r->saved_date) < 0) { - *error_field_r = "saved-date"; - return -1; - } return 0; }
--- a/src/doveadm/dsync/dsync-mail.h Fri Sep 26 00:32:03 2014 +0300 +++ b/src/doveadm/dsync/dsync-mail.h Mon Sep 29 14:16:06 2014 +0300 @@ -10,21 +10,24 @@ /* either GUID="" or uid=0 */ const char *guid; uint32_t uid; - - const char *pop3_uidl; - unsigned int pop3_order; - time_t received_date; time_t saved_date; - /* Input stream containing the message text, or NULL if all instances - of the message were already expunged from this mailbox. */ - struct istream *input; - /* If non-NULL, we're syncing within the dsync process using ibc-pipe. This mail can be used to mailbox_copy() the mail. */ struct mail *input_mail; /* Verify that this equals to input_mail->uid */ uint32_t input_mail_uid; + + /* TRUE if the following fields aren't set, because minimal_fill=TRUE + parameter was used. */ + bool minimal_fields; + + const char *pop3_uidl; + unsigned int pop3_order; + time_t received_date; + /* Input stream containing the message text, or NULL if all instances + of the message were already expunged from this mailbox. */ + struct istream *input; }; struct dsync_mail_request { @@ -80,8 +83,10 @@ dsync_mail_get_hash_headers(struct mailbox *box); int dsync_mail_get_hdr_hash(struct mail *mail, const char **hdr_hash_r); -int dsync_mail_fill(struct mail *mail, struct dsync_mail *dmail_r, - const char **error_field_r); +int dsync_mail_fill(struct mail *mail, bool minimal_fill, + struct dsync_mail *dmail_r, const char **error_field_r); +int dsync_mail_fill_nonminimal(struct mail *mail, struct dsync_mail *dmail_r, + const char **error_field_r); void dsync_mail_change_dup(pool_t pool, const struct dsync_mail_change *src, struct dsync_mail_change *dest_r);
--- a/src/doveadm/dsync/dsync-mailbox-export.c Fri Sep 26 00:32:03 2014 +0300 +++ b/src/doveadm/dsync/dsync-mailbox-export.c Mon Sep 29 14:16:06 2014 +0300 @@ -57,6 +57,7 @@ unsigned int body_search_initialized:1; unsigned int auto_export_mails:1; unsigned int mails_have_guids:1; + unsigned int minimal_dmail_fill:1; unsigned int return_all_mails:1; }; @@ -474,6 +475,8 @@ (flags & DSYNC_MAILBOX_EXPORTER_FLAG_AUTO_EXPORT_MAILS) != 0; exporter->mails_have_guids = (flags & DSYNC_MAILBOX_EXPORTER_FLAG_MAILS_HAVE_GUIDS) != 0; + exporter->minimal_dmail_fill = + (flags & DSYNC_MAILBOX_EXPORTER_FLAG_MINIMAL_DMAIL_FILL) != 0; p_array_init(&exporter->requested_uids, pool, 16); p_array_init(&exporter->search_uids, pool, 16); hash_table_create(&exporter->export_guids, pool, 0, str_hash, strcmp); @@ -657,6 +660,7 @@ const struct seq_range *uids; char *guid; const char *const_guid; + enum mail_fetch_field wanted_fields; struct dsync_mail_guid_instances *instances; const struct seq_range *range; unsigned int i, count; @@ -714,16 +718,16 @@ array_append_array(&exporter->search_uids, &exporter->requested_uids); array_clear(&exporter->requested_uids); + wanted_fields = MAIL_FETCH_GUID | MAIL_FETCH_SAVE_DATE; + if (!exporter->minimal_dmail_fill) { + wanted_fields |= MAIL_FETCH_RECEIVED_DATE | + MAIL_FETCH_UIDL_BACKEND | MAIL_FETCH_POP3_ORDER | + MAIL_FETCH_STREAM_HEADER | MAIL_FETCH_STREAM_BODY; + } exporter->search_count += seq_range_count(&sarg->value.seqset); exporter->search_ctx = mailbox_search_init(exporter->trans, search_args, NULL, - MAIL_FETCH_GUID | - MAIL_FETCH_UIDL_BACKEND | - MAIL_FETCH_POP3_ORDER | - MAIL_FETCH_RECEIVED_DATE | - MAIL_FETCH_SAVE_DATE | - MAIL_FETCH_STREAM_HEADER | - MAIL_FETCH_STREAM_BODY, NULL); + wanted_fields, NULL); mail_search_args_unref(&search_args); return array_count(&sarg->value.seqset) > 0 ? 1 : 0; } @@ -748,7 +752,8 @@ struct dsync_mail_guid_instances *instances; const char *error_field; - if (dsync_mail_fill(mail, &exporter->dsync_mail, &error_field) < 0) + if (dsync_mail_fill(mail, exporter->minimal_dmail_fill, + &exporter->dsync_mail, &error_field) < 0) return dsync_mail_error(exporter, mail, error_field); instances = *exporter->dsync_mail.guid == '\0' ? NULL :
--- a/src/doveadm/dsync/dsync-mailbox-export.h Fri Sep 26 00:32:03 2014 +0300 +++ b/src/doveadm/dsync/dsync-mailbox-export.h Mon Sep 29 14:16:06 2014 +0300 @@ -3,7 +3,8 @@ enum dsync_mailbox_exporter_flags { DSYNC_MAILBOX_EXPORTER_FLAG_AUTO_EXPORT_MAILS = 0x01, - DSYNC_MAILBOX_EXPORTER_FLAG_MAILS_HAVE_GUIDS = 0x02 + DSYNC_MAILBOX_EXPORTER_FLAG_MAILS_HAVE_GUIDS = 0x02, + DSYNC_MAILBOX_EXPORTER_FLAG_MINIMAL_DMAIL_FILL = 0x04 }; struct dsync_mailbox_exporter *
--- a/src/doveadm/dsync/dsync-mailbox-import.c Fri Sep 26 00:32:03 2014 +0300 +++ b/src/doveadm/dsync/dsync-mailbox-import.c Mon Sep 29 14:16:06 2014 +0300 @@ -1544,7 +1544,7 @@ if (!mail_set_uid(importer->mail, uid)) return 0; - if (dsync_mail_fill(importer->mail, dmail_r, &error_field) < 0) { + if (dsync_mail_fill(importer->mail, TRUE, dmail_r, &error_field) < 0) { errstr = mailbox_get_last_error(importer->mail->box, &error); if (error == MAIL_ERROR_EXPUNGED) return 0; @@ -1905,6 +1905,17 @@ return 0; } +static void +dsync_mailbox_save_set_nonminimal(struct mail_save_context *save_ctx, + const struct dsync_mail *mail) +{ + if (mail->pop3_uidl != NULL && *mail->pop3_uidl != '\0') + mailbox_save_set_pop3_uidl(save_ctx, mail->pop3_uidl); + if (mail->pop3_order > 0) + mailbox_save_set_pop3_order(save_ctx, mail->pop3_order); + mailbox_save_set_received_date(save_ctx, mail->received_date, 0); +} + static struct mail_save_context * dsync_mailbox_save_init(struct dsync_mailbox_importer *importer, const struct dsync_mail *mail, @@ -1919,11 +1930,9 @@ if (mail->saved_date != 0) mailbox_save_set_save_date(save_ctx, mail->saved_date); dsync_mailbox_save_set_metadata(importer, save_ctx, newmail->change); - if (mail->pop3_uidl != NULL && *mail->pop3_uidl != '\0') - mailbox_save_set_pop3_uidl(save_ctx, mail->pop3_uidl); - if (mail->pop3_order > 0) - mailbox_save_set_pop3_order(save_ctx, mail->pop3_order); - mailbox_save_set_received_date(save_ctx, mail->received_date, 0); + + if (!mail->minimal_fields) + dsync_mailbox_save_set_nonminimal(save_ctx, mail); return save_ctx; } @@ -1934,6 +1943,7 @@ struct importer_new_mail **all_newmails_forcopy) { struct mail_save_context *save_ctx; + struct istream *input; ssize_t ret; bool save_failed = FALSE; @@ -1960,22 +1970,42 @@ return; } /* fallback to saving from remote stream */ + if (mail->minimal_fields) { + struct dsync_mail mail2; + const char *error_field; - if (mail->input == NULL) { + i_assert(mail->input_mail != NULL); + + if (dsync_mail_fill_nonminimal(mail->input_mail, &mail2, + &error_field) < 0) { + i_error("Mailbox %s: Failed to read mail %s uid=%u: %s", + mailbox_get_vname(importer->box), + error_field, mail->uid, + mailbox_get_last_error(importer->box, NULL)); + importer->failed = TRUE; + return; + } + dsync_mailbox_save_set_nonminimal(save_ctx, &mail2); + input = mail2.input; + } else { + input = mail->input; + } + + if (input == NULL) { /* it was just expunged in remote, skip it */ mailbox_save_cancel(&save_ctx); return; } - i_stream_seek(mail->input, 0); - if (mailbox_save_begin(&save_ctx, mail->input) < 0) { + i_stream_seek(input, 0); + if (mailbox_save_begin(&save_ctx, input) < 0) { i_error("Mailbox %s: Saving failed: %s", mailbox_get_vname(importer->box), mailbox_get_last_error(importer->box, NULL)); importer->failed = TRUE; return; } - while ((ret = i_stream_read(mail->input)) > 0 || ret == -2) { + while ((ret = i_stream_read(input)) > 0 || ret == -2) { if (mailbox_save_continue(save_ctx) < 0) { save_failed = TRUE; ret = -1; @@ -1984,10 +2014,10 @@ } i_assert(ret == -1); - if (mail->input->stream_errno != 0) { + if (input->stream_errno != 0) { i_error("Mailbox %s: read(msg input) failed: %s", mailbox_get_vname(importer->box), - i_stream_get_error(mail->input)); + i_stream_get_error(input)); mailbox_save_cancel(&save_ctx); importer->failed = TRUE; } else if (save_failed) { @@ -1997,7 +2027,7 @@ mailbox_save_cancel(&save_ctx); importer->failed = TRUE; } else { - i_assert(mail->input->eof); + i_assert(input->eof); if (mailbox_save_finish(&save_ctx) < 0) { i_error("Mailbox %s: Saving failed: %s", mailbox_get_vname(importer->box),