Mercurial > dovecot > core-2.2
changeset 12609:391ba80cb125
imapc: Send UID FETCH commands with larger uidset parameter if possible.
This code assumes that server doesn't reorganize FETCH replies when it's
given an increasing UID range, which in theory IMAP server would be allowed
to do, but I doubt there are any.
| author | Timo Sirainen <tss@iki.fi> |
|---|---|
| date | Sun, 30 Jan 2011 20:47:43 +0200 |
| parents | b96efbad2fa4 |
| children | 4964b39c6ae4 |
| files | src/lib-storage/index/imapc/Makefile.am src/lib-storage/index/imapc/imapc-client-private.h src/lib-storage/index/imapc/imapc-client.c src/lib-storage/index/imapc/imapc-client.h src/lib-storage/index/imapc/imapc-connection.c src/lib-storage/index/imapc/imapc-connection.h src/lib-storage/index/imapc/imapc-mail.c src/lib-storage/index/imapc/imapc-mail.h src/lib-storage/index/imapc/imapc-mailbox.c src/lib-storage/index/imapc/imapc-search.c src/lib-storage/index/imapc/imapc-storage.c src/lib-storage/index/imapc/imapc-storage.h src/lib-storage/index/index-mail.c src/lib-storage/index/index-mail.h |
| diffstat | 14 files changed, 333 insertions(+), 70 deletions(-) [+] |
line wrap: on
line diff
--- a/src/lib-storage/index/imapc/Makefile.am Sun Jan 30 20:16:04 2011 +0200 +++ b/src/lib-storage/index/imapc/Makefile.am Sun Jan 30 20:47:43 2011 +0200 @@ -25,6 +25,7 @@ headers = \ imapc-connection.h \ imapc-list.h \ + imapc-mail.h \ imapc-seqmap.h \ imapc-storage.h \ imapc-sync.h
--- a/src/lib-storage/index/imapc/imapc-client-private.h Sun Jan 30 20:16:04 2011 +0200 +++ b/src/lib-storage/index/imapc/imapc-client-private.h Sun Jan 30 20:47:43 2011 +0200 @@ -18,6 +18,8 @@ ARRAY_DEFINE(conns, struct imapc_client_connection *); struct ioloop *ioloop; + + unsigned int stop_now:1; }; struct imapc_client_mailbox {
--- a/src/lib-storage/index/imapc/imapc-client.c Sun Jan 30 20:16:04 2011 +0200 +++ b/src/lib-storage/index/imapc/imapc-client.c Sun Jan 30 20:47:43 2011 +0200 @@ -70,15 +70,24 @@ { struct imapc_client_connection *const *connp; struct ioloop *prev_ioloop = current_ioloop; + bool handle_pending = client->stop_now; i_assert(client->ioloop == NULL); + client->stop_now = FALSE; + client->ioloop = io_loop_create(); + io_loop_set_running(client->ioloop); + array_foreach(&client->conns, connp) { imapc_connection_ioloop_changed((*connp)->conn); imapc_connection_connect((*connp)->conn); + if (handle_pending) + imapc_connection_input_pending((*connp)->conn); } - io_loop_run(client->ioloop); + + if (io_loop_is_running(client->ioloop)) + io_loop_run(client->ioloop); current_ioloop = prev_ioloop; array_foreach(&client->conns, connp) @@ -94,6 +103,12 @@ io_loop_stop(client->ioloop); } +void imapc_client_stop_now(struct imapc_client *client) +{ + client->stop_now = TRUE; + imapc_client_stop(client); +} + static struct imapc_client_connection * imapc_client_add_connection(struct imapc_client *client) {
--- a/src/lib-storage/index/imapc/imapc-client.h Sun Jan 30 20:16:04 2011 +0200 +++ b/src/lib-storage/index/imapc/imapc-client.h Sun Jan 30 20:47:43 2011 +0200 @@ -86,6 +86,9 @@ void imapc_client_run(struct imapc_client *client); void imapc_client_stop(struct imapc_client *client); +/* Stop immediately, don't finish even any already read pending replies. + They'll be finished when imapc_client_run() is again called. */ +void imapc_client_stop_now(struct imapc_client *client); struct imapc_client_mailbox * imapc_client_mailbox_open(struct imapc_client *client, const char *name,
--- a/src/lib-storage/index/imapc/imapc-connection.c Sun Jan 30 20:16:04 2011 +0200 +++ b/src/lib-storage/index/imapc/imapc-connection.c Sun Jan 30 20:47:43 2011 +0200 @@ -677,8 +677,6 @@ static void imapc_connection_input(struct imapc_connection *conn) { - int ret; - if (i_stream_read(conn->input) == -1) { /* disconnected */ i_error("imapc(%s): Server disconnected unexpectedly", @@ -687,15 +685,7 @@ return; } - o_stream_cork(conn->output); - do { - T_BEGIN { - ret = imapc_connection_input_one(conn); - } T_END; - } while (ret > 0); - - if (conn->output != NULL) - o_stream_uncork(conn->output); + imapc_connection_input_pending(conn); } static void imapc_connection_connected(struct imapc_connection *conn) @@ -800,6 +790,24 @@ } } +void imapc_connection_input_pending(struct imapc_connection *conn) +{ + int ret = 1; + + if (conn->input == NULL) + return; + + o_stream_cork(conn->output); + while (ret > 0 && !conn->client->stop_now) { + T_BEGIN { + ret = imapc_connection_input_one(conn); + } T_END; + } + + if (conn->output != NULL) + o_stream_uncork(conn->output); +} + static struct imapc_command * imapc_command_begin(imapc_command_callback_t *callback, void *context) {
--- a/src/lib-storage/index/imapc/imapc-connection.h Sun Jan 30 20:16:04 2011 +0200 +++ b/src/lib-storage/index/imapc/imapc-connection.h Sun Jan 30 20:47:43 2011 +0200 @@ -23,6 +23,7 @@ void imapc_connection_connect(struct imapc_connection *conn); void imapc_connection_ioloop_changed(struct imapc_connection *conn); +void imapc_connection_input_pending(struct imapc_connection *conn); void imapc_connection_cmd(struct imapc_connection *conn, const char *cmdline, imapc_command_callback_t *callback, void *context);
--- a/src/lib-storage/index/imapc/imapc-mail.c Sun Jan 30 20:16:04 2011 +0200 +++ b/src/lib-storage/index/imapc/imapc-mail.c Sun Jan 30 20:47:43 2011 +0200 @@ -3,22 +3,24 @@ #include "lib.h" #include "str.h" #include "istream.h" -#include "index-mail.h" +#include "imapc-mail.h" #include "imapc-client.h" #include "imapc-storage.h" -static void imapc_mail_set_seq(struct mail *_mail, uint32_t seq) +struct mail * +imapc_mail_alloc(struct mailbox_transaction_context *t, + enum mail_fetch_field wanted_fields, + struct mailbox_header_lookup_ctx *wanted_headers) { - index_mail_set_seq(_mail, seq); - imapc_mail_fetch(_mail); -} + struct imapc_mail *mail; + pool_t pool; -static bool imapc_mail_set_uid(struct mail *_mail, uint32_t uid) -{ - if (!index_mail_set_uid(_mail, uid)) - return FALSE; - imapc_mail_fetch(_mail); - return TRUE; + pool = pool_alloconly_create("mail", 2048); + mail = p_new(pool, struct imapc_mail, 1); + mail->imail.mail.pool = pool; + + index_mail_init(&mail->imail, t, wanted_fields, wanted_headers); + return &mail->imail.mail.mail; } static int imapc_mail_get_received_date(struct mail *_mail, time_t *date_r) @@ -26,8 +28,15 @@ struct index_mail *mail = (struct index_mail *)_mail; struct index_mail_data *data = &mail->data; - if (data->received_date == (time_t)-1) - return -1; + if (data->received_date == (time_t)-1) { + if (imapc_mail_fetch(_mail, MAIL_FETCH_RECEIVED_DATE) < 0) + return -1; + if (data->received_date == (time_t)-1) { + mail_storage_set_critical(_mail->box->storage, + "imapc: Remote server didn't send INTERNALDATE"); + return -1; + } + } *date_r = data->received_date; return 0; } @@ -37,8 +46,10 @@ struct index_mail *mail = (struct index_mail *)_mail; struct index_mail_data *data = &mail->data; - if (data->save_date == (time_t)-1) + if (data->save_date == (time_t)-1) { + /* FIXME */ return -1; + } *date_r = data->save_date; return 0; } @@ -90,9 +101,25 @@ { struct index_mail *mail = (struct index_mail *)_mail; struct index_mail_data *data = &mail->data; + enum mail_fetch_field fetch_field; - if (data->stream == NULL) - return -1; + if (data->stream == NULL) { + if (!mail->data.initialized) { + /* coming here from mail_set_seq() */ + return -1; + } + fetch_field = body_size != NULL || + (mail->wanted_fields & MAIL_FETCH_STREAM_BODY) != 0 ? + MAIL_FETCH_STREAM_BODY : MAIL_FETCH_STREAM_HEADER; + if (imapc_mail_fetch(_mail, fetch_field) < 0) + return -1; + + if (data->stream == NULL) { + mail_storage_set_critical(_mail->box->storage, + "imapc: Remote server didn't send BODY[]"); + return -1; + } + } return index_mail_init_stream(mail, hdr_size, body_size, stream_r); } @@ -100,8 +127,8 @@ struct mail_vfuncs imapc_mail_vfuncs = { index_mail_close, index_mail_free, - imapc_mail_set_seq, - imapc_mail_set_uid, + index_mail_set_seq, + index_mail_set_uid, index_mail_set_uid_cache_updates, index_mail_get_flags,
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-storage/index/imapc/imapc-mail.h Sun Jan 30 20:47:43 2011 +0200 @@ -0,0 +1,19 @@ +#ifndef IMAPC_MAIL_H +#define IMAPC_MAIL_H + +#include "index-mail.h" + +struct imapc_mail { + struct index_mail imail; + unsigned int searching:1; +}; + +extern struct mail_vfuncs imapc_mail_vfuncs; + +struct mail * +imapc_mail_alloc(struct mailbox_transaction_context *t, + enum mail_fetch_field wanted_fields, + struct mailbox_header_lookup_ctx *wanted_headers); +int imapc_mail_fetch(struct mail *mail, enum mail_fetch_field fields); + +#endif
--- a/src/lib-storage/index/imapc/imapc-mailbox.c Sun Jan 30 20:16:04 2011 +0200 +++ b/src/lib-storage/index/imapc/imapc-mailbox.c Sun Jan 30 20:47:43 2011 +0200 @@ -92,7 +92,7 @@ enum mail_flags flags; uint32_t uid, old_count; unsigned int i, j; - ARRAY_TYPE(const_string) keywords; + ARRAY_TYPE(const_string) keywords = ARRAY_INIT; bool seen_flags = FALSE; if (mbox == NULL || seq == 0 || !imap_arg_get_list(reply->args, &list))
--- a/src/lib-storage/index/imapc/imapc-search.c Sun Jan 30 20:16:04 2011 +0200 +++ b/src/lib-storage/index/imapc/imapc-search.c Sun Jan 30 20:47:43 2011 +0200 @@ -6,41 +6,94 @@ #include "str.h" #include "imap-arg.h" #include "imap-date.h" +#include "imap-util.h" #include "mail-user.h" -#include "index-mail.h" +#include "mail-search.h" +#include "index-search-private.h" +#include "imapc-mail.h" #include "imapc-client.h" #include "imapc-storage.h" -void imapc_mail_fetch(struct mail *mail) +struct imapc_search_context { + struct index_search_context ictx; + + /* non-NULL during _search_next_nonblock() */ + struct mail *cur_mail; + + /* sequences of messages we're next wanting to fetch. */ + ARRAY_TYPE(seq_range) next_seqs; + uint32_t next_pending_seq, saved_seq; + + unsigned int fetching:1; +}; + +static void imapc_search_fetch_callback(const struct imapc_command_reply *reply, + void *context) { + struct imapc_search_context *ctx = context; + struct imapc_mailbox *mbox = + (struct imapc_mailbox *)ctx->ictx.mail_ctx.transaction->box; + + if (reply->state == IMAPC_COMMAND_STATE_OK) + ; + else if (reply->state == IMAPC_COMMAND_STATE_NO) { + imapc_copy_error_from_reply(mbox->storage, MAIL_ERROR_PARAMS, + reply); + } else { + mail_storage_set_critical(&mbox->storage->storage, + "imapc: Command failed: %s", reply->text_full); + } + ctx->fetching = FALSE; + imapc_client_stop(mbox->storage->client); +} + +static bool +imapc_append_wanted_fields(string_t *str, enum mail_fetch_field fields, + bool want_headers) +{ + bool ret = FALSE; + + if ((fields & (MAIL_FETCH_STREAM_BODY | + MAIL_FETCH_MESSAGE_PARTS | + MAIL_FETCH_NUL_STATE | + MAIL_FETCH_IMAP_BODY | + MAIL_FETCH_IMAP_BODYSTRUCTURE | + MAIL_FETCH_PHYSICAL_SIZE | + MAIL_FETCH_VIRTUAL_SIZE)) != 0) { + str_append(str, "BODY.PEEK[] "); + ret = TRUE; + } else if (want_headers || + (fields & (MAIL_FETCH_STREAM_HEADER | + MAIL_FETCH_IMAP_ENVELOPE | + MAIL_FETCH_HEADER_MD5 | + MAIL_FETCH_DATE)) != 0) { + str_append(str, "BODY.PEEK[HEADER] "); + ret = TRUE; + } + + if ((fields & MAIL_FETCH_RECEIVED_DATE) != 0) { + str_append(str, "INTERNALDATE "); + ret = TRUE; + } + return ret; +} + +static void +imapc_search_send_fetch(struct imapc_search_context *ctx, + const ARRAY_TYPE(seq_range) *uids) +{ + struct mail *mail = ctx->cur_mail; struct mail_private *pmail = (struct mail_private *)mail; struct imapc_mailbox *mbox = (struct imapc_mailbox *)mail->box; string_t *str; - unsigned int orig_len; str = t_str_new(64); - str_printfa(str, "UID FETCH %u (", mail->uid); - orig_len = str_len(str); + str_append(str, "UID FETCH "); + imap_write_seq_range(str, uids); + str_append(str, " ("); - if ((pmail->wanted_fields & (MAIL_FETCH_STREAM_HEADER | - MAIL_FETCH_STREAM_BODY | - MAIL_FETCH_MESSAGE_PARTS | - MAIL_FETCH_NUL_STATE | - MAIL_FETCH_IMAP_BODY | - MAIL_FETCH_IMAP_BODYSTRUCTURE | - MAIL_FETCH_PHYSICAL_SIZE | - MAIL_FETCH_VIRTUAL_SIZE)) != 0) - str_append(str, "BODY.PEEK[] "); - else if ((pmail->wanted_fields & (MAIL_FETCH_IMAP_ENVELOPE | - MAIL_FETCH_HEADER_MD5 | - MAIL_FETCH_DATE)) != 0 || - pmail->wanted_headers != NULL) - str_append(str, "BODY.PEEK[HEADER] "); - - if ((pmail->wanted_fields & MAIL_FETCH_RECEIVED_DATE) != 0) - str_append(str, "INTERNALDATE "); - - if (str_len(str) == orig_len) { + if (!imapc_append_wanted_fields(str, pmail->wanted_fields, + pmail->wanted_headers != NULL)) { /* we don't need to fetch anything */ return; } @@ -48,11 +101,9 @@ str_truncate(str, str_len(str) - 1); str_append_c(str, ')'); - mbox->cur_fetch_mail = mail; - imapc_client_mailbox_cmdf(mbox->client_box, imapc_async_stop_callback, - mbox->storage, "%1s", str_c(str)); - imapc_client_run(mbox->storage->client); - mbox->cur_fetch_mail = NULL; + ctx->fetching = TRUE; + imapc_client_mailbox_cmdf(mbox->client_box, imapc_search_fetch_callback, + ctx, "%1s", str_c(str)); } struct mail_search_context * @@ -60,16 +111,122 @@ struct mail_search_args *args, const enum mail_sort_type *sort_program) { - return index_storage_search_init(t, args, sort_program); + struct imapc_search_context *ctx; + + ctx = i_new(struct imapc_search_context, 1); + index_storage_search_init_context(&ctx->ictx, t, args, sort_program); + i_array_init(&ctx->next_seqs, 64); + ctx->ictx.recheck_index_args = TRUE; + return &ctx->ictx.mail_ctx; +} + +int imapc_search_deinit(struct mail_search_context *_ctx) +{ + struct imapc_search_context *ctx = (struct imapc_search_context *)_ctx; + struct imapc_mailbox *mbox = + (struct imapc_mailbox *)_ctx->transaction->box; + + while (ctx->fetching) + imapc_client_run(mbox->storage->client); + + array_free(&ctx->next_seqs); + return index_storage_search_deinit(_ctx); } bool imapc_search_next_nonblock(struct mail_search_context *_ctx, struct mail *mail, bool *tryagain_r) { - if (!index_storage_search_next_nonblock(_ctx, mail, tryagain_r)) + struct imapc_search_context *ctx = (struct imapc_search_context *)_ctx; + struct imapc_mailbox *mbox = + (struct imapc_mailbox *)_ctx->transaction->box; + struct imapc_mail *imail = (struct imapc_mail *)mail; + bool ret; + + imail->searching = TRUE; + ctx->cur_mail = mail; + ret = index_storage_search_next_nonblock(_ctx, mail, tryagain_r); + ctx->cur_mail = NULL; + imail->searching = FALSE; + if (!ret) return FALSE; - imapc_mail_fetch(mail); + if (ctx->fetching) { + mbox->cur_fetch_mail = mail; + imapc_client_run(mbox->storage->client); + mbox->cur_fetch_mail = NULL; + } + return TRUE; +} + +static void imapc_get_short_uid_range(struct mailbox *box, + const ARRAY_TYPE(seq_range) *seqs, + ARRAY_TYPE(seq_range) *uids) +{ + const struct seq_range *range; + unsigned int i, count; + uint32_t uid1, uid2; + + range = array_get(seqs, &count); + for (i = 0; i < count; i++) { + mail_index_lookup_uid(box->view, range[i].seq1, &uid1); + mail_index_lookup_uid(box->view, range[i].seq2, &uid2); + seq_range_array_add_range(uids, uid1, uid2); + } +} + +static void imapc_search_update_next_seqs(struct imapc_search_context *ctx) +{ + struct mail_search_context *_ctx = &ctx->ictx.mail_ctx; + uint32_t prev_seq; + + /* add messages to the next_seqs list as long as the sequences + are incrementing */ + if (ctx->next_pending_seq == 0) + prev_seq = 0; + else { + prev_seq = ctx->next_pending_seq; + seq_range_array_add(&ctx->next_seqs, 0, prev_seq); + } + if (ctx->saved_seq != 0) + _ctx->seq = ctx->saved_seq; + while (index_storage_search_next_update_seq(_ctx)) { + mail_search_args_reset(_ctx->args->args, FALSE); + if (_ctx->seq < prev_seq) { + ctx->next_pending_seq = _ctx->seq; + break; + } + seq_range_array_add(&ctx->next_seqs, 0, _ctx->seq); + } + ctx->saved_seq = _ctx->seq; + if (array_count(&ctx->next_seqs) > 0) T_BEGIN { + ARRAY_TYPE(seq_range) uids; + + t_array_init(&uids, array_count(&ctx->next_seqs)*2); + imapc_get_short_uid_range(_ctx->transaction->box, + &ctx->next_seqs, &uids); + imapc_search_send_fetch(ctx, &uids); + } T_END; +} + +bool imapc_search_next_update_seq(struct mail_search_context *_ctx) +{ + struct imapc_search_context *ctx = (struct imapc_search_context *)_ctx; + struct seq_range *seqs; + unsigned int count; + + seqs = array_get_modifiable(&ctx->next_seqs, &count); + if (count == 0) { + imapc_search_update_next_seqs(ctx); + seqs = array_get_modifiable(&ctx->next_seqs, &count); + if (count == 0) + return FALSE; + } + + _ctx->seq = seqs[0].seq1; + if (seqs[0].seq1 < seqs[0].seq2) + seqs[0].seq1++; + else + array_delete(&ctx->next_seqs, 0, 1); return TRUE; } @@ -123,6 +280,8 @@ void imapc_fetch_mail_update(struct mail *mail, const struct imap_arg *args) { + struct imapc_mailbox *mbox = + (struct imapc_mailbox *)mail->transaction->box; struct index_mail *imail = (struct index_mail *)mail; const char *key, *value; unsigned int i; @@ -151,4 +310,31 @@ imail->data.received_date = t; } } + imapc_client_stop_now(mbox->storage->client); } + +int imapc_mail_fetch(struct mail *mail, enum mail_fetch_field fields) +{ + struct imapc_mailbox *mbox = + (struct imapc_mailbox *)mail->transaction->box; + struct imapc_simple_context sctx; + string_t *str; + + str = t_str_new(64); + str_printfa(str, "UID FETCH %u (", mail->uid); + + if (!imapc_append_wanted_fields(str, fields, FALSE)) + return 0; + + str_truncate(str, str_len(str) - 1); + str_append_c(str, ')'); + + sctx.storage = mbox->storage; + imapc_client_mailbox_cmdf(mbox->client_box, imapc_async_stop_callback, + mbox->storage, "%1s", str_c(str)); + + mbox->cur_fetch_mail = mail; + imapc_client_run(mbox->storage->client); + mbox->cur_fetch_mail = NULL; + return sctx.ret; +}
--- a/src/lib-storage/index/imapc/imapc-storage.c Sun Jan 30 20:16:04 2011 +0200 +++ b/src/lib-storage/index/imapc/imapc-storage.c Sun Jan 30 20:47:43 2011 +0200 @@ -6,7 +6,7 @@ #include "safe-mkstemp.h" #include "imap-arg.h" #include "imap-resp-code.h" -#include "index-mail.h" +#include "imapc-mail.h" #include "imapc-client.h" #include "imapc-list.h" #include "imapc-sync.h" @@ -544,11 +544,11 @@ index_transaction_commit, index_transaction_rollback, NULL, - index_mail_alloc, + imapc_mail_alloc, imapc_search_init, - index_storage_search_deinit, + imapc_search_deinit, imapc_search_next_nonblock, - index_storage_search_next_update_seq, + imapc_search_next_update_seq, imapc_save_alloc, imapc_save_begin, imapc_save_continue,
--- a/src/lib-storage/index/imapc/imapc-storage.h Sun Jan 30 20:16:04 2011 +0200 +++ b/src/lib-storage/index/imapc/imapc-storage.h Sun Jan 30 20:47:43 2011 +0200 @@ -60,8 +60,6 @@ int ret; }; -extern struct mail_vfuncs imapc_mail_vfuncs; - struct mail_save_context * imapc_save_alloc(struct mailbox_transaction_context *_t); int imapc_save_begin(struct mail_save_context *ctx, struct istream *input); @@ -75,13 +73,14 @@ struct mail_index_transaction_commit_result *result); void imapc_transaction_save_rollback(struct mail_save_context *ctx); -void imapc_mail_fetch(struct mail *mail); struct mail_search_context * imapc_search_init(struct mailbox_transaction_context *t, struct mail_search_args *args, const enum mail_sort_type *sort_program); +int imapc_search_deinit(struct mail_search_context *_ctx); bool imapc_search_next_nonblock(struct mail_search_context *_ctx, struct mail *mail, bool *tryagain_r); +bool imapc_search_next_update_seq(struct mail_search_context *_ctx); void imapc_fetch_mail_update(struct mail *mail, const struct imap_arg *args); void imapc_copy_error_from_reply(struct imapc_storage *storage,
--- a/src/lib-storage/index/index-mail.c Sun Jan 30 20:16:04 2011 +0200 +++ b/src/lib-storage/index/index-mail.c Sun Jan 30 20:47:43 2011 +0200 @@ -1340,6 +1340,7 @@ if (!_mail->saving && _mail->uid < hdr->next_uid) (void)mail_get_stream(_mail, NULL, NULL, &input); } + mail->data.initialized = TRUE; } bool index_mail_set_uid(struct mail *_mail, uint32_t uid)
--- a/src/lib-storage/index/index-mail.h Sun Jan 30 20:16:04 2011 +0200 +++ b/src/lib-storage/index/index-mail.h Sun Jan 30 20:47:43 2011 +0200 @@ -96,6 +96,7 @@ ARRAY_TYPE(keywords) keywords; ARRAY_TYPE(keyword_indexes) keyword_indexes; + unsigned int initialized:1; unsigned int save_sent_date:1; unsigned int sent_date_parsed:1; unsigned int save_envelope:1;