Mercurial > dovecot > core-2.2
changeset 7647:879208fdc7e3 HEAD
Implemented CONTEXT=SEARCH extension.
line wrap: on
line diff
--- a/configure.in Thu Jun 05 05:02:01 2008 +0300 +++ b/configure.in Thu Jun 05 05:13:08 2008 +0300 @@ -2119,7 +2119,7 @@ dnl ** capabilities dnl ** -capability="IMAP4rev1 SASL-IR SORT THREAD=REFERENCES MULTIAPPEND UNSELECT LITERAL+ IDLE CHILDREN NAMESPACE LOGIN-REFERRALS UIDPLUS LIST-EXTENDED I18NLEVEL=1 ENABLE CONDSTORE QRESYNC ESEARCH SEARCHRES WITHIN" +capability="IMAP4rev1 SASL-IR SORT THREAD=REFERENCES MULTIAPPEND UNSELECT LITERAL+ IDLE CHILDREN NAMESPACE LOGIN-REFERRALS UIDPLUS LIST-EXTENDED I18NLEVEL=1 ENABLE CONDSTORE QRESYNC ESEARCH SEARCHRES WITHIN CONTEXT=SEARCH" AC_DEFINE_UNQUOTED(CAPABILITY_STRING, "$capability", IMAP capabilities) CFLAGS="$CFLAGS $EXTRA_CFLAGS"
--- a/src/imap/Makefile.am Thu Jun 05 05:02:01 2008 +0300 +++ b/src/imap/Makefile.am Thu Jun 05 05:13:08 2008 +0300 @@ -40,6 +40,7 @@ cmds = \ cmd-append.c \ cmd-capability.c \ + cmd-cancelupdate.c \ cmd-check.c \ cmd-close.c \ cmd-copy.c \
--- a/src/imap/client.c Thu Jun 05 05:02:01 2008 +0300 +++ b/src/imap/client.c Thu Jun 05 05:13:08 2008 +0300 @@ -133,8 +133,10 @@ while (client->command_queue != NULL) client_command_cancel(client->command_queue); - if (client->mailbox != NULL) + if (client->mailbox != NULL) { + client_search_updates_free(client); mailbox_close(&client->mailbox); + } mail_namespaces_deinit(&client->namespaces); if (client->free_parser != NULL) @@ -157,6 +159,8 @@ if (array_is_created(&client->search_saved_uidset)) array_free(&client->search_saved_uidset); + if (array_is_created(&client->search_updates)) + array_free(&client->search_updates); pool_unref(&client->command_pool); i_free(client); @@ -826,6 +830,42 @@ } } +struct imap_search_update * +client_search_update_lookup(struct client *client, const char *tag, + unsigned int *idx_r) +{ + struct imap_search_update *updates; + unsigned int i, count; + + if (!array_is_created(&client->search_updates)) + return NULL; + + updates = array_get_modifiable(&client->search_updates, &count); + for (i = 0; i < count; i++) { + if (strcmp(updates[i].tag, tag) == 0) { + *idx_r = i; + return &updates[i]; + } + } + return NULL; +} + +void client_search_updates_free(struct client *client) +{ + struct imap_search_update *updates; + unsigned int i, count; + + if (!array_is_created(&client->search_updates)) + return; + + updates = array_get_modifiable(&client->search_updates, &count); + for (i = 0; i < count; i++) { + i_free(updates[i].tag); + mailbox_search_result_free(&updates[i].result); + } + array_clear(&client->search_updates); +} + void clients_init(void) { my_client = NULL;
--- a/src/imap/client.h Thu Jun 05 05:02:01 2008 +0300 +++ b/src/imap/client.h Thu Jun 05 05:13:08 2008 +0300 @@ -4,6 +4,10 @@ #include "commands.h" #define CLIENT_COMMAND_QUEUE_MAX_SIZE 4 +/* Maximum number of CONTEXT=SEARCH UPDATEs. Clients probably won't need more + than a few, so this is mainly to avoid more or less accidental pointless + resource usage. */ +#define CLIENT_MAX_SEARCH_UPDATES 10 struct client; struct mail_storage; @@ -20,6 +24,12 @@ unsigned int announce_count; }; +struct imap_search_update { + char *tag; + struct mail_search_result *result; + bool return_uids; +}; + enum client_command_state { /* Waiting for more input */ CLIENT_COMMAND_STATE_WAIT_INPUT, @@ -85,6 +95,8 @@ /* SEARCHRES extension: Last saved SEARCH result */ ARRAY_TYPE(seq_range) search_saved_uidset; + /* SEARCH=CONTEXT extension: Searches that get updated */ + ARRAY_DEFINE(search_updates, struct imap_search_update); /* client input/output is locked by this command */ struct client_command_context *input_lock; @@ -138,6 +150,11 @@ void client_enable(struct client *client, enum mailbox_feature features); +struct imap_search_update * +client_search_update_lookup(struct client *client, const char *tag, + unsigned int *idx_r); +void client_search_updates_free(struct client *client); + void clients_init(void); void clients_deinit(void);
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/imap/cmd-cancelupdate.c Thu Jun 05 05:13:08 2008 +0300 @@ -0,0 +1,44 @@ +/* Copyright (c) 2008 Dovecot authors, see the included COPYING file */ + +#include "common.h" +#include "commands.h" + +static bool client_search_update_cancel(struct client *client, const char *tag) +{ + struct imap_search_update *update; + unsigned int idx; + + update = client_search_update_lookup(client, tag, &idx); + if (update == NULL) + return FALSE; + + i_free(update->tag); + mailbox_search_result_free(&update->result); + array_delete(&client->search_updates, idx, 1); + return TRUE; +} + +bool cmd_cancelupdate(struct client_command_context *cmd) +{ + const struct imap_arg *args; + const char *str; + unsigned int i; + + if (!client_read_args(cmd, 0, 0, &args)) + return FALSE; + + for (i = 0; args[i].type == IMAP_ARG_STRING; i++) ; + if (args[i].type != IMAP_ARG_EOL || i == 0) { + client_send_tagline(cmd, "BAD Invalid parameters."); + return TRUE; + } + for (i = 0; args[i].type == IMAP_ARG_STRING; i++) { + str = IMAP_ARG_STR_NONULL(&args[i]); + if (!client_search_update_cancel(cmd->client, str)) { + client_send_tagline(cmd, "BAD Unknown tag."); + return TRUE; + } + } + client_send_tagline(cmd, "OK Updates cancelled."); + return TRUE; +}
--- a/src/imap/cmd-close.c Thu Jun 05 05:02:01 2008 +0300 +++ b/src/imap/cmd-close.c Thu Jun 05 05:13:08 2008 +0300 @@ -6,6 +6,7 @@ static void cmd_close_finish(struct client *client) { + client_search_updates_free(client); if (mailbox_close(&client->mailbox) < 0) { client_send_untagged_storage_error(client, mailbox_get_storage(client->mailbox));
--- a/src/imap/cmd-delete.c Thu Jun 05 05:02:01 2008 +0300 +++ b/src/imap/cmd-delete.c Thu Jun 05 05:13:08 2008 +0300 @@ -24,6 +24,7 @@ mailbox = client->mailbox; if (mailbox != NULL && strcmp(mailbox_get_name(mailbox), name) == 0) { /* deleting selected mailbox. close it first */ + client_search_updates_free(client); storage = mailbox_get_storage(mailbox); client->mailbox = NULL;
--- a/src/imap/cmd-logout.c Thu Jun 05 05:02:01 2008 +0300 +++ b/src/imap/cmd-logout.c Thu Jun 05 05:13:08 2008 +0300 @@ -12,6 +12,7 @@ o_stream_uncork(client->output); if (client->mailbox != NULL) { + client_search_updates_free(client); /* this could be done at client_disconnect() as well, but eg. mbox rewrite takes a while so the waiting is better to happen before "OK" message. */
--- a/src/imap/cmd-search.c Thu Jun 05 05:02:01 2008 +0300 +++ b/src/imap/cmd-search.c Thu Jun 05 05:13:08 2008 +0300 @@ -17,9 +17,11 @@ SEARCH_RETURN_ALL = 0x08, SEARCH_RETURN_COUNT = 0x10, SEARCH_RETURN_MODSEQ = 0x20, - SEARCH_RETURN_SAVE = 0x40 + SEARCH_RETURN_SAVE = 0x40, + SEARCH_RETURN_UPDATE = 0x80 #define SEARCH_RETURN_EXTRAS \ - (SEARCH_RETURN_ESEARCH | SEARCH_RETURN_MODSEQ | SEARCH_RETURN_SAVE) + (SEARCH_RETURN_ESEARCH | SEARCH_RETURN_MODSEQ | SEARCH_RETURN_SAVE | \ + SEARCH_RETURN_UPDATE) }; struct imap_search_context { @@ -66,12 +68,15 @@ *return_options_r |= SEARCH_RETURN_COUNT; else if (strcmp(name, "SAVE") == 0) *return_options_r |= SEARCH_RETURN_SAVE; + else if (strcmp(name, "UPDATE") == 0) + *return_options_r |= SEARCH_RETURN_UPDATE; else { client_send_command_error(cmd, "Unknown SEARCH return option"); return FALSE; } } + if (*return_options_r == 0) *return_options_r = SEARCH_RETURN_ALL; *return_options_r |= SEARCH_RETURN_ESEARCH; @@ -96,7 +101,36 @@ return FALSE; } -static struct imap_search_context * +static void imap_search_result_save(struct imap_search_context *ctx) +{ + struct client *client = ctx->cmd->client; + struct mail_search_result *result; + struct imap_search_update *update; + + if (!array_is_created(&client->search_updates)) + i_array_init(&client->search_updates, 32); + else if (array_count(&client->search_updates) >= + CLIENT_MAX_SEARCH_UPDATES) { + /* too many updates */ + string_t *str = t_str_new(256); + str_append(str, "* NO [NOUPDATE "); + imap_quote_append_string(str, ctx->cmd->tag, FALSE); + str_append_c(str, ']'); + client_send_line(client, str_c(str)); + ctx->return_options &= ~SEARCH_RETURN_UPDATE; + return; + } + result = mailbox_search_result_save(ctx->search_ctx, + MAILBOX_SEARCH_RESULT_FLAG_UPDATE | + MAILBOX_SEARCH_RESULT_FLAG_QUEUE_SYNC); + + update = array_append_space(&client->search_updates); + update->tag = i_strdup(ctx->cmd->tag); + update->result = result; + update->return_uids = ctx->cmd->uid; +} + +static void imap_search_init(struct imap_search_context *ctx, struct mail_search_args *sargs) { @@ -112,7 +146,8 @@ ctx->mail = mail_alloc(ctx->trans, 0, NULL); (void)gettimeofday(&ctx->start_time, NULL); i_array_init(&ctx->result, 128); - return ctx; + if ((ctx->return_options & SEARCH_RETURN_UPDATE) != 0) + imap_search_result_save(ctx); } static void imap_search_send_result_standard(struct imap_search_context *ctx) @@ -365,6 +400,7 @@ const struct imap_arg *args; enum search_return_options return_options; int ret, args_count; + unsigned int idx; const char *charset; args_count = imap_parser_read_args(cmd->parser, 0, 0, &args); @@ -408,6 +444,12 @@ i_array_init(&client->search_saved_uidset, 128); } + if ((return_options & SEARCH_RETURN_UPDATE) != 0 && + client_search_update_lookup(client, cmd->tag, &idx) != NULL) { + client_send_command_error(cmd, "Duplicate search update tag"); + return TRUE; + } + ctx = p_new(cmd->pool, struct imap_search_context, 1); ctx->cmd = cmd; ctx->return_options = return_options;
--- a/src/imap/cmd-select.c Thu Jun 05 05:02:01 2008 +0300 +++ b/src/imap/cmd-select.c Thu Jun 05 05:13:08 2008 +0300 @@ -358,6 +358,7 @@ client->changing_mailbox = TRUE; if (client->mailbox != NULL) { + client_search_updates_free(client); box = client->mailbox; client->mailbox = NULL;
--- a/src/imap/cmd-unselect.c Thu Jun 05 05:02:01 2008 +0300 +++ b/src/imap/cmd-unselect.c Thu Jun 05 05:13:08 2008 +0300 @@ -12,6 +12,8 @@ if (!client_verify_open_mailbox(cmd)) return TRUE; + client_search_updates_free(client); + i_assert(!client->changing_mailbox); client->mailbox = NULL;
--- a/src/imap/commands.c Thu Jun 05 05:02:01 2008 +0300 +++ b/src/imap/commands.c Thu Jun 05 05:13:08 2008 +0300 @@ -41,6 +41,7 @@ #define IMAP4REV1_COMMANDS_COUNT N_ELEMENTS(imap4rev1_commands) const struct command imap_ext_commands[] = { + { "CANCELUPDATE", cmd_cancelupdate,0 }, { "ENABLE", cmd_enable, 0 }, { "IDLE", cmd_idle, COMMAND_FLAG_BREAKS_SEQS }, { "NAMESPACE", cmd_namespace, 0 },
--- a/src/imap/commands.h Thu Jun 05 05:02:01 2008 +0300 +++ b/src/imap/commands.h Thu Jun 05 05:13:08 2008 +0300 @@ -77,6 +77,7 @@ bool cmd_uid(struct client_command_context *cmd); /* IMAP extensions: */ +bool cmd_cancelupdate(struct client_command_context *cmd); bool cmd_enable(struct client_command_context *cmd); bool cmd_idle(struct client_command_context *cmd); bool cmd_namespace(struct client_command_context *cmd);
--- a/src/imap/imap-sync.c Thu Jun 05 05:02:01 2008 +0300 +++ b/src/imap/imap-sync.c Thu Jun 05 05:13:08 2008 +0300 @@ -4,6 +4,7 @@ #include "str.h" #include "ostream.h" #include "mail-storage.h" +#include "imap-quote.h" #include "imap-util.h" #include "imap-sync.h" #include "commands.h" @@ -32,12 +33,101 @@ ARRAY_TYPE(seq_range) expunges; uint32_t seq; + ARRAY_TYPE(seq_range) search_adds, search_removes; + unsigned int messages_count; unsigned int failed:1; unsigned int no_newmail:1; }; +static void uids_to_seqs(struct mailbox *box, ARRAY_TYPE(seq_range) *uids) +{ + T_BEGIN { + ARRAY_TYPE(seq_range) seqs; + const struct seq_range *range; + unsigned int i, count; + uint32_t seq1, seq2; + + range = array_get(uids, &count); + t_array_init(&seqs, count); + for (i = 0; i < count; i++) { + mailbox_get_uids(box, range[i].seq1, range[i].seq2, + &seq1, &seq2); + /* since we have to notify about expunged messages, + we expect that all the referenced UIDs exist */ + i_assert(seq1 != 0); + i_assert(seq2 - seq1 == range[i].seq2 - range[i].seq1); + + seq_range_array_add_range(&seqs, seq1, seq2); + } + /* replace uids with seqs */ + array_clear(uids); + array_append_array(uids, &seqs); + + } T_END; +} + +static void +imap_sync_send_search_update(struct imap_sync_context *ctx, + const struct imap_search_update *update) +{ + string_t *cmd; + unsigned int pos; + + mailbox_search_result_sync(update->result, &ctx->search_removes, + &ctx->search_adds); + if (array_count(&ctx->search_adds) == 0 && + array_count(&ctx->search_removes) == 0) + return; + + cmd = t_str_new(256); + str_append(cmd, "* ESEARCH (TAG "); + imap_quote_append_string(cmd, update->tag, FALSE); + str_append(cmd, ") "); + if (update->return_uids) + str_append(cmd, "UID "); + else { + /* convert to sequences */ + uids_to_seqs(ctx->client->mailbox, &ctx->search_removes); + uids_to_seqs(ctx->client->mailbox, &ctx->search_adds); + } + pos = str_len(cmd); + + if (array_count(&ctx->search_removes) != 0) { + str_printfa(cmd, "REMOVEFROM (0 "); + imap_write_seq_range(cmd, &ctx->search_removes); + str_append(cmd, ")\r\n"); + o_stream_send(ctx->client->output, str_data(cmd), str_len(cmd)); + str_truncate(cmd, pos); + } + if (array_count(&ctx->search_adds) != 0) { + str_printfa(cmd, "ADDTO (0 "); + imap_write_seq_range(cmd, &ctx->search_adds); + str_append(cmd, ")\r\n"); + o_stream_send(ctx->client->output, str_data(cmd), str_len(cmd)); + } +} + +static void imap_sync_send_search_updates(struct imap_sync_context *ctx) +{ + const struct imap_search_update *updates; + unsigned int i, count; + + if (!array_is_created(&ctx->client->search_updates)) + return; + + if (!array_is_created(&ctx->search_removes)) { + i_array_init(&ctx->search_removes, 64); + i_array_init(&ctx->search_adds, 128); + } + + updates = array_get(&ctx->client->search_updates, &count); + for (i = 0; i < count; i++) T_BEGIN { + imap_sync_send_search_update(ctx, &updates[i]); + } T_END; +} + struct imap_sync_context * imap_sync_init(struct client *client, struct mailbox *box, enum imap_sync_flags imap_flags, enum mailbox_sync_flags flags) @@ -61,6 +151,10 @@ i_array_init(&ctx->expunges, 128); client_send_mailbox_flags(client, FALSE); + /* send search updates the first time after sync is initialized. + it now contains expunged messages that must be sent before + EXPUNGE replies. */ + imap_sync_send_search_updates(ctx); return ctx; } @@ -103,6 +197,14 @@ t_strdup_printf("* %u RECENT", status.recent)); } } + /* send search updates the second time after syncing in done. + now it contains added/removed messages. */ + imap_sync_send_search_updates(ctx); + + if (array_is_created(&ctx->search_removes)) { + array_free(&ctx->search_removes); + array_free(&ctx->search_adds); + } array_free(&ctx->tmp_keywords); i_free(ctx);
--- a/src/lib-storage/Makefile.am Thu Jun 05 05:02:01 2008 +0300 +++ b/src/lib-storage/Makefile.am Thu Jun 05 05:13:08 2008 +0300 @@ -17,6 +17,7 @@ mail-search-build.c \ mail-storage.c \ mailbox-list.c \ + mailbox-search-result.c \ mailbox-tree.c headers = \ @@ -29,6 +30,7 @@ mail-storage-private.h \ mailbox-list.h \ mailbox-list-private.h \ + mailbox-search-result-private.h \ mailbox-tree.h if INSTALL_HEADERS
--- a/src/lib-storage/index/Makefile.am Thu Jun 05 05:02:01 2008 +0300 +++ b/src/lib-storage/index/Makefile.am Thu Jun 05 05:13:08 2008 +0300 @@ -20,13 +20,15 @@ index-storage.c \ index-sync.c \ index-sync-changes.c \ + index-sync-search.c \ index-transaction.c headers = \ index-mail.h \ index-sort.h \ index-storage.h \ - index-sync-changes.h + index-sync-changes.h \ + index-sync-private.h if INSTALL_HEADERS pkginc_libdir=$(pkgincludedir)/src/lib-storage/index
--- a/src/lib-storage/index/index-search.c Thu Jun 05 05:02:01 2008 +0300 +++ b/src/lib-storage/index/index-search.c Thu Jun 05 05:13:08 2008 +0300 @@ -15,6 +15,7 @@ #include "index-mail.h" #include "index-sort.h" #include "mail-search.h" +#include "mailbox-search-result-private.h" #include <stdlib.h> #include <ctype.h> @@ -863,6 +864,7 @@ ctx->mail_ctx.args = args; ctx->mail_ctx.sort_program = index_sort_program_init(_t, sort_program); + i_array_init(&ctx->mail_ctx.results, 5); array_create(&ctx->mail_ctx.module_contexts, default_pool, sizeof(void *), 5); @@ -908,6 +910,7 @@ if (ctx->mail_ctx.sort_program != NULL) index_sort_program_deinit(&ctx->mail_ctx.sort_program); + array_free(&ctx->mail_ctx.results); array_free(&ctx->mail_ctx.module_contexts); i_free(ctx); return ret; @@ -971,6 +974,55 @@ ctx->last_notify = ioloop_timeval; } +static bool search_arg_is_static(struct mail_search_arg *arg) +{ + struct mail_search_arg *subarg; + + switch (arg->type) { + case SEARCH_OR: + case SEARCH_SUB: + /* they're static only if all subargs are static */ + subarg = arg->value.subargs; + for (; subarg != NULL; subarg = subarg->next) { + if (!search_arg_is_static(subarg)) + return FALSE; + } + return TRUE; + case SEARCH_SEQSET: /* changes between syncs */ + case SEARCH_FLAGS: + case SEARCH_KEYWORDS: + case SEARCH_MODSEQ: + break; + case SEARCH_ALL: + case SEARCH_UIDSET: + case SEARCH_BEFORE: + case SEARCH_ON: + case SEARCH_SINCE: + case SEARCH_SENTBEFORE: + case SEARCH_SENTON: + case SEARCH_SENTSINCE: + case SEARCH_SMALLER: + case SEARCH_LARGER: + case SEARCH_HEADER: + case SEARCH_HEADER_ADDRESS: + case SEARCH_HEADER_COMPRESS_LWSP: + case SEARCH_BODY: + case SEARCH_TEXT: + case SEARCH_BODY_FAST: + case SEARCH_TEXT_FAST: + return TRUE; + } + return FALSE; +} + +static void search_set_static_matches(struct mail_search_arg *arg) +{ + for (; arg != NULL; arg = arg->next) { + if (search_arg_is_static(arg)) + arg->result = 1; + } +} + int index_storage_search_next_nonblock(struct mail_search_context *_ctx, struct mail *mail, bool *tryagain_r) { @@ -999,6 +1051,14 @@ while ((ret = box->v.search_next_update_seq(_ctx)) > 0) { mail_set_seq(mail, _ctx->seq); + if (ctx->mail_ctx.update_result != NULL && + seq_range_exists(&ctx->mail_ctx.update_result->uids, + mail->uid)) { + /* updating an existing search result: we already know + that the static data matches. mark it as such. */ + search_set_static_matches(ctx->mail_ctx.args->args); + } + T_BEGIN { ret = search_match_next(ctx) ? 1 : 0; } T_END;
--- a/src/lib-storage/index/index-storage.c Thu Jun 05 05:02:01 2008 +0300 +++ b/src/lib-storage/index/index-storage.c Thu Jun 05 05:13:08 2008 +0300 @@ -421,6 +421,7 @@ ibox->box.file_create_gid = (gid_t)-1; } + p_array_init(&ibox->box.search_results, ibox->box.pool, 16); array_create(&ibox->box.module_contexts, ibox->box.pool, sizeof(void *), 5);
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-storage/index/index-sync-private.h Thu Jun 05 05:13:08 2008 +0300 @@ -0,0 +1,23 @@ +#ifndef INDEX_SYNC_PRIVATE_H +#define INDEX_SYNC_PRIVATE_H + +#include "index-storage.h" + +struct index_mailbox_sync_context { + struct mailbox_sync_context ctx; + struct index_mailbox *ibox; + struct mail_index_view_sync_ctx *sync_ctx; + uint32_t messages_count; + + ARRAY_TYPE(seq_range) flag_updates; + ARRAY_TYPE(seq_range) modseq_updates; + const ARRAY_TYPE(seq_range) *expunges; + unsigned int flag_update_idx, modseq_update_idx, expunge_pos; + + bool failed; +}; + +void index_sync_search_results_update(struct index_mailbox_sync_context *ctx); +void index_sync_search_results_expunge(struct index_mailbox_sync_context *ctx); + +#endif
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-storage/index/index-sync-search.c Thu Jun 05 05:13:08 2008 +0300 @@ -0,0 +1,243 @@ +/* Copyright (c) 2002-2008 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "seq-range-array.h" +#include "mail-search.h" +#include "mailbox-search-result-private.h" +#include "index-sync-private.h" + +static void +search_result_range_remove(struct mail_search_result *result, + const ARRAY_TYPE(seq_range) *search_seqs_range, + unsigned int *pos, + uint32_t *next_seq, uint32_t last_seq) +{ + struct index_mailbox *ibox = (struct index_mailbox *)result->box; + const struct seq_range *seqs; + unsigned int i, count; + uint32_t seq, uid; + + seq = *next_seq; + seqs = array_get(search_seqs_range, &count); + for (i = *pos; seqs[i].seq2 < last_seq;) { + i_assert(seqs[i].seq1 <= seq); + for (; seq <= seqs[i].seq2; seq++) { + mail_index_lookup_uid(ibox->view, seq, &uid); + mailbox_search_result_remove(result, uid); + } + i++; + i_assert(i < count); + seq = seqs[i].seq1; + } + + i_assert(seqs[i].seq1 <= seq && seqs[i].seq2 >= last_seq); + for (; seq <= last_seq; seq++) { + mail_index_lookup_uid(ibox->view, seq, &uid); + mailbox_search_result_remove(result, uid); + } + if (seq > seqs[i].seq2) { + /* finished this range */ + if (++i < count) + seq = seqs[i].seq1; + else { + /* this was the last searched message */ + seq = 0; + } + } + + *next_seq = seq; + *pos = i; +} + +static int +search_result_update_search(struct mail_search_result *result, + const ARRAY_TYPE(seq_range) *search_seqs_range) +{ + struct mailbox_transaction_context *t; + struct mail_search_context *search_ctx; + struct mail *mail; + const struct seq_range *search_seqs; + unsigned int seqcount, seqpos; + uint32_t next_seq; + int ret; + + search_seqs = array_get(search_seqs_range, &seqcount); + i_assert(seqcount > 0); + next_seq = search_seqs[0].seq1; + seqpos = 0; + + t = mailbox_transaction_begin(result->box, 0); + search_ctx = mailbox_search_init(t, result->search_args, NULL); + /* tell search that we're updating an existing search result, + so it can do some optimizations based on it */ + search_ctx->update_result = result; + + mail = mail_alloc(t, 0, NULL); + while (mailbox_search_next(search_ctx, mail) > 0) { + i_assert(next_seq != 0); + + if (next_seq != mail->seq) { + /* some messages in search_seqs didn't match. + make sure they don't exist in the search result. */ + search_result_range_remove(result, search_seqs_range, + &seqpos, &next_seq, + mail->seq-1); + i_assert(next_seq == mail->seq); + } + if (search_seqs[seqpos].seq2 > next_seq) { + next_seq++; + } else if (++seqpos < seqcount) { + next_seq = search_seqs[seqpos].seq1; + } else { + /* this was the last searched message */ + next_seq = 0; + } + /* match - make sure it exists in search result */ + mailbox_search_result_add(result, mail->uid); + } + mail_free(&mail); + ret = mailbox_search_deinit(&search_ctx); + + if (next_seq != 0 && ret == 0) { + /* last message(s) didn't match. make sure they don't exist + in the search result. */ + search_result_range_remove(result, search_seqs_range, &seqpos, + &next_seq, + search_seqs[seqcount-1].seq2); + } + + if (mailbox_transaction_commit(&t) < 0) + ret = -1; + return ret; +} + +static int +search_result_update_existing(struct index_mailbox_sync_context *ctx, + struct mail_search_result *result) +{ + /* @UNSAFE */ + const ARRAY_TYPE(seq_range) *merge_ranges[2]; + ARRAY_TYPE(seq_range) changes; + struct mail_search_arg search_arg; + unsigned int i, count, merge_count = 0; + int ret; + + /* get the changes we're interested in */ + if ((result->args_have_flags || result->args_have_keywords) && + array_is_created(&ctx->flag_updates)) + merge_ranges[merge_count++] = &ctx->flag_updates; + if (result->args_have_modseq && array_is_created(&ctx->modseq_updates)) + merge_ranges[merge_count++] = &ctx->modseq_updates; + i_assert(merge_count < N_ELEMENTS(merge_ranges)); + + for (i = 0, count = 0; i < merge_count; i++) + count += array_count(merge_ranges[i]); + if (count == 0) { + /* no changes */ + return 0; + } + + /* merge the changed sequences */ + t_array_init(&changes, count); + for (i = 0; i < merge_count; i++) + seq_range_array_merge(&changes, merge_ranges[i]); + + /* add a temporary search parameter to limit the search only to + the changed messages */ + memset(&search_arg, 0, sizeof(search_arg)); + search_arg.type = SEARCH_SEQSET; + search_arg.value.seqset = changes; + search_arg.next = result->search_args->args; + result->search_args->args = &search_arg; + ret = search_result_update_search(result, &changes); + i_assert(result->search_args->args == &search_arg); + result->search_args->args = search_arg.next; + return ret; +} + +static int +search_result_update_appends(struct index_mailbox_sync_context *ctx, + struct mail_search_result *result) +{ + struct mailbox_transaction_context *t; + struct mail_search_context *search_ctx; + struct mail *mail; + struct mail_search_arg search_arg; + uint32_t message_count; + int ret; + + message_count = mail_index_view_get_messages_count(ctx->ibox->view); + if (ctx->messages_count == message_count) { + /* no new messages */ + return 0; + } + + /* add a temporary search parameter to limit the search only to + the new messages */ + memset(&search_arg, 0, sizeof(search_arg)); + search_arg.type = SEARCH_SEQSET; + t_array_init(&search_arg.value.seqset, 1); + seq_range_array_add_range(&search_arg.value.seqset, + ctx->messages_count + 1, message_count); + search_arg.next = result->search_args->args; + result->search_args->args = &search_arg; + + /* add all messages matching the search to search result */ + t = mailbox_transaction_begin(result->box, 0); + search_ctx = mailbox_search_init(t, result->search_args, NULL); + + mail = mail_alloc(t, 0, NULL); + while (mailbox_search_next(search_ctx, mail) > 0) + mailbox_search_result_add(result, mail->uid); + mail_free(&mail); + + ret = mailbox_search_deinit(&search_ctx); + if (mailbox_transaction_commit(&t) < 0) + ret = -1; + + i_assert(result->search_args->args == &search_arg); + result->search_args->args = search_arg.next; + return ret; +} + +static void +search_result_update(struct index_mailbox_sync_context *ctx, + struct mail_search_result *result) +{ + if ((result->flags & MAILBOX_SEARCH_RESULT_FLAG_UPDATE) == 0) + return; + + T_BEGIN { + (void)search_result_update_existing(ctx, result); + } T_END; + (void)search_result_update_appends(ctx, result); +} + +void index_sync_search_results_update(struct index_mailbox_sync_context *ctx) +{ + struct mail_search_result *const *results; + unsigned int i, count; + + results = array_get(&ctx->ibox->box.search_results, &count); + for (i = 0; i < count; i++) + search_result_update(ctx, results[i]); +} + +void index_sync_search_results_expunge(struct index_mailbox_sync_context *ctx) +{ + const struct seq_range *seqs; + unsigned int i, count; + uint32_t seq, uid; + + if (ctx->expunges == NULL) + return; + + seqs = array_get(ctx->expunges, &count); + for (i = 0; i < count; i++) { + for (seq = seqs[i].seq1; seq <= seqs[i].seq2; seq++) { + mail_index_lookup_uid(ctx->ibox->view, seq, &uid); + mailbox_search_results_remove(&ctx->ibox->box, uid); + } + } +}
--- a/src/lib-storage/index/index-sync.c Thu Jun 05 05:02:01 2008 +0300 +++ b/src/lib-storage/index/index-sync.c Thu Jun 05 05:13:08 2008 +0300 @@ -4,22 +4,7 @@ #include "seq-range-array.h" #include "ioloop.h" #include "array.h" -#include "buffer.h" -#include "index-storage.h" - -struct index_mailbox_sync_context { - struct mailbox_sync_context ctx; - struct index_mailbox *ibox; - struct mail_index_view_sync_ctx *sync_ctx; - uint32_t messages_count; - - ARRAY_TYPE(seq_range) flag_updates; - ARRAY_TYPE(seq_range) modseq_updates; - const ARRAY_TYPE(seq_range) *expunges; - unsigned int flag_update_idx, modseq_update_idx, expunge_pos; - - bool failed; -}; +#include "index-sync-private.h" bool index_mailbox_want_full_sync(struct index_mailbox *ibox, enum mailbox_sync_flags flags) @@ -129,7 +114,7 @@ case MAIL_INDEX_SYNC_TYPE_KEYWORD_REMOVE: case MAIL_INDEX_SYNC_TYPE_KEYWORD_RESET: if (!mail_index_lookup_seq_range(ctx->ibox->view, - sync.uid1, sync.uid2, + sync.uid1, sync.uid2, &seq1, &seq2)) break; @@ -201,6 +186,7 @@ ctx->expunge_pos = array_count(ctx->expunges); } index_view_sync_recs_get(ctx); + index_sync_search_results_expunge(ctx); return &ctx->ctx; } @@ -362,6 +348,8 @@ if (ret == 0 && status_items != 0) mailbox_get_status(_ctx->box, status_items, status_r); + index_sync_search_results_update(ctx); + if (array_is_created(&ctx->flag_updates)) array_free(&ctx->flag_updates); if (array_is_created(&ctx->modseq_updates))
--- a/src/lib-storage/mail-search.c Thu Jun 05 05:02:01 2008 +0300 +++ b/src/lib-storage/mail-search.c Thu Jun 05 05:13:08 2008 +0300 @@ -125,6 +125,9 @@ void mail_search_args_deinit(struct mail_search_args *args) { + if (args->refcount > 1) + return; + mail_search_args_deinit_sub(args, args->args); }
--- a/src/lib-storage/mail-storage-private.h Thu Jun 05 05:02:01 2008 +0300 +++ b/src/lib-storage/mail-storage-private.h Thu Jun 05 05:13:08 2008 +0300 @@ -195,6 +195,9 @@ mailbox_notify_callback_t *notify_callback; void *notify_context; + /* Saved search results */ + ARRAY_DEFINE(search_results, struct mail_search_result *); + /* Module-specific contexts. See mail_storage_module_id. */ ARRAY_DEFINE(module_contexts, union mailbox_module_context *); @@ -291,6 +294,14 @@ struct mail_search_args *args; struct mail_search_sort_program *sort_program; + /* if non-NULL, specifies that a search resulting is being updated. + this can be used as a search optimization: if searched message + already exists in search result, it's not necessary to check if + static data matches. */ + struct mail_search_result *update_result; + /* add matches to these search results */ + ARRAY_DEFINE(results, struct mail_search_result *); + uint32_t seq; ARRAY_DEFINE(module_contexts, union mail_search_module_context *); };
--- a/src/lib-storage/mail-storage.c Thu Jun 05 05:02:01 2008 +0300 +++ b/src/lib-storage/mail-storage.c Thu Jun 05 05:13:08 2008 +0300 @@ -10,6 +10,7 @@ #include "mail-storage-private.h" #include "mail-namespace.h" #include "mail-search.h" +#include "mailbox-search-result-private.h" #include <stdlib.h> #include <time.h> @@ -646,6 +647,7 @@ int ret; *_ctx = NULL; + mailbox_search_results_initial_done(ctx); ret = ctx->transaction->box->v.search_deinit(ctx); mail_search_args_unref(&args); return ret; @@ -668,8 +670,13 @@ int mailbox_search_next_nonblock(struct mail_search_context *ctx, struct mail *mail, bool *tryagain_r) { - return ctx->transaction->box->v. + int ret; + + ret = ctx->transaction->box->v. search_next_nonblock(ctx, mail, tryagain_r); + if (ret > 0) + mailbox_search_results_add(ctx, mail->uid); + return ret; } struct mailbox_transaction_context *
--- a/src/lib-storage/mail-storage.h Thu Jun 05 05:02:01 2008 +0300 +++ b/src/lib-storage/mail-storage.h Thu Jun 05 05:13:08 2008 +0300 @@ -76,6 +76,14 @@ STATUS_HIGHESTMODSEQ = 0x80 }; +enum mailbox_search_result_flags { + /* Update search results whenever the mailbox view is synced. + Expunged messages are removed even without this flag. */ + MAILBOX_SEARCH_RESULT_FLAG_UPDATE = 0x01, + /* Queue changes so _sync() can be used. */ + MAILBOX_SEARCH_RESULT_FLAG_QUEUE_SYNC = 0x02 +}; + enum mail_sort_type { /* Maximum size for sort program (each one separately + END) */ #define MAX_SORT_PROGRAM_SIZE (7 + 1) @@ -159,6 +167,7 @@ struct mail_namespace; struct mail_storage; struct mail_search_args; +struct mail_search_result; struct mail_keywords; struct mail_save_context; struct mailbox; @@ -388,6 +397,23 @@ int mailbox_search_next_nonblock(struct mail_search_context *ctx, struct mail *mail, bool *tryagain_r); +/* Remember the search result for future use. This must be called before the + first mailbox_search_next*() call. */ +struct mail_search_result * +mailbox_search_result_save(struct mail_search_context *ctx, + enum mailbox_search_result_flags flags); +/* Free memory used by search result. */ +void mailbox_search_result_free(struct mail_search_result **result); +/* Return all messages' UIDs in the search result. */ +const ARRAY_TYPE(seq_range) * +mailbox_search_result_get(struct mail_search_result *result); +/* Return messages that have been removed and added since the last sync call. + This function must not be called if search result wasn't saved with + _QUEUE_SYNC flag. */ +void mailbox_search_result_sync(struct mail_search_result *result, + ARRAY_TYPE(seq_range) *removed_uids, + ARRAY_TYPE(seq_range) *added_uids); + /* Save a mail into mailbox. timezone_offset specifies the timezone in minutes in which received_date was originally given with. To use current time, set received_date to (time_t)-1.
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-storage/mailbox-search-result-private.h Thu Jun 05 05:13:08 2008 +0300 @@ -0,0 +1,28 @@ +#ifndef MAILBOX_SEARCH_RESULT_PRIVATE_H +#define MAILBOX_SEARCH_RESULT_PRIVATE_H + +#include "mail-storage.h" + +struct mail_search_result { + struct mailbox *box; + enum mailbox_search_result_flags flags; + struct mail_search_args *search_args; + + ARRAY_TYPE(seq_range) uids; + ARRAY_TYPE(seq_range) removed_uids, added_uids; + + unsigned int args_have_flags:1; + unsigned int args_have_keywords:1; + unsigned int args_have_modseq:1; +}; + +/* called when initial search is done. */ +void mailbox_search_results_initial_done(struct mail_search_context *ctx); + +void mailbox_search_result_add(struct mail_search_result *result, uint32_t uid); +void mailbox_search_result_remove(struct mail_search_result *result, + uint32_t uid); +void mailbox_search_results_add(struct mail_search_context *ctx, uint32_t uid); +void mailbox_search_results_remove(struct mailbox *box, uint32_t uid); + +#endif
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-storage/mailbox-search-result.c Thu Jun 05 05:13:08 2008 +0300 @@ -0,0 +1,163 @@ +/* Copyright (c) 2008 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "mail-storage-private.h" +#include "mail-search.h" +#include "mailbox-search-result-private.h" + +static void +mailbox_search_result_analyze_args(struct mail_search_result *result, + struct mail_search_arg *arg) +{ + for (; arg != NULL; arg = arg->next) { + switch (arg->type) { + case SEARCH_OR: + case SEARCH_SUB: + mailbox_search_result_analyze_args(result, + arg->value.subargs); + break; + case SEARCH_FLAGS: + result->args_have_flags = TRUE; + break; + case SEARCH_KEYWORDS: + result->args_have_keywords = TRUE; + break; + case SEARCH_MODSEQ: + result->args_have_modseq = TRUE; + break; + default: + break; + } + } +} + +struct mail_search_result * +mailbox_search_result_save(struct mail_search_context *ctx, + enum mailbox_search_result_flags flags) +{ + struct mail_search_result *result; + + result = i_new(struct mail_search_result, 1); + result->box = ctx->transaction->box; + result->flags = flags; + i_array_init(&result->uids, 32); + + if ((result->flags & MAILBOX_SEARCH_RESULT_FLAG_UPDATE) != 0) { + result->search_args = ctx->args; + mail_search_args_ref(result->search_args); + mailbox_search_result_analyze_args(result, ctx->args->args); + } + + array_append(&ctx->results, &result, 1); + array_append(&result->box->search_results, &result, 1); + return result; +} + +void mailbox_search_result_free(struct mail_search_result **_result) +{ + struct mail_search_result *result = *_result; + struct mail_search_result *const *results; + unsigned int i, count; + + *_result = NULL; + + results = array_get(&result->box->search_results, &count); + for (i = 0; i < count; i++) { + if (results[i] == result) { + array_delete(&result->box->search_results, i, 1); + break; + } + } + i_assert(i != count); + + if (result->search_args != NULL) + mail_search_args_unref(&result->search_args); + + array_free(&result->uids); + if (array_is_created(&result->removed_uids)) { + array_free(&result->removed_uids); + array_free(&result->added_uids); + } + i_free(result); +} + +static void +mailbox_search_result_initial_done(struct mail_search_result *result) +{ + if ((result->flags & MAILBOX_SEARCH_RESULT_FLAG_QUEUE_SYNC) != 0) { + i_array_init(&result->removed_uids, 32); + i_array_init(&result->added_uids, 32); + } +} + +void mailbox_search_results_initial_done(struct mail_search_context *ctx) +{ + struct mail_search_result *const *results; + unsigned int i, count; + + results = array_get(&ctx->results, &count); + for (i = 0; i < count; i++) + mailbox_search_result_initial_done(results[i]); +} + +void mailbox_search_result_add(struct mail_search_result *result, uint32_t uid) +{ + i_assert(uid > 0); + + if (seq_range_exists(&result->uids, uid)) + return; + + seq_range_array_add(&result->uids, 0, uid); + if (array_is_created(&result->added_uids)) + seq_range_array_add(&result->added_uids, 0, uid); +} + +void mailbox_search_result_remove(struct mail_search_result *result, + uint32_t uid) +{ + if (seq_range_array_remove(&result->uids, uid)) { + if (array_is_created(&result->removed_uids)) + seq_range_array_add(&result->removed_uids, 0, uid); + } +} + +void mailbox_search_results_add(struct mail_search_context *ctx, uint32_t uid) +{ + struct mail_search_result *const *results; + unsigned int i, count; + + results = array_get(&ctx->results, &count); + for (i = 0; i < count; i++) + mailbox_search_result_add(results[i], uid); +} + +void mailbox_search_results_remove(struct mailbox *box, uint32_t uid) +{ + struct mail_search_result *const *results; + unsigned int i, count; + + results = array_get(&box->search_results, &count); + for (i = 0; i < count; i++) + mailbox_search_result_remove(results[i], uid); +} + +const ARRAY_TYPE(seq_range) * +mailbox_search_result_get(struct mail_search_result *result) +{ + return &result->uids; +} + +void mailbox_search_result_sync(struct mail_search_result *result, + ARRAY_TYPE(seq_range) *removed_uids, + ARRAY_TYPE(seq_range) *added_uids) +{ + array_clear(removed_uids); + array_clear(added_uids); + + array_append_array(removed_uids, &result->removed_uids); + array_append_array(added_uids, &result->added_uids); + + array_clear(&result->removed_uids); + array_clear(&result->added_uids); +}