Mercurial > dovecot > original-hg > dovecot-1.2
changeset 4939:ff2272c228cc HEAD
Dovecot is now able to execute multiple commands at the same time.
Practically this means commands: FETCH, LIST, SEARCH and syncing output for
all commands. For example it's possible that doing two FETCH commands at the
same time makes their output mixed together.
Non-blocking SEARCH is done by doing search for 20 mails at a time, and then
checking if another command is pending.
Also added X-CANCEL <tag> command to cancel running commands.
author | Timo Sirainen <tss@iki.fi> |
---|---|
date | Wed, 20 Dec 2006 21:23:43 +0200 |
parents | e3539fafe74f |
children | 76aa8a360c2f |
files | src/imap/Makefile.am src/imap/client.c src/imap/client.h src/imap/cmd-append.c src/imap/cmd-fetch.c src/imap/cmd-idle.c src/imap/cmd-list.c src/imap/cmd-search.c src/imap/cmd-sort.c src/imap/cmd-thread.c src/imap/cmd-uid.c src/imap/commands.c src/imap/commands.h src/imap/imap-fetch.c src/imap/imap-fetch.h src/imap/imap-sync.c src/lib-storage/index/dbox/dbox-storage.c src/lib-storage/index/index-search.c src/lib-storage/index/index-storage.h src/lib-storage/index/maildir/maildir-storage.c src/lib-storage/index/mbox/mbox-storage.c src/lib-storage/mail-storage-private.h src/lib-storage/mail-storage.c src/lib-storage/mail-storage.h |
diffstat | 24 files changed, 497 insertions(+), 236 deletions(-) [+] |
line wrap: on
line diff
--- a/src/imap/Makefile.am Wed Dec 20 17:40:22 2006 +0200 +++ b/src/imap/Makefile.am Wed Dec 20 21:23:43 2006 +0200 @@ -17,6 +17,7 @@ # be a shared library so this wouldn't be needed.. unused_objects = \ ../lib/mountpoint.o \ + ../lib/unichar.o \ ../lib-mail/message-decoder.o libs = \ @@ -66,7 +67,8 @@ cmd-thread.c \ cmd-uid.c \ cmd-unselect.c \ - cmd-unsubscribe.c + cmd-unsubscribe.c \ + cmd-x-cancel.c imap_SOURCES = \ $(cmds) \
--- a/src/imap/client.c Wed Dec 20 17:40:22 2006 +0200 +++ b/src/imap/client.c Wed Dec 20 21:23:43 2006 +0200 @@ -36,13 +36,9 @@ o_stream_set_flush_callback(client->output, _client_output, client); client->io = io_add(fd_in, IO_READ, _client_input, client); - client->parser = imap_parser_create(client->input, client->output, - imap_max_line_length); client->last_input = ioloop_time; - client->cmd.pool = pool_alloconly_create("command pool", 8192); - client->cmd.client = client; - + client->command_pool = pool_alloconly_create("client command", 8192); client->keywords.pool = pool_alloconly_create("mailbox_keywords", 512); client->namespaces = namespaces; @@ -60,10 +56,22 @@ return client; } +void client_command_cancel(struct client_command_context *cmd) +{ + bool cmd_ret; + + cmd->cancel = TRUE; + cmd_ret = cmd->func(cmd); + if (!cmd_ret) { + if (cmd->client->output->closed) + i_panic("command didn't cancel itself: %s", cmd->name); + } else { + client_command_free(cmd); + } +} + void client_destroy(struct client *client, const char *reason) { - int ret; - i_assert(!client->destroyed); client->destroyed = TRUE; @@ -74,22 +82,23 @@ i_info("%s", reason); } - if (client->command_pending) { - /* try to deinitialize the command */ - i_assert(client->cmd.func != NULL); - i_stream_close(client->input); - o_stream_close(client->output); - client->input_pending = FALSE; + i_stream_close(client->input); + o_stream_close(client->output); - ret = client->cmd.func(&client->cmd); - i_assert(ret); - } + /* finish off all the queued commands. */ + if (client->output_lock != NULL) + client_command_cancel(client->output_lock); + if (client->input_lock != NULL) + client_command_cancel(client->input_lock); + while (client->command_queue != NULL) + client_command_cancel(client->command_queue); if (client->mailbox != NULL) mailbox_close(&client->mailbox); namespace_deinit(client->namespaces); - imap_parser_destroy(&client->parser); + if (client->free_parser != NULL) + imap_parser_destroy(&client->free_parser); if (client->io != NULL) io_remove(&client->io); @@ -104,7 +113,7 @@ } pool_unref(client->keywords.pool); - pool_unref(client->cmd.pool); + pool_unref(client->command_pool); i_free(client); /* quit the program */ @@ -161,7 +170,7 @@ struct client *client = cmd->client; const char *tag = cmd->tag; - if (client->output->closed) + if (client->output->closed || cmd->cancel) return; if (tag == NULL || *tag == '\0') @@ -181,7 +190,7 @@ bool fatal; if (msg == NULL) { - msg = imap_parser_get_error(client->parser, &fatal); + msg = imap_parser_get_error(cmd->parser, &fatal); if (fatal) { client_disconnect_with_error(client, msg); return; @@ -218,9 +227,12 @@ i_assert(count <= INT_MAX); - ret = imap_parser_read_args(cmd->client->parser, count, flags, args); + ret = imap_parser_read_args(cmd->parser, count, flags, args); if (ret >= (int)count) { /* all parameters read successfully */ + i_assert(cmd->client->input_lock == NULL || + cmd->client->input_lock == cmd); + cmd->client->input_lock = NULL; return TRUE; } else if (ret == -2) { /* need more data */ @@ -267,38 +279,97 @@ return i == count; } -void _client_reset_command(struct client *client) +static struct client_command_context * +client_command_new(struct client *client) { - pool_t pool; + struct client_command_context *cmd; + + cmd = p_new(client->command_pool, struct client_command_context, 1); + cmd->client = client; + cmd->pool = client->command_pool; + + if (client->free_parser != NULL) { + cmd->parser = client->free_parser; + client->free_parser = NULL; + } else { + cmd->parser = imap_parser_create(client->input, client->output, + imap_max_line_length); + } + + /* add to beginning of the queue */ + if (client->command_queue != NULL) { + client->command_queue->prev = cmd; + cmd->next = client->command_queue; + } + client->command_queue = cmd; + client->command_queue_size++; + + return cmd; +} + +void client_command_free(struct client_command_context *cmd) +{ + struct client *client = cmd->client; size_t size; /* reset input idle time because command output might have taken a long time and we don't want to disconnect client immediately then */ client->last_input = ioloop_time; - client->command_pending = FALSE; - if (client->io == NULL && !client->disconnected) { - i_assert(i_stream_get_fd(client->input) >= 0); - client->io = io_add(i_stream_get_fd(client->input), - IO_READ, _client_input, client); + if (cmd->cancel) { + cmd->cancel = FALSE; + client_send_tagline(cmd, "NO Command cancelled."); + } + + if (!cmd->param_error) + client->bad_counter = 0; + + if (client->input_lock == cmd) { + /* reset the input handler in case it was changed */ + client->input_lock = NULL; } - o_stream_set_flush_callback(client->output, _client_output, client); + if (client->output_lock == cmd) { + /* reset the output handler in case it was changed */ + o_stream_set_flush_callback(client->output, + _client_output, client); + client->output_lock = NULL; + } - pool = client->cmd.pool; - memset(&client->cmd, 0, sizeof(client->cmd)); + if (client->free_parser != NULL) + imap_parser_destroy(&cmd->parser); + else { + imap_parser_reset(cmd->parser); + client->free_parser = cmd->parser; + } - p_clear(pool); - client->cmd.pool = pool; - client->cmd.client = client; + client->command_queue_size--; + if (cmd->prev != NULL) + cmd->prev->next = cmd->next; + else + client->command_queue = cmd->next; + if (cmd->next != NULL) + cmd->next->prev = cmd->prev; + cmd = NULL; - imap_parser_reset(client->parser); + if (client->command_queue == NULL) { + /* no commands left in the queue, we can clear the pool */ + p_clear(client->command_pool); + } - /* if there's unread data in buffer, remember that there's input - pending and we should get around to calling client_input() soon. - This is mostly for APPEND/IDLE. */ - (void)i_stream_get_data(client->input, &size); - if (size > 0 && !client->destroyed) - client->input_pending = TRUE; + if (client->input_lock == NULL && !client->disconnected) { + if (client->io == NULL) { + i_assert(i_stream_get_fd(client->input) >= 0); + client->io = io_add(i_stream_get_fd(client->input), + IO_READ, _client_input, client); + } + + /* if there's unread data in buffer, handle it. */ + if (!client->handling_input) { + (void)i_stream_get_data(client->input, &size); + if (size > 0 && !client->destroyed) + _client_input(client); + } + } } /* Skip incoming data until newline is found, @@ -322,7 +393,7 @@ return !client->input_skip_line; } -static bool client_handle_input(struct client_command_context *cmd) +static bool client_command_input(struct client_command_context *cmd) { struct client *client = cmd->client; @@ -330,33 +401,25 @@ /* command is being executed - continue it */ if (cmd->func(cmd) || cmd->param_error) { /* command execution was finished */ - client->bad_counter = 0; - _client_reset_command(client); + client_command_free(cmd); return TRUE; } + + /* unfinished */ + if (cmd->output_pending) + o_stream_set_flush_pending(client->output, TRUE); return FALSE; } - if (client->input_skip_line) { - /* we're just waiting for new line.. */ - if (!client_skip_line(client)) - return FALSE; - - /* got the newline */ - _client_reset_command(client); - - /* pass through to parse next command */ - } - if (cmd->tag == NULL) { - cmd->tag = imap_parser_read_word(client->parser); + cmd->tag = imap_parser_read_word(cmd->parser); if (cmd->tag == NULL) return FALSE; /* need more data */ cmd->tag = p_strdup(cmd->pool, cmd->tag); } if (cmd->name == NULL) { - cmd->name = imap_parser_read_word(client->parser); + cmd->name = imap_parser_read_word(cmd->parser); if (cmd->name == NULL) return FALSE; /* need more data */ cmd->name = p_strdup(cmd->pool, cmd->name); @@ -369,44 +432,57 @@ cmd->func = command_find(cmd->name); } + client->input_skip_line = TRUE; if (cmd->func == NULL) { /* unknown command */ client_send_command_error(cmd, "Unknown command."); - client->input_skip_line = TRUE; - _client_reset_command(client); + cmd->param_error = TRUE; + client_command_free(cmd); + return TRUE; } else { i_assert(!client->disconnected); - client->input_skip_line = TRUE; - if (cmd->func(cmd) || cmd->param_error) { - /* command execution was finished. */ - client->bad_counter = 0; - _client_reset_command(client); - } else { - /* unfinished */ - if (client->command_pending) { - o_stream_set_flush_pending(client->output, - TRUE); - } + return client_command_input(cmd); + } +} + +static bool client_handle_next_command(struct client *client) +{ + size_t size; + + if (client->input_lock != NULL) + return client_command_input(client->input_lock); + + if (client->input_skip_line) { + /* first eat the previous command line */ + if (!client_skip_line(client)) return FALSE; - } + client->input_skip_line = FALSE; } - return TRUE; + /* don't bother creating a new client command before there's at least + some input */ + (void)i_stream_get_data(client->input, &size); + if (size == 0) + return FALSE; + + /* beginning a new command */ + if (client->command_queue_size >= CLIENT_COMMAND_QUEUE_MAX_SIZE || + client->output_lock != NULL) { + /* wait for some of the commands to finish */ + io_remove(&client->io); + return FALSE; + } + + client->input_lock = client_command_new(client); + return client_command_input(client->input_lock); } void _client_input(struct client *client) { - struct client_command_context *cmd = &client->cmd; + struct client_command_context *cmd; int ret; - if (client->command_pending) { - /* already processing one command. wait. */ - io_remove(&client->io); - return; - } - - client->input_pending = FALSE; client->last_input = ioloop_time; switch (i_stream_read(client->input)) { @@ -420,31 +496,49 @@ until newline is found. */ client->input_skip_line = TRUE; + cmd = client->input_lock != NULL ? client->input_lock : + client_command_new(client); + cmd->param_error = TRUE; client_send_command_error(cmd, "Too long argument."); - _client_reset_command(client); - break; + client_command_free(cmd); + return; } o_stream_cork(client->output); + client->handling_input = TRUE; do { t_push(); - ret = client_handle_input(cmd); + ret = client_handle_next_command(client); t_pop(); } while (ret); + client->handling_input = FALSE; o_stream_uncork(client->output); - if (client->command_pending) - client->input_pending = TRUE; - if (client->output->closed) client_destroy(client, NULL); } +static void client_output_cmd(struct client_command_context *cmd) +{ + struct client *client = cmd->client; + bool finished; + + /* continue processing command */ + finished = cmd->func(cmd) || cmd->param_error; + + if (!finished) { + if (cmd->output_pending) + o_stream_set_flush_pending(client->output, TRUE); + } else { + /* command execution was finished */ + client_command_free(cmd); + } +} + int _client_output(struct client *client) { - struct client_command_context *cmd = &client->cmd; + struct client_command_context *cmd; int ret; - bool finished; client->last_output = ioloop_time; @@ -453,30 +547,18 @@ return 1; } - if (!client->command_pending) - return 1; - - /* continue processing command */ o_stream_cork(client->output); - client->output_pending = TRUE; - finished = cmd->func(cmd) || cmd->param_error; - - /* a bit kludgy check. normally we would want to get back to this - output handler, but IDLE is a special case which has command - pending but without necessarily anything to write. */ - if (!finished && client->output_pending) - o_stream_set_flush_pending(client->output, TRUE); - + if (client->output_lock != NULL) + client_output_cmd(client->output_lock); + if (client->output_lock == NULL) { + cmd = client->command_queue; + for (; cmd != NULL; cmd = cmd->next) { + client_output_cmd(cmd); + if (client->output_lock != NULL) + break; + } + } o_stream_uncork(client->output); - - if (finished) { - /* command execution was finished */ - client->bad_counter = 0; - _client_reset_command(client); - - if (client->input_pending) - _client_input(client); - } return ret; } @@ -490,15 +572,14 @@ idle_time = ioloop_time - I_MAX(my_client->last_input, my_client->last_output); - if (my_client->command_pending && - o_stream_get_buffer_used_size(my_client->output) > 0 && + if (o_stream_get_buffer_used_size(my_client->output) > 0 && idle_time >= CLIENT_OUTPUT_TIMEOUT) { /* client isn't reading our output */ client_destroy(my_client, "Disconnected for inactivity " "in reading our output"); } else if (idle_time >= CLIENT_IDLE_TIMEOUT) { /* client isn't sending us anything */ - if (!my_client->command_pending) { + if (my_client->output_lock == NULL) { client_send_line(my_client, "* BYE Disconnected for inactivity."); }
--- a/src/imap/client.h Wed Dec 20 17:40:22 2006 +0200 +++ b/src/imap/client.h Wed Dec 20 21:23:43 2006 +0200 @@ -3,6 +3,8 @@ #include "commands.h" +#define CLIENT_COMMAND_QUEUE_MAX_SIZE 4 + struct client; struct mail_storage; struct imap_parser; @@ -15,6 +17,7 @@ }; struct client_command_context { + struct client_command_context *prev, *next; struct client *client; pool_t pool; @@ -24,8 +27,12 @@ command_func_t *func; void *context; + struct imap_parser *parser; + unsigned int uid:1; /* used UID command */ + unsigned int cancel:1; /* command is wanted to be cancelled */ unsigned int param_error:1; + unsigned int output_pending:1; }; struct client { @@ -43,14 +50,20 @@ time_t last_input, last_output; unsigned int bad_counter; - struct imap_parser *parser; - struct client_command_context cmd; + /* one parser is kept here to be used for new commands */ + struct imap_parser *free_parser; + /* command_pool is cleared when the command queue gets empty */ + pool_t command_pool; + struct client_command_context *command_queue; + unsigned int command_queue_size; + + /* client input/output is locked by this command */ + struct client_command_context *input_lock; + struct client_command_context *output_lock; unsigned int disconnected:1; unsigned int destroyed:1; - unsigned int command_pending:1; - unsigned int input_pending:1; - unsigned int output_pending:1; + unsigned int handling_input:1; unsigned int rawlog:1; unsigned int input_skip_line:1; /* skip all the data until we've found a new line */ @@ -88,7 +101,9 @@ void clients_init(void); void clients_deinit(void); -void _client_reset_command(struct client *client); +void client_command_cancel(struct client_command_context *cmd); +void client_command_free(struct client_command_context *cmd); + void _client_input(struct client *client); int _client_output(struct client *client);
--- a/src/imap/cmd-append.c Wed Dec 20 17:40:22 2006 +0200 +++ b/src/imap/cmd-append.c Wed Dec 20 21:23:43 2006 +0200 @@ -24,14 +24,17 @@ struct imap_parser *save_parser; struct mail_save_context *save_ctx; + + unsigned int message_input:1; }; static void cmd_append_finish(struct cmd_append_context *ctx); static bool cmd_append_continue_message(struct client_command_context *cmd); -static void client_input(struct client *client) +static void client_input(struct client_command_context *cmd) { - struct client_command_context *cmd = &client->cmd; + struct cmd_append_context *ctx = cmd->context; + struct client *client = cmd->client; client->last_input = ioloop_time; @@ -41,12 +44,12 @@ cmd_append_finish(cmd->context); /* Reset command so that client_destroy() doesn't try to call cmd_append_continue_message() anymore. */ - _client_reset_command(client); + client_command_free(cmd); client_destroy(client, "Disconnected in APPEND"); return; case -2: cmd_append_finish(cmd->context); - if (client->command_pending) { + if (ctx->message_input) { /* message data, this is handled internally by mailbox_save_continue() */ break; @@ -58,22 +61,13 @@ client->input_skip_line = TRUE; client_send_command_error(cmd, "Too long argument."); - _client_reset_command(client); + cmd->param_error = TRUE; + client_command_free(cmd); return; } - if (cmd->func(cmd)) { - /* command execution was finished. Note that if cmd_sync() - didn't finish, we didn't get here but the input handler - has already been moved. So don't do anything important - here.. - - reset command once again to reset cmd_sync()'s changes. */ - _client_reset_command(client); - - if (client->input_pending) - _client_input(client); - } + if (cmd->func(cmd)) + client_command_free(cmd); } /* Returns -1 = error, 0 = need more data, 1 = successful. flags and @@ -112,25 +106,18 @@ static void cmd_append_finish(struct cmd_append_context *ctx) { - io_remove(&ctx->client->io); + imap_parser_destroy(&ctx->save_parser); - imap_parser_destroy(&ctx->save_parser); + i_assert(ctx->client->input_lock == ctx->cmd); if (ctx->input != NULL) i_stream_unref(&ctx->input); - if (ctx->save_ctx != NULL) mailbox_save_cancel(&ctx->save_ctx); - if (ctx->t != NULL) mailbox_transaction_rollback(&ctx->t); - if (ctx->box != ctx->cmd->client->mailbox && ctx->box != NULL) mailbox_close(&ctx->box); - - ctx->client->bad_counter = 0; - o_stream_set_flush_callback(ctx->client->output, - _client_output, ctx->client); } static bool cmd_append_continue_cancel(struct client_command_context *cmd) @@ -138,6 +125,11 @@ struct cmd_append_context *ctx = cmd->context; size_t size; + if (cmd->cancel) { + cmd_append_finish(ctx); + return TRUE; + } + (void)i_stream_read(ctx->input); (void)i_stream_get_data(ctx->input, &size); i_stream_skip(ctx->input, size); @@ -163,7 +155,7 @@ ctx->client->input->v_offset, ctx->msg_size); - ctx->client->command_pending = TRUE; + ctx->message_input = TRUE; ctx->cmd->func = cmd_append_continue_cancel; ctx->cmd->context = ctx; return cmd_append_continue_cancel(ctx->cmd); @@ -183,6 +175,11 @@ int ret, timezone_offset; bool nonsync; + if (cmd->cancel) { + cmd_append_finish(ctx); + return TRUE; + } + /* if error occurs, the CRLF is already read. */ client->input_skip_line = FALSE; @@ -293,7 +290,7 @@ o_stream_uncork(client->output); } - client->command_pending = TRUE; + ctx->message_input = TRUE; cmd->func = cmd_append_continue_message; return cmd_append_continue_message(cmd); } @@ -305,6 +302,11 @@ size_t size; bool failed; + if (cmd->cancel) { + cmd_append_finish(ctx); + return TRUE; + } + if (ctx->save_ctx != NULL) { if (mailbox_save_continue(ctx->save_ctx) < 0) { /* we still have to finish reading the message @@ -349,7 +351,7 @@ } /* prepare for next message */ - client->command_pending = FALSE; + ctx->message_input = FALSE; imap_parser_reset(ctx->save_parser); cmd->func = cmd_append_continue_parsing; return cmd_append_continue_parsing(cmd); @@ -395,6 +397,9 @@ if (!client_read_string_args(cmd, 1, &mailbox)) return FALSE; + /* we keep the input locked all the time */ + client->input_lock = cmd; + ctx = p_new(cmd->pool, struct cmd_append_context, 1); ctx->cmd = cmd; ctx->client = client; @@ -417,7 +422,7 @@ io_remove(&client->io); client->io = io_add(i_stream_get_fd(client->input), IO_READ, - client_input, client); + client_input, cmd); /* append is special because we're only waiting on client input, not client output, so disable the standard output handler until we're finished */
--- a/src/imap/cmd-fetch.c Wed Dec 20 17:40:22 2006 +0200 +++ b/src/imap/cmd-fetch.c Wed Dec 20 21:23:43 2006 +0200 @@ -118,13 +118,9 @@ struct imap_fetch_context *ctx = cmd->context; int ret; - if (cmd->client->output->closed) - ret = -1; - else { - if ((ret = imap_fetch(ctx)) == 0) { - /* unfinished */ - return FALSE; - } + if ((ret = imap_fetch(ctx)) == 0) { + /* unfinished */ + return FALSE; } if (ret < 0) ctx->failed = TRUE; @@ -134,7 +130,6 @@ bool cmd_fetch(struct client_command_context *cmd) { - struct client *client = cmd->client; struct imap_fetch_context *ctx; struct imap_arg *args; struct mail_search_arg *search_arg; @@ -170,7 +165,8 @@ imap_fetch_begin(ctx, search_arg); if ((ret = imap_fetch(ctx)) == 0) { /* unfinished */ - client->command_pending = TRUE; + cmd->output_pending = TRUE; + cmd->func = cmd_fetch_continue; cmd->context = ctx; return FALSE;
--- a/src/imap/cmd-idle.c Wed Dec 20 17:40:22 2006 +0200 +++ b/src/imap/cmd-idle.c Wed Dec 20 21:23:43 2006 +0200 @@ -62,12 +62,7 @@ client_send_tagline(ctx->cmd, "BAD Expected DONE."); o_stream_uncork(client->output); - - client->bad_counter = 0; - _client_reset_command(client); - - if (client->input_pending) - _client_input(client); + client_command_free(ctx->cmd); } static void idle_client_input(struct cmd_idle_context *ctx) @@ -134,7 +129,7 @@ static void keepalive_timeout(struct cmd_idle_context *ctx) { - if (ctx->client->output_pending) { + if (ctx->client->output_lock != NULL) { /* it's busy sending output */ return; } @@ -166,6 +161,11 @@ struct client *client = cmd->client; struct cmd_idle_context *ctx = cmd->context; + if (cmd->cancel) { + idle_finish(ctx, FALSE); + return TRUE; + } + if (ctx->manual_cork) { /* we're coming from idle_callback instead of a normal I/O handler, so we'll have to do corking manually */ @@ -179,6 +179,7 @@ ctx->manual_cork = FALSE; o_stream_uncork(client->output); } + cmd->output_pending = TRUE; return FALSE; } @@ -201,7 +202,7 @@ so we return here instead of doing everything twice. */ return FALSE; } - client->output_pending = FALSE; + cmd->output_pending = FALSE; if (ctx->manual_cork) { ctx->manual_cork = FALSE; @@ -255,7 +256,6 @@ client->io = io_add(i_stream_get_fd(client->input), IO_READ, idle_client_input, ctx); - client->command_pending = TRUE; cmd->func = cmd_idle_continue; cmd->context = ctx;
--- a/src/imap/cmd-list.c Wed Dec 20 17:40:22 2006 +0200 +++ b/src/imap/cmd-list.c Wed Dec 20 21:23:43 2006 +0200 @@ -389,6 +389,13 @@ struct cmd_list_context *ctx = cmd->context; int ret; + if (cmd->cancel) { + if (ctx->list_iter != NULL) { + if (mailbox_list_iter_deinit(&ctx->list_iter) < 0) + mail_storage_set_list_error(ctx->ns->storage); + } + return TRUE; + } for (; ctx->ns != NULL; ctx->ns = ctx->ns->next) { if (ctx->list_iter == NULL) list_namespace_init(cmd, ctx); @@ -497,7 +504,7 @@ cmd->context = ctx; if (!cmd_list_continue(cmd)) { /* unfinished */ - client->command_pending = TRUE; + cmd->output_pending = TRUE; cmd->func = cmd_list_continue; return FALSE; }
--- a/src/imap/cmd-search.c Wed Dec 20 17:40:22 2006 +0200 +++ b/src/imap/cmd-search.c Wed Dec 20 21:23:43 2006 +0200 @@ -6,61 +6,121 @@ #include "commands.h" #include "imap-search.h" -#define STRBUF_SIZE 1024 +#define OUTBUF_SIZE 65536 + +struct imap_search_context { + struct mailbox_transaction_context *trans; + struct mail_search_context *search_ctx; + struct mail *mail; -static bool imap_search(struct client_command_context *cmd, const char *charset, - struct mail_search_arg *sargs) + struct timeout *to; + string_t *output_buf; + + unsigned int output_sent:1; +}; + +static struct imap_search_context * +imap_search_init(struct client_command_context *cmd, const char *charset, + struct mail_search_arg *sargs) { - struct client *client = cmd->client; - struct mail_search_context *ctx; - struct mailbox_transaction_context *trans; - struct mail *mail; - string_t *str; + struct imap_search_context *ctx; + + ctx = p_new(cmd->pool, struct imap_search_context, 1); + ctx->trans = mailbox_transaction_begin(cmd->client->mailbox, 0); + ctx->search_ctx = mailbox_search_init(ctx->trans, charset, sargs, NULL); + ctx->mail = mail_alloc(ctx->trans, 0, NULL); + + ctx->output_buf = str_new(default_pool, OUTBUF_SIZE); + str_append(ctx->output_buf, "* SEARCH"); + return ctx; +} + +static int imap_search_deinit(struct client_command_context *cmd, + struct imap_search_context *ctx) +{ int ret; - bool uid, first = TRUE; + + mail_free(&ctx->mail); + ret = mailbox_search_deinit(&ctx->search_ctx); - str = t_str_new(STRBUF_SIZE); - uid = cmd->uid; + if (mailbox_transaction_commit(&ctx->trans, 0) < 0) + ret = -1; - trans = mailbox_transaction_begin(client->mailbox, 0); - ctx = mailbox_search_init(trans, charset, sargs, NULL); + if (ctx->output_sent || (ret == 0 && !cmd->cancel)) { + str_append(ctx->output_buf, "\r\n"); + o_stream_send(cmd->client->output, + str_data(ctx->output_buf), + str_len(ctx->output_buf)); + } + if (ctx->to != NULL) + timeout_remove(&ctx->to); + str_free(&ctx->output_buf); + return ret; +} - str_append(str, "* SEARCH"); - mail = mail_alloc(trans, 0, NULL); - while ((ret = mailbox_search_next(ctx, mail)) > 0) { - if (str_len(str) >= STRBUF_SIZE-MAX_INT_STRLEN) { - /* flush */ - o_stream_send(client->output, - str_data(str), str_len(str)); - str_truncate(str, 0); - first = FALSE; +static bool cmd_search_more(struct client_command_context *cmd) +{ + struct imap_search_context *ctx = cmd->context; + bool tryagain; + int ret; + + if (cmd->cancel) { + (void)imap_search_deinit(cmd, ctx); + return TRUE; + } + + while ((ret = mailbox_search_next_nonblock(ctx->search_ctx, ctx->mail, + &tryagain)) > 0) { + if (str_len(ctx->output_buf) >= OUTBUF_SIZE - MAX_INT_STRLEN) { + /* flush. this also causes us to lock the output. */ + cmd->client->output_lock = cmd; + o_stream_send(cmd->client->output, + str_data(ctx->output_buf), + str_len(ctx->output_buf)); + str_truncate(ctx->output_buf, 0); + ctx->output_sent = TRUE; } - str_printfa(str, " %u", uid ? mail->uid : mail->seq); + str_printfa(ctx->output_buf, " %u", + cmd->uid ? ctx->mail->uid : ctx->mail->seq); } - mail_free(&mail); + if (tryagain) + return FALSE; - ret = mailbox_search_deinit(&ctx); + if (imap_search_deinit(cmd, ctx) < 0) + ret = -1; + cmd->context = NULL; - if (mailbox_transaction_commit(&trans, 0) < 0) - ret = -1; + if (ret < 0) { + client_send_storage_error(cmd, + mailbox_get_storage(cmd->client->mailbox)); + return TRUE; + } else { + return cmd_sync(cmd, MAILBOX_SYNC_FLAG_FAST | + (cmd->uid ? 0 : MAILBOX_SYNC_FLAG_NO_EXPUNGES), + 0, "OK Search completed."); + } +} - if (!first || ret == 0) { - str_append(str, "\r\n"); - o_stream_send(client->output, str_data(str), str_len(str)); +static void cmd_search_more_callback(struct client_command_context *cmd) +{ + if (cmd_search_more(cmd)) + client_command_free(cmd); + else { + if (cmd->output_pending) + o_stream_set_flush_pending(cmd->client->output, TRUE); } - return ret == 0; } bool cmd_search(struct client_command_context *cmd) { - struct client *client = cmd->client; + struct imap_search_context *ctx; struct mail_search_arg *sargs; struct imap_arg *args; int args_count; const char *error, *charset; - args_count = imap_parser_read_args(client->parser, 0, 0, &args); + args_count = imap_parser_read_args(cmd->parser, 0, 0, &args); if (args_count < 1) { if (args_count == -2) return FALSE; @@ -69,6 +129,7 @@ "Missing SEARCH arguments."); return TRUE; } + cmd->client->input_lock = NULL; if (!client_verify_open_mailbox(cmd)) return TRUE; @@ -90,19 +151,21 @@ charset = NULL; } - sargs = imap_search_args_build(cmd->pool, client->mailbox, + sargs = imap_search_args_build(cmd->pool, cmd->client->mailbox, args, &error); if (sargs == NULL) { /* error in search arguments */ client_send_tagline(cmd, t_strconcat("NO ", error, NULL)); - } else if (imap_search(cmd, charset, sargs)) { - return cmd_sync(cmd, MAILBOX_SYNC_FLAG_FAST | - (cmd->uid ? 0 : MAILBOX_SYNC_FLAG_NO_EXPUNGES), - 0, "OK Search completed."); - } else { - client_send_storage_error(cmd, - mailbox_get_storage(client->mailbox)); + return TRUE; } - return TRUE; + ctx = imap_search_init(cmd, charset, sargs); + cmd->func = cmd_search_more; + cmd->context = ctx; + + if (cmd_search_more(cmd)) + return TRUE; + + ctx->to = timeout_add(0, cmd_search_more_callback, cmd); + return FALSE; }
--- a/src/imap/cmd-sort.c Wed Dec 20 17:40:22 2006 +0200 +++ b/src/imap/cmd-sort.c Wed Dec 20 21:23:43 2006 +0200 @@ -91,9 +91,10 @@ pool_t pool; const char *error, *charset; - args_count = imap_parser_read_args(client->parser, 0, 0, &args); + args_count = imap_parser_read_args(cmd->parser, 0, 0, &args); if (args_count == -2) return FALSE; + client->input_lock = NULL; if (args_count < 3) { client_send_command_error(cmd, args_count < 0 ? NULL :
--- a/src/imap/cmd-thread.c Wed Dec 20 17:40:22 2006 +0200 +++ b/src/imap/cmd-thread.c Wed Dec 20 21:23:43 2006 +0200 @@ -16,9 +16,10 @@ pool_t pool; const char *error, *charset, *str; - args_count = imap_parser_read_args(client->parser, 0, 0, &args); + args_count = imap_parser_read_args(cmd->parser, 0, 0, &args); if (args_count == -2) return FALSE; + client->input_lock = NULL; if (args_count < 3) { client_send_command_error(cmd, args_count < 0 ? NULL :
--- a/src/imap/cmd-uid.c Wed Dec 20 17:40:22 2006 +0200 +++ b/src/imap/cmd-uid.c Wed Dec 20 21:23:43 2006 +0200 @@ -8,7 +8,7 @@ const char *cmd_name; /* UID <command> <args> */ - cmd_name = imap_parser_read_word(cmd->client->parser); + cmd_name = imap_parser_read_word(cmd->parser); if (cmd_name == NULL) return FALSE;
--- a/src/imap/commands.c Wed Dec 20 17:40:22 2006 +0200 +++ b/src/imap/commands.c Wed Dec 20 21:23:43 2006 +0200 @@ -48,7 +48,8 @@ { "UID EXPUNGE", cmd_uid_expunge }, { "UID SORT", cmd_sort }, { "UID THREAD", cmd_thread }, - { "UNSELECT", cmd_unselect } + { "UNSELECT", cmd_unselect }, + { "X-CANCEL", cmd_x_cancel } }; #define IMAP_EXT_COMMANDS_COUNT \ (sizeof(imap_ext_commands) / sizeof(imap_ext_commands[0]))
--- a/src/imap/commands.h Wed Dec 20 17:40:22 2006 +0200 +++ b/src/imap/commands.h Wed Dec 20 21:23:43 2006 +0200 @@ -71,6 +71,7 @@ bool cmd_thread(struct client_command_context *cmd); bool cmd_uid_expunge(struct client_command_context *cmd); bool cmd_unselect(struct client_command_context *cmd); +bool cmd_x_cancel(struct client_command_context *cmd); /* private: */ bool _cmd_list_full(struct client_command_context *cmd, bool lsub);
--- a/src/imap/imap-fetch.c Wed Dec 20 17:40:22 2006 +0200 +++ b/src/imap/imap-fetch.c Wed Dec 20 21:23:43 2006 +0200 @@ -205,6 +205,7 @@ int imap_fetch(struct imap_fetch_context *ctx) { + struct client *client = ctx->client; const struct imap_fetch_context_handler *handlers; unsigned int size; int ret; @@ -229,16 +230,29 @@ ctx->cur_handler++; } + /* assume initially that we're locking it */ + i_assert(client->output_lock == NULL || + client->output_lock == ctx->cmd); + client->output_lock = ctx->cmd; + handlers = array_get(&ctx->handlers, &size); for (;;) { - if (o_stream_get_buffer_used_size(ctx->client->output) >= + if (o_stream_get_buffer_used_size(client->output) >= CLIENT_OUTPUT_OPTIMAL_SIZE) { - ret = o_stream_flush(ctx->client->output); - if (ret <= 0) + ret = o_stream_flush(client->output); + if (ret <= 0) { + if (!ctx->line_partial) { + /* last line was fully sent */ + client->output_lock = NULL; + } return ret; + } } if (ctx->cur_mail == NULL) { + if (ctx->cmd->cancel) + return 1; + if (ctx->cur_input != NULL) i_stream_unref(&ctx->cur_input); @@ -258,6 +272,7 @@ !handlers[ctx->cur_handler].buffered) { /* first non-buffered handler. flush the buffer. */ + ctx->line_partial = TRUE; if (imap_fetch_flush_buffer(ctx) < 0) return -1; } @@ -268,8 +283,13 @@ handlers[ctx->cur_handler].context); t_pop(); - if (ret == 0) + if (ret == 0) { + if (!ctx->line_partial) { + /* last line was fully sent */ + client->output_lock = NULL; + } return 0; + } if (ret < 0) { if (ctx->cur_mail->expunged) { @@ -293,7 +313,8 @@ } ctx->line_finished = TRUE; - if (o_stream_send(ctx->client->output, ")\r\n", 3) < 0) + ctx->line_partial = FALSE; + if (o_stream_send(client->output, ")\r\n", 3) < 0) return -1; ctx->cur_mail = NULL;
--- a/src/imap/imap-fetch.h Wed Dec 20 17:40:22 2006 +0200 +++ b/src/imap/imap-fetch.h Wed Dec 20 21:23:43 2006 +0200 @@ -57,6 +57,7 @@ unsigned int cur_have_eoh:1; unsigned int cur_append_eoh:1; unsigned int first:1; + unsigned int line_partial:1; unsigned int line_finished:1; unsigned int partial_fetch:1; unsigned int failed:1;
--- a/src/imap/imap-sync.c Wed Dec 20 17:40:22 2006 +0200 +++ b/src/imap/imap-sync.c Wed Dec 20 21:23:43 2006 +0200 @@ -204,8 +204,12 @@ struct cmd_sync_context *ctx = cmd->context; int ret; - if ((ret = imap_sync_more(ctx->sync_ctx)) == 0) - return FALSE; + if (cmd->cancel) + ret = 0; + else { + if ((ret = imap_sync_more(ctx->sync_ctx)) == 0) + return FALSE; + } if (ret < 0) ctx->sync_ctx->failed = TRUE; @@ -215,16 +219,21 @@ mailbox_get_storage(cmd->client->mailbox)); } - client_send_tagline(cmd, ctx->tagline); + if (!cmd->cancel) + client_send_tagline(cmd, ctx->tagline); return TRUE; } bool cmd_sync(struct client_command_context *cmd, enum mailbox_sync_flags flags, enum imap_sync_flags imap_flags, const char *tagline) { - struct cmd_sync_context *ctx; + struct client *client = cmd->client; + struct cmd_sync_context *ctx; - if (cmd->client->mailbox == NULL) { + i_assert(client->output_lock == cmd || client->output_lock == NULL); + + if (client->mailbox == NULL || + mailbox_transaction_get_count(client->mailbox) > 0) { client_send_tagline(cmd, tagline); return TRUE; } @@ -241,11 +250,14 @@ ctx = p_new(cmd->pool, struct cmd_sync_context, 1); ctx->tagline = p_strdup(cmd->pool, tagline); - ctx->sync_ctx = imap_sync_init(cmd->client, cmd->client->mailbox, + ctx->sync_ctx = imap_sync_init(client, client->mailbox, imap_flags, flags); cmd->func = cmd_sync_continue; cmd->context = ctx; - cmd->client->command_pending = TRUE; + cmd->output_pending = TRUE; + if (client->input_lock == cmd) + client->input_lock = NULL; + client->output_lock = NULL; return cmd_sync_continue(cmd); }
--- a/src/lib-storage/index/dbox/dbox-storage.c Wed Dec 20 17:40:22 2006 +0200 +++ b/src/lib-storage/index/dbox/dbox-storage.c Wed Dec 20 21:23:43 2006 +0200 @@ -628,7 +628,7 @@ index_header_lookup_deinit, index_storage_search_init, index_storage_search_deinit, - index_storage_search_next, + index_storage_search_next_nonblock, index_storage_search_next_update_seq, dbox_save_init, dbox_save_continue,
--- a/src/lib-storage/index/index-search.c Wed Dec 20 17:40:22 2006 +0200 +++ b/src/lib-storage/index/index-search.c Wed Dec 20 21:23:43 2006 +0200 @@ -21,6 +21,8 @@ #define TXT_UNKNOWN_CHARSET "[BADCHARSET] Unknown charset" #define TXT_INVALID_SEARCH_KEY "Invalid search key" +#define SEARCH_NONBLOCK_COUNT 20 + struct index_search_context { struct mail_search_context mail_ctx; struct mail_index_view *view; @@ -952,15 +954,21 @@ return TRUE; } -int index_storage_search_next(struct mail_search_context *_ctx, - struct mail *mail) +int index_storage_search_next_nonblock(struct mail_search_context *_ctx, + struct mail *mail, bool *tryagain_r) { struct index_search_context *ctx = (struct index_search_context *)_ctx; struct mailbox *box = _ctx->transaction->box; + unsigned int count = 0; int ret; - if (ctx->sorted) + *tryagain_r = FALSE; + + if (ctx->sorted) { + /* everything searched at this point already. just returning + matches from sort list */ return index_sort_list_next(ctx->mail_ctx.sort_program, mail); + } ctx->mail = mail; ctx->imail = (struct index_mail *)mail; @@ -989,6 +997,11 @@ break; } } + + if (++count == SEARCH_NONBLOCK_COUNT) { + *tryagain_r = TRUE; + return 0; + } } if (ret < 0) ctx->failed = TRUE; @@ -996,10 +1009,13 @@ ctx->imail = NULL; if (ctx->mail_ctx.sort_program != NULL && ret == 0) { + /* finished searching the messages. now sort them and start + returning the messages. */ ctx->sorted = TRUE; if (index_sort_list_finish(ctx->mail_ctx.sort_program) < 0) return -1; - return index_sort_list_next(ctx->mail_ctx.sort_program, mail); + return index_storage_search_next_nonblock(_ctx, mail, + tryagain_r); } return ret;
--- a/src/lib-storage/index/index-storage.h Wed Dec 20 17:40:22 2006 +0200 +++ b/src/lib-storage/index/index-storage.h Wed Dec 20 21:23:43 2006 +0200 @@ -179,6 +179,8 @@ int index_storage_search_deinit(struct mail_search_context *ctx); int index_storage_search_next(struct mail_search_context *ctx, struct mail *mail); +int index_storage_search_next_nonblock(struct mail_search_context *ctx, + struct mail *mail, bool *tryagain_r); int index_storage_search_next_update_seq(struct mail_search_context *ctx); void index_transaction_init(struct index_transaction_context *t,
--- a/src/lib-storage/index/maildir/maildir-storage.c Wed Dec 20 17:40:22 2006 +0200 +++ b/src/lib-storage/index/maildir/maildir-storage.c Wed Dec 20 21:23:43 2006 +0200 @@ -1013,7 +1013,7 @@ index_header_lookup_deinit, index_storage_search_init, index_storage_search_deinit, - index_storage_search_next, + index_storage_search_next_nonblock, index_storage_search_next_update_seq, maildir_save_init, maildir_save_continue,
--- a/src/lib-storage/index/mbox/mbox-storage.c Wed Dec 20 17:40:22 2006 +0200 +++ b/src/lib-storage/index/mbox/mbox-storage.c Wed Dec 20 21:23:43 2006 +0200 @@ -1093,7 +1093,7 @@ index_header_lookup_deinit, index_storage_search_init, index_storage_search_deinit, - index_storage_search_next, + index_storage_search_next_nonblock, index_storage_search_next_update_seq, mbox_save_init, mbox_save_continue,
--- a/src/lib-storage/mail-storage-private.h Wed Dec 20 17:40:22 2006 +0200 +++ b/src/lib-storage/mail-storage-private.h Wed Dec 20 21:23:43 2006 +0200 @@ -129,7 +129,8 @@ const char *charset, struct mail_search_arg *args, const enum mail_sort_type *sort_program); int (*search_deinit)(struct mail_search_context *ctx); - int (*search_next)(struct mail_search_context *ctx, struct mail *mail); + int (*search_next_nonblock)(struct mail_search_context *ctx, + struct mail *mail, bool *tryagain_r); /* Internal search function which updates ctx->seq */ int (*search_next_update_seq)(struct mail_search_context *ctx); @@ -159,6 +160,8 @@ /* private: */ pool_t pool; + unsigned int transaction_count; + /* Module-specific contexts. See mail_storage_module_id. */ ARRAY_DEFINE(module_contexts, void);
--- a/src/lib-storage/mail-storage.c Wed Dec 20 17:40:22 2006 +0200 +++ b/src/lib-storage/mail-storage.c Wed Dec 20 21:23:43 2006 +0200 @@ -541,13 +541,30 @@ int mailbox_search_next(struct mail_search_context *ctx, struct mail *mail) { - return ctx->transaction->box->v.search_next(ctx, mail); + bool tryagain; + int ret; + + while ((ret = mailbox_search_next_nonblock(ctx, mail, + &tryagain)) == 0) { + if (!tryagain) + break; + } + + return ret; +} + +int mailbox_search_next_nonblock(struct mail_search_context *ctx, + struct mail *mail, bool *tryagain_r) +{ + return ctx->transaction->box->v. + search_next_nonblock(ctx, mail, tryagain_r); } struct mailbox_transaction_context * mailbox_transaction_begin(struct mailbox *box, enum mailbox_transaction_flags flags) { + box->transaction_count++; return box->v.transaction_begin(box, flags); } @@ -556,6 +573,8 @@ { struct mailbox_transaction_context *t = *_t; + t->box->transaction_count--; + *_t = NULL; return t->box->v.transaction_commit(t, flags); } @@ -564,10 +583,17 @@ { struct mailbox_transaction_context *t = *_t; + t->box->transaction_count--; + *_t = NULL; t->box->v.transaction_rollback(t); } +unsigned int mailbox_transaction_get_count(struct mailbox *box) +{ + return box->transaction_count; +} + int mailbox_save_init(struct mailbox_transaction_context *t, enum mail_flags flags, struct mail_keywords *keywords, time_t received_date, int timezone_offset,
--- a/src/lib-storage/mail-storage.h Wed Dec 20 17:40:22 2006 +0200 +++ b/src/lib-storage/mail-storage.h Wed Dec 20 21:23:43 2006 +0200 @@ -324,6 +324,8 @@ int mailbox_transaction_commit(struct mailbox_transaction_context **t, enum mailbox_sync_flags flags); void mailbox_transaction_rollback(struct mailbox_transaction_context **t); +/* Return the number of active transactions for the mailbox. */ +unsigned int mailbox_transaction_get_count(struct mailbox *box); /* Build mail_keywords from NULL-terminated keywords list. */ struct mail_keywords * @@ -352,6 +354,11 @@ int mailbox_search_deinit(struct mail_search_context **ctx); /* Search the next message. Returns 1 if found, 0 if not, -1 if failure. */ int mailbox_search_next(struct mail_search_context *ctx, struct mail *mail); +/* Like mailbox_search_next(), but don't spend too much time searching. + Returns 1 if found, -1 if failure or 0 with tryagain_r=FALSE if + finished, and TRUE if more results will by calling the function again. */ +int mailbox_search_next_nonblock(struct mail_search_context *ctx, + struct mail *mail, bool *tryagain_r); /* Save a mail into mailbox. timezone_offset specifies the timezone in minutes in which received_date was originally given with. To use