Mercurial > dovecot > core-2.2
changeset 12763:345100da8c67
imapc: Many fixes related to syncing and error handling.
author | Timo Sirainen <tss@iki.fi> |
---|---|
date | Thu, 24 Feb 2011 15:17:47 +0200 |
parents | 74300385cce0 |
children | 0c1e221fc140 |
files | src/lib-storage/index/imapc/imapc-client.c src/lib-storage/index/imapc/imapc-client.h src/lib-storage/index/imapc/imapc-connection.c src/lib-storage/index/imapc/imapc-connection.h src/lib-storage/index/imapc/imapc-mailbox.c src/lib-storage/index/imapc/imapc-save.c src/lib-storage/index/imapc/imapc-seqmap.c src/lib-storage/index/imapc/imapc-seqmap.h src/lib-storage/index/imapc/imapc-storage.c src/lib-storage/index/imapc/imapc-storage.h src/lib-storage/index/imapc/imapc-sync.c src/lib-storage/index/imapc/imapc-sync.h |
diffstat | 12 files changed, 341 insertions(+), 122 deletions(-) [+] |
line wrap: on
line diff
--- a/src/lib-storage/index/imapc/imapc-client.c Thu Feb 24 13:19:05 2011 +0200 +++ b/src/lib-storage/index/imapc/imapc-client.c Thu Feb 24 15:17:47 2011 +0200 @@ -84,8 +84,10 @@ if (client->ssl_ctx != NULL) ssl_iostream_context_deinit(&client->ssl_ctx); - array_foreach_modifiable(&client->conns, connp) + array_foreach_modifiable(&client->conns, connp) { imapc_connection_deinit(&(*connp)->conn); + i_free(*connp); + } pool_unref(&client->pool); } @@ -212,6 +214,12 @@ return box; } +void imapc_client_mailbox_disconnect(struct imapc_client_mailbox *box) +{ + if (box->conn != NULL) + imapc_connection_disconnect(box->conn); +} + void imapc_client_mailbox_close(struct imapc_client_mailbox **_box) { struct imapc_client_mailbox *box = *_box; @@ -226,7 +234,8 @@ } } - imapc_connection_unselect(box); + if (box->conn != NULL) + imapc_connection_unselect(box); imapc_seqmap_deinit(&box->seqmap); i_free(box); } @@ -258,12 +267,43 @@ return ctx; } +static bool +imapc_client_mailbox_is_selected(struct imapc_client_mailbox *box, + struct imapc_command_reply *reply_r) +{ + struct imapc_client_mailbox *selected_box; + + selected_box = box->conn == NULL ? NULL : + imapc_connection_get_mailbox(box->conn); + if (selected_box == box) + return TRUE; + + memset(reply_r, 0, sizeof(*reply_r)); + reply_r->state = IMAPC_COMMAND_STATE_DISCONNECTED; + if (selected_box == NULL) { + reply_r->text_full = "Disconnected from server"; + } else { + i_error("imapc: Selected mailbox changed unexpectedly"); + reply_r->text_full = "Internal error"; + } + reply_r->text_without_resp = reply_r->text_full; + + box->conn = NULL; + return FALSE; +} + void imapc_client_mailbox_cmd(struct imapc_client_mailbox *box, const char *cmd, imapc_command_callback_t *callback, void *context) { struct imapc_client_command_context *ctx; + struct imapc_command_reply reply; + + if (!imapc_client_mailbox_is_selected(box, &reply)) { + callback(&reply, context); + return; + } ctx = imapc_client_mailbox_cmd_common(box, callback, context); imapc_connection_cmd(box->conn, cmd, imapc_client_mailbox_cmd_cb, ctx); @@ -275,6 +315,12 @@ { struct imapc_client_command_context *ctx; va_list args; + struct imapc_command_reply reply; + + if (!imapc_client_mailbox_is_selected(box, &reply)) { + callback(&reply, context); + return; + } ctx = imapc_client_mailbox_cmd_common(box, callback, context); va_start(args, cmd_fmt); @@ -291,7 +337,17 @@ void imapc_client_mailbox_idle(struct imapc_client_mailbox *box) { - imapc_connection_idle(box->conn); + struct imapc_command_reply reply; + + if (imapc_client_mailbox_is_selected(box, &reply)) + imapc_connection_idle(box->conn); +} + +bool imapc_client_mailbox_is_connected(struct imapc_client_mailbox *box) +{ + struct imapc_command_reply reply; + + return imapc_client_mailbox_is_selected(box, &reply); } enum imapc_capability
--- a/src/lib-storage/index/imapc/imapc-client.h Thu Feb 24 13:19:05 2011 +0200 +++ b/src/lib-storage/index/imapc/imapc-client.h Thu Feb 24 15:17:47 2011 +0200 @@ -123,6 +123,7 @@ imapc_command_callback_t *callback, void *context, void *untagged_box_context); void imapc_client_mailbox_close(struct imapc_client_mailbox **box); +void imapc_client_mailbox_disconnect(struct imapc_client_mailbox *box); void imapc_client_mailbox_cmd(struct imapc_client_mailbox *box, const char *cmd, imapc_command_callback_t *callback, @@ -135,6 +136,7 @@ imapc_client_mailbox_get_seqmap(struct imapc_client_mailbox *box); void imapc_client_mailbox_idle(struct imapc_client_mailbox *box); +bool imapc_client_mailbox_is_connected(struct imapc_client_mailbox *box); enum imapc_capability imapc_client_get_capabilities(struct imapc_client *client);
--- a/src/lib-storage/index/imapc/imapc-connection.c Thu Feb 24 13:19:05 2011 +0200 +++ b/src/lib-storage/index/imapc/imapc-connection.c Thu Feb 24 15:17:47 2011 +0200 @@ -22,14 +22,15 @@ #define IMAPC_DNS_LOOKUP_TIMEOUT_MSECS (1000*30) #define IMAPC_CONNECT_TIMEOUT_MSECS (1000*30) +#define IMAPC_COMMAND_TIMEOUT_MSECS (1000*60*5) #define IMAPC_MAX_INLINE_LITERAL_SIZE (1024*32) enum imapc_input_state { IMAPC_INPUT_STATE_NONE = 0, + IMAPC_INPUT_STATE_PLUS, IMAPC_INPUT_STATE_UNTAGGED, IMAPC_INPUT_STATE_UNTAGGED_NUM, - IMAPC_INPUT_STATE_TAGGED, - IMAPC_INPUT_STATE_SKIPLINE + IMAPC_INPUT_STATE_TAGGED }; struct imapc_command_stream { @@ -84,7 +85,9 @@ enum imapc_capability capabilities; char **capabilities_list; + /* commands pending in queue to be sent */ ARRAY_DEFINE(cmd_send_queue, struct imapc_command *); + /* commands that have been sent, waiting for their tagged reply */ ARRAY_DEFINE(cmd_wait_list, struct imapc_command *); unsigned int ips_count, prev_connect_idx; @@ -99,8 +102,6 @@ }; static int imapc_connection_ssl_init(struct imapc_connection *conn); -static void imapc_connection_disconnect(struct imapc_connection *conn); - static void imapc_command_free(struct imapc_command *cmd); static void imapc_command_send_more(struct imapc_connection *conn, struct imapc_command *cmd); @@ -147,39 +148,50 @@ conn->to = io_loop_move_timeout(&conn->to); } +static void +imapc_connection_abort_pending_commands(struct imapc_connection *conn, + const struct imapc_command_reply *reply) +{ + struct imapc_command *const *cmdp, *cmd; + + while (array_count(&conn->cmd_wait_list) > 0) { + cmdp = array_idx(&conn->cmd_wait_list, 0); + cmd = *cmdp; + array_delete(&conn->cmd_wait_list, 0, 1); + + if (cmd->callback != NULL) + cmd->callback(reply, cmd->context); + imapc_command_free(cmd); + } + while (array_count(&conn->cmd_send_queue) > 0) { + cmdp = array_idx(&conn->cmd_send_queue, 0); + cmd = *cmdp; + array_delete(&conn->cmd_send_queue, 0, 1); + + if (cmd->callback != NULL) + cmd->callback(reply, cmd->context); + imapc_command_free(cmd); + } +} + static void imapc_connection_set_state(struct imapc_connection *conn, enum imapc_connection_state state) { if (state == IMAPC_CONNECTION_STATE_DISCONNECTED) { - /* abort all pending commands */ struct imapc_command_reply reply; - struct imapc_command *const *cmdp, *cmd; memset(&reply, 0, sizeof(reply)); reply.state = IMAPC_COMMAND_STATE_DISCONNECTED; reply.text_without_resp = reply.text_full = "Disconnected from server"; + imapc_connection_abort_pending_commands(conn, &reply); conn->idling = FALSE; conn->idle_plus_waiting = FALSE; conn->idle_stopping = FALSE; - while (array_count(&conn->cmd_wait_list) > 0) { - cmdp = array_idx(&conn->cmd_wait_list, 0); - cmd = *cmdp; - array_delete(&conn->cmd_wait_list, 0, 1); - - cmd->callback(&reply, cmd->context); - imapc_command_free(cmd); - } - while (array_count(&conn->cmd_send_queue) > 0) { - cmdp = array_idx(&conn->cmd_send_queue, 0); - cmd = *cmdp; - array_delete(&conn->cmd_send_queue, 0, 1); - - cmd->callback(&reply, cmd->context); - imapc_command_free(cmd); - } + conn->selecting_box = NULL; + conn->selected_box = NULL; } if (state == IMAPC_CONNECTION_STATE_DONE) { if (array_count(&conn->cmd_send_queue) > 0) { @@ -215,7 +227,7 @@ literal->fd = -1; } -static void imapc_connection_disconnect(struct imapc_connection *conn) +void imapc_connection_disconnect(struct imapc_connection *conn) { if (conn->fd == -1) return; @@ -375,10 +387,22 @@ imapc_connection_read_line(struct imapc_connection *conn, const struct imap_arg **imap_args_r) { + const unsigned char *data; + size_t size; int ret; while ((ret = imapc_connection_read_line_more(conn, imap_args_r)) == 2) ; + + if (ret > 0) { + data = i_stream_get_data(conn->input, &size); + if (size >= 2 && data[0] == '\r' && data[1] == '\n') + i_stream_skip(conn->input, 2); + else if (size >= 1 && data[0] == '\n') + i_stream_skip(conn->input, 1); + else + i_panic("imapc: Missing LF from input line"); + } return ret; } @@ -506,29 +530,11 @@ conn->input_state = IMAPC_INPUT_STATE_NONE; conn->cur_tag = 0; conn->cur_num = 0; - imap_parser_reset(conn->parser); + if (conn->parser != NULL) + imap_parser_reset(conn->parser); imapc_connection_lfiles_free(conn); } -static int imapc_connection_skip_line(struct imapc_connection *conn) -{ - const unsigned char *data; - size_t i, data_size; - int ret = 0; - - data = i_stream_get_data(conn->input, &data_size); - for (i = 0; i < data_size; i++) { - if (data[i] == '\n') { - imapc_connection_input_reset(conn); - ret = 1; - i++; - break; - } - } - i_stream_skip(conn->input, i); - return ret; -} - static void imapc_connection_login_cb(const struct imapc_command_reply *reply, void *context) { @@ -749,11 +755,6 @@ conn->selected_box->untagged_box_context; } conn->client->untagged_callback(&reply, conn->client->untagged_context); - if (imap_arg_atom_equals(imap_args, "EXPUNGE") && - conn->selected_box != NULL) { - /* keep track of expunge map internally */ - imapc_seqmap_expunge(conn->selected_box->seqmap, conn->cur_num); - } imapc_connection_input_reset(conn); return 1; } @@ -761,23 +762,26 @@ static int imapc_connection_input_plus(struct imapc_connection *conn) { struct imapc_command *const *cmd_p; + const char *line; + + if ((line = i_stream_next_line(conn->input)) == NULL) + return 0; if (conn->idle_plus_waiting) { /* "+ idling" reply for IDLE command */ conn->idle_plus_waiting = FALSE; conn->idling = TRUE; - return imapc_connection_skip_line(conn); + } else if (array_count(&conn->cmd_send_queue) > 0) { + /* reply for literal */ + cmd_p = array_idx(&conn->cmd_send_queue, 0); + imapc_command_send_more(conn, *cmd_p); + } else { + imapc_connection_input_error(conn, "Unexpected '+': %s", line); + return -1; } - if (array_count(&conn->cmd_send_queue) == 0) { - imapc_connection_input_error(conn, "Unexpected '+'"); - return -1; - } - cmd_p = array_idx(&conn->cmd_send_queue, 0); - imapc_command_send_more(conn, *cmd_p); - - conn->input_state = IMAPC_INPUT_STATE_SKIPLINE; - return imapc_connection_skip_line(conn); + imapc_connection_input_reset(conn); + return 1; } static int imapc_connection_input_tagged(struct imapc_connection *conn) @@ -806,14 +810,12 @@ reply.state = IMAPC_COMMAND_STATE_OK; else if (strcasecmp(line, "no") == 0) reply.state = IMAPC_COMMAND_STATE_NO; - else if (strcasecmp(line, "bad") == 0) { - i_error("imapc(%s): Command failed with BAD: %u %s", - conn->name, conn->cur_tag, line); + else if (strcasecmp(line, "bad") == 0) reply.state = IMAPC_COMMAND_STATE_BAD; - } else { + else { imapc_connection_input_error(conn, - "Invalid state in tagged reply: %u %s", - conn->cur_tag, line); + "Invalid state in tagged reply: %u %s %s", + conn->cur_tag, line, reply.text_full); return -1; } @@ -849,15 +851,27 @@ } } } + if (array_count(&conn->cmd_wait_list) == 0 && + array_count(&conn->cmd_send_queue) == 0 && + conn->state == IMAPC_CONNECTION_STATE_DONE && conn->to != NULL) + timeout_remove(&conn->to); if (cmd == NULL) { imapc_connection_input_error(conn, - "Unknown tag in a reply: %u %s", conn->cur_tag, line); + "Unknown tag in a reply: %u %s %s", + conn->cur_tag, line, reply.text_full); return -1; } + if (reply.state == IMAPC_COMMAND_STATE_BAD) { + i_error("imapc(%s): Command '%s' failed with BAD: %u %s", + conn->name, str_c(cmd->data), conn->cur_tag, + reply.text_full); + } + imapc_connection_input_reset(conn); - cmd->callback(&reply, cmd->context); + if (cmd->callback != NULL) + cmd->callback(&reply, cmd->context); imapc_command_free(cmd); return 1; } @@ -876,15 +890,12 @@ if (tag == NULL) return 0; - if (strcmp(tag, "") == 0) { - /* FIXME: why do we get here.. */ - conn->input_state = IMAPC_INPUT_STATE_SKIPLINE; - return imapc_connection_skip_line(conn); - } else if (strcmp(tag, "*") == 0) { + if (strcmp(tag, "*") == 0) { conn->input_state = IMAPC_INPUT_STATE_UNTAGGED; conn->cur_num = 0; ret = imapc_connection_input_untagged(conn); } else if (strcmp(tag, "+") == 0) { + conn->input_state = IMAPC_INPUT_STATE_PLUS; ret = imapc_connection_input_plus(conn); } else { conn->input_state = IMAPC_INPUT_STATE_TAGGED; @@ -898,6 +909,9 @@ } } break; + case IMAPC_INPUT_STATE_PLUS: + ret = imapc_connection_input_plus(conn); + break; case IMAPC_INPUT_STATE_UNTAGGED: case IMAPC_INPUT_STATE_UNTAGGED_NUM: ret = imapc_connection_input_untagged(conn); @@ -905,9 +919,6 @@ case IMAPC_INPUT_STATE_TAGGED: ret = imapc_connection_input_tagged(conn); break; - case IMAPC_INPUT_STATE_SKIPLINE: - ret = imapc_connection_skip_line(conn); - break; } return ret; } @@ -1092,6 +1103,8 @@ if (conn->fd != -1) return; + imapc_connection_input_reset(conn); + if (conn->client->set.debug) i_debug("imapc(%s): Looking up IP address", conn->name); @@ -1116,6 +1129,9 @@ if (conn->input == NULL) return; + if (conn->to != NULL) + timeout_reset(conn->to); + o_stream_cork(conn->output); while (ret > 0 && !conn->client->stop_now && conn->input != NULL) { T_BEGIN { @@ -1134,7 +1150,7 @@ struct imapc_command *cmd; pool_t pool; - pool = pool_alloconly_create("imapc command", 1024); + pool = pool_alloconly_create("imapc command", 2048); cmd = p_new(pool, struct imapc_command, 1); cmd->pool = pool; cmd->callback = callback; @@ -1273,6 +1289,19 @@ } } +static void imapc_command_timeout(struct imapc_connection *conn) +{ + struct imapc_command *const *cmds; + unsigned int count; + + cmds = array_get(&conn->cmd_wait_list, &count); + i_assert(count > 0); + + i_error("imapc(%s): Command '%s' timed out, disconnecting", + conn->name, str_c(cmds[0]->data)); + imapc_connection_disconnect(conn); +} + static void imapc_command_send(struct imapc_connection *conn, struct imapc_command *cmd) { @@ -1286,6 +1315,14 @@ imapc_command_send_more(conn, cmd); break; case IMAPC_CONNECTION_STATE_DONE: + if (cmd->idle) { + if (conn->to != NULL) + timeout_remove(&conn->to); + } else if (conn->to == NULL) { + conn->to = timeout_add(IMAPC_COMMAND_TIMEOUT_MSECS, + imapc_command_timeout, conn); + } + array_append(&conn->cmd_send_queue, &cmd, 1); if (array_count(&conn->cmd_send_queue) == 1) imapc_command_send_more(conn, cmd); @@ -1436,11 +1473,47 @@ void imapc_connection_unselect(struct imapc_client_mailbox *box) { - i_assert(box->conn->selected_box == box || - box->conn->selecting_box == box); + struct imapc_command *const *cmdp; + struct imapc_command_reply reply; + + /* mailbox is being closed. if there are any pending commands, we must + finish them immediately so callbacks don't access any freed + contexts */ + memset(&reply, 0, sizeof(reply)); + reply.state = IMAPC_COMMAND_STATE_DISCONNECTED; + reply.text_without_resp = reply.text_full = "Closing mailbox"; - box->conn->selected_box = NULL; - box->conn->selecting_box = NULL; + array_foreach(&box->conn->cmd_wait_list, cmdp) { + if ((*cmdp)->callback != NULL) { + (*cmdp)->callback(&reply, (*cmdp)->context); + (*cmdp)->callback = NULL; + } + } + array_foreach(&box->conn->cmd_send_queue, cmdp) { + if ((*cmdp)->callback != NULL) { + (*cmdp)->callback(&reply, (*cmdp)->context); + (*cmdp)->callback = NULL; + } + } + + if (box->conn->selected_box == NULL && + box->conn->selecting_box == NULL) { + i_assert(box->conn->state == IMAPC_CONNECTION_STATE_DISCONNECTED); + } else { + i_assert(box->conn->selected_box == box || + box->conn->selecting_box == box); + + box->conn->selected_box = NULL; + box->conn->selecting_box = NULL; + } +} + +struct imapc_client_mailbox * +imapc_connection_get_mailbox(struct imapc_connection *conn) +{ + if (conn->selecting_box != NULL) + return conn->selecting_box; + return conn->selected_box; } static void
--- a/src/lib-storage/index/imapc/imapc-connection.h Thu Feb 24 13:19:05 2011 +0200 +++ b/src/lib-storage/index/imapc/imapc-connection.h Thu Feb 24 15:17:47 2011 +0200 @@ -22,6 +22,7 @@ void imapc_connection_deinit(struct imapc_connection **conn); void imapc_connection_connect(struct imapc_connection *conn); +void imapc_connection_disconnect(struct imapc_connection *conn); void imapc_connection_ioloop_changed(struct imapc_connection *conn); void imapc_connection_input_pending(struct imapc_connection *conn); @@ -43,6 +44,9 @@ enum imapc_capability imapc_connection_get_capabilities(struct imapc_connection *conn); +struct imapc_client_mailbox * +imapc_connection_get_mailbox(struct imapc_connection *conn); + void imapc_connection_idle(struct imapc_connection *conn); #endif
--- a/src/lib-storage/index/imapc/imapc-mailbox.c Thu Feb 24 13:19:05 2011 +0200 +++ b/src/lib-storage/index/imapc/imapc-mailbox.c Thu Feb 24 15:17:47 2011 +0200 @@ -10,13 +10,28 @@ #define NOTIFY_DELAY_MSECS 500 +static void imapc_mailbox_set_corrupted(struct imapc_mailbox *mbox, + const char *reason, ...) +{ + va_list va; + + va_start(va, reason); + i_error("imapc: Mailbox '%s' state corrupted: %s", + mbox->box.name, t_strdup_vprintf(reason, va)); + va_end(va); + + mail_index_mark_corrupted(mbox->box.index); + imapc_client_mailbox_disconnect(mbox->client_box); +} + static void imapc_mailbox_init_delayed_trans(struct imapc_mailbox *mbox) { if (mbox->delayed_sync_trans != NULL) return; + mbox->sync_view = mail_index_view_open(mbox->box.index); mbox->delayed_sync_trans = - mail_index_transaction_begin(mbox->box.view, + mail_index_transaction_begin(mbox->sync_view, MAIL_INDEX_TRANSACTION_FLAG_EXTERNAL); mbox->delayed_sync_view = mail_index_transaction_open_updated_view(mbox->delayed_sync_trans); @@ -44,6 +59,7 @@ static void imapc_untagged_exists(const struct imapc_untagged_reply *reply, struct imapc_mailbox *mbox) { + struct mail_index_view *view = mbox->delayed_sync_view; uint32_t rcount = reply->num; const struct mail_index_header *hdr; struct imapc_seqmap *seqmap; @@ -52,13 +68,16 @@ if (mbox == NULL) return; + if (view == NULL) + view = mbox->box.view; + seqmap = imapc_client_mailbox_get_seqmap(mbox->client_box); - next_lseq = mail_index_view_get_messages_count(mbox->box.view) + 1; + next_lseq = mail_index_view_get_messages_count(view) + 1; next_rseq = imapc_seqmap_lseq_to_rseq(seqmap, next_lseq); if (next_rseq > rcount) return; - hdr = mail_index_get_header(mbox->box.view); + hdr = mail_index_get_header(view); mbox->new_msgs = TRUE; imapc_client_mailbox_cmdf(mbox->client_box, imapc_newmsgs_callback, @@ -85,7 +104,7 @@ static void imapc_untagged_fetch(const struct imapc_untagged_reply *reply, struct imapc_mailbox *mbox) { - uint32_t seq = reply->num; + uint32_t lseq, rseq = reply->num; struct imapc_seqmap *seqmap; const struct imap_arg *list, *flags_list; const char *atom; @@ -96,7 +115,7 @@ ARRAY_TYPE(const_string) keywords = ARRAY_INIT; bool seen_flags = FALSE; - if (mbox == NULL || seq == 0 || !imap_arg_get_list(reply->args, &list)) + if (mbox == NULL || rseq == 0 || !imap_arg_get_list(reply->args, &list)) return; uid = 0; flags = 0; @@ -126,26 +145,33 @@ } } } + /* FIXME: need to do something about recent flags */ + flags &= ~MAIL_RECENT; seqmap = imapc_client_mailbox_get_seqmap(mbox->client_box); - seq = imapc_seqmap_rseq_to_lseq(seqmap, seq); + lseq = imapc_seqmap_rseq_to_lseq(seqmap, rseq); - if (mbox->cur_fetch_mail != NULL && mbox->cur_fetch_mail->seq == seq) { + if (mbox->cur_fetch_mail != NULL && mbox->cur_fetch_mail->seq == lseq) { i_assert(uid == 0 || mbox->cur_fetch_mail->uid == uid); imapc_fetch_mail_update(mbox->cur_fetch_mail, reply, list); } imapc_mailbox_init_delayed_trans(mbox); old_count = mail_index_view_get_messages_count(mbox->delayed_sync_view); - if (seq > old_count) { - if (uid == 0) + if (lseq > old_count) { + if (uid == 0 || lseq != old_count + 1) return; - i_assert(seq == old_count + 1); - mail_index_append(mbox->delayed_sync_trans, uid, &seq); + i_assert(lseq == old_count + 1); + mail_index_append(mbox->delayed_sync_trans, uid, &lseq); } - rec = mail_index_lookup(mbox->delayed_sync_view, seq); + rec = mail_index_lookup(mbox->delayed_sync_view, lseq); + if (rec->uid != uid && uid != 0) { + imapc_mailbox_set_corrupted(mbox, "Message UID changed %u -> %u", + rec->uid, uid); + return; + } if (seen_flags && rec->flags != flags) { - mail_index_update_flags(mbox->delayed_sync_trans, seq, + mail_index_update_flags(mbox->delayed_sync_trans, lseq, MODIFY_REPLACE, flags); } if (seen_flags) { @@ -154,7 +180,7 @@ (void)array_append_space(&keywords); kw = mail_index_keywords_create(mbox->box.index, array_idx(&keywords, 0)); - mail_index_update_keywords(mbox->delayed_sync_trans, seq, + mail_index_update_keywords(mbox->delayed_sync_trans, lseq, MODIFY_REPLACE, kw); mail_index_keywords_unref(&kw); } @@ -166,16 +192,28 @@ { struct imapc_seqmap *seqmap; uint32_t lseq, rseq = reply->num; - + if (mbox == NULL || rseq == 0) return; + imapc_mailbox_init_delayed_trans(mbox); + seqmap = imapc_client_mailbox_get_seqmap(mbox->client_box); lseq = imapc_seqmap_rseq_to_lseq(seqmap, rseq); - imapc_seqmap_expunge(seqmap, rseq); - imapc_mailbox_init_delayed_trans(mbox); - mail_index_expunge(mbox->delayed_sync_trans, lseq); + if (lseq <= mail_index_view_get_messages_count(mbox->sync_view)) { + /* expunging a message in index */ + imapc_seqmap_expunge(seqmap, rseq); + mail_index_expunge(mbox->delayed_sync_trans, lseq); + i_assert(array_count(&mbox->delayed_sync_trans->expunges) > 0); + } else if (lseq <= mail_index_view_get_messages_count(mbox->delayed_sync_view)) { + /* expunging a message that was added to transaction, + but not yet committed. expunging it here takes + effect immediately. */ + mail_index_expunge(mbox->delayed_sync_trans, lseq); + } else { + /* expunging a message whose UID wasn't known yet */ + } imapc_mailbox_idle_notify(mbox); }
--- a/src/lib-storage/index/imapc/imapc-save.c Thu Feb 24 13:19:05 2011 +0200 +++ b/src/lib-storage/index/imapc/imapc-save.c Thu Feb 24 15:17:47 2011 +0200 @@ -172,7 +172,7 @@ string_t *str = t_str_new(64); str_append(str, " ("); - imap_write_flags(str, _ctx->flags, NULL); + imap_write_flags(str, _ctx->flags & ~MAIL_RECENT, NULL); if (_ctx->keywords != NULL) imapc_append_keywords(str, _ctx->keywords); str_append_c(str, ')');
--- a/src/lib-storage/index/imapc/imapc-seqmap.c Thu Feb 24 13:19:05 2011 +0200 +++ b/src/lib-storage/index/imapc/imapc-seqmap.c Thu Feb 24 15:17:47 2011 +0200 @@ -36,6 +36,12 @@ array_clear(&seqmap->expunges); } +bool imapc_seqmap_is_reset(struct imapc_seqmap *seqmap) +{ + return array_count(&seqmap->queue) == 0 && + array_count(&seqmap->expunges) == 0; +} + void imapc_seqmap_expunge(struct imapc_seqmap *seqmap, uint32_t rseq) { i_assert(rseq > 0);
--- a/src/lib-storage/index/imapc/imapc-seqmap.h Thu Feb 24 13:19:05 2011 +0200 +++ b/src/lib-storage/index/imapc/imapc-seqmap.h Thu Feb 24 15:17:47 2011 +0200 @@ -15,6 +15,7 @@ /* Reset local and remote sequences to be equal. */ void imapc_seqmap_reset(struct imapc_seqmap *seqmap); +bool imapc_seqmap_is_reset(struct imapc_seqmap *seqmap); /* Mark given remote sequence expunged. */ void imapc_seqmap_expunge(struct imapc_seqmap *seqmap, uint32_t rseq);
--- a/src/lib-storage/index/imapc/imapc-storage.c Thu Feb 24 13:19:05 2011 +0200 +++ b/src/lib-storage/index/imapc/imapc-storage.c Thu Feb 24 15:17:47 2011 +0200 @@ -72,7 +72,7 @@ struct imapc_storage *storage; pool_t pool; - pool = pool_alloconly_create("imapc storage", 512+256); + pool = pool_alloconly_create("imapc storage", 2048); storage = p_new(pool, struct imapc_storage, 1); storage->storage = imapc_storage; storage->storage.pool = pool; @@ -341,6 +341,8 @@ if (mail_index_transaction_commit(&mbox->delayed_sync_trans) < 0) mail_storage_set_index_error(&mbox->box); } + if (mbox->sync_view != NULL) + mail_index_view_close(&mbox->sync_view); if (mbox->to_idle_delay != NULL) timeout_remove(&mbox->to_idle_delay); if (mbox->to_idle_check != NULL) @@ -523,6 +525,16 @@ } } +static bool imapc_is_inconsistent(struct mailbox *box) +{ + struct imapc_mailbox *mbox = (struct imapc_mailbox *)box; + + if (mail_index_view_is_inconsistent(box->view)) + return TRUE; + + return !imapc_client_mailbox_is_connected(mbox->client_box); +} + struct mail_storage imapc_storage = { .name = IMAPC_STORAGE_NAME, .class_flags = MAIL_STORAGE_CLASS_FLAG_NO_ROOT, @@ -577,6 +589,6 @@ imapc_save_finish, imapc_save_cancel, imapc_copy, - index_storage_is_inconsistent + imapc_is_inconsistent } };
--- a/src/lib-storage/index/imapc/imapc-storage.h Thu Feb 24 13:19:05 2011 +0200 +++ b/src/lib-storage/index/imapc/imapc-storage.h Thu Feb 24 15:17:47 2011 +0200 @@ -45,7 +45,7 @@ struct imapc_client_mailbox *client_box; struct mail_index_transaction *delayed_sync_trans; - struct mail_index_view *delayed_sync_view; + struct mail_index_view *sync_view, *delayed_sync_view; struct timeout *to_idle_check, *to_idle_delay; struct mail *cur_fetch_mail;
--- a/src/lib-storage/index/imapc/imapc-sync.c Thu Feb 24 13:19:05 2011 +0200 +++ b/src/lib-storage/index/imapc/imapc-sync.c Thu Feb 24 15:17:47 2011 +0200 @@ -22,9 +22,15 @@ else if (reply->state == IMAPC_COMMAND_STATE_NO) { /* maybe the message was expunged already. some servers fail STOREs with NO in such situation. */ + } else if (reply->state == IMAPC_COMMAND_STATE_DISCONNECTED) { + /* the disconnection is already logged, don't flood + the logs unnecessarily */ + mail_storage_set_internal_error(&ctx->mbox->storage->storage); + ctx->failed = TRUE; } else { mail_storage_set_critical(&ctx->mbox->storage->storage, "imapc: Sync command failed: %s", reply->text_full); + ctx->failed = TRUE; } if (--ctx->sync_command_count == 0) @@ -71,6 +77,7 @@ i_assert(sync_rec->type == MAIL_INDEX_SYNC_TYPE_FLAGS); if (sync_rec->add_flags != 0) { + i_assert((sync_rec->add_flags & MAIL_RECENT) == 0); str_printfa(str, "UID STORE %u:%u +FLAGS (", sync_rec->uid1, sync_rec->uid2); imap_write_flags(str, sync_rec->add_flags, NULL); @@ -79,6 +86,7 @@ } if (sync_rec->remove_flags != 0) { + i_assert((sync_rec->remove_flags & MAIL_RECENT) == 0); str_truncate(str, 0); str_printfa(str, "UID STORE %u:%u -FLAGS (", sync_rec->uid1, sync_rec->uid2); @@ -126,6 +134,7 @@ for (seq = seq1; seq <= seq2; seq++) { rec = mail_index_lookup(ctx->sync_view, seq); + i_assert((rec->flags & MAIL_RECENT) == 0); str_truncate(str, 0); str_printfa(str, "UID STORE %u FLAGS (", rec->uid); @@ -202,7 +211,7 @@ } T_END; imapc_sync_expunge_finish(ctx); - if (ctx->sync_command_count > 0) + while (ctx->sync_command_count > 0) imapc_client_run(ctx->mbox->storage->client); array_free(&ctx->expunged_uids); @@ -238,25 +247,28 @@ } i_assert(mbox->delayed_sync_trans == NULL); - mbox->delayed_sync_view = ctx->sync_view; + mbox->sync_view = ctx->sync_view; + mbox->delayed_sync_view = + mail_index_transaction_open_updated_view(ctx->trans); mbox->delayed_sync_trans = ctx->trans; imapc_sync_index(ctx); - mbox->delayed_sync_view = NULL; + mail_index_view_close(&mbox->delayed_sync_view); mbox->delayed_sync_trans = NULL; + mbox->sync_view = NULL; *ctx_r = ctx; return 0; } -static int imapc_sync_finish(struct imapc_sync_context **_ctx, bool success) +static int imapc_sync_finish(struct imapc_sync_context **_ctx) { struct imapc_sync_context *ctx = *_ctx; - int ret = success ? 0 : -1; + int ret = ctx->failed ? -1 : 0; *_ctx = NULL; - if (success) { + if (ret == 0) { if (mail_index_sync_commit(&ctx->index_sync_ctx) < 0) { mail_storage_set_index_error(&ctx->mbox->box); ret = -1; @@ -271,12 +283,25 @@ static int imapc_sync(struct imapc_mailbox *mbox) { struct imapc_sync_context *sync_ctx; + struct imapc_seqmap *seqmap; + + /* if there are any pending expunges, they're now committed. syncing + will return a view where they no longer exist, so reset the seqmap + before syncing. */ + seqmap = imapc_client_mailbox_get_seqmap(mbox->client_box); + imapc_seqmap_reset(seqmap); if (imapc_sync_begin(mbox, &sync_ctx, FALSE) < 0) return -1; + if (sync_ctx == NULL) + return 0; + if (imapc_sync_finish(&sync_ctx) < 0) + return -1; - return sync_ctx == NULL ? 0 : - imapc_sync_finish(&sync_ctx, TRUE); + /* syncing itself may have also seen new expunges, which are also now + committed and synced. reset the seqmap again. */ + imapc_seqmap_reset(seqmap); + return 0; } struct mailbox_sync_context * @@ -284,6 +309,7 @@ { struct imapc_mailbox *mbox = (struct imapc_mailbox *)box; enum imapc_capability capabilities; + bool changes = FALSE; int ret = 0; if (!box->opened) { @@ -305,26 +331,29 @@ mail_index_view_close(&mbox->delayed_sync_view); if (mbox->delayed_sync_trans != NULL) { if (mail_index_transaction_commit(&mbox->delayed_sync_trans) < 0) { - // FIXME: mark inconsistent mail_storage_set_index_error(&mbox->box); ret = -1; } + changes = TRUE; } + if (mbox->sync_view != NULL) + mail_index_view_close(&mbox->sync_view); - if (index_mailbox_want_full_sync(&mbox->box, flags) && ret == 0) + if ((changes || index_mailbox_want_full_sync(&mbox->box, flags)) && + ret == 0) ret = imapc_sync(mbox); + if (changes && ret < 0) { + /* we're now out of sync and can't safely continue */ + mail_index_mark_corrupted(mbox->box.index); + } return index_mailbox_sync_init(box, flags, ret < 0); } int imapc_mailbox_sync_deinit(struct mailbox_sync_context *ctx, struct mailbox_sync_status *status_r) { - struct index_mailbox_sync_context *ictx = - (struct index_mailbox_sync_context *)ctx; struct imapc_mailbox *mbox = (struct imapc_mailbox *)ctx->box; - enum mailbox_sync_flags flags = ictx->flags; - struct imapc_seqmap *seqmap; int ret; ret = index_mailbox_sync_deinit(ctx, status_r); @@ -333,10 +362,6 @@ if (mbox->client_box == NULL) return ret; - if ((flags & MAILBOX_SYNC_FLAG_NO_EXPUNGES) == 0 && ret == 0) { - seqmap = imapc_client_mailbox_get_seqmap(mbox->client_box); - imapc_seqmap_reset(seqmap); - } imapc_client_mailbox_idle(mbox->client_box); return ret; }
--- a/src/lib-storage/index/imapc/imapc-sync.h Thu Feb 24 13:19:05 2011 +0200 +++ b/src/lib-storage/index/imapc/imapc-sync.h Thu Feb 24 15:17:47 2011 +0200 @@ -12,6 +12,8 @@ const ARRAY_TYPE(keywords) *keywords; ARRAY_TYPE(seq_range) expunged_uids; unsigned int sync_command_count; + + unsigned int failed:1; }; struct mailbox_sync_context *