Mercurial > dovecot > core-2.2

changeset 12763:345100da8c67

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
imapc: Many fixes related to syncing and error handling.
author Timo Sirainen <tss@iki.fi>
date Thu, 24 Feb 2011 15:17:47 +0200
parents 74300385cce0
children 0c1e221fc140
files src/lib-storage/index/imapc/imapc-client.c src/lib-storage/index/imapc/imapc-client.h src/lib-storage/index/imapc/imapc-connection.c src/lib-storage/index/imapc/imapc-connection.h src/lib-storage/index/imapc/imapc-mailbox.c src/lib-storage/index/imapc/imapc-save.c src/lib-storage/index/imapc/imapc-seqmap.c src/lib-storage/index/imapc/imapc-seqmap.h src/lib-storage/index/imapc/imapc-storage.c src/lib-storage/index/imapc/imapc-storage.h src/lib-storage/index/imapc/imapc-sync.c src/lib-storage/index/imapc/imapc-sync.h
diffstat 12 files changed, 341 insertions(+), 122 deletions(-) [+]
line wrap: on
line diff
--- a/src/lib-storage/index/imapc/imapc-client.c	Thu Feb 24 13:19:05 2011 +0200
+++ b/src/lib-storage/index/imapc/imapc-client.c	Thu Feb 24 15:17:47 2011 +0200
@@ -84,8 +84,10 @@
 
 	if (client->ssl_ctx != NULL)
 		ssl_iostream_context_deinit(&client->ssl_ctx);
-	array_foreach_modifiable(&client->conns, connp)
+	array_foreach_modifiable(&client->conns, connp) {
 		imapc_connection_deinit(&(*connp)->conn);
+		i_free(*connp);
+	}
 	pool_unref(&client->pool);
 }
 
@@ -212,6 +214,12 @@
 	return box;
 }
 
+void imapc_client_mailbox_disconnect(struct imapc_client_mailbox *box)
+{
+	if (box->conn != NULL)
+		imapc_connection_disconnect(box->conn);
+}
+
 void imapc_client_mailbox_close(struct imapc_client_mailbox **_box)
 {
 	struct imapc_client_mailbox *box = *_box;
@@ -226,7 +234,8 @@
 		}
 	}
 
-	imapc_connection_unselect(box);
+	if (box->conn != NULL)
+		imapc_connection_unselect(box);
 	imapc_seqmap_deinit(&box->seqmap);
 	i_free(box);
 }
@@ -258,12 +267,43 @@
 	return ctx;
 }
 
+static bool
+imapc_client_mailbox_is_selected(struct imapc_client_mailbox *box,
+				 struct imapc_command_reply *reply_r)
+{
+	struct imapc_client_mailbox *selected_box;
+
+	selected_box = box->conn == NULL ? NULL :
+		imapc_connection_get_mailbox(box->conn);
+	if (selected_box == box)
+		return TRUE;
+
+	memset(reply_r, 0, sizeof(*reply_r));
+	reply_r->state = IMAPC_COMMAND_STATE_DISCONNECTED;
+	if (selected_box == NULL) {
+		reply_r->text_full = "Disconnected from server";
+	} else {
+		i_error("imapc: Selected mailbox changed unexpectedly");
+		reply_r->text_full = "Internal error";
+	}
+	reply_r->text_without_resp = reply_r->text_full;
+
+	box->conn = NULL;
+	return FALSE;
+}
+
 void imapc_client_mailbox_cmd(struct imapc_client_mailbox *box,
 			      const char *cmd,
 			      imapc_command_callback_t *callback,
 			      void *context)
 {
 	struct imapc_client_command_context *ctx;
+	struct imapc_command_reply reply;
+
+	if (!imapc_client_mailbox_is_selected(box, &reply)) {
+		callback(&reply, context);
+		return;
+	}
 
 	ctx = imapc_client_mailbox_cmd_common(box, callback, context);
 	imapc_connection_cmd(box->conn, cmd, imapc_client_mailbox_cmd_cb, ctx);
@@ -275,6 +315,12 @@
 {
 	struct imapc_client_command_context *ctx;
 	va_list args;
+	struct imapc_command_reply reply;
+
+	if (!imapc_client_mailbox_is_selected(box, &reply)) {
+		callback(&reply, context);
+		return;
+	}
 
 	ctx = imapc_client_mailbox_cmd_common(box, callback, context);
 	va_start(args, cmd_fmt);
@@ -291,7 +337,17 @@
 
 void imapc_client_mailbox_idle(struct imapc_client_mailbox *box)
 {
-	imapc_connection_idle(box->conn);
+	struct imapc_command_reply reply;
+
+	if (imapc_client_mailbox_is_selected(box, &reply))
+		imapc_connection_idle(box->conn);
+}
+
+bool imapc_client_mailbox_is_connected(struct imapc_client_mailbox *box)
+{
+	struct imapc_command_reply reply;
+
+	return imapc_client_mailbox_is_selected(box, &reply);
 }
 
 enum imapc_capability
--- a/src/lib-storage/index/imapc/imapc-client.h	Thu Feb 24 13:19:05 2011 +0200
+++ b/src/lib-storage/index/imapc/imapc-client.h	Thu Feb 24 15:17:47 2011 +0200
@@ -123,6 +123,7 @@
 			  imapc_command_callback_t *callback, void *context,
 			  void *untagged_box_context);
 void imapc_client_mailbox_close(struct imapc_client_mailbox **box);
+void imapc_client_mailbox_disconnect(struct imapc_client_mailbox *box);
 void imapc_client_mailbox_cmd(struct imapc_client_mailbox *box,
 			      const char *cmd,
 			      imapc_command_callback_t *callback,
@@ -135,6 +136,7 @@
 imapc_client_mailbox_get_seqmap(struct imapc_client_mailbox *box);
 
 void imapc_client_mailbox_idle(struct imapc_client_mailbox *box);
+bool imapc_client_mailbox_is_connected(struct imapc_client_mailbox *box);
 
 enum imapc_capability
 imapc_client_get_capabilities(struct imapc_client *client);
--- a/src/lib-storage/index/imapc/imapc-connection.c	Thu Feb 24 13:19:05 2011 +0200
+++ b/src/lib-storage/index/imapc/imapc-connection.c	Thu Feb 24 15:17:47 2011 +0200
@@ -22,14 +22,15 @@
 
 #define IMAPC_DNS_LOOKUP_TIMEOUT_MSECS (1000*30)
 #define IMAPC_CONNECT_TIMEOUT_MSECS (1000*30)
+#define IMAPC_COMMAND_TIMEOUT_MSECS (1000*60*5)
 #define IMAPC_MAX_INLINE_LITERAL_SIZE (1024*32)
 
 enum imapc_input_state {
 	IMAPC_INPUT_STATE_NONE = 0,
+	IMAPC_INPUT_STATE_PLUS,
 	IMAPC_INPUT_STATE_UNTAGGED,
 	IMAPC_INPUT_STATE_UNTAGGED_NUM,
-	IMAPC_INPUT_STATE_TAGGED,
-	IMAPC_INPUT_STATE_SKIPLINE
+	IMAPC_INPUT_STATE_TAGGED
 };
 
 struct imapc_command_stream {
@@ -84,7 +85,9 @@
 	enum imapc_capability capabilities;
 	char **capabilities_list;
 
+	/* commands pending in queue to be sent */
 	ARRAY_DEFINE(cmd_send_queue, struct imapc_command *);
+	/* commands that have been sent, waiting for their tagged reply */
 	ARRAY_DEFINE(cmd_wait_list, struct imapc_command *);
 
 	unsigned int ips_count, prev_connect_idx;
@@ -99,8 +102,6 @@
 };
 
 static int imapc_connection_ssl_init(struct imapc_connection *conn);
-static void imapc_connection_disconnect(struct imapc_connection *conn);
-
 static void imapc_command_free(struct imapc_command *cmd);
 static void imapc_command_send_more(struct imapc_connection *conn,
 				    struct imapc_command *cmd);
@@ -147,39 +148,50 @@
 		conn->to = io_loop_move_timeout(&conn->to);
 }
 
+static void
+imapc_connection_abort_pending_commands(struct imapc_connection *conn,
+					const struct imapc_command_reply *reply)
+{
+	struct imapc_command *const *cmdp, *cmd;
+
+	while (array_count(&conn->cmd_wait_list) > 0) {
+		cmdp = array_idx(&conn->cmd_wait_list, 0);
+		cmd = *cmdp;
+		array_delete(&conn->cmd_wait_list, 0, 1);
+
+		if (cmd->callback != NULL)
+			cmd->callback(reply, cmd->context);
+		imapc_command_free(cmd);
+	}
+	while (array_count(&conn->cmd_send_queue) > 0) {
+		cmdp = array_idx(&conn->cmd_send_queue, 0);
+		cmd = *cmdp;
+		array_delete(&conn->cmd_send_queue, 0, 1);
+
+		if (cmd->callback != NULL)
+			cmd->callback(reply, cmd->context);
+		imapc_command_free(cmd);
+	}
+}
+
 static void imapc_connection_set_state(struct imapc_connection *conn,
 				       enum imapc_connection_state state)
 {
 	if (state == IMAPC_CONNECTION_STATE_DISCONNECTED) {
-		/* abort all pending commands */
 		struct imapc_command_reply reply;
-		struct imapc_command *const *cmdp, *cmd;
 
 		memset(&reply, 0, sizeof(reply));
 		reply.state = IMAPC_COMMAND_STATE_DISCONNECTED;
 		reply.text_without_resp = reply.text_full =
 			"Disconnected from server";
+		imapc_connection_abort_pending_commands(conn, &reply);
 
 		conn->idling = FALSE;
 		conn->idle_plus_waiting = FALSE;
 		conn->idle_stopping = FALSE;
 
-		while (array_count(&conn->cmd_wait_list) > 0) {
-			cmdp = array_idx(&conn->cmd_wait_list, 0);
-			cmd = *cmdp;
-			array_delete(&conn->cmd_wait_list, 0, 1);
-
-			cmd->callback(&reply, cmd->context);
-			imapc_command_free(cmd);
-		}
-		while (array_count(&conn->cmd_send_queue) > 0) {
-			cmdp = array_idx(&conn->cmd_send_queue, 0);
-			cmd = *cmdp;
-			array_delete(&conn->cmd_send_queue, 0, 1);
-
-			cmd->callback(&reply, cmd->context);
-			imapc_command_free(cmd);
-		}
+		conn->selecting_box = NULL;
+		conn->selected_box = NULL;
 	}
 	if (state == IMAPC_CONNECTION_STATE_DONE) {
 		if (array_count(&conn->cmd_send_queue) > 0) {
@@ -215,7 +227,7 @@
 	literal->fd = -1;
 }
 
-static void imapc_connection_disconnect(struct imapc_connection *conn)
+void imapc_connection_disconnect(struct imapc_connection *conn)
 {
 	if (conn->fd == -1)
 		return;
@@ -375,10 +387,22 @@
 imapc_connection_read_line(struct imapc_connection *conn,
 			   const struct imap_arg **imap_args_r)
 {
+	const unsigned char *data;
+	size_t size;
 	int ret;
 
 	while ((ret = imapc_connection_read_line_more(conn, imap_args_r)) == 2)
 		;
+
+	if (ret > 0) {
+		data = i_stream_get_data(conn->input, &size);
+		if (size >= 2 && data[0] == '\r' && data[1] == '\n')
+			i_stream_skip(conn->input, 2);
+		else if (size >= 1 && data[0] == '\n')
+			i_stream_skip(conn->input, 1);
+		else
+			i_panic("imapc: Missing LF from input line");
+	}
 	return ret;
 }
 
@@ -506,29 +530,11 @@
 	conn->input_state = IMAPC_INPUT_STATE_NONE;
 	conn->cur_tag = 0;
 	conn->cur_num = 0;
-	imap_parser_reset(conn->parser);
+	if (conn->parser != NULL)
+		imap_parser_reset(conn->parser);
 	imapc_connection_lfiles_free(conn);
 }
 
-static int imapc_connection_skip_line(struct imapc_connection *conn)
-{
-	const unsigned char *data;
-	size_t i, data_size;
-	int ret = 0;
-
-	data = i_stream_get_data(conn->input, &data_size);
-	for (i = 0; i < data_size; i++) {
-		if (data[i] == '\n') {
-			imapc_connection_input_reset(conn);
-			ret = 1;
-			i++;
-			break;
-		}
-	}
-	i_stream_skip(conn->input, i);
-	return ret;
-}
-
 static void imapc_connection_login_cb(const struct imapc_command_reply *reply,
 				      void *context)
 {
@@ -749,11 +755,6 @@
 			conn->selected_box->untagged_box_context;
 	}
 	conn->client->untagged_callback(&reply, conn->client->untagged_context);
-	if (imap_arg_atom_equals(imap_args, "EXPUNGE") &&
-	    conn->selected_box != NULL) {
-		/* keep track of expunge map internally */
-		imapc_seqmap_expunge(conn->selected_box->seqmap, conn->cur_num);
-	}
 	imapc_connection_input_reset(conn);
 	return 1;
 }
@@ -761,23 +762,26 @@
 static int imapc_connection_input_plus(struct imapc_connection *conn)
 {
 	struct imapc_command *const *cmd_p;
+	const char *line;
+
+	if ((line = i_stream_next_line(conn->input)) == NULL)
+		return 0;
 
 	if (conn->idle_plus_waiting) {
 		/* "+ idling" reply for IDLE command */
 		conn->idle_plus_waiting = FALSE;
 		conn->idling = TRUE;
-		return imapc_connection_skip_line(conn);
+	} else if (array_count(&conn->cmd_send_queue) > 0) {
+		/* reply for literal */
+		cmd_p = array_idx(&conn->cmd_send_queue, 0);
+		imapc_command_send_more(conn, *cmd_p);
+	} else {
+		imapc_connection_input_error(conn, "Unexpected '+': %s", line);
+		return -1;
 	}
 
-	if (array_count(&conn->cmd_send_queue) == 0) {
-		imapc_connection_input_error(conn, "Unexpected '+'");
-		return -1;
-	}
-	cmd_p = array_idx(&conn->cmd_send_queue, 0);
-	imapc_command_send_more(conn, *cmd_p);
-
-	conn->input_state = IMAPC_INPUT_STATE_SKIPLINE;
-	return imapc_connection_skip_line(conn);
+	imapc_connection_input_reset(conn);
+	return 1;
 }
 
 static int imapc_connection_input_tagged(struct imapc_connection *conn)
@@ -806,14 +810,12 @@
 		reply.state = IMAPC_COMMAND_STATE_OK;
 	else if (strcasecmp(line, "no") == 0)
 		reply.state = IMAPC_COMMAND_STATE_NO;
-	else if (strcasecmp(line, "bad") == 0) {
-		i_error("imapc(%s): Command failed with BAD: %u %s",
-			conn->name, conn->cur_tag, line);
+	else if (strcasecmp(line, "bad") == 0)
 		reply.state = IMAPC_COMMAND_STATE_BAD;
-	} else {
+	else {
 		imapc_connection_input_error(conn,
-			"Invalid state in tagged reply: %u %s",
-			conn->cur_tag, line);
+			"Invalid state in tagged reply: %u %s %s",
+			conn->cur_tag, line, reply.text_full);
 		return -1;
 	}
 
@@ -849,15 +851,27 @@
 			}
 		}
 	}
+	if (array_count(&conn->cmd_wait_list) == 0 &&
+	    array_count(&conn->cmd_send_queue) == 0 &&
+	    conn->state == IMAPC_CONNECTION_STATE_DONE && conn->to != NULL)
+		timeout_remove(&conn->to);
 
 	if (cmd == NULL) {
 		imapc_connection_input_error(conn,
-			"Unknown tag in a reply: %u %s", conn->cur_tag, line);
+			"Unknown tag in a reply: %u %s %s",
+			conn->cur_tag, line, reply.text_full);
 		return -1;
 	}
 
+	if (reply.state == IMAPC_COMMAND_STATE_BAD) {
+		i_error("imapc(%s): Command '%s' failed with BAD: %u %s",
+			conn->name, str_c(cmd->data), conn->cur_tag,
+			reply.text_full);
+	}
+
 	imapc_connection_input_reset(conn);
-	cmd->callback(&reply, cmd->context);
+	if (cmd->callback != NULL)
+		cmd->callback(&reply, cmd->context);
 	imapc_command_free(cmd);
 	return 1;
 }
@@ -876,15 +890,12 @@
 		if (tag == NULL)
 			return 0;
 
-		if (strcmp(tag, "") == 0) {
-			/* FIXME: why do we get here.. */
-			conn->input_state = IMAPC_INPUT_STATE_SKIPLINE;
-			return imapc_connection_skip_line(conn);
-		} else if (strcmp(tag, "*") == 0) {
+		if (strcmp(tag, "*") == 0) {
 			conn->input_state = IMAPC_INPUT_STATE_UNTAGGED;
 			conn->cur_num = 0;
 			ret = imapc_connection_input_untagged(conn);
 		} else if (strcmp(tag, "+") == 0) {
+			conn->input_state = IMAPC_INPUT_STATE_PLUS;
 			ret = imapc_connection_input_plus(conn);
 		} else {
 			conn->input_state = IMAPC_INPUT_STATE_TAGGED;
@@ -898,6 +909,9 @@
 			}
 		}
 		break;
+	case IMAPC_INPUT_STATE_PLUS:
+		ret = imapc_connection_input_plus(conn);
+		break;
 	case IMAPC_INPUT_STATE_UNTAGGED:
 	case IMAPC_INPUT_STATE_UNTAGGED_NUM:
 		ret = imapc_connection_input_untagged(conn);
@@ -905,9 +919,6 @@
 	case IMAPC_INPUT_STATE_TAGGED:
 		ret = imapc_connection_input_tagged(conn);
 		break;
-	case IMAPC_INPUT_STATE_SKIPLINE:
-		ret = imapc_connection_skip_line(conn);
-		break;
 	}
 	return ret;
 }
@@ -1092,6 +1103,8 @@
 	if (conn->fd != -1)
 		return;
 
+	imapc_connection_input_reset(conn);
+
 	if (conn->client->set.debug)
 		i_debug("imapc(%s): Looking up IP address", conn->name);
 
@@ -1116,6 +1129,9 @@
 	if (conn->input == NULL)
 		return;
 
+	if (conn->to != NULL)
+		timeout_reset(conn->to);
+
 	o_stream_cork(conn->output);
 	while (ret > 0 && !conn->client->stop_now && conn->input != NULL) {
 		T_BEGIN {
@@ -1134,7 +1150,7 @@
 	struct imapc_command *cmd;
 	pool_t pool;
 
-	pool = pool_alloconly_create("imapc command", 1024);
+	pool = pool_alloconly_create("imapc command", 2048);
 	cmd = p_new(pool, struct imapc_command, 1);
 	cmd->pool = pool;
 	cmd->callback = callback;
@@ -1273,6 +1289,19 @@
 	}
 }
 
+static void imapc_command_timeout(struct imapc_connection *conn)
+{
+	struct imapc_command *const *cmds;
+	unsigned int count;
+
+	cmds = array_get(&conn->cmd_wait_list, &count);
+	i_assert(count > 0);
+
+	i_error("imapc(%s): Command '%s' timed out, disconnecting",
+		conn->name, str_c(cmds[0]->data));
+	imapc_connection_disconnect(conn);
+}
+
 static void imapc_command_send(struct imapc_connection *conn,
 			       struct imapc_command *cmd)
 {
@@ -1286,6 +1315,14 @@
 		imapc_command_send_more(conn, cmd);
 		break;
 	case IMAPC_CONNECTION_STATE_DONE:
+		if (cmd->idle) {
+			if (conn->to != NULL)
+				timeout_remove(&conn->to);
+		} else if (conn->to == NULL) {
+			conn->to = timeout_add(IMAPC_COMMAND_TIMEOUT_MSECS,
+					       imapc_command_timeout, conn);
+		}
+
 		array_append(&conn->cmd_send_queue, &cmd, 1);
 		if (array_count(&conn->cmd_send_queue) == 1)
 			imapc_command_send_more(conn, cmd);
@@ -1436,11 +1473,47 @@
 
 void imapc_connection_unselect(struct imapc_client_mailbox *box)
 {
-	i_assert(box->conn->selected_box == box ||
-		 box->conn->selecting_box == box);
+	struct imapc_command *const *cmdp;
+	struct imapc_command_reply reply;
+
+	/* mailbox is being closed. if there are any pending commands, we must
+	   finish them immediately so callbacks don't access any freed
+	   contexts */
+	memset(&reply, 0, sizeof(reply));
+	reply.state = IMAPC_COMMAND_STATE_DISCONNECTED;
+	reply.text_without_resp = reply.text_full = "Closing mailbox";
 
-	box->conn->selected_box = NULL;
-	box->conn->selecting_box = NULL;
+	array_foreach(&box->conn->cmd_wait_list, cmdp) {
+		if ((*cmdp)->callback != NULL) {
+			(*cmdp)->callback(&reply, (*cmdp)->context);
+			(*cmdp)->callback = NULL;
+		}
+	}
+	array_foreach(&box->conn->cmd_send_queue, cmdp) {
+		if ((*cmdp)->callback != NULL) {
+			(*cmdp)->callback(&reply, (*cmdp)->context);
+			(*cmdp)->callback = NULL;
+		}
+	}
+
+	if (box->conn->selected_box == NULL &&
+	    box->conn->selecting_box == NULL) {
+		i_assert(box->conn->state == IMAPC_CONNECTION_STATE_DISCONNECTED);
+	} else {
+		i_assert(box->conn->selected_box == box ||
+			 box->conn->selecting_box == box);
+
+		box->conn->selected_box = NULL;
+		box->conn->selecting_box = NULL;
+	}
+}
+
+struct imapc_client_mailbox *
+imapc_connection_get_mailbox(struct imapc_connection *conn)
+{
+	if (conn->selecting_box != NULL)
+		return conn->selecting_box;
+	return conn->selected_box;
 }
 
 static void
--- a/src/lib-storage/index/imapc/imapc-connection.h	Thu Feb 24 13:19:05 2011 +0200
+++ b/src/lib-storage/index/imapc/imapc-connection.h	Thu Feb 24 15:17:47 2011 +0200
@@ -22,6 +22,7 @@
 void imapc_connection_deinit(struct imapc_connection **conn);
 
 void imapc_connection_connect(struct imapc_connection *conn);
+void imapc_connection_disconnect(struct imapc_connection *conn);
 void imapc_connection_ioloop_changed(struct imapc_connection *conn);
 void imapc_connection_input_pending(struct imapc_connection *conn);
 
@@ -43,6 +44,9 @@
 enum imapc_capability
 imapc_connection_get_capabilities(struct imapc_connection *conn);
 
+struct imapc_client_mailbox *
+imapc_connection_get_mailbox(struct imapc_connection *conn);
+
 void imapc_connection_idle(struct imapc_connection *conn);
 
 #endif
--- a/src/lib-storage/index/imapc/imapc-mailbox.c	Thu Feb 24 13:19:05 2011 +0200
+++ b/src/lib-storage/index/imapc/imapc-mailbox.c	Thu Feb 24 15:17:47 2011 +0200
@@ -10,13 +10,28 @@
 
 #define NOTIFY_DELAY_MSECS 500
 
+static void imapc_mailbox_set_corrupted(struct imapc_mailbox *mbox,
+					const char *reason, ...)
+{
+	va_list va;
+
+	va_start(va, reason);
+	i_error("imapc: Mailbox '%s' state corrupted: %s",
+		mbox->box.name, t_strdup_vprintf(reason, va));
+	va_end(va);
+
+	mail_index_mark_corrupted(mbox->box.index);
+	imapc_client_mailbox_disconnect(mbox->client_box);
+}
+
 static void imapc_mailbox_init_delayed_trans(struct imapc_mailbox *mbox)
 {
 	if (mbox->delayed_sync_trans != NULL)
 		return;
 
+	mbox->sync_view = mail_index_view_open(mbox->box.index);
 	mbox->delayed_sync_trans =
-		mail_index_transaction_begin(mbox->box.view,
+		mail_index_transaction_begin(mbox->sync_view,
 					MAIL_INDEX_TRANSACTION_FLAG_EXTERNAL);
 	mbox->delayed_sync_view =
 		mail_index_transaction_open_updated_view(mbox->delayed_sync_trans);
@@ -44,6 +59,7 @@
 static void imapc_untagged_exists(const struct imapc_untagged_reply *reply,
 				  struct imapc_mailbox *mbox)
 {
+	struct mail_index_view *view = mbox->delayed_sync_view;
 	uint32_t rcount = reply->num;
 	const struct mail_index_header *hdr;
 	struct imapc_seqmap *seqmap;
@@ -52,13 +68,16 @@
 	if (mbox == NULL)
 		return;
 
+	if (view == NULL)
+		view = mbox->box.view;
+
 	seqmap = imapc_client_mailbox_get_seqmap(mbox->client_box);
-	next_lseq = mail_index_view_get_messages_count(mbox->box.view) + 1;
+	next_lseq = mail_index_view_get_messages_count(view) + 1;
 	next_rseq = imapc_seqmap_lseq_to_rseq(seqmap, next_lseq);
 	if (next_rseq > rcount)
 		return;
 
-	hdr = mail_index_get_header(mbox->box.view);
+	hdr = mail_index_get_header(view);
 
 	mbox->new_msgs = TRUE;
 	imapc_client_mailbox_cmdf(mbox->client_box, imapc_newmsgs_callback,
@@ -85,7 +104,7 @@
 static void imapc_untagged_fetch(const struct imapc_untagged_reply *reply,
 				 struct imapc_mailbox *mbox)
 {
-	uint32_t seq = reply->num;
+	uint32_t lseq, rseq = reply->num;
 	struct imapc_seqmap *seqmap;
 	const struct imap_arg *list, *flags_list;
 	const char *atom;
@@ -96,7 +115,7 @@
 	ARRAY_TYPE(const_string) keywords = ARRAY_INIT;
 	bool seen_flags = FALSE;
 
-	if (mbox == NULL || seq == 0 || !imap_arg_get_list(reply->args, &list))
+	if (mbox == NULL || rseq == 0 || !imap_arg_get_list(reply->args, &list))
 		return;
 
 	uid = 0; flags = 0;
@@ -126,26 +145,33 @@
 			}
 		}
 	}
+	/* FIXME: need to do something about recent flags */
+	flags &= ~MAIL_RECENT;
 
 	seqmap = imapc_client_mailbox_get_seqmap(mbox->client_box);
-	seq = imapc_seqmap_rseq_to_lseq(seqmap, seq);
+	lseq = imapc_seqmap_rseq_to_lseq(seqmap, rseq);
 
-	if (mbox->cur_fetch_mail != NULL && mbox->cur_fetch_mail->seq == seq) {
+	if (mbox->cur_fetch_mail != NULL && mbox->cur_fetch_mail->seq == lseq) {
 		i_assert(uid == 0 || mbox->cur_fetch_mail->uid == uid);
 		imapc_fetch_mail_update(mbox->cur_fetch_mail, reply, list);
 	}
 
 	imapc_mailbox_init_delayed_trans(mbox);
 	old_count = mail_index_view_get_messages_count(mbox->delayed_sync_view);
-	if (seq > old_count) {
-		if (uid == 0)
+	if (lseq > old_count) {
+		if (uid == 0 || lseq != old_count + 1)
 			return;
-		i_assert(seq == old_count + 1);
-		mail_index_append(mbox->delayed_sync_trans, uid, &seq);
+		i_assert(lseq == old_count + 1);
+		mail_index_append(mbox->delayed_sync_trans, uid, &lseq);
 	}
-	rec = mail_index_lookup(mbox->delayed_sync_view, seq);
+	rec = mail_index_lookup(mbox->delayed_sync_view, lseq);
+	if (rec->uid != uid && uid != 0) {
+		imapc_mailbox_set_corrupted(mbox, "Message UID changed %u -> %u",
+					    rec->uid, uid);
+		return;
+	}
 	if (seen_flags && rec->flags != flags) {
-		mail_index_update_flags(mbox->delayed_sync_trans, seq,
+		mail_index_update_flags(mbox->delayed_sync_trans, lseq,
 					MODIFY_REPLACE, flags);
 	}
 	if (seen_flags) {
@@ -154,7 +180,7 @@
 		(void)array_append_space(&keywords);
 		kw = mail_index_keywords_create(mbox->box.index,
 						array_idx(&keywords, 0));
-		mail_index_update_keywords(mbox->delayed_sync_trans, seq,
+		mail_index_update_keywords(mbox->delayed_sync_trans, lseq,
 					   MODIFY_REPLACE, kw);
 		mail_index_keywords_unref(&kw);
 	}
@@ -166,16 +192,28 @@
 {
 	struct imapc_seqmap *seqmap;
 	uint32_t lseq, rseq = reply->num;
-
+	
 	if (mbox == NULL || rseq == 0)
 		return;
 
+	imapc_mailbox_init_delayed_trans(mbox);
+
 	seqmap = imapc_client_mailbox_get_seqmap(mbox->client_box);
 	lseq = imapc_seqmap_rseq_to_lseq(seqmap, rseq);
-	imapc_seqmap_expunge(seqmap, rseq);
 
-	imapc_mailbox_init_delayed_trans(mbox);
-	mail_index_expunge(mbox->delayed_sync_trans, lseq);
+	if (lseq <= mail_index_view_get_messages_count(mbox->sync_view)) {
+		/* expunging a message in index */
+		imapc_seqmap_expunge(seqmap, rseq);
+		mail_index_expunge(mbox->delayed_sync_trans, lseq);
+		i_assert(array_count(&mbox->delayed_sync_trans->expunges) > 0);
+	} else if (lseq <= mail_index_view_get_messages_count(mbox->delayed_sync_view)) {
+		/* expunging a message that was added to transaction,
+		   but not yet committed. expunging it here takes
+		   effect immediately. */
+		mail_index_expunge(mbox->delayed_sync_trans, lseq);
+	} else {
+		/* expunging a message whose UID wasn't known yet */
+	}
 
 	imapc_mailbox_idle_notify(mbox);
 }
--- a/src/lib-storage/index/imapc/imapc-save.c	Thu Feb 24 13:19:05 2011 +0200
+++ b/src/lib-storage/index/imapc/imapc-save.c	Thu Feb 24 15:17:47 2011 +0200
@@ -172,7 +172,7 @@
 		string_t *str = t_str_new(64);
 
 		str_append(str, " (");
-		imap_write_flags(str, _ctx->flags, NULL);
+		imap_write_flags(str, _ctx->flags & ~MAIL_RECENT, NULL);
 		if (_ctx->keywords != NULL)
 			imapc_append_keywords(str, _ctx->keywords);
 		str_append_c(str, ')');
--- a/src/lib-storage/index/imapc/imapc-seqmap.c	Thu Feb 24 13:19:05 2011 +0200
+++ b/src/lib-storage/index/imapc/imapc-seqmap.c	Thu Feb 24 15:17:47 2011 +0200
@@ -36,6 +36,12 @@
 	array_clear(&seqmap->expunges);
 }
 
+bool imapc_seqmap_is_reset(struct imapc_seqmap *seqmap)
+{
+	return array_count(&seqmap->queue) == 0 &&
+		array_count(&seqmap->expunges) == 0;
+}
+
 void imapc_seqmap_expunge(struct imapc_seqmap *seqmap, uint32_t rseq)
 {
 	i_assert(rseq > 0);
--- a/src/lib-storage/index/imapc/imapc-seqmap.h	Thu Feb 24 13:19:05 2011 +0200
+++ b/src/lib-storage/index/imapc/imapc-seqmap.h	Thu Feb 24 15:17:47 2011 +0200
@@ -15,6 +15,7 @@
 
 /* Reset local and remote sequences to be equal. */
 void imapc_seqmap_reset(struct imapc_seqmap *seqmap);
+bool imapc_seqmap_is_reset(struct imapc_seqmap *seqmap);
 
 /* Mark given remote sequence expunged. */
 void imapc_seqmap_expunge(struct imapc_seqmap *seqmap, uint32_t rseq);
--- a/src/lib-storage/index/imapc/imapc-storage.c	Thu Feb 24 13:19:05 2011 +0200
+++ b/src/lib-storage/index/imapc/imapc-storage.c	Thu Feb 24 15:17:47 2011 +0200
@@ -72,7 +72,7 @@
 	struct imapc_storage *storage;
 	pool_t pool;
 
-	pool = pool_alloconly_create("imapc storage", 512+256);
+	pool = pool_alloconly_create("imapc storage", 2048);
 	storage = p_new(pool, struct imapc_storage, 1);
 	storage->storage = imapc_storage;
 	storage->storage.pool = pool;
@@ -341,6 +341,8 @@
 		if (mail_index_transaction_commit(&mbox->delayed_sync_trans) < 0)
 			mail_storage_set_index_error(&mbox->box);
 	}
+	if (mbox->sync_view != NULL)
+		mail_index_view_close(&mbox->sync_view);
 	if (mbox->to_idle_delay != NULL)
 		timeout_remove(&mbox->to_idle_delay);
 	if (mbox->to_idle_check != NULL)
@@ -523,6 +525,16 @@
 	}
 }
 
+static bool imapc_is_inconsistent(struct mailbox *box)
+{
+	struct imapc_mailbox *mbox = (struct imapc_mailbox *)box;
+
+	if (mail_index_view_is_inconsistent(box->view))
+		return TRUE;
+
+	return !imapc_client_mailbox_is_connected(mbox->client_box);
+}
+
 struct mail_storage imapc_storage = {
 	.name = IMAPC_STORAGE_NAME,
 	.class_flags = MAIL_STORAGE_CLASS_FLAG_NO_ROOT,
@@ -577,6 +589,6 @@
 		imapc_save_finish,
 		imapc_save_cancel,
 		imapc_copy,
-		index_storage_is_inconsistent
+		imapc_is_inconsistent
 	}
 };
--- a/src/lib-storage/index/imapc/imapc-storage.h	Thu Feb 24 13:19:05 2011 +0200
+++ b/src/lib-storage/index/imapc/imapc-storage.h	Thu Feb 24 15:17:47 2011 +0200
@@ -45,7 +45,7 @@
 	struct imapc_client_mailbox *client_box;
 
 	struct mail_index_transaction *delayed_sync_trans;
-	struct mail_index_view *delayed_sync_view;
+	struct mail_index_view *sync_view, *delayed_sync_view;
 	struct timeout *to_idle_check, *to_idle_delay;
 
 	struct mail *cur_fetch_mail;
--- a/src/lib-storage/index/imapc/imapc-sync.c	Thu Feb 24 13:19:05 2011 +0200
+++ b/src/lib-storage/index/imapc/imapc-sync.c	Thu Feb 24 15:17:47 2011 +0200
@@ -22,9 +22,15 @@
 	else if (reply->state == IMAPC_COMMAND_STATE_NO) {
 		/* maybe the message was expunged already.
 		   some servers fail STOREs with NO in such situation. */
+	} else if (reply->state == IMAPC_COMMAND_STATE_DISCONNECTED) {
+		/* the disconnection is already logged, don't flood
+		   the logs unnecessarily */
+		mail_storage_set_internal_error(&ctx->mbox->storage->storage);
+		ctx->failed = TRUE;
 	} else {
 		mail_storage_set_critical(&ctx->mbox->storage->storage,
 			"imapc: Sync command failed: %s", reply->text_full);
+		ctx->failed = TRUE;
 	}
 	
 	if (--ctx->sync_command_count == 0)
@@ -71,6 +77,7 @@
 	i_assert(sync_rec->type == MAIL_INDEX_SYNC_TYPE_FLAGS);
 
 	if (sync_rec->add_flags != 0) {
+		i_assert((sync_rec->add_flags & MAIL_RECENT) == 0);
 		str_printfa(str, "UID STORE %u:%u +FLAGS (",
 			    sync_rec->uid1, sync_rec->uid2);
 		imap_write_flags(str, sync_rec->add_flags, NULL);
@@ -79,6 +86,7 @@
 	}
 
 	if (sync_rec->remove_flags != 0) {
+		i_assert((sync_rec->remove_flags & MAIL_RECENT) == 0);
 		str_truncate(str, 0);
 		str_printfa(str, "UID STORE %u:%u -FLAGS (",
 			    sync_rec->uid1, sync_rec->uid2);
@@ -126,6 +134,7 @@
 
 	for (seq = seq1; seq <= seq2; seq++) {
 		rec = mail_index_lookup(ctx->sync_view, seq);
+		i_assert((rec->flags & MAIL_RECENT) == 0);
 
 		str_truncate(str, 0);
 		str_printfa(str, "UID STORE %u FLAGS (", rec->uid);
@@ -202,7 +211,7 @@
 	} T_END;
 
 	imapc_sync_expunge_finish(ctx);
-	if (ctx->sync_command_count > 0)
+	while (ctx->sync_command_count > 0)
 		imapc_client_run(ctx->mbox->storage->client);
 	array_free(&ctx->expunged_uids);
 
@@ -238,25 +247,28 @@
 	}
 
 	i_assert(mbox->delayed_sync_trans == NULL);
-	mbox->delayed_sync_view = ctx->sync_view;
+	mbox->sync_view = ctx->sync_view;
+	mbox->delayed_sync_view =
+		mail_index_transaction_open_updated_view(ctx->trans);
 	mbox->delayed_sync_trans = ctx->trans;
 
 	imapc_sync_index(ctx);
 
-	mbox->delayed_sync_view = NULL;
+	mail_index_view_close(&mbox->delayed_sync_view);
 	mbox->delayed_sync_trans = NULL;
+	mbox->sync_view = NULL;
 
 	*ctx_r = ctx;
 	return 0;
 }
 
-static int imapc_sync_finish(struct imapc_sync_context **_ctx, bool success)
+static int imapc_sync_finish(struct imapc_sync_context **_ctx)
 {
 	struct imapc_sync_context *ctx = *_ctx;
-	int ret = success ? 0 : -1;
+	int ret = ctx->failed ? -1 : 0;
 
 	*_ctx = NULL;
-	if (success) {
+	if (ret == 0) {
 		if (mail_index_sync_commit(&ctx->index_sync_ctx) < 0) {
 			mail_storage_set_index_error(&ctx->mbox->box);
 			ret = -1;
@@ -271,12 +283,25 @@
 static int imapc_sync(struct imapc_mailbox *mbox)
 {
 	struct imapc_sync_context *sync_ctx;
+	struct imapc_seqmap *seqmap;
+
+	/* if there are any pending expunges, they're now committed. syncing
+	   will return a view where they no longer exist, so reset the seqmap
+	   before syncing. */
+	seqmap = imapc_client_mailbox_get_seqmap(mbox->client_box);
+	imapc_seqmap_reset(seqmap);
 
 	if (imapc_sync_begin(mbox, &sync_ctx, FALSE) < 0)
 		return -1;
+	if (sync_ctx == NULL)
+		return 0;
+	if (imapc_sync_finish(&sync_ctx) < 0)
+		return -1;
 
-	return sync_ctx == NULL ? 0 :
-		imapc_sync_finish(&sync_ctx, TRUE);
+	/* syncing itself may have also seen new expunges, which are also now
+	   committed and synced. reset the seqmap again. */
+	imapc_seqmap_reset(seqmap);
+	return 0;
 }
 
 struct mailbox_sync_context *
@@ -284,6 +309,7 @@
 {
 	struct imapc_mailbox *mbox = (struct imapc_mailbox *)box;
 	enum imapc_capability capabilities;
+	bool changes = FALSE;
 	int ret = 0;
 
 	if (!box->opened) {
@@ -305,26 +331,29 @@
 		mail_index_view_close(&mbox->delayed_sync_view);
 	if (mbox->delayed_sync_trans != NULL) {
 		if (mail_index_transaction_commit(&mbox->delayed_sync_trans) < 0) {
-			// FIXME: mark inconsistent
 			mail_storage_set_index_error(&mbox->box);
 			ret = -1;
 		}
+		changes = TRUE;
 	}
+	if (mbox->sync_view != NULL)
+		mail_index_view_close(&mbox->sync_view);
 
-	if (index_mailbox_want_full_sync(&mbox->box, flags) && ret == 0)
+	if ((changes || index_mailbox_want_full_sync(&mbox->box, flags)) &&
+	    ret == 0)
 		ret = imapc_sync(mbox);
 
+	if (changes && ret < 0) {
+		/* we're now out of sync and can't safely continue */
+		mail_index_mark_corrupted(mbox->box.index);
+	}
 	return index_mailbox_sync_init(box, flags, ret < 0);
 }
 
 int imapc_mailbox_sync_deinit(struct mailbox_sync_context *ctx,
 			      struct mailbox_sync_status *status_r)
 {
-	struct index_mailbox_sync_context *ictx =
-		(struct index_mailbox_sync_context *)ctx;
 	struct imapc_mailbox *mbox = (struct imapc_mailbox *)ctx->box;
-	enum mailbox_sync_flags flags = ictx->flags;
-	struct imapc_seqmap *seqmap;
 	int ret;
 
 	ret = index_mailbox_sync_deinit(ctx, status_r);
@@ -333,10 +362,6 @@
 	if (mbox->client_box == NULL)
 		return ret;
 
-	if ((flags & MAILBOX_SYNC_FLAG_NO_EXPUNGES) == 0 && ret == 0) {
-		seqmap = imapc_client_mailbox_get_seqmap(mbox->client_box);
-		imapc_seqmap_reset(seqmap);
-	}
 	imapc_client_mailbox_idle(mbox->client_box);
 	return ret;
 }
--- a/src/lib-storage/index/imapc/imapc-sync.h	Thu Feb 24 13:19:05 2011 +0200
+++ b/src/lib-storage/index/imapc/imapc-sync.h	Thu Feb 24 15:17:47 2011 +0200
@@ -12,6 +12,8 @@
 	const ARRAY_TYPE(keywords) *keywords;
 	ARRAY_TYPE(seq_range) expunged_uids;
 	unsigned int sync_command_count;
+
+	unsigned int failed:1;
 };
 
 struct mailbox_sync_context *