changeset 20049:713cb90f4ce8

imapc: Added support for imapc_features=modseq If the remote server supports CONDSTORE or QRESYNC extensions we'll use the remote's MODSEQ and HIGHESTMODSEQ counts. There are some situations where the HIGHESTMODSEQ isn't updated exactly correctly on an open mailbox, so this feature shouldn't be fully relied on. It was primarily implemented for dsync+imapc support - both for preserving modseqs and also for HIGHESTMODSEQ lookups.
author Timo Sirainen <timo.sirainen@dovecot.fi>
date Fri, 22 Apr 2016 00:21:12 +0300
parents 551a444a60d8
children bdce227775cc
files src/lib-storage/index/imapc/imapc-mail.c src/lib-storage/index/imapc/imapc-mailbox.c src/lib-storage/index/imapc/imapc-settings.c src/lib-storage/index/imapc/imapc-settings.h src/lib-storage/index/imapc/imapc-storage.c src/lib-storage/index/imapc/imapc-storage.h src/lib-storage/index/imapc/imapc-sync.c
diffstat 7 files changed, 127 insertions(+), 9 deletions(-) [+]
line wrap: on
line diff
--- a/src/lib-storage/index/imapc/imapc-mail.c	Fri Apr 22 20:31:02 2016 +0300
+++ b/src/lib-storage/index/imapc/imapc-mail.c	Fri Apr 22 00:21:12 2016 +0300
@@ -96,6 +96,26 @@
 	return fix_broken_mail ? 0 : -1;
 }
 
+static uint64_t imapc_mail_get_modseq(struct mail *_mail)
+{
+	struct imapc_mailbox *mbox = (struct imapc_mailbox *)_mail->box;
+	struct imapc_msgmap *msgmap;
+	const uint64_t *modseqs;
+	unsigned int count;
+	uint32_t rseq;
+
+	if (!imapc_storage_has_modseqs(mbox->storage))
+		return index_mail_get_modseq(_mail);
+
+	msgmap = imapc_client_mailbox_get_msgmap(mbox->client_box);
+	if (imapc_msgmap_uid_to_rseq(msgmap, _mail->uid, &rseq)) {
+		modseqs = array_get(&mbox->rseq_modseqs, &count);
+		if (rseq <= count)
+			return modseqs[rseq-1];
+	}
+	return 1; /* unknown modseq */
+}
+
 static int imapc_mail_get_received_date(struct mail *_mail, time_t *date_r)
 {
 	struct index_mail *mail = (struct index_mail *)_mail;
@@ -561,7 +581,7 @@
 	index_mail_get_flags,
 	index_mail_get_keywords,
 	index_mail_get_keyword_indexes,
-	index_mail_get_modseq,
+	imapc_mail_get_modseq,
 	index_mail_get_pvt_modseq,
 	index_mail_get_parts,
 	index_mail_get_date,
--- a/src/lib-storage/index/imapc/imapc-mailbox.c	Fri Apr 22 20:31:02 2016 +0300
+++ b/src/lib-storage/index/imapc/imapc-mailbox.c	Fri Apr 22 00:21:12 2016 +0300
@@ -2,6 +2,7 @@
 
 #include "lib.h"
 #include "ioloop.h"
+#include "mail-index-modseq.h"
 #include "imap-arg.h"
 #include "imap-seqset.h"
 #include "imap-util.h"
@@ -282,11 +283,12 @@
 	uint32_t lseq, rseq = reply->num;
 	struct imapc_fetch_request *const *fetch_requestp;
 	struct imapc_mail *const *mailp;
-	const struct imap_arg *list, *flags_list;
+	const struct imap_arg *list, *flags_list, *modseq_list;
 	const char *atom, *guid = NULL;
 	const struct mail_index_record *rec = NULL;
 	enum mail_flags flags;
 	uint32_t fetch_uid, uid;
+	uint64_t modseq = 0;
 	unsigned int i, j;
 	ARRAY_TYPE(const_string) keywords = ARRAY_INIT;
 	bool seen_flags = FALSE, have_labels = FALSE;
@@ -319,6 +321,15 @@
 					array_append(&keywords, &atom, 1);
 				}
 			}
+		} else if (strcasecmp(atom, "MODSEQ") == 0 &&
+			   imapc_storage_has_modseqs(mbox->storage)) {
+			/* (modseq-number) */
+			if (!imap_arg_get_list(&list[i+1], &modseq_list))
+				return;
+			if (!imap_arg_get_atom(&modseq_list[0], &atom) ||
+			    str_to_uint64(atom, &modseq) < 0 ||
+			    modseq_list[1].type != IMAP_ARG_EOL)
+				return;
 		} else if (strcasecmp(atom, "X-GM-MSGID") == 0 &&
 			   !mbox->initial_sync_done) {
 			if (imap_arg_get_atom(&list[i+1], &atom))
@@ -414,6 +425,11 @@
 		}
 		mail_index_keywords_unref(&kw);
 	}
+	if (modseq != 0) {
+		if (mail_index_modseq_lookup(mbox->delayed_sync_view, lseq) < modseq)
+			mail_index_update_modseq(mbox->delayed_sync_trans, lseq, modseq);
+		array_idx_set(&mbox->rseq_modseqs, rseq-1, &modseq);
+	}
 	if (guid != NULL) {
 		struct index_mailbox_context *ibox = INDEX_STORAGE_CONTEXT(&mbox->box);
 		const enum index_cache_field guid_cache_idx =
@@ -454,6 +470,7 @@
 	}
 	uid = imapc_msgmap_rseq_to_uid(msgmap, rseq);
 	imapc_msgmap_expunge(msgmap, rseq);
+	array_delete(&mbox->rseq_modseqs, rseq-1, 1);
 
 	imapc_mailbox_init_delayed_trans(mbox);
 	if (mail_index_lookup_seq(mbox->sync_view, uid, &lseq))
@@ -579,6 +596,19 @@
 }
 
 static void
+imapc_resp_text_highestmodseq(const struct imapc_untagged_reply *reply,
+			      struct imapc_mailbox *mbox)
+{
+	uint64_t highestmodseq;
+
+	if (mbox == NULL ||
+	    str_to_uint64(reply->resp_text_value, &highestmodseq) < 0)
+		return;
+
+	mbox->sync_highestmodseq = highestmodseq;
+}
+
+static void
 imapc_resp_text_permanentflags(const struct imapc_untagged_reply *reply,
 			       struct imapc_mailbox *mbox)
 {
@@ -646,6 +676,8 @@
 					 imapc_resp_text_uidvalidity);
 	imapc_mailbox_register_resp_text(mbox, "UIDNEXT",
 					 imapc_resp_text_uidnext);
+	imapc_mailbox_register_resp_text(mbox, "HIGHESTMODSEQ",
+					 imapc_resp_text_highestmodseq);
 	imapc_mailbox_register_resp_text(mbox, "PERMANENTFLAGS",
 					 imapc_resp_text_permanentflags);
 }
--- a/src/lib-storage/index/imapc/imapc-settings.c	Fri Apr 22 20:31:02 2016 +0300
+++ b/src/lib-storage/index/imapc/imapc-settings.c	Fri Apr 22 00:21:12 2016 +0300
@@ -93,6 +93,7 @@
 	{ "proxyauth", IMAPC_FEATURE_PROXYAUTH },
 	{ "fetch-msn-workarounds", IMAPC_FEATURE_FETCH_MSN_WORKAROUNDS },
 	{ "fetch-fix-broken-mails", IMAPC_FEATURE_FETCH_FIX_BROKEN_MAILS },
+	{ "modseq", IMAPC_FEATURE_MODSEQ },
 	{ NULL, 0 }
 };
 
--- a/src/lib-storage/index/imapc/imapc-settings.h	Fri Apr 22 20:31:02 2016 +0300
+++ b/src/lib-storage/index/imapc/imapc-settings.h	Fri Apr 22 00:21:12 2016 +0300
@@ -14,7 +14,8 @@
 	IMAPC_FEATURE_NO_EXAMINE		= 0x40,
 	IMAPC_FEATURE_PROXYAUTH			= 0x80,
 	IMAPC_FEATURE_FETCH_MSN_WORKAROUNDS	= 0x100,
-	IMAPC_FEATURE_FETCH_FIX_BROKEN_MAILS	= 0x200
+	IMAPC_FEATURE_FETCH_FIX_BROKEN_MAILS	= 0x200,
+	IMAPC_FEATURE_MODSEQ			= 0x400
 };
 /* </settings checks> */
 
--- a/src/lib-storage/index/imapc/imapc-storage.c	Fri Apr 22 20:31:02 2016 +0300
+++ b/src/lib-storage/index/imapc/imapc-storage.c	Fri Apr 22 00:21:12 2016 +0300
@@ -55,6 +55,9 @@
 				  struct imapc_storage_client *client);
 static void imapc_untagged_namespace(const struct imapc_untagged_reply *reply,
 				     struct imapc_storage_client *client);
+static int imapc_mailbox_run_status(struct mailbox *box,
+				    enum mailbox_status_items items,
+				    struct mailbox_status *status_r);
 
 bool imap_resp_text_code_parse(const char *str, enum mail_error *error_r)
 {
@@ -72,6 +75,16 @@
 	return FALSE;
 }
 
+bool imapc_storage_has_modseqs(struct imapc_storage *storage)
+{
+	enum imapc_capability capa =
+		imapc_client_get_capabilities(storage->client->client);
+
+	return (capa & (IMAPC_CAPABILITY_CONDSTORE |
+			IMAPC_CAPABILITY_QRESYNC)) != 0 &&
+		IMAPC_HAS_FEATURE(storage, IMAPC_FEATURE_MODSEQ);
+}
+
 static struct mail_storage *imapc_storage_alloc(void)
 {
 	struct imapc_storage *storage;
@@ -603,6 +616,13 @@
 		return -1;
 	}
 
+	if (imapc_storage_has_modseqs(mbox->storage)) {
+		if (!array_is_created(&mbox->rseq_modseqs))
+			i_array_init(&mbox->rseq_modseqs, 32);
+		else
+			array_clear(&mbox->rseq_modseqs);
+	}
+
 	if (imapc_mailbox_select(mbox) < 0) {
 		mailbox_close(box);
 		return -1;
@@ -635,6 +655,8 @@
 		if (mail_index_transaction_commit(&mbox->delayed_sync_trans) < 0)
 			mailbox_set_index_error(&mbox->box);
 	}
+	if (array_is_created(&mbox->rseq_modseqs))
+		array_free(&mbox->rseq_modseqs);
 	if (mbox->sync_view != NULL)
 		mail_index_view_close(&mbox->sync_view);
 	if (mbox->to_idle_delay != NULL)
@@ -727,6 +749,9 @@
 			status->uidvalidity = num;
 		else if (strcasecmp(key, "UNSEEN") == 0)
 			status->unseen = num;
+		else if (strcasecmp(key, "HIGHESTMODSEQ") == 0 &&
+			 imapc_storage_has_modseqs(storage))
+			status->highest_modseq = num;
 	}
 }
 
@@ -765,15 +790,32 @@
 	}
 }
 
-static void imapc_mailbox_get_selected_status(struct imapc_mailbox *mbox,
-					      enum mailbox_status_items items,
-					      struct mailbox_status *status_r)
+static int imapc_mailbox_get_selected_status(struct imapc_mailbox *mbox,
+					     enum mailbox_status_items items,
+					     struct mailbox_status *status_r)
 {
+	int ret = 0;
+
 	index_storage_get_open_status(&mbox->box, items, status_r);
 	if ((items & STATUS_PERMANENT_FLAGS) != 0)
 		status_r->permanent_flags = mbox->permanent_flags;
 	if ((items & STATUS_FIRST_RECENT_UID) != 0)
 		status_r->first_recent_uid = mbox->highest_nonrecent_uid + 1;
+	if ((items & STATUS_HIGHESTMODSEQ) != 0) {
+		/* FIXME: this doesn't work perfectly. we're now just returning
+		   the HIGHESTMODSEQ from the current index, which may or may
+		   not be correct. with QRESYNC enabled we could be returning
+		   sync_highestmodseq, but that would require implementing
+		   VANISHED replies. and without QRESYNC we'd have to issue
+		   STATUS (HIGHESTMODSEQ), which isn't efficient since we get
+		   here constantly (after every IMAP command). */
+	}
+	if (imapc_storage_has_modseqs(mbox->storage)) {
+		/* even if local indexes are only in memory, we still
+		   have modseqs on the IMAP server itself. */
+		status_r->nonpermanent_modseqs = FALSE;
+	}
+	return ret;
 }
 
 static int imapc_mailbox_delete(struct mailbox *box)
@@ -802,6 +844,9 @@
 		str_append(str, " UIDVALIDITY");
 	if ((items & STATUS_UNSEEN) != 0)
 		str_append(str, " UNSEEN");
+	if ((items & STATUS_HIGHESTMODSEQ) != 0 &&
+	    imapc_storage_has_modseqs(mbox->storage))
+		str_append(str, " HIGHESTMODSEQ");
 
 	if (str_len(str) == 0) {
 		/* nothing requested */
@@ -833,14 +878,17 @@
 		status_r->have_guids = TRUE;
 
 	if (box->opened) {
-		imapc_mailbox_get_selected_status(mbox, items, status_r);
+		if (imapc_mailbox_get_selected_status(mbox, items, status_r) < 0) {
+			/* can't do anything about this */
+		}
 	} else if ((items & (STATUS_FIRST_UNSEEN_SEQ | STATUS_KEYWORDS |
 			     STATUS_PERMANENT_FLAGS |
 			     STATUS_FIRST_RECENT_UID)) != 0) {
 		/* getting these requires opening the mailbox */
 		if (mailbox_open(box) < 0)
 			return -1;
-		imapc_mailbox_get_selected_status(mbox, items, status_r);
+		if (imapc_mailbox_get_selected_status(mbox, items, status_r) < 0)
+			return -1;
 	} else {
 		if (imapc_mailbox_run_status(box, items, status_r) < 0)
 			return -1;
--- a/src/lib-storage/index/imapc/imapc-storage.h	Fri Apr 22 20:31:02 2016 +0300
+++ b/src/lib-storage/index/imapc/imapc-storage.h	Fri Apr 22 00:21:12 2016 +0300
@@ -108,9 +108,11 @@
 	enum mail_flags permanent_flags;
 	uint32_t highest_nonrecent_uid;
 
+	ARRAY(uint64_t) rseq_modseqs;
 	ARRAY_TYPE(uint32_t) delayed_expunged_uids;
 	uint32_t sync_uid_validity;
 	uint32_t sync_uid_next;
+	uint64_t sync_highestmodseq;
 	uint32_t sync_fetch_first_uid;
 	uint32_t sync_next_lseq;
 	uint32_t sync_next_rseq;
@@ -165,6 +167,7 @@
 void imapc_mail_cache_free(struct imapc_mail_cache *cache);
 int imapc_mailbox_select(struct imapc_mailbox *mbox);
 
+bool imapc_storage_has_modseqs(struct imapc_storage *storage);
 bool imap_resp_text_code_parse(const char *str, enum mail_error *error_r);
 void imapc_copy_error_from_reply(struct imapc_storage *storage,
 				 enum mail_error default_error,
--- a/src/lib-storage/index/imapc/imapc-sync.c	Fri Apr 22 20:31:02 2016 +0300
+++ b/src/lib-storage/index/imapc/imapc-sync.c	Fri Apr 22 00:21:12 2016 +0300
@@ -5,6 +5,7 @@
 #include "str.h"
 #include "imap-util.h"
 #include "mail-cache.h"
+#include "mail-index-modseq.h"
 #include "index-sync-private.h"
 #include "imapc-client.h"
 #include "imapc-msgmap.h"
@@ -245,6 +246,13 @@
 	}
 }
 
+static void imapc_sync_highestmodseq(struct imapc_sync_context *ctx)
+{
+	if (imapc_storage_has_modseqs(ctx->mbox->storage) &&
+	    mail_index_modseq_get_highest(ctx->sync_view) < ctx->mbox->sync_highestmodseq)
+		mail_index_update_highest_modseq(ctx->trans, ctx->mbox->sync_highestmodseq);
+}
+
 static void
 imapc_initial_sync_check(struct imapc_sync_context *ctx, bool nooped)
 {
@@ -311,6 +319,10 @@
 	string_t *cmd = t_str_new(64);
 
 	str_printfa(cmd, "UID FETCH %u:* (FLAGS", first_uid);
+	if (imapc_storage_has_modseqs(ctx->mbox->storage)) {
+		str_append(cmd, " MODSEQ");
+		mail_index_modseq_enable(ctx->mbox->box.index);
+	}
 	if (IMAPC_BOX_HAS_FEATURE(ctx->mbox, IMAPC_FEATURE_GMAIL_MIGRATION)) {
 		enum mailbox_info_flags flags;
 
@@ -393,8 +405,9 @@
 		imapc_mailbox_run(mbox);
 	array_free(&ctx->expunged_uids);
 
-	/* add uidnext after all appends */
+	/* add uidnext & highestmodseq after all appends */
 	imapc_sync_uid_next(ctx);
+	imapc_sync_highestmodseq(ctx);
 
 	if (!ctx->failed)
 		imapc_sync_expunge_eom(ctx);