Mercurial > dovecot > core-2.2

changeset 22450:1ff8f91dafba

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
pop3-migration: Try to assign UIDLs based on dovecot.index.cache Add the UIDLs to (imapc) mailbox cache after they've been assigned. Try to read them from the cache on the next sync to avoid reading mails' headers. This can be disabled with pop3_migration_skip_uidl_cache=yes, just in case there's some kind of a problem with it.
author Timo Sirainen <timo.sirainen@dovecot.fi>
date Fri, 25 Aug 2017 13:38:35 +0300
parents 56e8c501d8c1
children e2b9cadf91e4
files src/plugins/pop3-migration/pop3-migration-plugin.c
diffstat 1 files changed, 126 insertions(+), 6 deletions(-) [+]
line wrap: on
line diff
--- a/src/plugins/pop3-migration/pop3-migration-plugin.c	Fri Aug 25 13:36:11 2017 +0300
+++ b/src/plugins/pop3-migration/pop3-migration-plugin.c	Fri Aug 25 13:38:35 2017 +0300
@@ -57,6 +57,7 @@
 	unsigned int pop3_all_hdr_sha1_set:1;
 	unsigned int ignore_missing_uidls:1;
 	unsigned int skip_size_check:1;
+	unsigned int skip_uidl_cache:1;
 };
 
 struct pop3_migration_mailbox {
@@ -113,6 +114,18 @@
 	return 0;
 }
 
+static int pop3_uidl_map_uidl_cmp(const struct pop3_uidl_map *map1,
+				  const struct pop3_uidl_map *map2)
+{
+	return strcmp(map1->pop3_uidl, map2->pop3_uidl);
+}
+
+static int imap_msg_map_uidl_cmp(const struct imap_msg_map *map1,
+				 const struct imap_msg_map *map2)
+{
+	return null_strcmp(map1->pop3_uidl, map2->pop3_uidl);
+}
+
 static int pop3_uidl_map_hdr_cmp(const struct pop3_uidl_map *map1,
 				 const struct pop3_uidl_map *map2)
 {
@@ -525,8 +538,11 @@
 static int imap_map_read(struct mailbox *box)
 {
 	struct pop3_migration_mailbox *mbox = POP3_MIGRATION_CONTEXT(box);
+	struct index_mailbox_context *ibox = INDEX_STORAGE_CONTEXT(box);
 	struct pop3_migration_mail_storage *mstorage =
 		POP3_MIGRATION_CONTEXT(box->storage);
+	const unsigned int uidl_cache_idx =
+		ibox->cache_fields[MAIL_CACHE_POP3_UIDL].idx;
 	struct mailbox_status status;
         struct mailbox_transaction_context *t;
 	struct mail_search_args *search_args;
@@ -534,6 +550,7 @@
 	struct mail *mail;
 	struct imap_msg_map *map;
 	uoff_t psize = (uoff_t)-1;
+	string_t *uidl;
 	int ret = 0;
 
 	mailbox_get_open_status(box, STATUS_MESSAGES, &status);
@@ -549,6 +566,7 @@
 				  MAIL_FETCH_PHYSICAL_SIZE, NULL);
 	mail_search_args_unref(&search_args);
 
+	uidl = t_str_new(64);
 	while (mailbox_search_next(ctx, &mail)) {
 		if (mstorage->skip_size_check)
 			;
@@ -560,9 +578,16 @@
 			break;
 		}
 
+		if (!mstorage->skip_uidl_cache) {
+			str_truncate(uidl, 0);
+			(void)mail_cache_lookup_field(mail->transaction->cache_view,
+						      uidl, mail->seq, uidl_cache_idx);
+		}
+
 		map = array_append_space(&mbox->imap_msg_map);
 		map->uid = mail->uid;
 		map->psize = psize;
+		map->pop3_uidl = p_strdup_empty(box->pool, str_c(uidl));
 	}
 
 	if (mailbox_search_deinit(&ctx) < 0) {
@@ -582,6 +607,45 @@
 				   mbox->first_unfound_idx+1);
 }
 
+static void pop3_uidl_assign_cached(struct mailbox *box)
+{
+	struct pop3_migration_mailbox *mbox = POP3_MIGRATION_CONTEXT(box);
+	struct pop3_migration_mail_storage *mstorage =
+		POP3_MIGRATION_CONTEXT(box->storage);
+	struct pop3_uidl_map *pop3_map;
+	struct imap_msg_map *imap_map;
+	unsigned int imap_idx, pop3_idx, pop3_count, imap_count;
+	int ret;
+
+	if (mstorage->skip_uidl_cache)
+		return;
+
+	array_sort(&mstorage->pop3_uidl_map, pop3_uidl_map_uidl_cmp);
+	array_sort(&mbox->imap_msg_map, imap_msg_map_uidl_cmp);
+
+	pop3_map = array_get_modifiable(&mstorage->pop3_uidl_map, &pop3_count);
+	imap_map = array_get_modifiable(&mbox->imap_msg_map, &imap_count);
+
+	/* see if we can match the messages using sizes */
+	for (imap_idx = pop3_idx = 0; imap_idx < imap_count; imap_idx++) {
+		if (imap_map[imap_idx].pop3_uidl == NULL)
+			continue;
+
+		ret = 1;
+		for (; pop3_idx < pop3_count; pop3_idx++) {
+			ret = strcmp(imap_map[imap_idx].pop3_uidl,
+				     pop3_map[pop3_idx].pop3_uidl);
+			if (ret >= 0)
+				break;
+		}
+		if (ret == 0) {
+			imap_map[imap_idx].pop3_seq =
+				pop3_map[pop3_idx].pop3_seq;
+			pop3_map[pop3_idx].imap_uid = imap_map[imap_idx].uid;
+		}
+	}
+}
+
 static bool pop3_uidl_assign_by_size(struct mailbox *box)
 {
 	struct pop3_migration_mailbox *mbox = POP3_MIGRATION_CONTEXT(box);
@@ -590,9 +654,7 @@
 	struct pop3_uidl_map *pop3_map;
 	struct imap_msg_map *imap_map;
 	unsigned int i, pop3_count, imap_count, count;
-
-	if (mstorage->skip_size_check)
-		return FALSE;
+	unsigned int size_match = 0, uidl_match = 0;
 
 	pop3_map = array_get_modifiable(&mstorage->pop3_uidl_map, &pop3_count);
 	imap_map = array_get_modifiable(&mbox->imap_msg_map, &imap_count);
@@ -600,20 +662,34 @@
 
 	/* see if we can match the messages using sizes */
 	for (i = 0; i < count; i++) {
-		if (pop3_map[i].size != imap_map[i].psize)
+		if (imap_map[i].pop3_uidl != NULL) {
+			/* some of the UIDLs were already found cached. */
+			if (strcmp(pop3_map[i].pop3_uidl, imap_map[i].pop3_uidl) == 0) {
+				uidl_match++;
+				continue;
+			}
+			/* mismatch - can't trust the sizes */
+			break;
+		}
+
+		if (pop3_map[i].size != imap_map[i].psize ||
+		    mstorage->skip_size_check)
 			break;
 		if (i+1 < count && pop3_map[i].size == pop3_map[i+1].size) {
 			/* two messages with same size, don't trust them */
 			break;
 		}
 
+		size_match++;
 		pop3_map[i].imap_uid = imap_map[i].uid;
 		imap_map[i].pop3_uidl = pop3_map[i].pop3_uidl;
 		imap_map[i].pop3_seq = pop3_map[i].pop3_seq;
 	}
 	mbox->first_unfound_idx = i;
-	if (box->storage->user->mail_debug)
-		i_debug("pop3_migration: %u/%u mails matched by size", i, count);
+	if (box->storage->user->mail_debug) {
+		i_debug("pop3_migration: cached uidls=%u, size matches=%u, total=%u",
+			uidl_match, size_match, count);
+	}
 	return i == count;
 }
 
@@ -708,9 +784,42 @@
 	return 0;
 }
 
+static void imap_uidls_add_to_cache(struct mailbox *box)
+{
+	struct pop3_migration_mailbox *mbox = POP3_MIGRATION_CONTEXT(box);
+	struct mailbox_transaction_context *t;
+	struct mail *mail;
+	struct index_mail *imail;
+	struct imap_msg_map *imap_map;
+	unsigned int i, count;
+	unsigned int field_idx;
+
+	t = mailbox_transaction_begin(box, 0);
+	mail = mail_alloc(t, 0, NULL);
+	imail = (struct index_mail *)mail;
+	field_idx = imail->ibox->cache_fields[MAIL_CACHE_POP3_UIDL].idx;
+
+	imap_map = array_get_modifiable(&mbox->imap_msg_map, &count);
+	for (i = 0; i < count; i++) {
+		if (imap_map[i].pop3_uidl == NULL)
+			continue;
+
+		if (!mail_set_uid(mail, imap_map[i].uid))
+			i_unreached();
+		if (mail_cache_field_can_add(t->cache_trans, mail->seq, field_idx)) {
+			index_mail_cache_add_idx(imail, field_idx,
+				imap_map[i].pop3_uidl, strlen(imap_map[i].pop3_uidl)+1);
+		}
+	}
+	mail_free(&mail);
+	(void)mailbox_transaction_commit(&t);
+}
+
 static int pop3_migration_uidl_sync(struct mailbox *box)
 {
 	struct pop3_migration_mailbox *mbox = POP3_MIGRATION_CONTEXT(box);
+	struct pop3_migration_mail_storage *mstorage =
+		POP3_MIGRATION_CONTEXT(box->storage);
 	struct mailbox *pop3_box;
 
 	pop3_box = pop3_mailbox_alloc(box->storage);
@@ -723,6 +832,11 @@
 		return -1;
 	}
 
+	pop3_uidl_assign_cached(box);
+
+	array_sort(&mstorage->pop3_uidl_map, pop3_uidl_map_pop3_seq_cmp);
+	array_sort(&mbox->imap_msg_map, imap_msg_map_uid_cmp);
+
 	if (!pop3_uidl_assign_by_size(box)) {
 		/* everything wasn't assigned, figure out the rest with
 		   header hashes */
@@ -732,6 +846,9 @@
 		}
 	}
 
+	if (!mstorage->skip_uidl_cache)
+		imap_uidls_add_to_cache(box);
+
 	mbox->uidl_synced = TRUE;
 	mailbox_free(&pop3_box);
 	return 0;
@@ -899,6 +1016,9 @@
 	mstorage->skip_size_check =
 		mail_user_plugin_getenv(storage->user,
 			"pop3_migration_skip_size_check") != NULL;
+	mstorage->skip_uidl_cache =
+		mail_user_plugin_getenv(storage->user,
+			"pop3_migration_skip_uidl_cache") != NULL;
 
 	MODULE_CONTEXT_SET(storage, pop3_migration_storage_module, mstorage);
 }