changeset 17845:d2e1b3f6d13b

dsync: Added DSYNC_BRAIN_FLAG_NO_MAIL_PREFETCH to avoid opening mails unnecessarily.
author Timo Sirainen <tss@iki.fi>
date Mon, 29 Sep 2014 14:16:06 +0300
parents 35c3194900b7
children 475ff24ac365
files src/doveadm/dsync/dsync-brain-mailbox.c src/doveadm/dsync/dsync-brain-private.h src/doveadm/dsync/dsync-brain.c src/doveadm/dsync/dsync-brain.h src/doveadm/dsync/dsync-ibc-stream.c src/doveadm/dsync/dsync-mail.c src/doveadm/dsync/dsync-mail.h src/doveadm/dsync/dsync-mailbox-export.c src/doveadm/dsync/dsync-mailbox-export.h src/doveadm/dsync/dsync-mailbox-import.c
diffstat 10 files changed, 103 insertions(+), 40 deletions(-) [+]
line wrap: on
line diff
--- a/src/doveadm/dsync/dsync-brain-mailbox.c	Fri Sep 26 00:32:03 2014 +0300
+++ b/src/doveadm/dsync/dsync-brain-mailbox.c	Mon Sep 29 14:16:06 2014 +0300
@@ -304,6 +304,8 @@
 	    (brain->local_dsync_box.have_save_guids ||
 	     (brain->backup_send && brain->local_dsync_box.have_guids)))
 		exporter_flags |= DSYNC_MAILBOX_EXPORTER_FLAG_MAILS_HAVE_GUIDS;
+	if (brain->no_mail_prefetch)
+		exporter_flags |= DSYNC_MAILBOX_EXPORTER_FLAG_MINIMAL_DMAIL_FILL;
 
 	brain->box_exporter = brain->backup_recv ? NULL :
 		dsync_mailbox_export_init(brain->box, brain->log_scan,
--- a/src/doveadm/dsync/dsync-brain-private.h	Fri Sep 26 00:32:03 2014 +0300
+++ b/src/doveadm/dsync/dsync-brain-private.h	Mon Sep 29 14:16:06 2014 +0300
@@ -100,6 +100,7 @@
 	unsigned int sync_visible_namespaces:1;
 	unsigned int no_mail_sync:1;
 	unsigned int no_backup_overwrite:1;
+	unsigned int no_mail_prefetch:1;
 	unsigned int changes_during_sync:1;
 	unsigned int require_full_resync:1;
 	unsigned int verbose_proctitle:1;
--- a/src/doveadm/dsync/dsync-brain.c	Fri Sep 26 00:32:03 2014 +0300
+++ b/src/doveadm/dsync/dsync-brain.c	Mon Sep 29 14:16:06 2014 +0300
@@ -137,6 +137,8 @@
 	brain->no_mail_sync = (flags & DSYNC_BRAIN_FLAG_NO_MAIL_SYNC) != 0;
 	brain->no_backup_overwrite =
 		(flags & DSYNC_BRAIN_FLAG_NO_BACKUP_OVERWRITE) != 0;
+	brain->no_mail_prefetch =
+		(flags & DSYNC_BRAIN_FLAG_NO_MAIL_PREFETCH) != 0;
 }
 
 struct dsync_brain *
--- a/src/doveadm/dsync/dsync-brain.h	Fri Sep 26 00:32:03 2014 +0300
+++ b/src/doveadm/dsync/dsync-brain.h	Mon Sep 29 14:16:06 2014 +0300
@@ -22,7 +22,11 @@
 	DSYNC_BRAIN_FLAG_NO_BACKUP_OVERWRITE	= 0x40,
 	/* Run storage purge on the remote after syncing.
 	   Useful with e.g. a nightly doveadm backup. */
-	DSYNC_BRAIN_FLAG_PURGE_REMOTE		= 0x80
+	DSYNC_BRAIN_FLAG_PURGE_REMOTE		= 0x80,
+	/* Don't prefetch mail bodies until they're actually needed. This works
+	   only with pipe ibc. It's useful if most of the mails can be copied
+	   directly within filesystem without having to read them. */
+	DSYNC_BRAIN_FLAG_NO_MAIL_PREFETCH	= 0x100
 };
 
 enum dsync_brain_sync_type {
--- a/src/doveadm/dsync/dsync-ibc-stream.c	Fri Sep 26 00:32:03 2014 +0300
+++ b/src/doveadm/dsync/dsync-ibc-stream.c	Mon Sep 29 14:16:06 2014 +0300
@@ -1681,6 +1681,7 @@
 	struct dsync_serializer_encoder *encoder;
 	string_t *str = t_str_new(128);
 
+	i_assert(!mail->minimal_fields);
 	i_assert(ibc->value_output == NULL);
 
 	str_append_c(str, items[ITEM_MAIL].chr);
--- a/src/doveadm/dsync/dsync-mail.c	Fri Sep 26 00:32:03 2014 +0300
+++ b/src/doveadm/dsync/dsync-mail.c	Mon Sep 29 14:16:06 2014 +0300
@@ -60,10 +60,10 @@
 	return ret;
 }
 
-int dsync_mail_fill(struct mail *mail, struct dsync_mail *dmail_r,
-		    const char **error_field_r)
+int dsync_mail_fill(struct mail *mail, bool minimal_fill,
+		    struct dsync_mail *dmail_r, const char **error_field_r)
 {
-	const char *guid, *str;
+	const char *guid;
 
 	memset(dmail_r, 0, sizeof(*dmail_r));
 
@@ -76,6 +76,22 @@
 
 	dmail_r->input_mail = mail;
 	dmail_r->input_mail_uid = mail->uid;
+
+	if (mail_get_save_date(mail, &dmail_r->saved_date) < 0) {
+		*error_field_r = "saved-date";
+		return -1;
+	}
+	if (!minimal_fill)
+		return dsync_mail_fill_nonminimal(mail, dmail_r, error_field_r);
+	dmail_r->minimal_fields = TRUE;
+	return 0;
+}
+
+int dsync_mail_fill_nonminimal(struct mail *mail, struct dsync_mail *dmail_r,
+			       const char **error_field_r)
+{
+	const char *str;
+
 	if (mail_get_stream(mail, NULL, NULL, &dmail_r->input) < 0) {
 		*error_field_r = "body";
 		return -1;
@@ -97,10 +113,6 @@
 		*error_field_r = "received-date";
 		return -1;
 	}
-	if (mail_get_save_date(mail, &dmail_r->saved_date) < 0) {
-		*error_field_r = "saved-date";
-		return -1;
-	}
 	return 0;
 }
 
--- a/src/doveadm/dsync/dsync-mail.h	Fri Sep 26 00:32:03 2014 +0300
+++ b/src/doveadm/dsync/dsync-mail.h	Mon Sep 29 14:16:06 2014 +0300
@@ -10,21 +10,24 @@
 	/* either GUID="" or uid=0 */
 	const char *guid;
 	uint32_t uid;
-
-	const char *pop3_uidl;
-	unsigned int pop3_order;
-	time_t received_date;
 	time_t saved_date;
 
-	/* Input stream containing the message text, or NULL if all instances
-	   of the message were already expunged from this mailbox. */
-	struct istream *input;
-
 	/* If non-NULL, we're syncing within the dsync process using ibc-pipe.
 	   This mail can be used to mailbox_copy() the mail. */
 	struct mail *input_mail;
 	/* Verify that this equals to input_mail->uid */
 	uint32_t input_mail_uid;
+
+	/* TRUE if the following fields aren't set, because minimal_fill=TRUE
+	   parameter was used. */
+	bool minimal_fields;
+
+	const char *pop3_uidl;
+	unsigned int pop3_order;
+	time_t received_date;
+	/* Input stream containing the message text, or NULL if all instances
+	   of the message were already expunged from this mailbox. */
+	struct istream *input;
 };
 
 struct dsync_mail_request {
@@ -80,8 +83,10 @@
 dsync_mail_get_hash_headers(struct mailbox *box);
 
 int dsync_mail_get_hdr_hash(struct mail *mail, const char **hdr_hash_r);
-int dsync_mail_fill(struct mail *mail, struct dsync_mail *dmail_r,
-		    const char **error_field_r);
+int dsync_mail_fill(struct mail *mail, bool minimal_fill,
+		    struct dsync_mail *dmail_r, const char **error_field_r);
+int dsync_mail_fill_nonminimal(struct mail *mail, struct dsync_mail *dmail_r,
+			       const char **error_field_r);
 
 void dsync_mail_change_dup(pool_t pool, const struct dsync_mail_change *src,
 			   struct dsync_mail_change *dest_r);
--- a/src/doveadm/dsync/dsync-mailbox-export.c	Fri Sep 26 00:32:03 2014 +0300
+++ b/src/doveadm/dsync/dsync-mailbox-export.c	Mon Sep 29 14:16:06 2014 +0300
@@ -57,6 +57,7 @@
 	unsigned int body_search_initialized:1;
 	unsigned int auto_export_mails:1;
 	unsigned int mails_have_guids:1;
+	unsigned int minimal_dmail_fill:1;
 	unsigned int return_all_mails:1;
 };
 
@@ -474,6 +475,8 @@
 		(flags & DSYNC_MAILBOX_EXPORTER_FLAG_AUTO_EXPORT_MAILS) != 0;
 	exporter->mails_have_guids =
 		(flags & DSYNC_MAILBOX_EXPORTER_FLAG_MAILS_HAVE_GUIDS) != 0;
+	exporter->minimal_dmail_fill =
+		(flags & DSYNC_MAILBOX_EXPORTER_FLAG_MINIMAL_DMAIL_FILL) != 0;
 	p_array_init(&exporter->requested_uids, pool, 16);
 	p_array_init(&exporter->search_uids, pool, 16);
 	hash_table_create(&exporter->export_guids, pool, 0, str_hash, strcmp);
@@ -657,6 +660,7 @@
 	const struct seq_range *uids;
 	char *guid;
 	const char *const_guid;
+	enum mail_fetch_field wanted_fields;
 	struct dsync_mail_guid_instances *instances;
 	const struct seq_range *range;
 	unsigned int i, count;
@@ -714,16 +718,16 @@
 	array_append_array(&exporter->search_uids, &exporter->requested_uids);
 	array_clear(&exporter->requested_uids);
 
+	wanted_fields = MAIL_FETCH_GUID | MAIL_FETCH_SAVE_DATE;
+	if (!exporter->minimal_dmail_fill) {
+		wanted_fields |= MAIL_FETCH_RECEIVED_DATE |
+			MAIL_FETCH_UIDL_BACKEND | MAIL_FETCH_POP3_ORDER |
+			MAIL_FETCH_STREAM_HEADER | MAIL_FETCH_STREAM_BODY;
+	}
 	exporter->search_count += seq_range_count(&sarg->value.seqset);
 	exporter->search_ctx =
 		mailbox_search_init(exporter->trans, search_args, NULL,
-				    MAIL_FETCH_GUID |
-				    MAIL_FETCH_UIDL_BACKEND |
-				    MAIL_FETCH_POP3_ORDER |
-				    MAIL_FETCH_RECEIVED_DATE |
-				    MAIL_FETCH_SAVE_DATE |
-				    MAIL_FETCH_STREAM_HEADER |
-				    MAIL_FETCH_STREAM_BODY, NULL);
+				    wanted_fields, NULL);
 	mail_search_args_unref(&search_args);
 	return array_count(&sarg->value.seqset) > 0 ? 1 : 0;
 }
@@ -748,7 +752,8 @@
 	struct dsync_mail_guid_instances *instances;
 	const char *error_field;
 
-	if (dsync_mail_fill(mail, &exporter->dsync_mail, &error_field) < 0)
+	if (dsync_mail_fill(mail, exporter->minimal_dmail_fill,
+			    &exporter->dsync_mail, &error_field) < 0)
 		return dsync_mail_error(exporter, mail, error_field);
 
 	instances = *exporter->dsync_mail.guid == '\0' ? NULL :
--- a/src/doveadm/dsync/dsync-mailbox-export.h	Fri Sep 26 00:32:03 2014 +0300
+++ b/src/doveadm/dsync/dsync-mailbox-export.h	Mon Sep 29 14:16:06 2014 +0300
@@ -3,7 +3,8 @@
 
 enum dsync_mailbox_exporter_flags {
 	DSYNC_MAILBOX_EXPORTER_FLAG_AUTO_EXPORT_MAILS	= 0x01,
-	DSYNC_MAILBOX_EXPORTER_FLAG_MAILS_HAVE_GUIDS	= 0x02
+	DSYNC_MAILBOX_EXPORTER_FLAG_MAILS_HAVE_GUIDS	= 0x02,
+	DSYNC_MAILBOX_EXPORTER_FLAG_MINIMAL_DMAIL_FILL	= 0x04
 };
 
 struct dsync_mailbox_exporter *
--- a/src/doveadm/dsync/dsync-mailbox-import.c	Fri Sep 26 00:32:03 2014 +0300
+++ b/src/doveadm/dsync/dsync-mailbox-import.c	Mon Sep 29 14:16:06 2014 +0300
@@ -1544,7 +1544,7 @@
 	if (!mail_set_uid(importer->mail, uid))
 		return 0;
 
-	if (dsync_mail_fill(importer->mail, dmail_r, &error_field) < 0) {
+	if (dsync_mail_fill(importer->mail, TRUE, dmail_r, &error_field) < 0) {
 		errstr = mailbox_get_last_error(importer->mail->box, &error);
 		if (error == MAIL_ERROR_EXPUNGED)
 			return 0;
@@ -1905,6 +1905,17 @@
 	return 0;
 }
 
+static void
+dsync_mailbox_save_set_nonminimal(struct mail_save_context *save_ctx,
+				  const struct dsync_mail *mail)
+{
+	if (mail->pop3_uidl != NULL && *mail->pop3_uidl != '\0')
+		mailbox_save_set_pop3_uidl(save_ctx, mail->pop3_uidl);
+	if (mail->pop3_order > 0)
+		mailbox_save_set_pop3_order(save_ctx, mail->pop3_order);
+	mailbox_save_set_received_date(save_ctx, mail->received_date, 0);
+}
+
 static struct mail_save_context *
 dsync_mailbox_save_init(struct dsync_mailbox_importer *importer,
 			const struct dsync_mail *mail,
@@ -1919,11 +1930,9 @@
 	if (mail->saved_date != 0)
 		mailbox_save_set_save_date(save_ctx, mail->saved_date);
 	dsync_mailbox_save_set_metadata(importer, save_ctx, newmail->change);
-	if (mail->pop3_uidl != NULL && *mail->pop3_uidl != '\0')
-		mailbox_save_set_pop3_uidl(save_ctx, mail->pop3_uidl);
-	if (mail->pop3_order > 0)
-		mailbox_save_set_pop3_order(save_ctx, mail->pop3_order);
-	mailbox_save_set_received_date(save_ctx, mail->received_date, 0);
+
+	if (!mail->minimal_fields)
+		dsync_mailbox_save_set_nonminimal(save_ctx, mail);
 	return save_ctx;
 }
 
@@ -1934,6 +1943,7 @@
 			struct importer_new_mail **all_newmails_forcopy)
 {
 	struct mail_save_context *save_ctx;
+	struct istream *input;
 	ssize_t ret;
 	bool save_failed = FALSE;
 
@@ -1960,22 +1970,42 @@
 		return;
 	}
 	/* fallback to saving from remote stream */
+	if (mail->minimal_fields) {
+		struct dsync_mail mail2;
+		const char *error_field;
 
-	if (mail->input == NULL) {
+		i_assert(mail->input_mail != NULL);
+
+		if (dsync_mail_fill_nonminimal(mail->input_mail, &mail2,
+					       &error_field) < 0) {
+			i_error("Mailbox %s: Failed to read mail %s uid=%u: %s",
+				mailbox_get_vname(importer->box),
+				error_field, mail->uid,
+				mailbox_get_last_error(importer->box, NULL));
+			importer->failed = TRUE;
+			return;
+		}
+		dsync_mailbox_save_set_nonminimal(save_ctx, &mail2);
+		input = mail2.input;
+	} else {
+		input = mail->input;
+	}
+
+	if (input == NULL) {
 		/* it was just expunged in remote, skip it */
 		mailbox_save_cancel(&save_ctx);
 		return;
 	}
 
-	i_stream_seek(mail->input, 0);
-	if (mailbox_save_begin(&save_ctx, mail->input) < 0) {
+	i_stream_seek(input, 0);
+	if (mailbox_save_begin(&save_ctx, input) < 0) {
 		i_error("Mailbox %s: Saving failed: %s",
 			mailbox_get_vname(importer->box),
 			mailbox_get_last_error(importer->box, NULL));
 		importer->failed = TRUE;
 		return;
 	}
-	while ((ret = i_stream_read(mail->input)) > 0 || ret == -2) {
+	while ((ret = i_stream_read(input)) > 0 || ret == -2) {
 		if (mailbox_save_continue(save_ctx) < 0) {
 			save_failed = TRUE;
 			ret = -1;
@@ -1984,10 +2014,10 @@
 	}
 	i_assert(ret == -1);
 
-	if (mail->input->stream_errno != 0) {
+	if (input->stream_errno != 0) {
 		i_error("Mailbox %s: read(msg input) failed: %s",
 			mailbox_get_vname(importer->box),
-			i_stream_get_error(mail->input));
+			i_stream_get_error(input));
 		mailbox_save_cancel(&save_ctx);
 		importer->failed = TRUE;
 	} else if (save_failed) {
@@ -1997,7 +2027,7 @@
 		mailbox_save_cancel(&save_ctx);
 		importer->failed = TRUE;
 	} else {
-		i_assert(mail->input->eof);
+		i_assert(input->eof);
 		if (mailbox_save_finish(&save_ctx) < 0) {
 			i_error("Mailbox %s: Saving failed: %s",
 				mailbox_get_vname(importer->box),