# HG changeset patch
# User Timo Sirainen <timo.sirainen@dovecot.fi>
# Date 1467372624 -10800
# Node ID d54651ba988a255e87925d13a058b284d2cd0552
# Parent  6c40ae42ba1400a3daf898fcb8924d411e71d966
doveadm sync/backup: Added -S <max size> parameter to skip too large mails.

diff -r 6c40ae42ba14 -r d54651ba988a src/doveadm/doveadm-dsync.c
--- a/src/doveadm/doveadm-dsync.c	Wed Jul 13 09:51:08 2016 +0300
+++ b/src/doveadm/doveadm-dsync.c	Fri Jul 01 14:30:24 2016 +0300
@@ -67,6 +67,7 @@
 	ARRAY_TYPE(const_string) exclude_mailboxes;
 	ARRAY_TYPE(const_string) namespace_prefixes;
 	time_t sync_since_timestamp;
+	uoff_t sync_max_size;
 	unsigned int io_timeout_secs;
 
 	const char *remote_name;
@@ -566,6 +567,7 @@
 			"%s ", net_ip2addr(&_ctx->cur_client_ip));
 	}
 	set.sync_since_timestamp = ctx->sync_since_timestamp;
+	set.sync_max_size = ctx->sync_max_size;
 	set.sync_box = ctx->mailbox;
 	set.sync_flag = ctx->sync_flags;
 	set.virtual_all_box = ctx->virtual_all_box;
@@ -1036,6 +1038,10 @@
 		if (mail_parse_human_timestamp(optarg, &ctx->sync_since_timestamp) < 0)
 			i_fatal("Invalid -t parameter: %s", optarg);
 		break;
+	case 'S':
+		if (settings_parse_size(optarg, &ctx->sync_max_size) < 0)
+			i_fatal("Invalid -S parameter: %s", optarg);
+		break;
 	case 'T':
 		if (str_to_uint(optarg, &ctx->io_timeout_secs) < 0)
 			i_fatal("Invalid -T parameter: %s", optarg);
diff -r 6c40ae42ba14 -r d54651ba988a src/doveadm/dsync/dsync-brain-mailbox.c
--- a/src/doveadm/dsync/dsync-brain-mailbox.c	Wed Jul 13 09:51:08 2016 +0300
+++ b/src/doveadm/dsync/dsync-brain-mailbox.c	Fri Jul 01 14:30:24 2016 +0300
@@ -233,6 +233,7 @@
 					  remote_dsync_box->highest_modseq,
 					  remote_dsync_box->highest_pvt_modseq,
 					  brain->sync_since_timestamp,
+					  brain->sync_max_size,
 					  brain->sync_flag,
 					  import_flags);
 }
@@ -322,6 +323,8 @@
 		exporter_flags |= DSYNC_MAILBOX_EXPORTER_FLAG_MINIMAL_DMAIL_FILL;
 	if (brain->sync_since_timestamp > 0)
 		exporter_flags |= DSYNC_MAILBOX_EXPORTER_FLAG_TIMESTAMPS;
+	if (brain->sync_max_size > 0)
+		exporter_flags |= DSYNC_MAILBOX_EXPORTER_FLAG_VSIZES;
 	if (brain->hdr_hash_v2)
 		exporter_flags |= DSYNC_MAILBOX_EXPORTER_FLAG_HDR_HASH_V2;
 	if (remote_dsync_box->messages_count == 0) {
diff -r 6c40ae42ba14 -r d54651ba988a src/doveadm/dsync/dsync-brain-private.h
--- a/src/doveadm/dsync/dsync-brain-private.h	Wed Jul 13 09:51:08 2016 +0300
+++ b/src/doveadm/dsync/dsync-brain-private.h	Fri Jul 01 14:30:24 2016 +0300
@@ -58,6 +58,7 @@
 	const char *const *exclude_mailboxes;
 	enum dsync_brain_sync_type sync_type;
 	time_t sync_since_timestamp;
+	uoff_t sync_max_size;
 	const char *sync_flag;
 	char alt_char;
 
diff -r 6c40ae42ba14 -r d54651ba988a src/doveadm/dsync/dsync-brain.c
--- a/src/doveadm/dsync/dsync-brain.c	Wed Jul 13 09:51:08 2016 +0300
+++ b/src/doveadm/dsync/dsync-brain.c	Fri Jul 01 14:30:24 2016 +0300
@@ -192,6 +192,7 @@
 	brain->alt_char = set->mailbox_alt_char == '\0' ? '_' :
 		set->mailbox_alt_char;
 	brain->sync_since_timestamp = set->sync_since_timestamp;
+	brain->sync_max_size = set->sync_max_size;
 	brain->sync_flag = p_strdup(brain->pool, set->sync_flag);
 	brain->sync_box = p_strdup(brain->pool, set->sync_box);
 	brain->exclude_mailboxes = set->exclude_mailboxes == NULL ? NULL :
@@ -230,6 +231,7 @@
 	ibc_set.virtual_all_box = set->virtual_all_box;
 	ibc_set.exclude_mailboxes = set->exclude_mailboxes;
 	ibc_set.sync_since_timestamp = set->sync_since_timestamp;
+	ibc_set.sync_max_size = set->sync_max_size;
 	ibc_set.sync_flags = set->sync_flag;
 	memcpy(ibc_set.sync_box_guid, set->sync_box_guid,
 	       sizeof(ibc_set.sync_box_guid));
@@ -485,6 +487,7 @@
 	brain->exclude_mailboxes = ibc_set->exclude_mailboxes == NULL ? NULL :
 		p_strarray_dup(brain->pool, ibc_set->exclude_mailboxes);
 	brain->sync_since_timestamp = ibc_set->sync_since_timestamp;
+	brain->sync_max_size = ibc_set->sync_max_size;
 	brain->sync_flag = p_strdup(brain->pool, ibc_set->sync_flags);
 	memcpy(brain->sync_box_guid, ibc_set->sync_box_guid,
 	       sizeof(brain->sync_box_guid));
diff -r 6c40ae42ba14 -r d54651ba988a src/doveadm/dsync/dsync-brain.h
--- a/src/doveadm/dsync/dsync-brain.h	Wed Jul 13 09:51:08 2016 +0300
+++ b/src/doveadm/dsync/dsync-brain.h	Fri Jul 01 14:30:24 2016 +0300
@@ -69,6 +69,8 @@
 	char mailbox_alt_char;
 	/* Sync only mails with received timestamp at least this high. */
 	time_t sync_since_timestamp;
+	/* Don't sync mails larger than this. */
+	uoff_t sync_max_size;
 	/* Sync only mails which contains / doesn't contain this flag.
 	   '-' at the beginning means this flag must not exist. */
 	const char *sync_flag;
diff -r 6c40ae42ba14 -r d54651ba988a src/doveadm/dsync/dsync-ibc-pipe.c
--- a/src/doveadm/dsync/dsync-ibc-pipe.c	Wed Jul 13 09:51:08 2016 +0300
+++ b/src/doveadm/dsync/dsync-ibc-pipe.c	Fri Jul 01 14:30:24 2016 +0300
@@ -180,6 +180,7 @@
 	memcpy(item->u.set.sync_box_guid, set->sync_box_guid,
 	       sizeof(item->u.set.sync_box_guid));
 	item->u.set.sync_since_timestamp = set->sync_since_timestamp;
+	item->u.set.sync_max_size = set->sync_max_size;
 	item->u.set.sync_flags = p_strdup(item->pool, set->sync_flags);
 }
 
diff -r 6c40ae42ba14 -r d54651ba988a src/doveadm/dsync/dsync-ibc-stream.c
--- a/src/doveadm/dsync/dsync-ibc-stream.c	Wed Jul 13 09:51:08 2016 +0300
+++ b/src/doveadm/dsync/dsync-ibc-stream.c	Fri Jul 01 14:30:24 2016 +0300
@@ -77,7 +77,8 @@
 	  	"debug sync_visible_namespaces exclude_mailboxes  "
 	  	"send_mail_requests backup_send backup_recv lock_timeout "
 	  	"no_mail_sync no_mailbox_renames no_backup_overwrite purge_remote "
-		"no_notify sync_since_timestamp sync_flags virtual_all_box"
+		"no_notify sync_since_timestamp sync_max_size sync_flags "
+	  	"virtual_all_box"
 	},
 	{ .name = "mailbox_state",
 	  .chr = 'S',
@@ -710,6 +711,10 @@
 		dsync_serializer_encode_add(encoder, "sync_since_timestamp",
 			t_strdup_printf("%ld", (long)set->sync_since_timestamp));
 	}
+	if (set->sync_max_size > 0) {
+		dsync_serializer_encode_add(encoder, "sync_max_size",
+			t_strdup_printf("%llu", (unsigned long long)set->sync_max_size));
+	}
 	if (set->sync_flags != NULL) {
 		dsync_serializer_encode_add(encoder, "sync_flags",
 					    set->sync_flags);
@@ -822,6 +827,14 @@
 			return DSYNC_IBC_RECV_RET_TRYAGAIN;
 		}
 	}
+	if (dsync_deserializer_decode_try(decoder, "sync_max_size", &value)) {
+		if (str_to_uoff(value, &set->sync_max_size) < 0 ||
+		    set->sync_max_size == 0) {
+			dsync_ibc_input_error(ibc, decoder,
+				"Invalid sync_max_size: %s", value);
+			return DSYNC_IBC_RECV_RET_TRYAGAIN;
+		}
+	}
 	if (dsync_deserializer_decode_try(decoder, "sync_flags", &value))
 		set->sync_flags = p_strdup(pool, value);
 	if (dsync_deserializer_decode_try(decoder, "send_mail_requests", &value))
@@ -1634,6 +1647,10 @@
 		dsync_serializer_encode_add(encoder, "received_timestamp",
 			t_strdup_printf("%lx", (unsigned long)change->received_timestamp));
 	}
+	if (change->virtual_size > 0) {
+		dsync_serializer_encode_add(encoder, "virtual_size",
+			t_strdup_printf("%llx", (unsigned long long)change->virtual_size));
+	}
 
 	dsync_serializer_encode_finish(&encoder, str);
 	dsync_ibc_stream_send_string(ibc, str);
@@ -1741,6 +1758,11 @@
 		dsync_ibc_input_error(ibc, decoder, "Invalid received_timestamp");
 		return DSYNC_IBC_RECV_RET_TRYAGAIN;
 	}
+	if (dsync_deserializer_decode_try(decoder, "virtual_size", &value) &&
+	    str_to_uoff(value, &change->virtual_size) < 0) {
+		dsync_ibc_input_error(ibc, decoder, "Invalid virtual_size");
+		return DSYNC_IBC_RECV_RET_TRYAGAIN;
+	}
 
 	*change_r = change;
 	return DSYNC_IBC_RECV_RET_OK;
diff -r 6c40ae42ba14 -r d54651ba988a src/doveadm/dsync/dsync-ibc.h
--- a/src/doveadm/dsync/dsync-ibc.h	Wed Jul 13 09:51:08 2016 +0300
+++ b/src/doveadm/dsync/dsync-ibc.h	Fri Jul 01 14:30:24 2016 +0300
@@ -58,6 +58,8 @@
 	const char *const *exclude_mailboxes;
 	/* Sync only mails with received timestamp at least this high. */
 	time_t sync_since_timestamp;
+	/* Don't sync mails larger than this. */
+	uoff_t sync_max_size;
 	/* Sync only mails with specified flags. */
 	const char *sync_flags;
 
diff -r 6c40ae42ba14 -r d54651ba988a src/doveadm/dsync/dsync-mail.c
--- a/src/doveadm/dsync/dsync-mail.c	Wed Jul 13 09:51:08 2016 +0300
+++ b/src/doveadm/dsync/dsync-mail.c	Fri Jul 01 14:30:24 2016 +0300
@@ -160,4 +160,5 @@
 	const_string_array_dup(pool, &src->keyword_changes,
 			       &dest_r->keyword_changes);
 	dest_r->received_timestamp = src->received_timestamp;
+	dest_r->virtual_size = src->virtual_size;
 }
diff -r 6c40ae42ba14 -r d54651ba988a src/doveadm/dsync/dsync-mail.h
--- a/src/doveadm/dsync/dsync-mail.h	Wed Jul 13 09:51:08 2016 +0300
+++ b/src/doveadm/dsync/dsync-mail.h	Fri Jul 01 14:30:24 2016 +0300
@@ -81,6 +81,9 @@
 
 	/* Received timestamp for saves, if brain.sync_since_timestamp is set */
 	time_t received_timestamp;
+	/* Mail's size for saves if brain.sync_max_size is set,
+	   (uoff_t)-1 otherwise. */
+	uoff_t virtual_size;
 };
 
 struct mailbox_header_lookup_ctx *
diff -r 6c40ae42ba14 -r d54651ba988a src/doveadm/dsync/dsync-mailbox-export.c
--- a/src/doveadm/dsync/dsync-mailbox-export.c	Wed Jul 13 09:51:08 2016 +0300
+++ b/src/doveadm/dsync/dsync-mailbox-export.c	Fri Jul 01 14:30:24 2016 +0300
@@ -63,6 +63,7 @@
 	unsigned int minimal_dmail_fill:1;
 	unsigned int return_all_mails:1;
 	unsigned int export_received_timestamps:1;
+	unsigned int export_virtual_sizes:1;
 	unsigned int no_hdr_hashes:1;
 };
 
@@ -284,12 +285,15 @@
 	const char *guid, *hdr_hash;
 	enum mail_fetch_field wanted_fields = MAIL_FETCH_GUID;
 	time_t received_timestamp = 0;
+	uoff_t virtual_size = (uoff_t)-1;
 	int ret;
 
 	/* update wanted fields in case we didn't already set them for the
 	   search */
 	if (exporter->export_received_timestamps)
 		wanted_fields |= MAIL_FETCH_RECEIVED_DATE;
+	if (exporter->export_virtual_sizes)
+		wanted_fields |= MAIL_FETCH_VIRTUAL_SIZE;
 	mail_add_temp_wanted_fields(mail, wanted_fields,
 				    exporter->wanted_headers);
 
@@ -306,12 +310,18 @@
 			received_timestamp = 1;
 		}
 	}
+	if (exporter->export_received_timestamps) {
+		if (mail_get_virtual_size(mail, &virtual_size) < 0)
+			return dsync_mail_error(exporter, mail, "virtual-size");
+		i_assert(virtual_size != (uoff_t)-1);
+	}
 
 	change = export_save_change_get(exporter, mail->uid);
 	change->guid = *guid == '\0' ? "" :
 		p_strdup(exporter->pool, guid);
 	change->hdr_hash = p_strdup(exporter->pool, hdr_hash);
 	change->received_timestamp = received_timestamp;
+	change->virtual_size = virtual_size;
 	search_update_flag_changes(exporter, mail, change);
 
 	export_add_mail_instance(exporter, change, mail->seq);
@@ -508,6 +518,8 @@
 		(flags & DSYNC_MAILBOX_EXPORTER_FLAG_MINIMAL_DMAIL_FILL) != 0;
 	exporter->export_received_timestamps =
 		(flags & DSYNC_MAILBOX_EXPORTER_FLAG_TIMESTAMPS) != 0;
+	exporter->export_virtual_sizes =
+		(flags & DSYNC_MAILBOX_EXPORTER_FLAG_VSIZES) != 0;
 	exporter->hdr_hash_version =
 		(flags & DSYNC_MAILBOX_EXPORTER_FLAG_HDR_HASH_V2) ? 2 : 1;
 	exporter->no_hdr_hashes =
diff -r 6c40ae42ba14 -r d54651ba988a src/doveadm/dsync/dsync-mailbox-export.h
--- a/src/doveadm/dsync/dsync-mailbox-export.h	Wed Jul 13 09:51:08 2016 +0300
+++ b/src/doveadm/dsync/dsync-mailbox-export.h	Fri Jul 01 14:30:24 2016 +0300
@@ -7,7 +7,8 @@
 	DSYNC_MAILBOX_EXPORTER_FLAG_MINIMAL_DMAIL_FILL	= 0x04,
 	DSYNC_MAILBOX_EXPORTER_FLAG_TIMESTAMPS		= 0x08,
 	DSYNC_MAILBOX_EXPORTER_FLAG_HDR_HASH_V2		= 0x10,
-	DSYNC_MAILBOX_EXPORTER_FLAG_NO_HDR_HASHES	= 0x20
+	DSYNC_MAILBOX_EXPORTER_FLAG_NO_HDR_HASHES	= 0x20,
+	DSYNC_MAILBOX_EXPORTER_FLAG_VSIZES		= 0x40,
 };
 
 struct dsync_mailbox_exporter *
diff -r 6c40ae42ba14 -r d54651ba988a src/doveadm/dsync/dsync-mailbox-import.c
--- a/src/doveadm/dsync/dsync-mailbox-import.c	Wed Jul 13 09:51:08 2016 +0300
+++ b/src/doveadm/dsync/dsync-mailbox-import.c	Fri Jul 01 14:30:24 2016 +0300
@@ -62,6 +62,7 @@
 	uint32_t remote_first_recent_uid;
 	uint64_t remote_highest_modseq, remote_highest_pvt_modseq;
 	time_t sync_since_timestamp;
+	uoff_t sync_max_size;
 	enum mailbox_transaction_flags transaction_flags;
 	unsigned int hdr_hash_version;
 
@@ -216,7 +217,8 @@
 			  uint32_t remote_first_recent_uid,
 			  uint64_t remote_highest_modseq,
 			  uint64_t remote_highest_pvt_modseq,
-			  time_t sync_since_timestamp, const char *sync_flag,
+			  time_t sync_since_timestamp, uoff_t sync_max_size,
+			  const char *sync_flag,
 			  enum dsync_mailbox_import_flags flags)
 {
 	struct dsync_mailbox_importer *importer;
@@ -239,6 +241,7 @@
 	importer->remote_highest_modseq = remote_highest_modseq;
 	importer->remote_highest_pvt_modseq = remote_highest_pvt_modseq;
 	importer->sync_since_timestamp = sync_since_timestamp;
+	importer->sync_max_size = sync_max_size;
 	importer->stateful_import = importer->last_common_uid_found;
 	if (sync_flag != NULL) {
 		if (sync_flag[0] == '-') {
@@ -1349,6 +1352,14 @@
 			return FALSE;
 		}
 	}
+	if (importer->sync_max_size > 0) {
+		i_assert(change->virtual_size != (uoff_t)-1);
+		if (change->virtual_size < importer->sync_max_size) {
+			/* mail is too large - skip it */
+			*result_r = "Ignoring missing local mail with too large size";
+			return FALSE;
+		}
+	}
 	if (importer->sync_flag != 0) {
 		bool have_flag = (change->final_flags & importer->sync_flag) != 0;
 
@@ -1634,9 +1645,9 @@
 {
 	int ret;
 
-	i_assert(importer->sync_since_timestamp == 0 ||
-		 change->received_timestamp > 0 ||
-		 change->type == DSYNC_MAIL_CHANGE_TYPE_EXPUNGE);
+	i_assert(change->type == DSYNC_MAIL_CHANGE_TYPE_EXPUNGE ||
+		 ((change->received_timestamp > 0 || importer->sync_since_timestamp == 0) &&
+		  (change->virtual_size != (uoff_t)-1 || importer->sync_max_size == 0)));
 
 	/* try to find the matching local mail */
 	if (!importer_next_mail(importer, change->uid)) {
diff -r 6c40ae42ba14 -r d54651ba988a src/doveadm/dsync/dsync-mailbox-import.h
--- a/src/doveadm/dsync/dsync-mailbox-import.h	Wed Jul 13 09:51:08 2016 +0300
+++ b/src/doveadm/dsync/dsync-mailbox-import.h	Fri Jul 01 14:30:24 2016 +0300
@@ -32,7 +32,8 @@
 			  uint32_t remote_first_recent_uid,
 			  uint64_t remote_highest_modseq,
 			  uint64_t remote_highest_pvt_modseq,
-			  time_t sync_since_timestamp, const char *sync_flag,
+			  time_t sync_since_timestamp, uoff_t sync_max_size,
+			  const char *sync_flag,
 			  enum dsync_mailbox_import_flags flags);
 int dsync_mailbox_import_attribute(struct dsync_mailbox_importer *importer,
 				   const struct dsync_mailbox_attribute *attr);