Mercurial > dovecot > core-2.2

changeset 22549:400ff84f109d

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
dsync: Add hashed_headers setting This makes it possible to configure them
author Aki Tuomi <aki.tuomi@dovecot.fi>
date Fri, 22 Sep 2017 13:30:43 +0300
parents b4f0f49d1a93
children 73e9c31c22dd
files src/doveadm/doveadm-dsync.c src/doveadm/doveadm-settings.c src/doveadm/doveadm-settings.h src/doveadm/dsync/dsync-brain-mailbox.c src/doveadm/dsync/dsync-brain-private.h src/doveadm/dsync/dsync-brain.c src/doveadm/dsync/dsync-brain.h src/doveadm/dsync/dsync-ibc-stream.c src/doveadm/dsync/dsync-ibc.h src/doveadm/dsync/dsync-mail.c src/doveadm/dsync/dsync-mail.h src/doveadm/dsync/dsync-mailbox-export.c src/doveadm/dsync/dsync-mailbox-export.h src/doveadm/dsync/dsync-mailbox-import.c src/doveadm/dsync/dsync-mailbox-import.h
diffstat 15 files changed, 69 insertions(+), 24 deletions(-) [+]
line wrap: on
line diff
--- a/src/doveadm/doveadm-dsync.c	Fri Sep 22 14:08:45 2017 +0300
+++ b/src/doveadm/doveadm-dsync.c	Fri Sep 22 13:30:43 2017 +0300
@@ -591,7 +591,13 @@
 	set.import_commit_msgs_interval = ctx->import_commit_msgs_interval;
 	set.state = ctx->state_input;
 	set.mailbox_alt_char = doveadm_settings->dsync_alt_char[0];
-
+	if (*doveadm_settings->dsync_hashed_headers == '\0') {
+		i_error("dsync_hashed_headers must not be empty");
+		ctx->ctx.exit_code = EX_USAGE;
+		return -1;
+	}
+	set.hashed_headers =
+		t_strsplit_spaces(doveadm_settings->dsync_hashed_headers, " ,");
 	if (array_count(&ctx->exclude_mailboxes) > 0) {
 		/* array is NULL-terminated in init() */
 		set.exclude_mailboxes = array_idx(&ctx->exclude_mailboxes, 0);
--- a/src/doveadm/doveadm-settings.c	Fri Sep 22 14:08:45 2017 +0300
+++ b/src/doveadm/doveadm-settings.c	Fri Sep 22 13:30:43 2017 +0300
@@ -74,6 +74,7 @@
 	DEF(SET_STR, dsync_features),
 	DEF(SET_UINT, dsync_commit_msgs_interval),
 	DEF(SET_STR, doveadm_http_rawlog_dir),
+	DEF(SET_STR, dsync_hashed_headers),
 
 	{ SET_STRLIST, "plugin", offsetof(struct doveadm_settings, plugin_envs), NULL },
 
@@ -96,6 +97,7 @@
 	.dsync_alt_char = "_",
 	.dsync_remote_cmd = "ssh -l%{login} %{host} doveadm dsync-server -u%u -U",
 	.dsync_features = "",
+	.dsync_hashed_headers = "Date Message-ID",
 	.dsync_commit_msgs_interval = 100,
 	.ssl_client_ca_dir = "",
 	.ssl_client_ca_file = "",
@@ -181,6 +183,10 @@
 	fix_base_path(set, pool, &set->auth_socket_path);
 	fix_base_path(set, pool, &set->doveadm_socket_path);
 #endif
+	if (*set->dsync_hashed_headers == '\0') {
+		*error_r = "dsync_hashed_headers must not be empty";
+		return FALSE;
+	}
 	if (*set->dsync_alt_char == '\0') {
 		*error_r = "dsync_alt_char must not be empty";
 		return FALSE;
--- a/src/doveadm/doveadm-settings.h	Fri Sep 22 14:08:45 2017 +0300
+++ b/src/doveadm/doveadm-settings.h	Fri Sep 22 13:30:43 2017 +0300
@@ -29,6 +29,7 @@
 	const char *director_username_hash;
 	const char *doveadm_api_key;
 	const char *dsync_features;
+	const char *dsync_hashed_headers;
 	unsigned int dsync_commit_msgs_interval;
 	const char *doveadm_http_rawlog_dir;
 	enum dsync_features parsed_features;
--- a/src/doveadm/dsync/dsync-brain-mailbox.c	Fri Sep 22 14:08:45 2017 +0300
+++ b/src/doveadm/dsync/dsync-brain-mailbox.c	Fri Sep 22 13:30:43 2017 +0300
@@ -235,7 +235,8 @@
 					  brain->sync_max_size,
 					  brain->sync_flag,
 					  brain->import_commit_msgs_interval,
-					  import_flags, brain->hdr_hash_version);
+					  import_flags, brain->hdr_hash_version,
+					  brain->hashed_headers);
 }
 
 int dsync_brain_sync_mailbox_open(struct dsync_brain *brain,
@@ -338,7 +339,8 @@
 		dsync_mailbox_export_init(brain->box, brain->log_scan,
 					  last_common_uid,
 					  exporter_flags,
-					  brain->hdr_hash_version);
+					  brain->hdr_hash_version,
+					  brain->hashed_headers);
 	dsync_brain_sync_mailbox_init_remote(brain, remote_dsync_box);
 	return 1;
 }
--- a/src/doveadm/dsync/dsync-brain-private.h	Fri Sep 22 14:08:45 2017 +0300
+++ b/src/doveadm/dsync/dsync-brain-private.h	Fri Sep 22 13:30:43 2017 +0300
@@ -101,6 +101,7 @@
 
 	const char *changes_during_sync;
 	enum mail_error mail_error;
+	const char *const *hashed_headers;
 
 	unsigned int master_brain:1;
 	unsigned int mail_requests:1;
--- a/src/doveadm/dsync/dsync-brain.c	Fri Sep 22 14:08:45 2017 +0300
+++ b/src/doveadm/dsync/dsync-brain.c	Fri Sep 22 13:30:43 2017 +0300
@@ -223,6 +223,8 @@
 	brain->lock_timeout = set->lock_timeout_secs;
 	brain->import_commit_msgs_interval = set->import_commit_msgs_interval;
 	brain->master_brain = TRUE;
+	brain->hashed_headers =
+		(const char*const*)p_strarray_dup(brain->pool, set->hashed_headers);
 	dsync_brain_set_flags(brain, flags);
 
 	if (set->virtual_all_box != NULL)
@@ -262,6 +264,7 @@
 	ibc_set.hdr_hash_v2 = TRUE;
 	ibc_set.lock_timeout = set->lock_timeout_secs;
 	ibc_set.import_commit_msgs_interval = set->import_commit_msgs_interval;
+	ibc_set.hashed_headers = set->hashed_headers;
 	/* reverse the backup direction for the slave */
 	ibc_set.brain_flags = flags & ~(DSYNC_BRAIN_FLAG_BACKUP_SEND |
 					DSYNC_BRAIN_FLAG_BACKUP_RECV);
@@ -512,6 +515,9 @@
 	brain->sync_type = ibc_set->sync_type;
 
 	dsync_brain_set_flags(brain, ibc_set->brain_flags);
+	if (ibc_set->hashed_headers != NULL)
+		brain->hashed_headers =
+			p_strarray_dup(brain->pool, (const char*const*)ibc_set->hashed_headers);
 	/* this flag is only set on the remote slave brain */
 	brain->purge = (ibc_set->brain_flags &
 			DSYNC_BRAIN_FLAG_PURGE_REMOTE) != 0;
--- a/src/doveadm/dsync/dsync-brain.h	Fri Sep 22 14:08:45 2017 +0300
+++ b/src/doveadm/dsync/dsync-brain.h	Fri Sep 22 13:30:43 2017 +0300
@@ -76,6 +76,8 @@
 	/* Sync only mails which contains / doesn't contain this flag.
 	   '-' at the beginning means this flag must not exist. */
 	const char *sync_flag;
+	/* Headers to hash (defaults to Date, Message-ID) */
+	const char *const *hashed_headers;
 
 	/* If non-zero, use dsync lock file for this user */
 	unsigned int lock_timeout_secs;
--- a/src/doveadm/dsync/dsync-ibc-stream.c	Fri Sep 22 14:08:45 2017 +0300
+++ b/src/doveadm/dsync/dsync-ibc-stream.c	Fri Sep 22 13:30:43 2017 +0300
@@ -79,7 +79,8 @@
 	  	"send_mail_requests backup_send backup_recv lock_timeout "
 	  	"no_mail_sync no_mailbox_renames no_backup_overwrite purge_remote "
 		"no_notify sync_since_timestamp sync_max_size sync_flags sync_until_timestamp "
-	  	"virtual_all_box empty_hdr_workaround import_commit_msgs_interval"
+		"virtual_all_box empty_hdr_workaround import_commit_msgs_interval "
+		"hashed_headers"
 	},
 	{ .name = "mailbox_state",
 	  .chr = 'S',
@@ -751,7 +752,15 @@
 		dsync_serializer_encode_add(encoder, "no_notify", "");
 	if ((set->brain_flags & DSYNC_BRAIN_FLAG_EMPTY_HDR_WORKAROUND) != 0)
 		dsync_serializer_encode_add(encoder, "empty_hdr_workaround", "");
-
+	/* this can be NULL in slave */
+	string_t *str2 = t_str_new(32);
+	if (set->hashed_headers != NULL) {
+		for(const char *const *ptr = set->hashed_headers; *ptr != NULL; ptr++) {
+			str_append_tabescaped(str2, *ptr);
+			str_append_c(str2, '\t');
+		}
+	}
+	dsync_serializer_encode_add(encoder, "hashed_headers", str_c(str2));
 	dsync_serializer_encode_finish(&encoder, str);
 	dsync_ibc_stream_send_string(ibc, str);
 }
@@ -885,6 +894,8 @@
 		set->brain_flags |= DSYNC_BRAIN_FLAG_NO_NOTIFY;
 	if (dsync_deserializer_decode_try(decoder, "empty_hdr_workaround", &value))
 		set->brain_flags |= DSYNC_BRAIN_FLAG_EMPTY_HDR_WORKAROUND;
+	if (dsync_deserializer_decode_try(decoder, "hashed_headers", &value))
+		set->hashed_headers = (const char*const*)p_strsplit_tabescaped(pool, value);
 	set->hdr_hash_v2 = ibc->minor_version >= DSYNC_PROTOCOL_MINOR_HAVE_HDR_HASH_V2;
 	set->hdr_hash_v3 = ibc->minor_version >= DSYNC_PROTOCOL_MINOR_HAVE_HDR_HASH_V3;
 
--- a/src/doveadm/dsync/dsync-ibc.h	Fri Sep 22 14:08:45 2017 +0300
+++ b/src/doveadm/dsync/dsync-ibc.h	Fri Sep 22 13:30:43 2017 +0300
@@ -64,6 +64,8 @@
 	uoff_t sync_max_size;
 	/* Sync only mails with specified flags. */
 	const char *sync_flags;
+	/* Hashed headers */
+	const char *const *hashed_headers;
 
 	enum dsync_brain_sync_type sync_type;
 	enum dsync_brain_flags brain_flags;
--- a/src/doveadm/dsync/dsync-mail.c	Fri Sep 22 14:08:45 2017 +0300
+++ b/src/doveadm/dsync/dsync-mail.c	Fri Sep 22 13:30:43 2017 +0300
@@ -11,22 +11,14 @@
 #include "mail-storage.h"
 #include "dsync-mail.h"
 
-/* These should be good enough to identify all normal mails. Received: header
-   would make it even better, but those can be somewhat large. Also these
-   fields can be looked up using IMAP ENVELOPE, which is more efficient in
-   some IMAP servers. */
-static const char *hashed_headers[] = {
-	"Date", "Message-ID", NULL
-};
-
 struct mailbox_header_lookup_ctx *
-dsync_mail_get_hash_headers(struct mailbox *box)
+dsync_mail_get_hash_headers(struct mailbox *box, const char *const *hashed_headers)
 {
 	return mailbox_header_lookup_init(box, hashed_headers);
 }
 
 int dsync_mail_get_hdr_hash(struct mail *mail, unsigned int version,
-			    const char **hdr_hash_r)
+			    const char *const *hashed_headers, const char **hdr_hash_r)
 {
 	struct istream *hdr_input, *input;
 	struct mailbox_header_lookup_ctx *hdr_ctx;
--- a/src/doveadm/dsync/dsync-mail.h	Fri Sep 22 14:08:45 2017 +0300
+++ b/src/doveadm/dsync/dsync-mail.h	Fri Sep 22 13:30:43 2017 +0300
@@ -87,10 +87,10 @@
 };
 
 struct mailbox_header_lookup_ctx *
-dsync_mail_get_hash_headers(struct mailbox *box);
+dsync_mail_get_hash_headers(struct mailbox *box, const char *const *hashed_headers);
 
 int dsync_mail_get_hdr_hash(struct mail *mail, unsigned int version,
-			    const char **hdr_hash_r);
+			    const char *const *hashed_headers, const char **hdr_hash_r);
 static inline bool dsync_mail_hdr_hash_is_empty(const char *hdr_hash)
 {
 	/* md5(\n) */
--- a/src/doveadm/dsync/dsync-mailbox-export.c	Fri Sep 22 14:08:45 2017 +0300
+++ b/src/doveadm/dsync/dsync-mailbox-export.c	Fri Sep 22 13:30:43 2017 +0300
@@ -30,6 +30,8 @@
 	unsigned int search_pos, search_count;
 	unsigned int hdr_hash_version;
 
+	const char *const *hashed_headers;
+
 	/* GUID => instances */
 	HASH_TABLE(char *, struct dsync_mail_guid_instances *) export_guids;
 	ARRAY_TYPE(seq_range) requested_uids;
@@ -169,7 +171,8 @@
 			*hdr_hash_r = "";
 			return 1;
 		}
-		if (dsync_mail_get_hdr_hash(mail, exporter->hdr_hash_version, hdr_hash_r) < 0)
+		if (dsync_mail_get_hdr_hash(mail, exporter->hdr_hash_version,
+					    exporter->hashed_headers, hdr_hash_r) < 0)
 			return dsync_mail_error(exporter, mail, "hdr-stream");
 		return 1;
 	} else if (**guid_r == '\0') {
@@ -499,7 +502,8 @@
 			  struct dsync_transaction_log_scan *log_scan,
 			  uint32_t last_common_uid,
 			  enum dsync_mailbox_exporter_flags flags,
-			  unsigned int hdr_hash_version)
+			  unsigned int hdr_hash_version,
+			  const char *const *hashed_headers)
 {
 	struct dsync_mailbox_exporter *exporter;
 	pool_t pool;
@@ -524,6 +528,8 @@
 	exporter->hdr_hash_version = hdr_hash_version;
 	exporter->no_hdr_hashes =
 		(flags & DSYNC_MAILBOX_EXPORTER_FLAG_NO_HDR_HASHES) != 0;
+	exporter->hashed_headers = hashed_headers;
+
 	p_array_init(&exporter->requested_uids, pool, 16);
 	p_array_init(&exporter->search_uids, pool, 16);
 	hash_table_create(&exporter->export_guids, pool, 0, str_hash, strcmp);
@@ -531,7 +537,8 @@
 	p_array_init(&exporter->expunged_guids, pool, 16);
 
 	if (!exporter->mails_have_guids && !exporter->no_hdr_hashes)
-		exporter->wanted_headers = dsync_mail_get_hash_headers(box);
+		exporter->wanted_headers =
+			dsync_mail_get_hash_headers(box, exporter->hashed_headers);
 
 	/* first scan transaction log and save any expunges and flag changes */
 	dsync_mailbox_export_log_scan(exporter, log_scan);
--- a/src/doveadm/dsync/dsync-mailbox-export.h	Fri Sep 22 14:08:45 2017 +0300
+++ b/src/doveadm/dsync/dsync-mailbox-export.h	Fri Sep 22 13:30:43 2017 +0300
@@ -15,7 +15,8 @@
 			  struct dsync_transaction_log_scan *log_scan,
 			  uint32_t last_common_uid,
 			  enum dsync_mailbox_exporter_flags flags,
-			  unsigned int hdr_hash_version);
+			  unsigned int hdr_hash_version,
+			  const char *const *hashed_headers);
 /* Returns 1 if attribute was returned, 0 if no more attributes, -1 on error */
 int dsync_mailbox_export_next_attr(struct dsync_mailbox_exporter *exporter,
 				   const struct dsync_mailbox_attribute **attr_r);
--- a/src/doveadm/dsync/dsync-mailbox-import.c	Fri Sep 22 14:08:45 2017 +0300
+++ b/src/doveadm/dsync/dsync-mailbox-import.c	Fri Sep 22 13:30:43 2017 +0300
@@ -69,6 +69,8 @@
 	unsigned int hdr_hash_version;
 	unsigned int commit_msgs_interval;
 
+	const char *const *hashed_headers;
+
 	enum mail_flags sync_flag;
 	const char *sync_keyword;
 	bool sync_flag_dontwant;
@@ -227,7 +229,8 @@
 			  const char *sync_flag,
 			  unsigned int commit_msgs_interval,
 			  enum dsync_mailbox_import_flags flags,
-			  unsigned int hdr_hash_version)
+			  unsigned int hdr_hash_version,
+			  const char *const *hashed_headers)
 {
 	struct dsync_mailbox_importer *importer;
 	struct mailbox_status status;
@@ -252,6 +255,8 @@
 	importer->sync_until_timestamp = sync_until_timestamp;
 	importer->sync_max_size = sync_max_size;
 	importer->stateful_import = importer->last_common_uid_found;
+	importer->hashed_headers = hashed_headers;
+
 	if (sync_flag != NULL) {
 		if (sync_flag[0] == '-') {
 			importer->sync_flag_dontwant = TRUE;
@@ -659,6 +664,7 @@
 	} else {
 		if (dsync_mail_get_hdr_hash(importer->cur_mail,
 					    importer->hdr_hash_version,
+					    importer->hashed_headers,
 					    &hdr_hash) < 0) {
 			dsync_mail_error(importer, importer->cur_mail,
 					 "header hash");
@@ -1575,7 +1581,8 @@
 	}
 
 	if (dsync_mail_get_hdr_hash(importer->cur_mail,
-				    importer->hdr_hash_version, &hdr_hash) < 0) {
+				    importer->hdr_hash_version,
+				    importer->hashed_headers, &hdr_hash) < 0) {
 		dsync_mail_error(importer, importer->cur_mail, "hdr-stream");
 		*result_r = "Error fetching header stream";
 		return -1;
--- a/src/doveadm/dsync/dsync-mailbox-import.h	Fri Sep 22 14:08:45 2017 +0300
+++ b/src/doveadm/dsync/dsync-mailbox-import.h	Fri Sep 22 13:30:43 2017 +0300
@@ -37,7 +37,8 @@
 			  const char *sync_flag,
 			  unsigned int commit_msgs_interval,
 			  enum dsync_mailbox_import_flags flags,
-			  unsigned int hdr_hash_version);
+			  unsigned int hdr_hash_version,
+			  const char *const *hashed_headers);
 int dsync_mailbox_import_attribute(struct dsync_mailbox_importer *importer,
 				   const struct dsync_mailbox_attribute *attr);
 int dsync_mailbox_import_change(struct dsync_mailbox_importer *importer,