Mercurial > dovecot > core-2.2
changeset 20595:40ce04c672a4
dsync: Add support for features
Add empty_header_workaround as first feature
| author | Aki Tuomi <aki.tuomi@dovecot.fi> |
|---|---|
| date | Fri, 05 Aug 2016 15:21:29 +0300 |
| parents | 620e747d25e1 |
| children | 9a2ac2a69648 |
| files | src/doveadm/doveadm-dsync.c src/doveadm/doveadm-settings.c src/doveadm/doveadm-settings.h src/doveadm/doveadm.c src/doveadm/dsync/dsync-brain-mailbox.c src/doveadm/dsync/dsync-brain-private.h src/doveadm/dsync/dsync-brain.c src/doveadm/dsync/dsync-brain.h src/doveadm/dsync/dsync-ibc-stream.c src/doveadm/dsync/dsync-mail.h src/doveadm/dsync/dsync-mailbox-import.c src/doveadm/dsync/dsync-mailbox-import.h |
| diffstat | 12 files changed, 91 insertions(+), 4 deletions(-) [+] |
line wrap: on
line diff
--- a/src/doveadm/doveadm-dsync.c Sat Apr 09 18:21:23 2016 -0500 +++ b/src/doveadm/doveadm-dsync.c Fri Aug 05 15:21:29 2016 +0300 @@ -17,6 +17,7 @@ #include "settings-parser.h" #include "imap-util.h" #include "master-service.h" +#include "master-service-settings.h" #include "master-service-ssl-settings.h" #include "mail-storage.h" #include "mail-storage-service.h" @@ -103,6 +104,7 @@ unsigned int local_location_from_arg:1; unsigned int replicator_notify:1; unsigned int exited:1; + unsigned int empty_hdr_workaround:1; }; static bool legacy_dsync = FALSE; @@ -571,6 +573,7 @@ set.lock_timeout_secs = ctx->lock_timeout; set.state = ctx->state_input; set.mailbox_alt_char = doveadm_settings->dsync_alt_char[0]; + if (array_count(&ctx->exclude_mailboxes) > 0) { /* array is NULL-terminated in init() */ set.exclude_mailboxes = array_idx(&ctx->exclude_mailboxes, 0); @@ -620,6 +623,8 @@ brain_flags |= DSYNC_BRAIN_FLAG_NO_MAIL_SYNC; if (ctx->oneway) brain_flags |= DSYNC_BRAIN_FLAG_NO_BACKUP_OVERWRITE; + if (ctx->empty_hdr_workaround) + brain_flags |= DSYNC_BRAIN_FLAG_EMPTY_HDR_WORKAROUND; if (doveadm_debug) brain_flags |= DSYNC_BRAIN_FLAG_DEBUG; @@ -1062,6 +1067,8 @@ DOVEADM_PRINT_HEADER_FLAG_HIDE_TITLE); p_array_init(&ctx->exclude_mailboxes, ctx->ctx.pool, 4); p_array_init(&ctx->namespace_prefixes, ctx->ctx.pool, 4); + if ((doveadm_settings->parsed_features & DSYNC_FEATURE_EMPTY_HDR_WORKAROUND) != 0) + ctx->empty_hdr_workaround = TRUE; return &ctx->ctx; } @@ -1177,6 +1184,7 @@ ctx->ctx.v.parse_arg = cmd_mailbox_dsync_server_parse_arg; ctx->ctx.v.run = cmd_dsync_server_run; ctx->sync_type = DSYNC_BRAIN_SYNC_TYPE_CHANGED; + ctx->fd_in = STDIN_FILENO; ctx->fd_out = STDOUT_FILENO; return &ctx->ctx;
--- a/src/doveadm/doveadm-settings.c Sat Apr 09 18:21:23 2016 -0500 +++ b/src/doveadm/doveadm-settings.c Fri Aug 05 15:21:29 2016 +0300 @@ -71,6 +71,7 @@ DEF(SET_STR, ssl_client_ca_file), DEF(SET_STR, director_username_hash), DEF(SET_STR, doveadm_api_key), + DEF(SET_STR, dsync_features), { SET_STRLIST, "plugin", offsetof(struct doveadm_settings, plugin_envs), NULL }, @@ -92,6 +93,7 @@ .doveadm_allowed_commands = "", .dsync_alt_char = "_", .dsync_remote_cmd = "ssh -l%{login} %{host} doveadm dsync-server -u%u -U", + .dsync_features = "", .ssl_client_ca_dir = "", .ssl_client_ca_file = "", .director_username_hash = "%Lu", @@ -129,6 +131,43 @@ } /* <settings checks> */ +struct dsync_feature_list { + const char *name; + enum dsync_features num; +}; + +static const struct dsync_feature_list dsync_feature_list[] = { + { "empty_header_workaround", DSYNC_FEATURE_EMPTY_HDR_WORKAROUND }, + { NULL, 0 } +}; + +static int +dsync_settings_parse_features(struct doveadm_settings *set, + const char **error_r) +{ + enum dsync_features features = 0; + const struct dsync_feature_list *list; + const char *const *str; + + str = t_strsplit_spaces(set->dsync_features, " ,"); + for (; *str != NULL; str++) { + list = dsync_feature_list; + for (; list->name != NULL; list++) { + if (strcasecmp(*str, list->name) == 0) { + features |= list->num; + break; + } + } + if (list->name == NULL) { + *error_r = t_strdup_printf("dsync_features: " + "Unknown feature: %s", *str); + return -1; + } + } + set->parsed_features = features; + return 0; +} + static bool doveadm_settings_check(void *_set, pool_t pool ATTR_UNUSED, const char **error_r) { @@ -142,6 +181,8 @@ *error_r = "dsync_alt_char must not be empty"; return FALSE; } + if (dsync_settings_parse_features(set, error_r) != 0) + return FALSE; return TRUE; } /* </settings checks> */
--- a/src/doveadm/doveadm-settings.h Sat Apr 09 18:21:23 2016 -0500 +++ b/src/doveadm/doveadm-settings.h Fri Aug 05 15:21:29 2016 +0300 @@ -3,6 +3,12 @@ #include "net.h" +/* <settings checks> */ +enum dsync_features { + DSYNC_FEATURE_EMPTY_HDR_WORKAROUND = 0x1, +}; +/* </settings checks> */ + struct doveadm_settings { const char *base_dir; const char *libexec_dir; @@ -22,7 +28,8 @@ const char *ssl_client_ca_file; const char *director_username_hash; const char *doveadm_api_key; - + const char *dsync_features; + enum dsync_features parsed_features; ARRAY(const char *) plugin_envs; };
--- a/src/doveadm/doveadm.c Sat Apr 09 18:21:23 2016 -0500 +++ b/src/doveadm/doveadm.c Fri Aug 05 15:21:29 2016 +0300 @@ -269,6 +269,8 @@ set = master_service_settings_get_others(master_service)[0]; doveadm_settings = settings_dup(&doveadm_setting_parser_info, set, pool_datastack_create()); + + doveadm_settings->parsed_features = set->parsed_features; /* copy this value by hand */ } static struct doveadm_cmd *doveadm_cmdline_commands[] = {
--- a/src/doveadm/dsync/dsync-brain-mailbox.c Sat Apr 09 18:21:23 2016 -0500 +++ b/src/doveadm/dsync/dsync-brain-mailbox.c Fri Aug 05 15:21:29 2016 +0300 @@ -220,6 +220,8 @@ import_flags |= DSYNC_MAILBOX_IMPORT_FLAG_NO_NOTIFY; if (brain->hdr_hash_v2) import_flags |= DSYNC_MAILBOX_IMPORT_FLAG_HDR_HASH_V2; + if (brain->empty_hdr_workaround) + import_flags |= DSYNC_MAILBOX_IMPORT_FLAG_EMPTY_HDR_WORKAROUND; brain->box_importer = brain->backup_send ? NULL : dsync_mailbox_import_init(brain->box, brain->virtual_all_box,
--- a/src/doveadm/dsync/dsync-brain-private.h Sat Apr 09 18:21:23 2016 -0500 +++ b/src/doveadm/dsync/dsync-brain-private.h Fri Aug 05 15:21:29 2016 +0300 @@ -114,6 +114,7 @@ unsigned int no_notify:1; unsigned int hdr_hash_v2:1; unsigned int failed:1; + unsigned int empty_hdr_workaround:1; }; extern const char *dsync_box_state_names[DSYNC_BOX_STATE_DONE+1];
--- a/src/doveadm/dsync/dsync-brain.c Sat Apr 09 18:21:23 2016 -0500 +++ b/src/doveadm/dsync/dsync-brain.c Fri Aug 05 15:21:29 2016 +0300 @@ -143,6 +143,7 @@ brain->no_mailbox_renames = (flags & DSYNC_BRAIN_FLAG_NO_MAILBOX_RENAMES) != 0; brain->no_notify = (flags & DSYNC_BRAIN_FLAG_NO_NOTIFY) != 0; + brain->empty_hdr_workaround = (flags & DSYNC_BRAIN_FLAG_EMPTY_HDR_WORKAROUND) != 0; } static void
--- a/src/doveadm/dsync/dsync-brain.h Sat Apr 09 18:21:23 2016 -0500 +++ b/src/doveadm/dsync/dsync-brain.h Fri Aug 05 15:21:29 2016 +0300 @@ -32,7 +32,9 @@ be removed once the renaming logic has no more bugs.. */ DSYNC_BRAIN_FLAG_NO_MAILBOX_RENAMES = 0x200, /* Add MAILBOX_TRANSACTION_FLAG_NO_NOTIFY to transactions. */ - DSYNC_BRAIN_FLAG_NO_NOTIFY = 0x400 + DSYNC_BRAIN_FLAG_NO_NOTIFY = 0x400, + /* Workaround missing Date/Message-ID headers */ + DSYNC_BRAIN_FLAG_EMPTY_HDR_WORKAROUND = 0x800, }; enum dsync_brain_sync_type {
--- a/src/doveadm/dsync/dsync-ibc-stream.c Sat Apr 09 18:21:23 2016 -0500 +++ b/src/doveadm/dsync/dsync-ibc-stream.c Fri Aug 05 15:21:29 2016 +0300 @@ -734,6 +734,8 @@ dsync_serializer_encode_add(encoder, "purge_remote", ""); if ((set->brain_flags & DSYNC_BRAIN_FLAG_NO_NOTIFY) != 0) dsync_serializer_encode_add(encoder, "no_notify", ""); + if ((set->brain_flags & DSYNC_BRAIN_FLAG_EMPTY_HDR_WORKAROUND) != 0) + dsync_serializer_encode_add(encoder, "empty_hdr_workaround", ""); dsync_serializer_encode_finish(&encoder, str); dsync_ibc_stream_send_string(ibc, str); @@ -842,6 +844,8 @@ set->brain_flags |= DSYNC_BRAIN_FLAG_PURGE_REMOTE; if (dsync_deserializer_decode_try(decoder, "no_notify", &value)) set->brain_flags |= DSYNC_BRAIN_FLAG_NO_NOTIFY; + if (dsync_deserializer_decode_try(decoder, "empty_hdr_workaround", &value)) + set->brain_flags |= DSYNC_BRAIN_FLAG_EMPTY_HDR_WORKAROUND; set->hdr_hash_v2 = ibc->minor_version >= DSYNC_PROTOCOL_MINOR_HAVE_HDR_HASH_V2; *set_r = set;
--- a/src/doveadm/dsync/dsync-mail.h Sat Apr 09 18:21:23 2016 -0500 +++ b/src/doveadm/dsync/dsync-mail.h Fri Aug 05 15:21:29 2016 +0300 @@ -88,6 +88,12 @@ int dsync_mail_get_hdr_hash(struct mail *mail, unsigned int version, const char **hdr_hash_r); +static inline bool dsync_mail_hdr_hash_is_empty(const char *hdr_hash) +{ + /* md5(\n) */ + return strcmp(hdr_hash, "68b329da9893e34099c7d8ad5cb9c940") == 0; +} + int dsync_mail_fill(struct mail *mail, bool minimal_fill, struct dsync_mail *dmail_r, const char **error_field_r); int dsync_mail_fill_nonminimal(struct mail *mail, struct dsync_mail *dmail_r,
--- a/src/doveadm/dsync/dsync-mailbox-import.c Sat Apr 09 18:21:23 2016 -0500 +++ b/src/doveadm/dsync/dsync-mailbox-import.c Fri Aug 05 15:21:29 2016 +0300 @@ -122,6 +122,7 @@ unsigned int mails_have_guids:1; unsigned int mails_use_guid128:1; unsigned int delete_mailbox:1; + unsigned int empty_hdr_workaround:1; }; static const char *dsync_mail_change_type_names[] = { @@ -278,6 +279,8 @@ (flags & DSYNC_MAILBOX_IMPORT_FLAG_MAILS_USE_GUID128) != 0; importer->hdr_hash_version = (flags & DSYNC_MAILBOX_IMPORT_FLAG_HDR_HASH_V2) != 0 ? 2 : 1; + importer->empty_hdr_workaround = + (flags & DSYNC_MAILBOX_IMPORT_FLAG_EMPTY_HDR_WORKAROUND) != 0; mailbox_get_open_status(importer->box, STATUS_UIDNEXT | STATUS_HIGHESTMODSEQ | STATUS_HIGHESTPVTMODSEQ, @@ -784,7 +787,16 @@ i_assert(save_change->type != DSYNC_MAIL_CHANGE_TYPE_EXPUNGE); } - diff = importer_mail_cmp(&m1, &m2); + if (importer->empty_hdr_workaround && !importer->mails_have_guids && + importer->cur_mail != NULL && save_change != NULL && + (dsync_mail_hdr_hash_is_empty(m1.guid) || + dsync_mail_hdr_hash_is_empty(m2.guid))) { + /* one of the headers is empty. assume it's broken and that + the header matches what we have currently. */ + diff = 0; + } else { + diff = importer_mail_cmp(&m1, &m2); + } if (diff < 0) { /* add a record for local mail */ i_assert(importer->cur_mail != NULL);
--- a/src/doveadm/dsync/dsync-mailbox-import.h Sat Apr 09 18:21:23 2016 -0500 +++ b/src/doveadm/dsync/dsync-mailbox-import.h Fri Aug 05 15:21:29 2016 +0300 @@ -11,7 +11,8 @@ DSYNC_MAILBOX_IMPORT_FLAG_MAILS_HAVE_GUIDS = 0x10, DSYNC_MAILBOX_IMPORT_FLAG_MAILS_USE_GUID128 = 0x20, DSYNC_MAILBOX_IMPORT_FLAG_NO_NOTIFY = 0x40, - DSYNC_MAILBOX_IMPORT_FLAG_HDR_HASH_V2 = 0x80 + DSYNC_MAILBOX_IMPORT_FLAG_HDR_HASH_V2 = 0x80, + DSYNC_MAILBOX_IMPORT_FLAG_EMPTY_HDR_WORKAROUND = 0x100 }; struct mailbox;