dovecot/core-2.2: src/doveadm/dsync/dsync-mailbox-import.c annotate

annotate src/doveadm/dsync/dsync-mailbox-import.c @ 22713:cb108f786fb4

Updated copyright notices to include the year 2018.

author	Stephan Bosch <stephan.bosch@dovecot.fi>
date	Mon, 01 Jan 2018 22:42:08 +0100
parents	400ff84f109d
children	e33ad4efec00

rev	line source
22713 cb108f786fb4 Updated copyright notices to include the year 2018. Stephan Bosch <stephan.bosch@dovecot.fi> parents: 22549 diff changeset	1 /* Copyright (c) 2013-2018 Dovecot authors, see the included COPYING file */
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	3 #include "lib.h"
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	4 #include "array.h"
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	5 #include "hash.h"
16792 4a969a2e9399 dsync: Added some more debug logging. Timo Sirainen <tss@iki.fi> parents: 16762 diff changeset	6 #include "str.h"
15819 9c095a50ba3e dsync: Improved unexpected error logging. Timo Sirainen <tss@iki.fi> parents: 15781 diff changeset	7 #include "hex-binary.h"
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	8 #include "istream.h"
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	9 #include "seq-range-array.h"
16792 4a969a2e9399 dsync: Added some more debug logging. Timo Sirainen <tss@iki.fi> parents: 16762 diff changeset	10 #include "imap-util.h"
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	11 #include "mail-storage-private.h"
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	12 #include "mail-search-build.h"
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	13 #include "dsync-transaction-log-scan.h"
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	14 #include "dsync-mail.h"
16025 c6082de4bf5b dsync: Added support for syncing mailbox attributes. Timo Sirainen <tss@iki.fi> parents: 15841 diff changeset	15 #include "dsync-mailbox.h"
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	16 #include "dsync-mailbox-import.h"
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	17
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	18 struct importer_mail {
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	19 const char *guid;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	20 uint32_t uid;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	21 };
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	22
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	23 struct importer_new_mail {
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	24 /* linked list of mails for this GUID */
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	25 struct importer_new_mail *next;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	26 /* if non-NULL, this mail exists in both local and remote. this link
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	27 points to the other side. */
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	28 struct importer_new_mail *link;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	29
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	30 const char *guid;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	31 struct dsync_mail_change *change;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	32
15766 4260244e57e2 dsync: Various importer fixes. Handle unexpectedly wrong mailbox state better. Timo Sirainen <tss@iki.fi> parents: 15754 diff changeset	33 /* the final UID for the message */
4260244e57e2 dsync: Various importer fixes. Handle unexpectedly wrong mailbox state better. Timo Sirainen <tss@iki.fi> parents: 15754 diff changeset	34 uint32_t final_uid;
4260244e57e2 dsync: Various importer fixes. Handle unexpectedly wrong mailbox state better. Timo Sirainen <tss@iki.fi> parents: 15754 diff changeset	35 /* the original local UID, or 0 if exists only remotely */
4260244e57e2 dsync: Various importer fixes. Handle unexpectedly wrong mailbox state better. Timo Sirainen <tss@iki.fi> parents: 15754 diff changeset	36 uint32_t local_uid;
4260244e57e2 dsync: Various importer fixes. Handle unexpectedly wrong mailbox state better. Timo Sirainen <tss@iki.fi> parents: 15754 diff changeset	37 /* the original remote UID, or 0 if exists only remotely */
4260244e57e2 dsync: Various importer fixes. Handle unexpectedly wrong mailbox state better. Timo Sirainen <tss@iki.fi> parents: 15754 diff changeset	38 uint32_t remote_uid;
18180 39d00448490f dsync: If same GUID already exists in storage, try to copy it instead of recreating the mail. Timo Sirainen <tss@iki.fi> parents: 18178 diff changeset	39 /* UID for the mail in the virtual \All mailbox */
39d00448490f dsync: If same GUID already exists in storage, try to copy it instead of recreating the mail. Timo Sirainen <tss@iki.fi> parents: 18178 diff changeset	40 uint32_t virtual_all_uid;
39d00448490f dsync: If same GUID already exists in storage, try to copy it instead of recreating the mail. Timo Sirainen <tss@iki.fi> parents: 18178 diff changeset	41
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	42 unsigned int uid_in_local:1;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	43 unsigned int uid_is_usable:1;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	44 unsigned int skip:1;
15772 360661d99c42 dsync: Various importer fixes. Timo Sirainen <tss@iki.fi> parents: 15770 diff changeset	45 unsigned int expunged:1;
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	46 unsigned int copy_failed:1;
22043 81e013b3207d dsync: Try to commit transactions every dsync_commit_msgs_interval messages Timo Sirainen <timo.sirainen@dovecot.fi> parents: 21814 diff changeset	47 unsigned int saved:1;
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	48 };
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	49
15668 583ea6b63c5d dsync: Added a quick way to debug unexpected changes. Timo Sirainen <tss@iki.fi> parents: 15667 diff changeset	50 /* for quickly testing that two-way sync doesn't actually do any unexpected
583ea6b63c5d dsync: Added a quick way to debug unexpected changes. Timo Sirainen <tss@iki.fi> parents: 15667 diff changeset	51 modifications. */
583ea6b63c5d dsync: Added a quick way to debug unexpected changes. Timo Sirainen <tss@iki.fi> parents: 15667 diff changeset	52 #define IMPORTER_DEBUG_CHANGE(importer) /i_assert(!importer->master_brain)/
583ea6b63c5d dsync: Added a quick way to debug unexpected changes. Timo Sirainen <tss@iki.fi> parents: 15667 diff changeset	53
14918 8eae4e205c82 Hash table API is now (mostly) type safe. Timo Sirainen <tss@iki.fi> parents: 14917 diff changeset	54 HASH_TABLE_DEFINE_TYPE(guid_new_mail, const char , struct importer_new_mail );
14923 96fd2c3bf932 Reverted "support for non-pointers" part of the hash table API changes. Timo Sirainen <tss@iki.fi> parents: 14922 diff changeset	55 HASH_TABLE_DEFINE_TYPE(uid_new_mail, void , struct importer_new_mail );
14918 8eae4e205c82 Hash table API is now (mostly) type safe. Timo Sirainen <tss@iki.fi> parents: 14917 diff changeset	56
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	57 struct dsync_mailbox_importer {
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	58 pool_t pool;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	59 struct mailbox *box;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	60 uint32_t last_common_uid;
15496 5d90e2aa1ba8 dsync: Use private modseqs to support syncing private flags in shared mailboxes. Timo Sirainen <tss@iki.fi> parents: 15257 diff changeset	61 uint64_t last_common_modseq, last_common_pvt_modseq;
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	62 uint32_t remote_uid_next;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	63 uint32_t remote_first_recent_uid;
15496 5d90e2aa1ba8 dsync: Use private modseqs to support syncing private flags in shared mailboxes. Timo Sirainen <tss@iki.fi> parents: 15257 diff changeset	64 uint64_t remote_highest_modseq, remote_highest_pvt_modseq;
18177 f393f63764e0 dsync: Added -t <timestamp> parameter to save only mails newer than <timestamp> Timo Sirainen <tss@iki.fi> parents: 18137 diff changeset	65 time_t sync_since_timestamp;
20831 0c4e5c2725a3 doveadm-sync: Add end-date support Aki Tuomi <aki.tuomi@dovecot.fi> parents: 20633 diff changeset	66 time_t sync_until_timestamp;
20633 d54651ba988a doveadm sync/backup: Added -S <max size> parameter to skip too large mails. Timo Sirainen <timo.sirainen@dovecot.fi> parents: 20605 diff changeset	67 uoff_t sync_max_size;
19318 f9a143c630a5 dsync: Added DSYNC_BRAIN_FLAG_NO_NOTIFY to enable MAILBOX_TRANSACTION_FLAG_NO_NOTIFY Timo Sirainen <tss@iki.fi> parents: 19228 diff changeset	68 enum mailbox_transaction_flags transaction_flags;
19646 25f06710e671 dsync: When comparing headers' hashes to match messages, try to normalize the input. Timo Sirainen <timo.sirainen@dovecot.fi> parents: 19552 diff changeset	69 unsigned int hdr_hash_version;
22043 81e013b3207d dsync: Try to commit transactions every dsync_commit_msgs_interval messages Timo Sirainen <timo.sirainen@dovecot.fi> parents: 21814 diff changeset	70 unsigned int commit_msgs_interval;
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	71
22549 400ff84f109d dsync: Add hashed_headers setting Aki Tuomi <aki.tuomi@dovecot.fi> parents: 22264 diff changeset	72 const char const hashed_headers;
400ff84f109d dsync: Add hashed_headers setting Aki Tuomi <aki.tuomi@dovecot.fi> parents: 22264 diff changeset	73
18181 35e4a6ae8d85 dsync: Added -F parameter to sync only mails with[out] specific flag. Timo Sirainen <tss@iki.fi> parents: 18180 diff changeset	74 enum mail_flags sync_flag;
35e4a6ae8d85 dsync: Added -F parameter to sync only mails with[out] specific flag. Timo Sirainen <tss@iki.fi> parents: 18180 diff changeset	75 const char *sync_keyword;
35e4a6ae8d85 dsync: Added -F parameter to sync only mails with[out] specific flag. Timo Sirainen <tss@iki.fi> parents: 18180 diff changeset	76 bool sync_flag_dontwant;
35e4a6ae8d85 dsync: Added -F parameter to sync only mails with[out] specific flag. Timo Sirainen <tss@iki.fi> parents: 18180 diff changeset	77
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	78 struct mailbox_transaction_context trans, ext_trans;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	79 struct mail_search_context *search_ctx;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	80 struct mail mail, ext_mail;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	81
18180 39d00448490f dsync: If same GUID already exists in storage, try to copy it instead of recreating the mail. Timo Sirainen <tss@iki.fi> parents: 18178 diff changeset	82 struct mailbox *virtual_all_box;
39d00448490f dsync: If same GUID already exists in storage, try to copy it instead of recreating the mail. Timo Sirainen <tss@iki.fi> parents: 18178 diff changeset	83 struct mailbox_transaction_context *virtual_trans;
39d00448490f dsync: If same GUID already exists in storage, try to copy it instead of recreating the mail. Timo Sirainen <tss@iki.fi> parents: 18178 diff changeset	84 struct mail *virtual_mail;
39d00448490f dsync: If same GUID already exists in storage, try to copy it instead of recreating the mail. Timo Sirainen <tss@iki.fi> parents: 18178 diff changeset	85
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	86 struct mail *cur_mail;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	87 const char *cur_guid;
15753 f58e7b386c6e dsync: Fixes for syncing without GUIDs. Timo Sirainen <tss@iki.fi> parents: 15743 diff changeset	88 const char *cur_hdr_hash;
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	89
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	90 /* UID => struct dsync_mail_change */
14918 8eae4e205c82 Hash table API is now (mostly) type safe. Timo Sirainen <tss@iki.fi> parents: 14917 diff changeset	91 HASH_TABLE_TYPE(dsync_uid_mail_change) local_changes;
16025 c6082de4bf5b dsync: Added support for syncing mailbox attributes. Timo Sirainen <tss@iki.fi> parents: 15841 diff changeset	92 HASH_TABLE_TYPE(dsync_attr_change) local_attr_changes;
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	93
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	94 ARRAY_TYPE(seq_range) maybe_expunge_uids;
14920 a097ef0a9d6d Array API changed: ARRAY_DEFINE(name, type) -> ARRAY(type) name Timo Sirainen <tss@iki.fi> parents: 14918 diff changeset	95 ARRAY(struct dsync_mail_change *) maybe_saves;
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	96
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	97 /* GUID => struct importer_new_mail */
14918 8eae4e205c82 Hash table API is now (mostly) type safe. Timo Sirainen <tss@iki.fi> parents: 14917 diff changeset	98 HASH_TABLE_TYPE(guid_new_mail) import_guids;
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	99 /* UID => struct importer_new_mail */
14918 8eae4e205c82 Hash table API is now (mostly) type safe. Timo Sirainen <tss@iki.fi> parents: 14917 diff changeset	100 HASH_TABLE_TYPE(uid_new_mail) import_uids;
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	101
14920 a097ef0a9d6d Array API changed: ARRAY_DEFINE(name, type) -> ARRAY(type) name Timo Sirainen <tss@iki.fi> parents: 14918 diff changeset	102 ARRAY(struct importer_new_mail *) newmails;
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	103 ARRAY_TYPE(uint32_t) wanted_uids;
16255 548e59794f2e dsync: Commit large transactions every 100 new messages. Timo Sirainen <tss@iki.fi> parents: 16254 diff changeset	104 ARRAY_TYPE(uint32_t) saved_uids;
15824 953594b35bb3 dsync: Fixes to handling local changes during dsync. Timo Sirainen <tss@iki.fi> parents: 15820 diff changeset	105 uint32_t highest_wanted_uid;
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	106
14920 a097ef0a9d6d Array API changed: ARRAY_DEFINE(name, type) -> ARRAY(type) name Timo Sirainen <tss@iki.fi> parents: 14918 diff changeset	107 ARRAY(struct dsync_mail_request) mail_requests;
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	108 unsigned int mail_request_idx;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	109
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	110 uint32_t prev_uid, next_local_seq, local_uid_next;
15496 5d90e2aa1ba8 dsync: Use private modseqs to support syncing private flags in shared mailboxes. Timo Sirainen <tss@iki.fi> parents: 15257 diff changeset	111 uint64_t local_initial_highestmodseq, local_initial_highestpvtmodseq;
16539 b5876fa03b0e dsync: If verbose_proctitle=yes, show the current state in it. Timo Sirainen <tss@iki.fi> parents: 16259 diff changeset	112 unsigned int import_pos, import_count;
22043 81e013b3207d dsync: Try to commit transactions every dsync_commit_msgs_interval messages Timo Sirainen <timo.sirainen@dovecot.fi> parents: 21814 diff changeset	113 unsigned int first_unsaved_idx, saves_since_commit;
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	114
18371 b900b50085fc dsync: Use storage's mail_error to choose the doveadm exit code. Timo Sirainen <tss@iki.fi> parents: 18183 diff changeset	115 enum mail_error mail_error;
b900b50085fc dsync: Use storage's mail_error to choose the doveadm exit code. Timo Sirainen <tss@iki.fi> parents: 18183 diff changeset	116
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	117 unsigned int failed:1;
20074 7a29a0930b63 dsync: If state is invalid, exit with code 2 instead of tempfail. Timo Sirainen <timo.sirainen@dovecot.fi> parents: 20072 diff changeset	118 unsigned int require_full_resync:1;
15611 cc77431b09b4 dsync: Don't wrongly log errors about missing message bodies. Timo Sirainen <tss@iki.fi> parents: 15591 diff changeset	119 unsigned int debug:1;
15766 4260244e57e2 dsync: Various importer fixes. Handle unexpectedly wrong mailbox state better. Timo Sirainen <tss@iki.fi> parents: 15754 diff changeset	120 unsigned int stateful_import:1;
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	121 unsigned int last_common_uid_found:1;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	122 unsigned int cur_uid_has_change:1;
15661 940414cea18d dsync backup: Old unwanted messages weren't deleted as they should have. Timo Sirainen <tss@iki.fi> parents: 15658 diff changeset	123 unsigned int cur_mail_skip:1;
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	124 unsigned int local_expunged_guids_set:1;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	125 unsigned int new_uids_assigned:1;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	126 unsigned int want_mail_requests:1;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	127 unsigned int master_brain:1;
15257 60f0cb48fdb2 doveadm backup: Revert all local changes. Timo Sirainen <tss@iki.fi> parents: 15233 diff changeset	128 unsigned int revert_local_changes:1;
15753 f58e7b386c6e dsync: Fixes for syncing without GUIDs. Timo Sirainen <tss@iki.fi> parents: 15743 diff changeset	129 unsigned int mails_have_guids:1;
16977 8a7ecddca4ad dsync: Support syncing storages with 128bit GUIDs <-> string GUIDs. Timo Sirainen <tss@iki.fi> parents: 16940 diff changeset	130 unsigned int mails_use_guid128:1;
17859 c02918434721 doveadm backup: Locally expunged mails weren't always added back. Timo Sirainen <tss@iki.fi> parents: 17845 diff changeset	131 unsigned int delete_mailbox:1;
20595 40ce04c672a4 dsync: Add support for features Aki Tuomi <aki.tuomi@dovecot.fi> parents: 20262 diff changeset	132 unsigned int empty_hdr_workaround:1;
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	133 };
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	134
20208 f1a5362b8a85 dsync: Debug logging improvement. Timo Sirainen <timo.sirainen@dovecot.fi> parents: 20191 diff changeset	135 static const char *dsync_mail_change_type_names[] = {
f1a5362b8a85 dsync: Debug logging improvement. Timo Sirainen <timo.sirainen@dovecot.fi> parents: 20191 diff changeset	136 "save", "expunge", "flag-change"
f1a5362b8a85 dsync: Debug logging improvement. Timo Sirainen <timo.sirainen@dovecot.fi> parents: 20191 diff changeset	137 };
f1a5362b8a85 dsync: Debug logging improvement. Timo Sirainen <timo.sirainen@dovecot.fi> parents: 20191 diff changeset	138
18180 39d00448490f dsync: If same GUID already exists in storage, try to copy it instead of recreating the mail. Timo Sirainen <tss@iki.fi> parents: 18178 diff changeset	139 static bool dsync_mailbox_save_newmails(struct dsync_mailbox_importer *importer,
15766 4260244e57e2 dsync: Various importer fixes. Handle unexpectedly wrong mailbox state better. Timo Sirainen <tss@iki.fi> parents: 15754 diff changeset	140 const struct dsync_mail *mail,
18180 39d00448490f dsync: If same GUID already exists in storage, try to copy it instead of recreating the mail. Timo Sirainen <tss@iki.fi> parents: 18178 diff changeset	141 struct importer_new_mail *all_newmails,
39d00448490f dsync: If same GUID already exists in storage, try to copy it instead of recreating the mail. Timo Sirainen <tss@iki.fi> parents: 18178 diff changeset	142 bool remote_mail);
16255 548e59794f2e dsync: Commit large transactions every 100 new messages. Timo Sirainen <tss@iki.fi> parents: 16254 diff changeset	143 static int dsync_mailbox_import_commit(struct dsync_mailbox_importer *importer,
548e59794f2e dsync: Commit large transactions every 100 new messages. Timo Sirainen <tss@iki.fi> parents: 16254 diff changeset	144 bool final);
15766 4260244e57e2 dsync: Various importer fixes. Handle unexpectedly wrong mailbox state better. Timo Sirainen <tss@iki.fi> parents: 15754 diff changeset	145
17272 767acd575d06 Compiler warning fixes Timo Sirainen <tss@iki.fi> parents: 17261 diff changeset	146 static void ATTR_FORMAT(2, 3)
16792 4a969a2e9399 dsync: Added some more debug logging. Timo Sirainen <tss@iki.fi> parents: 16762 diff changeset	147 imp_debug(struct dsync_mailbox_importer importer, const char fmt, ...)
4a969a2e9399 dsync: Added some more debug logging. Timo Sirainen <tss@iki.fi> parents: 16762 diff changeset	148 {
4a969a2e9399 dsync: Added some more debug logging. Timo Sirainen <tss@iki.fi> parents: 16762 diff changeset	149 va_list args;
4a969a2e9399 dsync: Added some more debug logging. Timo Sirainen <tss@iki.fi> parents: 16762 diff changeset	150
4a969a2e9399 dsync: Added some more debug logging. Timo Sirainen <tss@iki.fi> parents: 16762 diff changeset	151 if (importer->debug) T_BEGIN {
4a969a2e9399 dsync: Added some more debug logging. Timo Sirainen <tss@iki.fi> parents: 16762 diff changeset	152 va_start(args, fmt);
4a969a2e9399 dsync: Added some more debug logging. Timo Sirainen <tss@iki.fi> parents: 16762 diff changeset	153 i_debug("brain %c: Import %s: %s",
4a969a2e9399 dsync: Added some more debug logging. Timo Sirainen <tss@iki.fi> parents: 16762 diff changeset	154 importer->master_brain ? 'M' : 'S',
4a969a2e9399 dsync: Added some more debug logging. Timo Sirainen <tss@iki.fi> parents: 16762 diff changeset	155 mailbox_get_vname(importer->box),
4a969a2e9399 dsync: Added some more debug logging. Timo Sirainen <tss@iki.fi> parents: 16762 diff changeset	156 t_strdup_vprintf(fmt, args));
4a969a2e9399 dsync: Added some more debug logging. Timo Sirainen <tss@iki.fi> parents: 16762 diff changeset	157 va_end(args);
4a969a2e9399 dsync: Added some more debug logging. Timo Sirainen <tss@iki.fi> parents: 16762 diff changeset	158 } T_END;
4a969a2e9399 dsync: Added some more debug logging. Timo Sirainen <tss@iki.fi> parents: 16762 diff changeset	159 }
4a969a2e9399 dsync: Added some more debug logging. Timo Sirainen <tss@iki.fi> parents: 16762 diff changeset	160
4a969a2e9399 dsync: Added some more debug logging. Timo Sirainen <tss@iki.fi> parents: 16762 diff changeset	161 static void
20072 18d1fcb14277 dsync: Improved detecting state state string. Timo Sirainen <timo.sirainen@dovecot.fi> parents: 19696 diff changeset	162 dsync_import_unexpected_state(struct dsync_mailbox_importer *importer,
18d1fcb14277 dsync: Improved detecting state state string. Timo Sirainen <timo.sirainen@dovecot.fi> parents: 19696 diff changeset	163 const char *error)
18d1fcb14277 dsync: Improved detecting state state string. Timo Sirainen <timo.sirainen@dovecot.fi> parents: 19696 diff changeset	164 {
18d1fcb14277 dsync: Improved detecting state state string. Timo Sirainen <timo.sirainen@dovecot.fi> parents: 19696 diff changeset	165 if (!importer->stateful_import) {
18d1fcb14277 dsync: Improved detecting state state string. Timo Sirainen <timo.sirainen@dovecot.fi> parents: 19696 diff changeset	166 i_error("Mailbox %s: %s", mailbox_get_vname(importer->box),
18d1fcb14277 dsync: Improved detecting state state string. Timo Sirainen <timo.sirainen@dovecot.fi> parents: 19696 diff changeset	167 error);
18d1fcb14277 dsync: Improved detecting state state string. Timo Sirainen <timo.sirainen@dovecot.fi> parents: 19696 diff changeset	168 } else {
18d1fcb14277 dsync: Improved detecting state state string. Timo Sirainen <timo.sirainen@dovecot.fi> parents: 19696 diff changeset	169 i_warning("Mailbox %s doesn't match previous state: %s "
18d1fcb14277 dsync: Improved detecting state state string. Timo Sirainen <timo.sirainen@dovecot.fi> parents: 19696 diff changeset	170 "(dsync must be run again without the state)",
18d1fcb14277 dsync: Improved detecting state state string. Timo Sirainen <timo.sirainen@dovecot.fi> parents: 19696 diff changeset	171 mailbox_get_vname(importer->box), error);
18d1fcb14277 dsync: Improved detecting state state string. Timo Sirainen <timo.sirainen@dovecot.fi> parents: 19696 diff changeset	172 }
20074 7a29a0930b63 dsync: If state is invalid, exit with code 2 instead of tempfail. Timo Sirainen <timo.sirainen@dovecot.fi> parents: 20072 diff changeset	173 importer->require_full_resync = TRUE;
20072 18d1fcb14277 dsync: Improved detecting state state string. Timo Sirainen <timo.sirainen@dovecot.fi> parents: 19696 diff changeset	174 }
18d1fcb14277 dsync: Improved detecting state state string. Timo Sirainen <timo.sirainen@dovecot.fi> parents: 19696 diff changeset	175
18d1fcb14277 dsync: Improved detecting state state string. Timo Sirainen <timo.sirainen@dovecot.fi> parents: 19696 diff changeset	176 static void
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	177 dsync_mailbox_import_search_init(struct dsync_mailbox_importer *importer)
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	178 {
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	179 struct mail_search_args *search_args;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	180 struct mail_search_arg *sarg;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	181
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	182 search_args = mail_search_build_init();
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	183 sarg = mail_search_build_add(search_args, SEARCH_UIDSET);
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	184 p_array_init(&sarg->value.seqset, search_args->pool, 128);
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	185 seq_range_array_add_range(&sarg->value.seqset,
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	186 importer->last_common_uid+1, (uint32_t)-1);
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	187
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	188 importer->search_ctx =
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	189 mailbox_search_init(importer->trans, search_args, NULL,
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	190 0, NULL);
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	191 mail_search_args_unref(&search_args);
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	192
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	193 if (mailbox_search_next(importer->search_ctx, &importer->cur_mail))
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	194 importer->next_local_seq = importer->cur_mail->seq;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	195 /* this flag causes cur_guid to be looked up later */
15661 940414cea18d dsync backup: Old unwanted messages weren't deleted as they should have. Timo Sirainen <tss@iki.fi> parents: 15658 diff changeset	196 importer->cur_mail_skip = TRUE;
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	197 }
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	198
16255 548e59794f2e dsync: Commit large transactions every 100 new messages. Timo Sirainen <tss@iki.fi> parents: 16254 diff changeset	199 static void
548e59794f2e dsync: Commit large transactions every 100 new messages. Timo Sirainen <tss@iki.fi> parents: 16254 diff changeset	200 dsync_mailbox_import_transaction_begin(struct dsync_mailbox_importer *importer)
548e59794f2e dsync: Commit large transactions every 100 new messages. Timo Sirainen <tss@iki.fi> parents: 16254 diff changeset	201 {
548e59794f2e dsync: Commit large transactions every 100 new messages. Timo Sirainen <tss@iki.fi> parents: 16254 diff changeset	202 const enum mailbox_transaction_flags ext_trans_flags =
19318 f9a143c630a5 dsync: Added DSYNC_BRAIN_FLAG_NO_NOTIFY to enable MAILBOX_TRANSACTION_FLAG_NO_NOTIFY Timo Sirainen <tss@iki.fi> parents: 19228 diff changeset	203 importer->transaction_flags \|
16255 548e59794f2e dsync: Commit large transactions every 100 new messages. Timo Sirainen <tss@iki.fi> parents: 16254 diff changeset	204 MAILBOX_TRANSACTION_FLAG_EXTERNAL \|
548e59794f2e dsync: Commit large transactions every 100 new messages. Timo Sirainen <tss@iki.fi> parents: 16254 diff changeset	205 MAILBOX_TRANSACTION_FLAG_ASSIGN_UIDS;
548e59794f2e dsync: Commit large transactions every 100 new messages. Timo Sirainen <tss@iki.fi> parents: 16254 diff changeset	206
548e59794f2e dsync: Commit large transactions every 100 new messages. Timo Sirainen <tss@iki.fi> parents: 16254 diff changeset	207 importer->trans = mailbox_transaction_begin(importer->box,
19318 f9a143c630a5 dsync: Added DSYNC_BRAIN_FLAG_NO_NOTIFY to enable MAILBOX_TRANSACTION_FLAG_NO_NOTIFY Timo Sirainen <tss@iki.fi> parents: 19228 diff changeset	208 importer->transaction_flags);
16255 548e59794f2e dsync: Commit large transactions every 100 new messages. Timo Sirainen <tss@iki.fi> parents: 16254 diff changeset	209 importer->ext_trans = mailbox_transaction_begin(importer->box,
548e59794f2e dsync: Commit large transactions every 100 new messages. Timo Sirainen <tss@iki.fi> parents: 16254 diff changeset	210 ext_trans_flags);
548e59794f2e dsync: Commit large transactions every 100 new messages. Timo Sirainen <tss@iki.fi> parents: 16254 diff changeset	211 importer->mail = mail_alloc(importer->trans, 0, NULL);
548e59794f2e dsync: Commit large transactions every 100 new messages. Timo Sirainen <tss@iki.fi> parents: 16254 diff changeset	212 importer->ext_mail = mail_alloc(importer->ext_trans, 0, NULL);
548e59794f2e dsync: Commit large transactions every 100 new messages. Timo Sirainen <tss@iki.fi> parents: 16254 diff changeset	213 }
548e59794f2e dsync: Commit large transactions every 100 new messages. Timo Sirainen <tss@iki.fi> parents: 16254 diff changeset	214
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	215 struct dsync_mailbox_importer *
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	216 dsync_mailbox_import_init(struct mailbox *box,
18180 39d00448490f dsync: If same GUID already exists in storage, try to copy it instead of recreating the mail. Timo Sirainen <tss@iki.fi> parents: 18178 diff changeset	217 struct mailbox *virtual_all_box,
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	218 struct dsync_transaction_log_scan *log_scan,
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	219 uint32_t last_common_uid,
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	220 uint64_t last_common_modseq,
15496 5d90e2aa1ba8 dsync: Use private modseqs to support syncing private flags in shared mailboxes. Timo Sirainen <tss@iki.fi> parents: 15257 diff changeset	221 uint64_t last_common_pvt_modseq,
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	222 uint32_t remote_uid_next,
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	223 uint32_t remote_first_recent_uid,
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	224 uint64_t remote_highest_modseq,
15496 5d90e2aa1ba8 dsync: Use private modseqs to support syncing private flags in shared mailboxes. Timo Sirainen <tss@iki.fi> parents: 15257 diff changeset	225 uint64_t remote_highest_pvt_modseq,
20831 0c4e5c2725a3 doveadm-sync: Add end-date support Aki Tuomi <aki.tuomi@dovecot.fi> parents: 20633 diff changeset	226 time_t sync_since_timestamp,
0c4e5c2725a3 doveadm-sync: Add end-date support Aki Tuomi <aki.tuomi@dovecot.fi> parents: 20633 diff changeset	227 time_t sync_until_timestamp,
0c4e5c2725a3 doveadm-sync: Add end-date support Aki Tuomi <aki.tuomi@dovecot.fi> parents: 20633 diff changeset	228 uoff_t sync_max_size,
20633 d54651ba988a doveadm sync/backup: Added -S <max size> parameter to skip too large mails. Timo Sirainen <timo.sirainen@dovecot.fi> parents: 20605 diff changeset	229 const char *sync_flag,
22043 81e013b3207d dsync: Try to commit transactions every dsync_commit_msgs_interval messages Timo Sirainen <timo.sirainen@dovecot.fi> parents: 21814 diff changeset	230 unsigned int commit_msgs_interval,
22264 e95435889161 dsync: Use header hashing version 3 Timo Sirainen <timo.sirainen@dovecot.fi> parents: 22043 diff changeset	231 enum dsync_mailbox_import_flags flags,
22549 400ff84f109d dsync: Add hashed_headers setting Aki Tuomi <aki.tuomi@dovecot.fi> parents: 22264 diff changeset	232 unsigned int hdr_hash_version,
400ff84f109d dsync: Add hashed_headers setting Aki Tuomi <aki.tuomi@dovecot.fi> parents: 22264 diff changeset	233 const char const hashed_headers)
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	234 {
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	235 struct dsync_mailbox_importer *importer;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	236 struct mailbox_status status;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	237 pool_t pool;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	238
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	239 pool = pool_alloconly_create(MEMPOOL_GROWING"dsync mailbox importer",
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	240 10240);
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	241 importer = p_new(pool, struct dsync_mailbox_importer, 1);
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	242 importer->pool = pool;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	243 importer->box = box;
18180 39d00448490f dsync: If same GUID already exists in storage, try to copy it instead of recreating the mail. Timo Sirainen <tss@iki.fi> parents: 18178 diff changeset	244 importer->virtual_all_box = virtual_all_box;
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	245 importer->last_common_uid = last_common_uid;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	246 importer->last_common_modseq = last_common_modseq;
15496 5d90e2aa1ba8 dsync: Use private modseqs to support syncing private flags in shared mailboxes. Timo Sirainen <tss@iki.fi> parents: 15257 diff changeset	247 importer->last_common_pvt_modseq = last_common_pvt_modseq;
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	248 importer->last_common_uid_found =
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	249 last_common_uid != 0 \|\| last_common_modseq != 0;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	250 importer->remote_uid_next = remote_uid_next;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	251 importer->remote_first_recent_uid = remote_first_recent_uid;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	252 importer->remote_highest_modseq = remote_highest_modseq;
15496 5d90e2aa1ba8 dsync: Use private modseqs to support syncing private flags in shared mailboxes. Timo Sirainen <tss@iki.fi> parents: 15257 diff changeset	253 importer->remote_highest_pvt_modseq = remote_highest_pvt_modseq;
18177 f393f63764e0 dsync: Added -t <timestamp> parameter to save only mails newer than <timestamp> Timo Sirainen <tss@iki.fi> parents: 18137 diff changeset	254 importer->sync_since_timestamp = sync_since_timestamp;
20831 0c4e5c2725a3 doveadm-sync: Add end-date support Aki Tuomi <aki.tuomi@dovecot.fi> parents: 20633 diff changeset	255 importer->sync_until_timestamp = sync_until_timestamp;
20633 d54651ba988a doveadm sync/backup: Added -S <max size> parameter to skip too large mails. Timo Sirainen <timo.sirainen@dovecot.fi> parents: 20605 diff changeset	256 importer->sync_max_size = sync_max_size;
15766 4260244e57e2 dsync: Various importer fixes. Handle unexpectedly wrong mailbox state better. Timo Sirainen <tss@iki.fi> parents: 15754 diff changeset	257 importer->stateful_import = importer->last_common_uid_found;
22549 400ff84f109d dsync: Add hashed_headers setting Aki Tuomi <aki.tuomi@dovecot.fi> parents: 22264 diff changeset	258 importer->hashed_headers = hashed_headers;
400ff84f109d dsync: Add hashed_headers setting Aki Tuomi <aki.tuomi@dovecot.fi> parents: 22264 diff changeset	259
18181 35e4a6ae8d85 dsync: Added -F parameter to sync only mails with[out] specific flag. Timo Sirainen <tss@iki.fi> parents: 18180 diff changeset	260 if (sync_flag != NULL) {
35e4a6ae8d85 dsync: Added -F parameter to sync only mails with[out] specific flag. Timo Sirainen <tss@iki.fi> parents: 18180 diff changeset	261 if (sync_flag[0] == '-') {
35e4a6ae8d85 dsync: Added -F parameter to sync only mails with[out] specific flag. Timo Sirainen <tss@iki.fi> parents: 18180 diff changeset	262 importer->sync_flag_dontwant = TRUE;
35e4a6ae8d85 dsync: Added -F parameter to sync only mails with[out] specific flag. Timo Sirainen <tss@iki.fi> parents: 18180 diff changeset	263 sync_flag++;
35e4a6ae8d85 dsync: Added -F parameter to sync only mails with[out] specific flag. Timo Sirainen <tss@iki.fi> parents: 18180 diff changeset	264 }
35e4a6ae8d85 dsync: Added -F parameter to sync only mails with[out] specific flag. Timo Sirainen <tss@iki.fi> parents: 18180 diff changeset	265 if (sync_flag[0] == '\\')
35e4a6ae8d85 dsync: Added -F parameter to sync only mails with[out] specific flag. Timo Sirainen <tss@iki.fi> parents: 18180 diff changeset	266 importer->sync_flag = imap_parse_system_flag(sync_flag);
35e4a6ae8d85 dsync: Added -F parameter to sync only mails with[out] specific flag. Timo Sirainen <tss@iki.fi> parents: 18180 diff changeset	267 else
35e4a6ae8d85 dsync: Added -F parameter to sync only mails with[out] specific flag. Timo Sirainen <tss@iki.fi> parents: 18180 diff changeset	268 importer->sync_keyword = p_strdup(pool, sync_flag);
35e4a6ae8d85 dsync: Added -F parameter to sync only mails with[out] specific flag. Timo Sirainen <tss@iki.fi> parents: 18180 diff changeset	269 }
22043 81e013b3207d dsync: Try to commit transactions every dsync_commit_msgs_interval messages Timo Sirainen <timo.sirainen@dovecot.fi> parents: 21814 diff changeset	270 importer->commit_msgs_interval = commit_msgs_interval;
19318 f9a143c630a5 dsync: Added DSYNC_BRAIN_FLAG_NO_NOTIFY to enable MAILBOX_TRANSACTION_FLAG_NO_NOTIFY Timo Sirainen <tss@iki.fi> parents: 19228 diff changeset	271 importer->transaction_flags = MAILBOX_TRANSACTION_FLAG_SYNC;
f9a143c630a5 dsync: Added DSYNC_BRAIN_FLAG_NO_NOTIFY to enable MAILBOX_TRANSACTION_FLAG_NO_NOTIFY Timo Sirainen <tss@iki.fi> parents: 19228 diff changeset	272 if ((flags & DSYNC_MAILBOX_IMPORT_FLAG_NO_NOTIFY) != 0)
f9a143c630a5 dsync: Added DSYNC_BRAIN_FLAG_NO_NOTIFY to enable MAILBOX_TRANSACTION_FLAG_NO_NOTIFY Timo Sirainen <tss@iki.fi> parents: 19228 diff changeset	273 importer->transaction_flags \|= MAILBOX_TRANSACTION_FLAG_NO_NOTIFY;
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	274
14918 8eae4e205c82 Hash table API is now (mostly) type safe. Timo Sirainen <tss@iki.fi> parents: 14917 diff changeset	275 hash_table_create(&importer->import_guids, pool, 0, str_hash, strcmp);
8eae4e205c82 Hash table API is now (mostly) type safe. Timo Sirainen <tss@iki.fi> parents: 14917 diff changeset	276 hash_table_create_direct(&importer->import_uids, pool, 0);
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	277 i_array_init(&importer->maybe_expunge_uids, 16);
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	278 i_array_init(&importer->maybe_saves, 128);
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	279 i_array_init(&importer->newmails, 128);
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	280 i_array_init(&importer->wanted_uids, 128);
16255 548e59794f2e dsync: Commit large transactions every 100 new messages. Timo Sirainen <tss@iki.fi> parents: 16254 diff changeset	281 i_array_init(&importer->saved_uids, 128);
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	282
16255 548e59794f2e dsync: Commit large transactions every 100 new messages. Timo Sirainen <tss@iki.fi> parents: 16254 diff changeset	283 dsync_mailbox_import_transaction_begin(importer);
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	284
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	285 if ((flags & DSYNC_MAILBOX_IMPORT_FLAG_WANT_MAIL_REQUESTS) != 0) {
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	286 i_array_init(&importer->mail_requests, 128);
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	287 importer->want_mail_requests = TRUE;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	288 }
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	289 importer->master_brain =
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	290 (flags & DSYNC_MAILBOX_IMPORT_FLAG_MASTER_BRAIN) != 0;
15257 60f0cb48fdb2 doveadm backup: Revert all local changes. Timo Sirainen <tss@iki.fi> parents: 15233 diff changeset	291 importer->revert_local_changes =
60f0cb48fdb2 doveadm backup: Revert all local changes. Timo Sirainen <tss@iki.fi> parents: 15233 diff changeset	292 (flags & DSYNC_MAILBOX_IMPORT_FLAG_REVERT_LOCAL_CHANGES) != 0;
15611 cc77431b09b4 dsync: Don't wrongly log errors about missing message bodies. Timo Sirainen <tss@iki.fi> parents: 15591 diff changeset	293 importer->debug = (flags & DSYNC_MAILBOX_IMPORT_FLAG_DEBUG) != 0;
15753 f58e7b386c6e dsync: Fixes for syncing without GUIDs. Timo Sirainen <tss@iki.fi> parents: 15743 diff changeset	294 importer->mails_have_guids =
f58e7b386c6e dsync: Fixes for syncing without GUIDs. Timo Sirainen <tss@iki.fi> parents: 15743 diff changeset	295 (flags & DSYNC_MAILBOX_IMPORT_FLAG_MAILS_HAVE_GUIDS) != 0;
16977 8a7ecddca4ad dsync: Support syncing storages with 128bit GUIDs <-> string GUIDs. Timo Sirainen <tss@iki.fi> parents: 16940 diff changeset	296 importer->mails_use_guid128 =
8a7ecddca4ad dsync: Support syncing storages with 128bit GUIDs <-> string GUIDs. Timo Sirainen <tss@iki.fi> parents: 16940 diff changeset	297 (flags & DSYNC_MAILBOX_IMPORT_FLAG_MAILS_USE_GUID128) != 0;
22264 e95435889161 dsync: Use header hashing version 3 Timo Sirainen <timo.sirainen@dovecot.fi> parents: 22043 diff changeset	298 importer->hdr_hash_version = hdr_hash_version;
20595 40ce04c672a4 dsync: Add support for features Aki Tuomi <aki.tuomi@dovecot.fi> parents: 20262 diff changeset	299 importer->empty_hdr_workaround =
40ce04c672a4 dsync: Add support for features Aki Tuomi <aki.tuomi@dovecot.fi> parents: 20262 diff changeset	300 (flags & DSYNC_MAILBOX_IMPORT_FLAG_EMPTY_HDR_WORKAROUND) != 0;
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	301
15496 5d90e2aa1ba8 dsync: Use private modseqs to support syncing private flags in shared mailboxes. Timo Sirainen <tss@iki.fi> parents: 15257 diff changeset	302 mailbox_get_open_status(importer->box, STATUS_UIDNEXT \|
5d90e2aa1ba8 dsync: Use private modseqs to support syncing private flags in shared mailboxes. Timo Sirainen <tss@iki.fi> parents: 15257 diff changeset	303 STATUS_HIGHESTMODSEQ \| STATUS_HIGHESTPVTMODSEQ,
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	304 &status);
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	305 importer->local_uid_next = status.uidnext;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	306 importer->local_initial_highestmodseq = status.highest_modseq;
15496 5d90e2aa1ba8 dsync: Use private modseqs to support syncing private flags in shared mailboxes. Timo Sirainen <tss@iki.fi> parents: 15257 diff changeset	307 importer->local_initial_highestpvtmodseq = status.highest_pvt_modseq;
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	308 dsync_mailbox_import_search_init(importer);
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	309
20072 18d1fcb14277 dsync: Improved detecting state state string. Timo Sirainen <timo.sirainen@dovecot.fi> parents: 19696 diff changeset	310 if (!importer->stateful_import)
18d1fcb14277 dsync: Improved detecting state state string. Timo Sirainen <timo.sirainen@dovecot.fi> parents: 19696 diff changeset	311 ;
18d1fcb14277 dsync: Improved detecting state state string. Timo Sirainen <timo.sirainen@dovecot.fi> parents: 19696 diff changeset	312 else if (importer->local_uid_next <= last_common_uid) {
18d1fcb14277 dsync: Improved detecting state state string. Timo Sirainen <timo.sirainen@dovecot.fi> parents: 19696 diff changeset	313 dsync_import_unexpected_state(importer, t_strdup_printf(
18d1fcb14277 dsync: Improved detecting state state string. Timo Sirainen <timo.sirainen@dovecot.fi> parents: 19696 diff changeset	314 "local UIDNEXT %u <= last common UID %u",
18d1fcb14277 dsync: Improved detecting state state string. Timo Sirainen <timo.sirainen@dovecot.fi> parents: 19696 diff changeset	315 importer->local_uid_next, last_common_uid));
18d1fcb14277 dsync: Improved detecting state state string. Timo Sirainen <timo.sirainen@dovecot.fi> parents: 19696 diff changeset	316 } else if (importer->local_initial_highestmodseq < last_common_modseq) {
18d1fcb14277 dsync: Improved detecting state state string. Timo Sirainen <timo.sirainen@dovecot.fi> parents: 19696 diff changeset	317 dsync_import_unexpected_state(importer, t_strdup_printf(
18d1fcb14277 dsync: Improved detecting state state string. Timo Sirainen <timo.sirainen@dovecot.fi> parents: 19696 diff changeset	318 "local HIGHESTMODSEQ %llu < last common HIGHESTMODSEQ %llu",
18d1fcb14277 dsync: Improved detecting state state string. Timo Sirainen <timo.sirainen@dovecot.fi> parents: 19696 diff changeset	319 (unsigned long long)importer->local_initial_highestmodseq,
18d1fcb14277 dsync: Improved detecting state state string. Timo Sirainen <timo.sirainen@dovecot.fi> parents: 19696 diff changeset	320 (unsigned long long)last_common_modseq));
18d1fcb14277 dsync: Improved detecting state state string. Timo Sirainen <timo.sirainen@dovecot.fi> parents: 19696 diff changeset	321 } else if (importer->local_initial_highestpvtmodseq < last_common_pvt_modseq) {
18d1fcb14277 dsync: Improved detecting state state string. Timo Sirainen <timo.sirainen@dovecot.fi> parents: 19696 diff changeset	322 dsync_import_unexpected_state(importer, t_strdup_printf(
18d1fcb14277 dsync: Improved detecting state state string. Timo Sirainen <timo.sirainen@dovecot.fi> parents: 19696 diff changeset	323 "local HIGHESTMODSEQ %llu < last common HIGHESTMODSEQ %llu",
18d1fcb14277 dsync: Improved detecting state state string. Timo Sirainen <timo.sirainen@dovecot.fi> parents: 19696 diff changeset	324 (unsigned long long)importer->local_initial_highestpvtmodseq,
18d1fcb14277 dsync: Improved detecting state state string. Timo Sirainen <timo.sirainen@dovecot.fi> parents: 19696 diff changeset	325 (unsigned long long)last_common_pvt_modseq));
18d1fcb14277 dsync: Improved detecting state state string. Timo Sirainen <timo.sirainen@dovecot.fi> parents: 19696 diff changeset	326 }
18d1fcb14277 dsync: Improved detecting state state string. Timo Sirainen <timo.sirainen@dovecot.fi> parents: 19696 diff changeset	327
14926 9c69df65af7b Simplified hash table union now that it again works only with pointers. Timo Sirainen <tss@iki.fi> parents: 14923 diff changeset	328 importer->local_changes = dsync_transaction_log_scan_get_hash(log_scan);
16025 c6082de4bf5b dsync: Added support for syncing mailbox attributes. Timo Sirainen <tss@iki.fi> parents: 15841 diff changeset	329 importer->local_attr_changes = dsync_transaction_log_scan_get_attr_hash(log_scan);
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	330 return importer;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	331 }
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	332
16025 c6082de4bf5b dsync: Added support for syncing mailbox attributes. Timo Sirainen <tss@iki.fi> parents: 15841 diff changeset	333 static int
c6082de4bf5b dsync: Added support for syncing mailbox attributes. Timo Sirainen <tss@iki.fi> parents: 15841 diff changeset	334 dsync_mailbox_import_lookup_attr(struct dsync_mailbox_importer *importer,
c6082de4bf5b dsync: Added support for syncing mailbox attributes. Timo Sirainen <tss@iki.fi> parents: 15841 diff changeset	335 enum mail_attribute_type type, const char *key,
16052 0e5a359b7b7f lib-storage: Mailbox attributes can now be accessed via istreams. Timo Sirainen <tss@iki.fi> parents: 16046 diff changeset	336 struct dsync_mailbox_attribute **attr_r)
16025 c6082de4bf5b dsync: Added support for syncing mailbox attributes. Timo Sirainen <tss@iki.fi> parents: 15841 diff changeset	337 {
c6082de4bf5b dsync: Added support for syncing mailbox attributes. Timo Sirainen <tss@iki.fi> parents: 15841 diff changeset	338 struct dsync_mailbox_attribute lookup_attr, *attr;
c6082de4bf5b dsync: Added support for syncing mailbox attributes. Timo Sirainen <tss@iki.fi> parents: 15841 diff changeset	339 const struct dsync_mailbox_attribute *attr_change;
c6082de4bf5b dsync: Added support for syncing mailbox attributes. Timo Sirainen <tss@iki.fi> parents: 15841 diff changeset	340 struct mail_attribute_value value;
c6082de4bf5b dsync: Added support for syncing mailbox attributes. Timo Sirainen <tss@iki.fi> parents: 15841 diff changeset	341
c6082de4bf5b dsync: Added support for syncing mailbox attributes. Timo Sirainen <tss@iki.fi> parents: 15841 diff changeset	342 *attr_r = NULL;
c6082de4bf5b dsync: Added support for syncing mailbox attributes. Timo Sirainen <tss@iki.fi> parents: 15841 diff changeset	343
16052 0e5a359b7b7f lib-storage: Mailbox attributes can now be accessed via istreams. Timo Sirainen <tss@iki.fi> parents: 16046 diff changeset	344 if (mailbox_attribute_get_stream(importer->trans, type, key, &value) < 0) {
16025 c6082de4bf5b dsync: Added support for syncing mailbox attributes. Timo Sirainen <tss@iki.fi> parents: 15841 diff changeset	345 i_error("Mailbox %s: Failed to get attribute %s: %s",
c6082de4bf5b dsync: Added support for syncing mailbox attributes. Timo Sirainen <tss@iki.fi> parents: 15841 diff changeset	346 mailbox_get_vname(importer->box), key,
21814 759962e70148 global: Log internal storage error on failure Martti Rannanjärvi <martti.rannanjarvi@dovecot.fi> parents: 21390 diff changeset	347 mailbox_get_last_internal_error(importer->box,
759962e70148 global: Log internal storage error on failure Martti Rannanjärvi <martti.rannanjarvi@dovecot.fi> parents: 21390 diff changeset	348 &importer->mail_error));
16025 c6082de4bf5b dsync: Added support for syncing mailbox attributes. Timo Sirainen <tss@iki.fi> parents: 15841 diff changeset	349 importer->failed = TRUE;
c6082de4bf5b dsync: Added support for syncing mailbox attributes. Timo Sirainen <tss@iki.fi> parents: 15841 diff changeset	350 return -1;
c6082de4bf5b dsync: Added support for syncing mailbox attributes. Timo Sirainen <tss@iki.fi> parents: 15841 diff changeset	351 }
c6082de4bf5b dsync: Added support for syncing mailbox attributes. Timo Sirainen <tss@iki.fi> parents: 15841 diff changeset	352
c6082de4bf5b dsync: Added support for syncing mailbox attributes. Timo Sirainen <tss@iki.fi> parents: 15841 diff changeset	353 lookup_attr.type = type;
c6082de4bf5b dsync: Added support for syncing mailbox attributes. Timo Sirainen <tss@iki.fi> parents: 15841 diff changeset	354 lookup_attr.key = key;
c6082de4bf5b dsync: Added support for syncing mailbox attributes. Timo Sirainen <tss@iki.fi> parents: 15841 diff changeset	355
c6082de4bf5b dsync: Added support for syncing mailbox attributes. Timo Sirainen <tss@iki.fi> parents: 15841 diff changeset	356 attr_change = hash_table_lookup(importer->local_attr_changes,
c6082de4bf5b dsync: Added support for syncing mailbox attributes. Timo Sirainen <tss@iki.fi> parents: 15841 diff changeset	357 &lookup_attr);
16052 0e5a359b7b7f lib-storage: Mailbox attributes can now be accessed via istreams. Timo Sirainen <tss@iki.fi> parents: 16046 diff changeset	358 if (attr_change == NULL &&
0e5a359b7b7f lib-storage: Mailbox attributes can now be accessed via istreams. Timo Sirainen <tss@iki.fi> parents: 16046 diff changeset	359 value.value == NULL && value.value_stream == NULL) {
16025 c6082de4bf5b dsync: Added support for syncing mailbox attributes. Timo Sirainen <tss@iki.fi> parents: 15841 diff changeset	360 /* we have no knowledge of this attribute */
c6082de4bf5b dsync: Added support for syncing mailbox attributes. Timo Sirainen <tss@iki.fi> parents: 15841 diff changeset	361 return 0;
c6082de4bf5b dsync: Added support for syncing mailbox attributes. Timo Sirainen <tss@iki.fi> parents: 15841 diff changeset	362 }
c6082de4bf5b dsync: Added support for syncing mailbox attributes. Timo Sirainen <tss@iki.fi> parents: 15841 diff changeset	363 attr = t_new(struct dsync_mailbox_attribute, 1);
c6082de4bf5b dsync: Added support for syncing mailbox attributes. Timo Sirainen <tss@iki.fi> parents: 15841 diff changeset	364 attr->type = type;
c6082de4bf5b dsync: Added support for syncing mailbox attributes. Timo Sirainen <tss@iki.fi> parents: 15841 diff changeset	365 attr->key = key;
c6082de4bf5b dsync: Added support for syncing mailbox attributes. Timo Sirainen <tss@iki.fi> parents: 15841 diff changeset	366 attr->value = value.value;
16052 0e5a359b7b7f lib-storage: Mailbox attributes can now be accessed via istreams. Timo Sirainen <tss@iki.fi> parents: 16046 diff changeset	367 attr->value_stream = value.value_stream;
16025 c6082de4bf5b dsync: Added support for syncing mailbox attributes. Timo Sirainen <tss@iki.fi> parents: 15841 diff changeset	368 attr->last_change = value.last_change;
c6082de4bf5b dsync: Added support for syncing mailbox attributes. Timo Sirainen <tss@iki.fi> parents: 15841 diff changeset	369 if (attr_change != NULL) {
16052 0e5a359b7b7f lib-storage: Mailbox attributes can now be accessed via istreams. Timo Sirainen <tss@iki.fi> parents: 16046 diff changeset	370 attr->deleted = attr_change->deleted &&
0e5a359b7b7f lib-storage: Mailbox attributes can now be accessed via istreams. Timo Sirainen <tss@iki.fi> parents: 16046 diff changeset	371 !DSYNC_ATTR_HAS_VALUE(attr);
16025 c6082de4bf5b dsync: Added support for syncing mailbox attributes. Timo Sirainen <tss@iki.fi> parents: 15841 diff changeset	372 attr->modseq = attr_change->modseq;
c6082de4bf5b dsync: Added support for syncing mailbox attributes. Timo Sirainen <tss@iki.fi> parents: 15841 diff changeset	373 }
c6082de4bf5b dsync: Added support for syncing mailbox attributes. Timo Sirainen <tss@iki.fi> parents: 15841 diff changeset	374 *attr_r = attr;
c6082de4bf5b dsync: Added support for syncing mailbox attributes. Timo Sirainen <tss@iki.fi> parents: 15841 diff changeset	375 return 0;
c6082de4bf5b dsync: Added support for syncing mailbox attributes. Timo Sirainen <tss@iki.fi> parents: 15841 diff changeset	376 }
c6082de4bf5b dsync: Added support for syncing mailbox attributes. Timo Sirainen <tss@iki.fi> parents: 15841 diff changeset	377
16052 0e5a359b7b7f lib-storage: Mailbox attributes can now be accessed via istreams. Timo Sirainen <tss@iki.fi> parents: 16046 diff changeset	378 static int
0e5a359b7b7f lib-storage: Mailbox attributes can now be accessed via istreams. Timo Sirainen <tss@iki.fi> parents: 16046 diff changeset	379 dsync_istreams_cmp(struct istream input1, struct istream input2, int *cmp_r)
0e5a359b7b7f lib-storage: Mailbox attributes can now be accessed via istreams. Timo Sirainen <tss@iki.fi> parents: 16046 diff changeset	380 {
0e5a359b7b7f lib-storage: Mailbox attributes can now be accessed via istreams. Timo Sirainen <tss@iki.fi> parents: 16046 diff changeset	381 const unsigned char data1, data2;
0e5a359b7b7f lib-storage: Mailbox attributes can now be accessed via istreams. Timo Sirainen <tss@iki.fi> parents: 16046 diff changeset	382 size_t size1, size2, size;
0e5a359b7b7f lib-storage: Mailbox attributes can now be accessed via istreams. Timo Sirainen <tss@iki.fi> parents: 16046 diff changeset	383
16088 b839613c1573 Compiler warning fix Timo Sirainen <tss@iki.fi> parents: 16055 diff changeset	384 cmp_r = -1; / quiet gcc */
b839613c1573 Compiler warning fix Timo Sirainen <tss@iki.fi> parents: 16055 diff changeset	385
16052 0e5a359b7b7f lib-storage: Mailbox attributes can now be accessed via istreams. Timo Sirainen <tss@iki.fi> parents: 16046 diff changeset	386 for (;;) {
0e5a359b7b7f lib-storage: Mailbox attributes can now be accessed via istreams. Timo Sirainen <tss@iki.fi> parents: 16046 diff changeset	387 (void)i_stream_read_data(input1, &data1, &size1, 0);
0e5a359b7b7f lib-storage: Mailbox attributes can now be accessed via istreams. Timo Sirainen <tss@iki.fi> parents: 16046 diff changeset	388 (void)i_stream_read_data(input2, &data2, &size2, 0);
0e5a359b7b7f lib-storage: Mailbox attributes can now be accessed via istreams. Timo Sirainen <tss@iki.fi> parents: 16046 diff changeset	389
0e5a359b7b7f lib-storage: Mailbox attributes can now be accessed via istreams. Timo Sirainen <tss@iki.fi> parents: 16046 diff changeset	390 if (size1 == 0 \|\| size2 == 0)
0e5a359b7b7f lib-storage: Mailbox attributes can now be accessed via istreams. Timo Sirainen <tss@iki.fi> parents: 16046 diff changeset	391 break;
0e5a359b7b7f lib-storage: Mailbox attributes can now be accessed via istreams. Timo Sirainen <tss@iki.fi> parents: 16046 diff changeset	392 size = I_MIN(size1, size2);
0e5a359b7b7f lib-storage: Mailbox attributes can now be accessed via istreams. Timo Sirainen <tss@iki.fi> parents: 16046 diff changeset	393 *cmp_r = memcmp(data1, data2, size);
0e5a359b7b7f lib-storage: Mailbox attributes can now be accessed via istreams. Timo Sirainen <tss@iki.fi> parents: 16046 diff changeset	394 if (*cmp_r != 0)
0e5a359b7b7f lib-storage: Mailbox attributes can now be accessed via istreams. Timo Sirainen <tss@iki.fi> parents: 16046 diff changeset	395 return 0;
0e5a359b7b7f lib-storage: Mailbox attributes can now be accessed via istreams. Timo Sirainen <tss@iki.fi> parents: 16046 diff changeset	396 i_stream_skip(input1, size);
0e5a359b7b7f lib-storage: Mailbox attributes can now be accessed via istreams. Timo Sirainen <tss@iki.fi> parents: 16046 diff changeset	397 i_stream_skip(input2, size);
0e5a359b7b7f lib-storage: Mailbox attributes can now be accessed via istreams. Timo Sirainen <tss@iki.fi> parents: 16046 diff changeset	398 }
0e5a359b7b7f lib-storage: Mailbox attributes can now be accessed via istreams. Timo Sirainen <tss@iki.fi> parents: 16046 diff changeset	399 if (input1->stream_errno != 0) {
0e5a359b7b7f lib-storage: Mailbox attributes can now be accessed via istreams. Timo Sirainen <tss@iki.fi> parents: 16046 diff changeset	400 errno = input1->stream_errno;
16940 38f404297728 dsync: Use i_stream_get_error() instead of just errno in stream error messages. Timo Sirainen <tss@iki.fi> parents: 16832 diff changeset	401 i_error("read(%s) failed: %s", i_stream_get_name(input1),
38f404297728 dsync: Use i_stream_get_error() instead of just errno in stream error messages. Timo Sirainen <tss@iki.fi> parents: 16832 diff changeset	402 i_stream_get_error(input1));
16052 0e5a359b7b7f lib-storage: Mailbox attributes can now be accessed via istreams. Timo Sirainen <tss@iki.fi> parents: 16046 diff changeset	403 return -1;
0e5a359b7b7f lib-storage: Mailbox attributes can now be accessed via istreams. Timo Sirainen <tss@iki.fi> parents: 16046 diff changeset	404 }
0e5a359b7b7f lib-storage: Mailbox attributes can now be accessed via istreams. Timo Sirainen <tss@iki.fi> parents: 16046 diff changeset	405 if (input2->stream_errno != 0) {
0e5a359b7b7f lib-storage: Mailbox attributes can now be accessed via istreams. Timo Sirainen <tss@iki.fi> parents: 16046 diff changeset	406 errno = input2->stream_errno;
16940 38f404297728 dsync: Use i_stream_get_error() instead of just errno in stream error messages. Timo Sirainen <tss@iki.fi> parents: 16832 diff changeset	407 i_error("read(%s) failed: %s", i_stream_get_name(input2),
38f404297728 dsync: Use i_stream_get_error() instead of just errno in stream error messages. Timo Sirainen <tss@iki.fi> parents: 16832 diff changeset	408 i_stream_get_error(input2));
16052 0e5a359b7b7f lib-storage: Mailbox attributes can now be accessed via istreams. Timo Sirainen <tss@iki.fi> parents: 16046 diff changeset	409 return -1;
0e5a359b7b7f lib-storage: Mailbox attributes can now be accessed via istreams. Timo Sirainen <tss@iki.fi> parents: 16046 diff changeset	410 }
0e5a359b7b7f lib-storage: Mailbox attributes can now be accessed via istreams. Timo Sirainen <tss@iki.fi> parents: 16046 diff changeset	411 if (size1 == 0 && size2 == 0)
0e5a359b7b7f lib-storage: Mailbox attributes can now be accessed via istreams. Timo Sirainen <tss@iki.fi> parents: 16046 diff changeset	412 *cmp_r = 0;
0e5a359b7b7f lib-storage: Mailbox attributes can now be accessed via istreams. Timo Sirainen <tss@iki.fi> parents: 16046 diff changeset	413 else
0e5a359b7b7f lib-storage: Mailbox attributes can now be accessed via istreams. Timo Sirainen <tss@iki.fi> parents: 16046 diff changeset	414 *cmp_r = size1 == 0 ? -1 : 1;
0e5a359b7b7f lib-storage: Mailbox attributes can now be accessed via istreams. Timo Sirainen <tss@iki.fi> parents: 16046 diff changeset	415 return 0;
0e5a359b7b7f lib-storage: Mailbox attributes can now be accessed via istreams. Timo Sirainen <tss@iki.fi> parents: 16046 diff changeset	416 }
0e5a359b7b7f lib-storage: Mailbox attributes can now be accessed via istreams. Timo Sirainen <tss@iki.fi> parents: 16046 diff changeset	417
0e5a359b7b7f lib-storage: Mailbox attributes can now be accessed via istreams. Timo Sirainen <tss@iki.fi> parents: 16046 diff changeset	418 static int
0e5a359b7b7f lib-storage: Mailbox attributes can now be accessed via istreams. Timo Sirainen <tss@iki.fi> parents: 16046 diff changeset	419 dsync_attributes_cmp_values(const struct dsync_mailbox_attribute *attr1,
0e5a359b7b7f lib-storage: Mailbox attributes can now be accessed via istreams. Timo Sirainen <tss@iki.fi> parents: 16046 diff changeset	420 const struct dsync_mailbox_attribute *attr2,
0e5a359b7b7f lib-storage: Mailbox attributes can now be accessed via istreams. Timo Sirainen <tss@iki.fi> parents: 16046 diff changeset	421 int *cmp_r)
0e5a359b7b7f lib-storage: Mailbox attributes can now be accessed via istreams. Timo Sirainen <tss@iki.fi> parents: 16046 diff changeset	422 {
0e5a359b7b7f lib-storage: Mailbox attributes can now be accessed via istreams. Timo Sirainen <tss@iki.fi> parents: 16046 diff changeset	423 struct istream input1, input2;
0e5a359b7b7f lib-storage: Mailbox attributes can now be accessed via istreams. Timo Sirainen <tss@iki.fi> parents: 16046 diff changeset	424 int ret;
0e5a359b7b7f lib-storage: Mailbox attributes can now be accessed via istreams. Timo Sirainen <tss@iki.fi> parents: 16046 diff changeset	425
17345 24d27f7da7b2 dsync: Make static analyzer happier (hopefully) Timo Sirainen <tss@iki.fi> parents: 17296 diff changeset	426 i_assert(attr1->value_stream != NULL \|\| attr1->value != NULL);
24d27f7da7b2 dsync: Make static analyzer happier (hopefully) Timo Sirainen <tss@iki.fi> parents: 17296 diff changeset	427 i_assert(attr2->value_stream != NULL \|\| attr2->value != NULL);
24d27f7da7b2 dsync: Make static analyzer happier (hopefully) Timo Sirainen <tss@iki.fi> parents: 17296 diff changeset	428
16052 0e5a359b7b7f lib-storage: Mailbox attributes can now be accessed via istreams. Timo Sirainen <tss@iki.fi> parents: 16046 diff changeset	429 if (attr1->value != NULL && attr2->value != NULL) {
0e5a359b7b7f lib-storage: Mailbox attributes can now be accessed via istreams. Timo Sirainen <tss@iki.fi> parents: 16046 diff changeset	430 *cmp_r = strcmp(attr1->value, attr2->value);
0e5a359b7b7f lib-storage: Mailbox attributes can now be accessed via istreams. Timo Sirainen <tss@iki.fi> parents: 16046 diff changeset	431 return 0;
0e5a359b7b7f lib-storage: Mailbox attributes can now be accessed via istreams. Timo Sirainen <tss@iki.fi> parents: 16046 diff changeset	432 }
0e5a359b7b7f lib-storage: Mailbox attributes can now be accessed via istreams. Timo Sirainen <tss@iki.fi> parents: 16046 diff changeset	433 /* at least one of them is a stream. make both of them streams. */
0e5a359b7b7f lib-storage: Mailbox attributes can now be accessed via istreams. Timo Sirainen <tss@iki.fi> parents: 16046 diff changeset	434 input1 = attr1->value_stream != NULL ? attr1->value_stream :
0e5a359b7b7f lib-storage: Mailbox attributes can now be accessed via istreams. Timo Sirainen <tss@iki.fi> parents: 16046 diff changeset	435 i_stream_create_from_data(attr1->value, strlen(attr1->value));
0e5a359b7b7f lib-storage: Mailbox attributes can now be accessed via istreams. Timo Sirainen <tss@iki.fi> parents: 16046 diff changeset	436 input2 = attr2->value_stream != NULL ? attr2->value_stream :
0e5a359b7b7f lib-storage: Mailbox attributes can now be accessed via istreams. Timo Sirainen <tss@iki.fi> parents: 16046 diff changeset	437 i_stream_create_from_data(attr2->value, strlen(attr2->value));
0e5a359b7b7f lib-storage: Mailbox attributes can now be accessed via istreams. Timo Sirainen <tss@iki.fi> parents: 16046 diff changeset	438 i_stream_seek(input1, 0);
0e5a359b7b7f lib-storage: Mailbox attributes can now be accessed via istreams. Timo Sirainen <tss@iki.fi> parents: 16046 diff changeset	439 i_stream_seek(input2, 0);
0e5a359b7b7f lib-storage: Mailbox attributes can now be accessed via istreams. Timo Sirainen <tss@iki.fi> parents: 16046 diff changeset	440 ret = dsync_istreams_cmp(input1, input2, cmp_r);
0e5a359b7b7f lib-storage: Mailbox attributes can now be accessed via istreams. Timo Sirainen <tss@iki.fi> parents: 16046 diff changeset	441 if (attr1->value_stream == NULL)
0e5a359b7b7f lib-storage: Mailbox attributes can now be accessed via istreams. Timo Sirainen <tss@iki.fi> parents: 16046 diff changeset	442 i_stream_unref(&input1);
0e5a359b7b7f lib-storage: Mailbox attributes can now be accessed via istreams. Timo Sirainen <tss@iki.fi> parents: 16046 diff changeset	443 if (attr2->value_stream == NULL)
0e5a359b7b7f lib-storage: Mailbox attributes can now be accessed via istreams. Timo Sirainen <tss@iki.fi> parents: 16046 diff changeset	444 i_stream_unref(&input2);
0e5a359b7b7f lib-storage: Mailbox attributes can now be accessed via istreams. Timo Sirainen <tss@iki.fi> parents: 16046 diff changeset	445 return ret;
0e5a359b7b7f lib-storage: Mailbox attributes can now be accessed via istreams. Timo Sirainen <tss@iki.fi> parents: 16046 diff changeset	446 }
0e5a359b7b7f lib-storage: Mailbox attributes can now be accessed via istreams. Timo Sirainen <tss@iki.fi> parents: 16046 diff changeset	447
0e5a359b7b7f lib-storage: Mailbox attributes can now be accessed via istreams. Timo Sirainen <tss@iki.fi> parents: 16046 diff changeset	448 static int
0e5a359b7b7f lib-storage: Mailbox attributes can now be accessed via istreams. Timo Sirainen <tss@iki.fi> parents: 16046 diff changeset	449 dsync_attributes_cmp(const struct dsync_mailbox_attribute *attr,
0e5a359b7b7f lib-storage: Mailbox attributes can now be accessed via istreams. Timo Sirainen <tss@iki.fi> parents: 16046 diff changeset	450 const struct dsync_mailbox_attribute *local_attr,
0e5a359b7b7f lib-storage: Mailbox attributes can now be accessed via istreams. Timo Sirainen <tss@iki.fi> parents: 16046 diff changeset	451 int *cmp_r)
0e5a359b7b7f lib-storage: Mailbox attributes can now be accessed via istreams. Timo Sirainen <tss@iki.fi> parents: 16046 diff changeset	452 {
0e5a359b7b7f lib-storage: Mailbox attributes can now be accessed via istreams. Timo Sirainen <tss@iki.fi> parents: 16046 diff changeset	453 if (DSYNC_ATTR_HAS_VALUE(attr) &&
0e5a359b7b7f lib-storage: Mailbox attributes can now be accessed via istreams. Timo Sirainen <tss@iki.fi> parents: 16046 diff changeset	454 !DSYNC_ATTR_HAS_VALUE(local_attr)) {
0e5a359b7b7f lib-storage: Mailbox attributes can now be accessed via istreams. Timo Sirainen <tss@iki.fi> parents: 16046 diff changeset	455 /* remote has a value and local doesn't -> use it */
16055 8b5c098cbd0f dsync: Fixed attribute value comparisons. Timo Sirainen <tss@iki.fi> parents: 16052 diff changeset	456 *cmp_r = 1;
8b5c098cbd0f dsync: Fixed attribute value comparisons. Timo Sirainen <tss@iki.fi> parents: 16052 diff changeset	457 return 0;
16052 0e5a359b7b7f lib-storage: Mailbox attributes can now be accessed via istreams. Timo Sirainen <tss@iki.fi> parents: 16046 diff changeset	458 } else if (!DSYNC_ATTR_HAS_VALUE(attr) &&
0e5a359b7b7f lib-storage: Mailbox attributes can now be accessed via istreams. Timo Sirainen <tss@iki.fi> parents: 16046 diff changeset	459 DSYNC_ATTR_HAS_VALUE(local_attr)) {
0e5a359b7b7f lib-storage: Mailbox attributes can now be accessed via istreams. Timo Sirainen <tss@iki.fi> parents: 16046 diff changeset	460 /* remote doesn't have a value, bt local does -> skip */
16055 8b5c098cbd0f dsync: Fixed attribute value comparisons. Timo Sirainen <tss@iki.fi> parents: 16052 diff changeset	461 *cmp_r = -1;
8b5c098cbd0f dsync: Fixed attribute value comparisons. Timo Sirainen <tss@iki.fi> parents: 16052 diff changeset	462 return 0;
16052 0e5a359b7b7f lib-storage: Mailbox attributes can now be accessed via istreams. Timo Sirainen <tss@iki.fi> parents: 16046 diff changeset	463 }
0e5a359b7b7f lib-storage: Mailbox attributes can now be accessed via istreams. Timo Sirainen <tss@iki.fi> parents: 16046 diff changeset	464
0e5a359b7b7f lib-storage: Mailbox attributes can now be accessed via istreams. Timo Sirainen <tss@iki.fi> parents: 16046 diff changeset	465 return dsync_attributes_cmp_values(attr, local_attr, cmp_r);
0e5a359b7b7f lib-storage: Mailbox attributes can now be accessed via istreams. Timo Sirainen <tss@iki.fi> parents: 16046 diff changeset	466 }
0e5a359b7b7f lib-storage: Mailbox attributes can now be accessed via istreams. Timo Sirainen <tss@iki.fi> parents: 16046 diff changeset	467
18018 4155055df7a9 dsync: Added debug logging for attribute importing Timo Sirainen <tss@iki.fi> parents: 17982 diff changeset	468 static int
4155055df7a9 dsync: Added debug logging for attribute importing Timo Sirainen <tss@iki.fi> parents: 17982 diff changeset	469 dsync_mailbox_import_attribute_real(struct dsync_mailbox_importer *importer,
4155055df7a9 dsync: Added debug logging for attribute importing Timo Sirainen <tss@iki.fi> parents: 17982 diff changeset	470 const struct dsync_mailbox_attribute *attr,
18999 d5cddaa5ca92 dsync: Fixed memory leaks when importing attributes whose values were in istreams. Timo Sirainen <tss@iki.fi> parents: 18371 diff changeset	471 const struct dsync_mailbox_attribute *local_attr,
18018 4155055df7a9 dsync: Added debug logging for attribute importing Timo Sirainen <tss@iki.fi> parents: 17982 diff changeset	472 const char **result_r)
16025 c6082de4bf5b dsync: Added support for syncing mailbox attributes. Timo Sirainen <tss@iki.fi> parents: 15841 diff changeset	473 {
16046 3e4c4f9c230b dsync: Set last_change timestamp for unset attributes. Timo Sirainen <tss@iki.fi> parents: 16044 diff changeset	474 struct mail_attribute_value value;
16259 8da591260f7c dsync: Don't fail the sync if attribute couldn't be set. Timo Sirainen <tss@iki.fi> parents: 16256 diff changeset	475 int cmp;
16052 0e5a359b7b7f lib-storage: Mailbox attributes can now be accessed via istreams. Timo Sirainen <tss@iki.fi> parents: 16046 diff changeset	476 bool ignore = FALSE;
16025 c6082de4bf5b dsync: Added support for syncing mailbox attributes. Timo Sirainen <tss@iki.fi> parents: 15841 diff changeset	477
16052 0e5a359b7b7f lib-storage: Mailbox attributes can now be accessed via istreams. Timo Sirainen <tss@iki.fi> parents: 16046 diff changeset	478 i_assert(DSYNC_ATTR_HAS_VALUE(attr) \|\| attr->deleted);
16025 c6082de4bf5b dsync: Added support for syncing mailbox attributes. Timo Sirainen <tss@iki.fi> parents: 15841 diff changeset	479
16052 0e5a359b7b7f lib-storage: Mailbox attributes can now be accessed via istreams. Timo Sirainen <tss@iki.fi> parents: 16046 diff changeset	480 if (attr->deleted &&
0e5a359b7b7f lib-storage: Mailbox attributes can now be accessed via istreams. Timo Sirainen <tss@iki.fi> parents: 16046 diff changeset	481 (local_attr == NULL \|\| !DSYNC_ATTR_HAS_VALUE(local_attr))) {
0e5a359b7b7f lib-storage: Mailbox attributes can now be accessed via istreams. Timo Sirainen <tss@iki.fi> parents: 16046 diff changeset	482 /* attribute doesn't exist on either side -> ignore */
18018 4155055df7a9 dsync: Added debug logging for attribute importing Timo Sirainen <tss@iki.fi> parents: 17982 diff changeset	483 *result_r = "Nonexistent in both sides";
16052 0e5a359b7b7f lib-storage: Mailbox attributes can now be accessed via istreams. Timo Sirainen <tss@iki.fi> parents: 16046 diff changeset	484 return 0;
0e5a359b7b7f lib-storage: Mailbox attributes can now be accessed via istreams. Timo Sirainen <tss@iki.fi> parents: 16046 diff changeset	485 }
16025 c6082de4bf5b dsync: Added support for syncing mailbox attributes. Timo Sirainen <tss@iki.fi> parents: 15841 diff changeset	486 if (local_attr == NULL) {
c6082de4bf5b dsync: Added support for syncing mailbox attributes. Timo Sirainen <tss@iki.fi> parents: 15841 diff changeset	487 /* we haven't seen this locally -> use whatever remote has */
18018 4155055df7a9 dsync: Added debug logging for attribute importing Timo Sirainen <tss@iki.fi> parents: 17982 diff changeset	488 *result_r = "Nonexistent locally";
16025 c6082de4bf5b dsync: Added support for syncing mailbox attributes. Timo Sirainen <tss@iki.fi> parents: 15841 diff changeset	489 } else if (local_attr->modseq <= importer->last_common_modseq &&
c6082de4bf5b dsync: Added support for syncing mailbox attributes. Timo Sirainen <tss@iki.fi> parents: 15841 diff changeset	490 attr->modseq > importer->last_common_modseq &&
c6082de4bf5b dsync: Added support for syncing mailbox attributes. Timo Sirainen <tss@iki.fi> parents: 15841 diff changeset	491 importer->last_common_modseq > 0) {
c6082de4bf5b dsync: Added support for syncing mailbox attributes. Timo Sirainen <tss@iki.fi> parents: 15841 diff changeset	492 /* we're doing incremental syncing, and we can see that the
c6082de4bf5b dsync: Added support for syncing mailbox attributes. Timo Sirainen <tss@iki.fi> parents: 15841 diff changeset	493 attribute was changed remotely, but not locally -> use it */
18018 4155055df7a9 dsync: Added debug logging for attribute importing Timo Sirainen <tss@iki.fi> parents: 17982 diff changeset	494 *result_r = "Changed remotely";
16025 c6082de4bf5b dsync: Added support for syncing mailbox attributes. Timo Sirainen <tss@iki.fi> parents: 15841 diff changeset	495 } else if (local_attr->modseq > importer->last_common_modseq &&
c6082de4bf5b dsync: Added support for syncing mailbox attributes. Timo Sirainen <tss@iki.fi> parents: 15841 diff changeset	496 attr->modseq <= importer->last_common_modseq &&
c6082de4bf5b dsync: Added support for syncing mailbox attributes. Timo Sirainen <tss@iki.fi> parents: 15841 diff changeset	497 importer->last_common_modseq > 0) {
c6082de4bf5b dsync: Added support for syncing mailbox attributes. Timo Sirainen <tss@iki.fi> parents: 15841 diff changeset	498 /* we're doing incremental syncing, and we can see that the
c6082de4bf5b dsync: Added support for syncing mailbox attributes. Timo Sirainen <tss@iki.fi> parents: 15841 diff changeset	499 attribute was changed locally, but not remotely -> ignore */
18018 4155055df7a9 dsync: Added debug logging for attribute importing Timo Sirainen <tss@iki.fi> parents: 17982 diff changeset	500 *result_r = "Changed locally";
16052 0e5a359b7b7f lib-storage: Mailbox attributes can now be accessed via istreams. Timo Sirainen <tss@iki.fi> parents: 16046 diff changeset	501 ignore = TRUE;
16025 c6082de4bf5b dsync: Added support for syncing mailbox attributes. Timo Sirainen <tss@iki.fi> parents: 15841 diff changeset	502 } else if (attr->last_change > local_attr->last_change) {
c6082de4bf5b dsync: Added support for syncing mailbox attributes. Timo Sirainen <tss@iki.fi> parents: 15841 diff changeset	503 /* remote has a newer timestamp -> use it */
18018 4155055df7a9 dsync: Added debug logging for attribute importing Timo Sirainen <tss@iki.fi> parents: 17982 diff changeset	504 *result_r = "Remote has newer timestamp";
16025 c6082de4bf5b dsync: Added support for syncing mailbox attributes. Timo Sirainen <tss@iki.fi> parents: 15841 diff changeset	505 } else if (attr->last_change < local_attr->last_change) {
c6082de4bf5b dsync: Added support for syncing mailbox attributes. Timo Sirainen <tss@iki.fi> parents: 15841 diff changeset	506 /* remote has an older timestamp -> ignore */
18018 4155055df7a9 dsync: Added debug logging for attribute importing Timo Sirainen <tss@iki.fi> parents: 17982 diff changeset	507 *result_r = "Local has newer timestamp";
16052 0e5a359b7b7f lib-storage: Mailbox attributes can now be accessed via istreams. Timo Sirainen <tss@iki.fi> parents: 16046 diff changeset	508 ignore = TRUE;
16025 c6082de4bf5b dsync: Added support for syncing mailbox attributes. Timo Sirainen <tss@iki.fi> parents: 15841 diff changeset	509 } else {
c6082de4bf5b dsync: Added support for syncing mailbox attributes. Timo Sirainen <tss@iki.fi> parents: 15841 diff changeset	510 /* the timestamps are the same. now we're down to guessing
16052 0e5a359b7b7f lib-storage: Mailbox attributes can now be accessed via istreams. Timo Sirainen <tss@iki.fi> parents: 16046 diff changeset	511 the right answer, unless the values are actually equal,
0e5a359b7b7f lib-storage: Mailbox attributes can now be accessed via istreams. Timo Sirainen <tss@iki.fi> parents: 16046 diff changeset	512 so check that first. next try to use modseqs, but if even
0e5a359b7b7f lib-storage: Mailbox attributes can now be accessed via istreams. Timo Sirainen <tss@iki.fi> parents: 16046 diff changeset	513 they are the same, fallback to just picking one based on the
16025 c6082de4bf5b dsync: Added support for syncing mailbox attributes. Timo Sirainen <tss@iki.fi> parents: 15841 diff changeset	514 value. */
16052 0e5a359b7b7f lib-storage: Mailbox attributes can now be accessed via istreams. Timo Sirainen <tss@iki.fi> parents: 16046 diff changeset	515 if (dsync_attributes_cmp(attr, local_attr, &cmp) < 0) {
18371 b900b50085fc dsync: Use storage's mail_error to choose the doveadm exit code. Timo Sirainen <tss@iki.fi> parents: 18183 diff changeset	516 importer->mail_error = MAIL_ERROR_TEMP;
16052 0e5a359b7b7f lib-storage: Mailbox attributes can now be accessed via istreams. Timo Sirainen <tss@iki.fi> parents: 16046 diff changeset	517 importer->failed = TRUE;
0e5a359b7b7f lib-storage: Mailbox attributes can now be accessed via istreams. Timo Sirainen <tss@iki.fi> parents: 16046 diff changeset	518 return -1;
0e5a359b7b7f lib-storage: Mailbox attributes can now be accessed via istreams. Timo Sirainen <tss@iki.fi> parents: 16046 diff changeset	519 }
0e5a359b7b7f lib-storage: Mailbox attributes can now be accessed via istreams. Timo Sirainen <tss@iki.fi> parents: 16046 diff changeset	520 if (cmp == 0) {
0e5a359b7b7f lib-storage: Mailbox attributes can now be accessed via istreams. Timo Sirainen <tss@iki.fi> parents: 16046 diff changeset	521 /* identical scripts */
18018 4155055df7a9 dsync: Added debug logging for attribute importing Timo Sirainen <tss@iki.fi> parents: 17982 diff changeset	522 *result_r = "Unchanged value";
16052 0e5a359b7b7f lib-storage: Mailbox attributes can now be accessed via istreams. Timo Sirainen <tss@iki.fi> parents: 16046 diff changeset	523 return 0;
0e5a359b7b7f lib-storage: Mailbox attributes can now be accessed via istreams. Timo Sirainen <tss@iki.fi> parents: 16046 diff changeset	524 }
0e5a359b7b7f lib-storage: Mailbox attributes can now be accessed via istreams. Timo Sirainen <tss@iki.fi> parents: 16046 diff changeset	525
16025 c6082de4bf5b dsync: Added support for syncing mailbox attributes. Timo Sirainen <tss@iki.fi> parents: 15841 diff changeset	526 if (attr->modseq > local_attr->modseq) {
c6082de4bf5b dsync: Added support for syncing mailbox attributes. Timo Sirainen <tss@iki.fi> parents: 15841 diff changeset	527 /* remote has a higher modseq -> use it */
18018 4155055df7a9 dsync: Added debug logging for attribute importing Timo Sirainen <tss@iki.fi> parents: 17982 diff changeset	528 *result_r = "Remote has newer modseq";
16025 c6082de4bf5b dsync: Added support for syncing mailbox attributes. Timo Sirainen <tss@iki.fi> parents: 15841 diff changeset	529 } else if (attr->modseq < local_attr->modseq) {
c6082de4bf5b dsync: Added support for syncing mailbox attributes. Timo Sirainen <tss@iki.fi> parents: 15841 diff changeset	530 /* remote has an older modseq -> ignore */
18018 4155055df7a9 dsync: Added debug logging for attribute importing Timo Sirainen <tss@iki.fi> parents: 17982 diff changeset	531 *result_r = "Local has newer modseq";
16052 0e5a359b7b7f lib-storage: Mailbox attributes can now be accessed via istreams. Timo Sirainen <tss@iki.fi> parents: 16046 diff changeset	532 ignore = TRUE;
18018 4155055df7a9 dsync: Added debug logging for attribute importing Timo Sirainen <tss@iki.fi> parents: 17982 diff changeset	533 } else if (cmp < 0) {
4155055df7a9 dsync: Added debug logging for attribute importing Timo Sirainen <tss@iki.fi> parents: 17982 diff changeset	534 ignore = TRUE;
4155055df7a9 dsync: Added debug logging for attribute importing Timo Sirainen <tss@iki.fi> parents: 17982 diff changeset	535 *result_r = "Value changed, but unknown which is newer - picking local";
16025 c6082de4bf5b dsync: Added support for syncing mailbox attributes. Timo Sirainen <tss@iki.fi> parents: 15841 diff changeset	536 } else {
18018 4155055df7a9 dsync: Added debug logging for attribute importing Timo Sirainen <tss@iki.fi> parents: 17982 diff changeset	537 *result_r = "Value changed, but unknown which is newer - picking remote";
16025 c6082de4bf5b dsync: Added support for syncing mailbox attributes. Timo Sirainen <tss@iki.fi> parents: 15841 diff changeset	538 }
c6082de4bf5b dsync: Added support for syncing mailbox attributes. Timo Sirainen <tss@iki.fi> parents: 15841 diff changeset	539 }
18999 d5cddaa5ca92 dsync: Fixed memory leaks when importing attributes whose values were in istreams. Timo Sirainen <tss@iki.fi> parents: 18371 diff changeset	540 if (ignore)
16052 0e5a359b7b7f lib-storage: Mailbox attributes can now be accessed via istreams. Timo Sirainen <tss@iki.fi> parents: 16046 diff changeset	541 return 0;
16044 2396eb0a3e3f lib-storage: mailbox_attribute_set() now uses struct mail_attribute_value. Timo Sirainen <tss@iki.fi> parents: 16025 diff changeset	542
21389 59437f8764c6 global: Replaced all instances of memset(p, 0, sizeof(p)) with the new i_zero() macro. Stephan Bosch <stephan.bosch@dovecot.fi>* parents: 21267 diff changeset	543 i_zero(&value);
16046 3e4c4f9c230b dsync: Set last_change timestamp for unset attributes. Timo Sirainen <tss@iki.fi> parents: 16044 diff changeset	544 value.value = attr->value;
16052 0e5a359b7b7f lib-storage: Mailbox attributes can now be accessed via istreams. Timo Sirainen <tss@iki.fi> parents: 16046 diff changeset	545 value.value_stream = attr->value_stream;
16046 3e4c4f9c230b dsync: Set last_change timestamp for unset attributes. Timo Sirainen <tss@iki.fi> parents: 16044 diff changeset	546 value.last_change = attr->last_change;
16259 8da591260f7c dsync: Don't fail the sync if attribute couldn't be set. Timo Sirainen <tss@iki.fi> parents: 16256 diff changeset	547 if (mailbox_attribute_set(importer->trans, attr->type,
8da591260f7c dsync: Don't fail the sync if attribute couldn't be set. Timo Sirainen <tss@iki.fi> parents: 16256 diff changeset	548 attr->key, &value) < 0) {
16025 c6082de4bf5b dsync: Added support for syncing mailbox attributes. Timo Sirainen <tss@iki.fi> parents: 15841 diff changeset	549 i_error("Mailbox %s: Failed to set attribute %s: %s",
c6082de4bf5b dsync: Added support for syncing mailbox attributes. Timo Sirainen <tss@iki.fi> parents: 15841 diff changeset	550 mailbox_get_vname(importer->box), attr->key,
21814 759962e70148 global: Log internal storage error on failure Martti Rannanjärvi <martti.rannanjarvi@dovecot.fi> parents: 21390 diff changeset	551 mailbox_get_last_internal_error(importer->box, NULL));
16259 8da591260f7c dsync: Don't fail the sync if attribute couldn't be set. Timo Sirainen <tss@iki.fi> parents: 16256 diff changeset	552 /* the attributes aren't vital, don't fail everything just
8da591260f7c dsync: Don't fail the sync if attribute couldn't be set. Timo Sirainen <tss@iki.fi> parents: 16256 diff changeset	553 because of them. */
16025 c6082de4bf5b dsync: Added support for syncing mailbox attributes. Timo Sirainen <tss@iki.fi> parents: 15841 diff changeset	554 }
16259 8da591260f7c dsync: Don't fail the sync if attribute couldn't be set. Timo Sirainen <tss@iki.fi> parents: 16256 diff changeset	555 return 0;
16025 c6082de4bf5b dsync: Added support for syncing mailbox attributes. Timo Sirainen <tss@iki.fi> parents: 15841 diff changeset	556 }
c6082de4bf5b dsync: Added support for syncing mailbox attributes. Timo Sirainen <tss@iki.fi> parents: 15841 diff changeset	557
18018 4155055df7a9 dsync: Added debug logging for attribute importing Timo Sirainen <tss@iki.fi> parents: 17982 diff changeset	558 int dsync_mailbox_import_attribute(struct dsync_mailbox_importer *importer,
4155055df7a9 dsync: Added debug logging for attribute importing Timo Sirainen <tss@iki.fi> parents: 17982 diff changeset	559 const struct dsync_mailbox_attribute *attr)
4155055df7a9 dsync: Added debug logging for attribute importing Timo Sirainen <tss@iki.fi> parents: 17982 diff changeset	560 {
18999 d5cddaa5ca92 dsync: Fixed memory leaks when importing attributes whose values were in istreams. Timo Sirainen <tss@iki.fi> parents: 18371 diff changeset	561 struct dsync_mailbox_attribute *local_attr;
19000 de8af4d857d3 dsync: Compiler warning fix to previous change. Timo Sirainen <tss@iki.fi> parents: 18999 diff changeset	562 const char *result = "";
18018 4155055df7a9 dsync: Added debug logging for attribute importing Timo Sirainen <tss@iki.fi> parents: 17982 diff changeset	563 int ret;
4155055df7a9 dsync: Added debug logging for attribute importing Timo Sirainen <tss@iki.fi> parents: 17982 diff changeset	564
18999 d5cddaa5ca92 dsync: Fixed memory leaks when importing attributes whose values were in istreams. Timo Sirainen <tss@iki.fi> parents: 18371 diff changeset	565 if (dsync_mailbox_import_lookup_attr(importer, attr->type,
d5cddaa5ca92 dsync: Fixed memory leaks when importing attributes whose values were in istreams. Timo Sirainen <tss@iki.fi> parents: 18371 diff changeset	566 attr->key, &local_attr) < 0)
d5cddaa5ca92 dsync: Fixed memory leaks when importing attributes whose values were in istreams. Timo Sirainen <tss@iki.fi> parents: 18371 diff changeset	567 ret = -1;
d5cddaa5ca92 dsync: Fixed memory leaks when importing attributes whose values were in istreams. Timo Sirainen <tss@iki.fi> parents: 18371 diff changeset	568 else {
d5cddaa5ca92 dsync: Fixed memory leaks when importing attributes whose values were in istreams. Timo Sirainen <tss@iki.fi> parents: 18371 diff changeset	569 ret = dsync_mailbox_import_attribute_real(importer, attr,
d5cddaa5ca92 dsync: Fixed memory leaks when importing attributes whose values were in istreams. Timo Sirainen <tss@iki.fi> parents: 18371 diff changeset	570 local_attr, &result);
d5cddaa5ca92 dsync: Fixed memory leaks when importing attributes whose values were in istreams. Timo Sirainen <tss@iki.fi> parents: 18371 diff changeset	571 if (local_attr != NULL && local_attr->value_stream != NULL)
d5cddaa5ca92 dsync: Fixed memory leaks when importing attributes whose values were in istreams. Timo Sirainen <tss@iki.fi> parents: 18371 diff changeset	572 i_stream_unref(&local_attr->value_stream);
d5cddaa5ca92 dsync: Fixed memory leaks when importing attributes whose values were in istreams. Timo Sirainen <tss@iki.fi> parents: 18371 diff changeset	573 }
18018 4155055df7a9 dsync: Added debug logging for attribute importing Timo Sirainen <tss@iki.fi> parents: 17982 diff changeset	574 imp_debug(importer, "Import attribute %s: %s", attr->key,
4155055df7a9 dsync: Added debug logging for attribute importing Timo Sirainen <tss@iki.fi> parents: 17982 diff changeset	575 ret < 0 ? "failed" : result);
4155055df7a9 dsync: Added debug logging for attribute importing Timo Sirainen <tss@iki.fi> parents: 17982 diff changeset	576 return ret;
4155055df7a9 dsync: Added debug logging for attribute importing Timo Sirainen <tss@iki.fi> parents: 17982 diff changeset	577 }
4155055df7a9 dsync: Added debug logging for attribute importing Timo Sirainen <tss@iki.fi> parents: 17982 diff changeset	578
14682 d0d7b810646b Make sure we check all the functions' return values. Minor API changes to simplify this. Timo Sirainen <tss@iki.fi> parents: 14676 diff changeset	579 static void dsync_mail_error(struct dsync_mailbox_importer *importer,
d0d7b810646b Make sure we check all the functions' return values. Minor API changes to simplify this. Timo Sirainen <tss@iki.fi> parents: 14676 diff changeset	580 struct mail mail, const char field)
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	581 {
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	582 const char *errstr;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	583 enum mail_error error;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	584
21814 759962e70148 global: Log internal storage error on failure Martti Rannanjärvi <martti.rannanjarvi@dovecot.fi> parents: 21390 diff changeset	585 errstr = mailbox_get_last_internal_error(importer->box, &error);
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	586 if (error == MAIL_ERROR_EXPUNGED)
14682 d0d7b810646b Make sure we check all the functions' return values. Minor API changes to simplify this. Timo Sirainen <tss@iki.fi> parents: 14676 diff changeset	587 return;
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	588
15766 4260244e57e2 dsync: Various importer fixes. Handle unexpectedly wrong mailbox state better. Timo Sirainen <tss@iki.fi> parents: 15754 diff changeset	589 i_error("Mailbox %s: Can't lookup %s for UID=%u: %s",
4260244e57e2 dsync: Various importer fixes. Handle unexpectedly wrong mailbox state better. Timo Sirainen <tss@iki.fi> parents: 15754 diff changeset	590 mailbox_get_vname(mail->box), field, mail->uid, errstr);
18371 b900b50085fc dsync: Use storage's mail_error to choose the doveadm exit code. Timo Sirainen <tss@iki.fi> parents: 18183 diff changeset	591 importer->mail_error = error;
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	592 importer->failed = TRUE;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	593 }
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	594
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	595 static bool
16977 8a7ecddca4ad dsync: Support syncing storages with 128bit GUIDs <-> string GUIDs. Timo Sirainen <tss@iki.fi> parents: 16940 diff changeset	596 dsync_mail_change_guid_equals(struct dsync_mailbox_importer *importer,
8a7ecddca4ad dsync: Support syncing storages with 128bit GUIDs <-> string GUIDs. Timo Sirainen <tss@iki.fi> parents: 16940 diff changeset	597 const struct dsync_mail_change *change,
15819 9c095a50ba3e dsync: Improved unexpected error logging. Timo Sirainen <tss@iki.fi> parents: 15781 diff changeset	598 const char guid, const char *cmp_guid_r)
15667 1b18ff11effc dsync: Fixes to handling storage formats that don't use 128bit GUIDs. Timo Sirainen <tss@iki.fi> parents: 15664 diff changeset	599 {
1b18ff11effc dsync: Fixes to handling storage formats that don't use 128bit GUIDs. Timo Sirainen <tss@iki.fi> parents: 15664 diff changeset	600 guid_128_t guid_128, change_guid_128;
1b18ff11effc dsync: Fixes to handling storage formats that don't use 128bit GUIDs. Timo Sirainen <tss@iki.fi> parents: 15664 diff changeset	601
16995 df2b27885c9d dsync: Fix to 128bit GUID syncing Timo Sirainen <tss@iki.fi> parents: 16977 diff changeset	602 if (change->type == DSYNC_MAIL_CHANGE_TYPE_EXPUNGE) {
df2b27885c9d dsync: Fix to 128bit GUID syncing Timo Sirainen <tss@iki.fi> parents: 16977 diff changeset	603 if (guid_128_from_string(change->guid, change_guid_128) < 0)
df2b27885c9d dsync: Fix to 128bit GUID syncing Timo Sirainen <tss@iki.fi> parents: 16977 diff changeset	604 i_unreached();
df2b27885c9d dsync: Fix to 128bit GUID syncing Timo Sirainen <tss@iki.fi> parents: 16977 diff changeset	605 } else if (importer->mails_use_guid128) {
df2b27885c9d dsync: Fix to 128bit GUID syncing Timo Sirainen <tss@iki.fi> parents: 16977 diff changeset	606 mail_generate_guid_128_hash(change->guid, change_guid_128);
df2b27885c9d dsync: Fix to 128bit GUID syncing Timo Sirainen <tss@iki.fi> parents: 16977 diff changeset	607 } else {
15819 9c095a50ba3e dsync: Improved unexpected error logging. Timo Sirainen <tss@iki.fi> parents: 15781 diff changeset	608 if (cmp_guid_r != NULL)
9c095a50ba3e dsync: Improved unexpected error logging. Timo Sirainen <tss@iki.fi> parents: 15781 diff changeset	609 *cmp_guid_r = change->guid;
15667 1b18ff11effc dsync: Fixes to handling storage formats that don't use 128bit GUIDs. Timo Sirainen <tss@iki.fi> parents: 15664 diff changeset	610 return strcmp(change->guid, guid) == 0;
15819 9c095a50ba3e dsync: Improved unexpected error logging. Timo Sirainen <tss@iki.fi> parents: 15781 diff changeset	611 }
15667 1b18ff11effc dsync: Fixes to handling storage formats that don't use 128bit GUIDs. Timo Sirainen <tss@iki.fi> parents: 15664 diff changeset	612
1b18ff11effc dsync: Fixes to handling storage formats that don't use 128bit GUIDs. Timo Sirainen <tss@iki.fi> parents: 15664 diff changeset	613 mail_generate_guid_128_hash(guid, guid_128);
15819 9c095a50ba3e dsync: Improved unexpected error logging. Timo Sirainen <tss@iki.fi> parents: 15781 diff changeset	614 if (memcmp(change_guid_128, guid_128, GUID_128_SIZE) != 0) {
9c095a50ba3e dsync: Improved unexpected error logging. Timo Sirainen <tss@iki.fi> parents: 15781 diff changeset	615 if (cmp_guid_r != NULL) {
16977 8a7ecddca4ad dsync: Support syncing storages with 128bit GUIDs <-> string GUIDs. Timo Sirainen <tss@iki.fi> parents: 16940 diff changeset	616 *cmp_guid_r = t_strdup_printf("%s(guid128, orig=%s)",
15819 9c095a50ba3e dsync: Improved unexpected error logging. Timo Sirainen <tss@iki.fi> parents: 15781 diff changeset	617 binary_to_hex(change_guid_128, sizeof(change_guid_128)),
9c095a50ba3e dsync: Improved unexpected error logging. Timo Sirainen <tss@iki.fi> parents: 15781 diff changeset	618 change->guid);
9c095a50ba3e dsync: Improved unexpected error logging. Timo Sirainen <tss@iki.fi> parents: 15781 diff changeset	619 }
9c095a50ba3e dsync: Improved unexpected error logging. Timo Sirainen <tss@iki.fi> parents: 15781 diff changeset	620 return FALSE;
9c095a50ba3e dsync: Improved unexpected error logging. Timo Sirainen <tss@iki.fi> parents: 15781 diff changeset	621 }
9c095a50ba3e dsync: Improved unexpected error logging. Timo Sirainen <tss@iki.fi> parents: 15781 diff changeset	622 return TRUE;
15667 1b18ff11effc dsync: Fixes to handling storage formats that don't use 128bit GUIDs. Timo Sirainen <tss@iki.fi> parents: 15664 diff changeset	623 }
1b18ff11effc dsync: Fixes to handling storage formats that don't use 128bit GUIDs. Timo Sirainen <tss@iki.fi> parents: 15664 diff changeset	624
15770 516a337480f4 dsync: Avoid possibly long function recursion. Timo Sirainen <tss@iki.fi> parents: 15766 diff changeset	625 static int
516a337480f4 dsync: Avoid possibly long function recursion. Timo Sirainen <tss@iki.fi> parents: 15766 diff changeset	626 importer_try_next_mail(struct dsync_mailbox_importer *importer,
516a337480f4 dsync: Avoid possibly long function recursion. Timo Sirainen <tss@iki.fi> parents: 15766 diff changeset	627 uint32_t wanted_uid)
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	628 {
15770 516a337480f4 dsync: Avoid possibly long function recursion. Timo Sirainen <tss@iki.fi> parents: 15766 diff changeset	629 struct mail_private *pmail;
15753 f58e7b386c6e dsync: Fixes for syncing without GUIDs. Timo Sirainen <tss@iki.fi> parents: 15743 diff changeset	630 const char *hdr_hash;
f58e7b386c6e dsync: Fixes for syncing without GUIDs. Timo Sirainen <tss@iki.fi> parents: 15743 diff changeset	631
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	632 if (importer->cur_mail == NULL) {
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	633 /* end of search */
15770 516a337480f4 dsync: Avoid possibly long function recursion. Timo Sirainen <tss@iki.fi> parents: 15766 diff changeset	634 return -1;
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	635 }
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	636 while (importer->cur_mail->seq < importer->next_local_seq \|\|
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	637 importer->cur_mail->uid < wanted_uid) {
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	638 if (!importer->cur_uid_has_change &&
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	639 !importer->last_common_uid_found) {
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	640 /* this message exists locally, but remote didn't send
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	641 expunge-change for it. if the message's
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	642 uid <= last-common-uid, it should be deleted */
14676 69ba6977bed8 seq_range_array_add() API changed. Added other functions to provide the less common use cases. Timo Sirainen <tss@iki.fi> parents: 14675 diff changeset	643 seq_range_array_add(&importer->maybe_expunge_uids,
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	644 importer->cur_mail->uid);
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	645 }
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	646
15661 940414cea18d dsync backup: Old unwanted messages weren't deleted as they should have. Timo Sirainen <tss@iki.fi> parents: 15658 diff changeset	647 importer->cur_mail_skip = FALSE;
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	648 if (!mailbox_search_next(importer->search_ctx,
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	649 &importer->cur_mail)) {
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	650 importer->cur_mail = NULL;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	651 importer->cur_guid = NULL;
15753 f58e7b386c6e dsync: Fixes for syncing without GUIDs. Timo Sirainen <tss@iki.fi> parents: 15743 diff changeset	652 importer->cur_hdr_hash = NULL;
15770 516a337480f4 dsync: Avoid possibly long function recursion. Timo Sirainen <tss@iki.fi> parents: 15766 diff changeset	653 return -1;
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	654 }
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	655 importer->cur_uid_has_change = FALSE;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	656 }
17464 203b3a6f508f Removed pointless NULL checks. Timo Sirainen <tss@iki.fi> parents: 17345 diff changeset	657 importer->cur_uid_has_change = importer->cur_mail->uid == wanted_uid;
15753 f58e7b386c6e dsync: Fixes for syncing without GUIDs. Timo Sirainen <tss@iki.fi> parents: 15743 diff changeset	658 if (importer->mails_have_guids) {
f58e7b386c6e dsync: Fixes for syncing without GUIDs. Timo Sirainen <tss@iki.fi> parents: 15743 diff changeset	659 if (mail_get_special(importer->cur_mail, MAIL_FETCH_GUID,
f58e7b386c6e dsync: Fixes for syncing without GUIDs. Timo Sirainen <tss@iki.fi> parents: 15743 diff changeset	660 &importer->cur_guid) < 0) {
f58e7b386c6e dsync: Fixes for syncing without GUIDs. Timo Sirainen <tss@iki.fi> parents: 15743 diff changeset	661 dsync_mail_error(importer, importer->cur_mail, "GUID");
15770 516a337480f4 dsync: Avoid possibly long function recursion. Timo Sirainen <tss@iki.fi> parents: 15766 diff changeset	662 return 0;
15753 f58e7b386c6e dsync: Fixes for syncing without GUIDs. Timo Sirainen <tss@iki.fi> parents: 15743 diff changeset	663 }
f58e7b386c6e dsync: Fixes for syncing without GUIDs. Timo Sirainen <tss@iki.fi> parents: 15743 diff changeset	664 } else {
f58e7b386c6e dsync: Fixes for syncing without GUIDs. Timo Sirainen <tss@iki.fi> parents: 15743 diff changeset	665 if (dsync_mail_get_hdr_hash(importer->cur_mail,
19646 25f06710e671 dsync: When comparing headers' hashes to match messages, try to normalize the input. Timo Sirainen <timo.sirainen@dovecot.fi> parents: 19552 diff changeset	666 importer->hdr_hash_version,
22549 400ff84f109d dsync: Add hashed_headers setting Aki Tuomi <aki.tuomi@dovecot.fi> parents: 22264 diff changeset	667 importer->hashed_headers,
15753 f58e7b386c6e dsync: Fixes for syncing without GUIDs. Timo Sirainen <tss@iki.fi> parents: 15743 diff changeset	668 &hdr_hash) < 0) {
f58e7b386c6e dsync: Fixes for syncing without GUIDs. Timo Sirainen <tss@iki.fi> parents: 15743 diff changeset	669 dsync_mail_error(importer, importer->cur_mail,
f58e7b386c6e dsync: Fixes for syncing without GUIDs. Timo Sirainen <tss@iki.fi> parents: 15743 diff changeset	670 "header hash");
15770 516a337480f4 dsync: Avoid possibly long function recursion. Timo Sirainen <tss@iki.fi> parents: 15766 diff changeset	671 return 0;
516a337480f4 dsync: Avoid possibly long function recursion. Timo Sirainen <tss@iki.fi> parents: 15766 diff changeset	672 }
516a337480f4 dsync: Avoid possibly long function recursion. Timo Sirainen <tss@iki.fi> parents: 15766 diff changeset	673 pmail = (struct mail_private *)importer->cur_mail;
516a337480f4 dsync: Avoid possibly long function recursion. Timo Sirainen <tss@iki.fi> parents: 15766 diff changeset	674 importer->cur_hdr_hash = p_strdup(pmail->pool, hdr_hash);
16117 494d1a69fc1c dsync: Crashfix when importing from storage without GUIDs. Timo Sirainen <tss@iki.fi> parents: 16088 diff changeset	675 importer->cur_guid = "";
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	676 }
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	677 /* make sure next_local_seq gets updated in case we came here
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	678 because of min_uid */
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	679 importer->next_local_seq = importer->cur_mail->seq;
15770 516a337480f4 dsync: Avoid possibly long function recursion. Timo Sirainen <tss@iki.fi> parents: 15766 diff changeset	680 return 1;
516a337480f4 dsync: Avoid possibly long function recursion. Timo Sirainen <tss@iki.fi> parents: 15766 diff changeset	681 }
516a337480f4 dsync: Avoid possibly long function recursion. Timo Sirainen <tss@iki.fi> parents: 15766 diff changeset	682
516a337480f4 dsync: Avoid possibly long function recursion. Timo Sirainen <tss@iki.fi> parents: 15766 diff changeset	683 static bool
516a337480f4 dsync: Avoid possibly long function recursion. Timo Sirainen <tss@iki.fi> parents: 15766 diff changeset	684 importer_next_mail(struct dsync_mailbox_importer *importer, uint32_t wanted_uid)
516a337480f4 dsync: Avoid possibly long function recursion. Timo Sirainen <tss@iki.fi> parents: 15766 diff changeset	685 {
516a337480f4 dsync: Avoid possibly long function recursion. Timo Sirainen <tss@iki.fi> parents: 15766 diff changeset	686 int ret;
516a337480f4 dsync: Avoid possibly long function recursion. Timo Sirainen <tss@iki.fi> parents: 15766 diff changeset	687
17261 ddf374a36057 dsync: Fixed high data stack memory usage Timo Sirainen <tss@iki.fi> parents: 17174 diff changeset	688 for (;;) {
ddf374a36057 dsync: Fixed high data stack memory usage Timo Sirainen <tss@iki.fi> parents: 17174 diff changeset	689 T_BEGIN {
ddf374a36057 dsync: Fixed high data stack memory usage Timo Sirainen <tss@iki.fi> parents: 17174 diff changeset	690 ret = importer_try_next_mail(importer, wanted_uid);
ddf374a36057 dsync: Fixed high data stack memory usage Timo Sirainen <tss@iki.fi> parents: 17174 diff changeset	691 } T_END;
ddf374a36057 dsync: Fixed high data stack memory usage Timo Sirainen <tss@iki.fi> parents: 17174 diff changeset	692 if (ret != 0 \|\| importer->failed)
ddf374a36057 dsync: Fixed high data stack memory usage Timo Sirainen <tss@iki.fi> parents: 17174 diff changeset	693 break;
15770 516a337480f4 dsync: Avoid possibly long function recursion. Timo Sirainen <tss@iki.fi> parents: 15766 diff changeset	694 importer->next_local_seq = importer->cur_mail->seq + 1;
17261 ddf374a36057 dsync: Fixed high data stack memory usage Timo Sirainen <tss@iki.fi> parents: 17174 diff changeset	695 }
15770 516a337480f4 dsync: Avoid possibly long function recursion. Timo Sirainen <tss@iki.fi> parents: 15766 diff changeset	696 return ret > 0;
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	697 }
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	698
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	699 static int
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	700 importer_mail_cmp(const struct importer_mail *m1,
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	701 const struct importer_mail *m2)
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	702 {
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	703 int ret;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	704
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	705 if (m1->guid == NULL)
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	706 return 1;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	707 if (m2->guid == NULL)
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	708 return -1;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	709
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	710 ret = strcmp(m1->guid, m2->guid);
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	711 if (ret != 0)
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	712 return ret;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	713
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	714 if (m1->uid < m2->uid)
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	715 return -1;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	716 if (m1->uid > m2->uid)
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	717 return 1;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	718 return 0;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	719 }
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	720
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	721 static void newmail_link(struct dsync_mailbox_importer *importer,
15753 f58e7b386c6e dsync: Fixes for syncing without GUIDs. Timo Sirainen <tss@iki.fi> parents: 15743 diff changeset	722 struct importer_new_mail *newmail, uint32_t remote_uid)
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	723 {
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	724 struct importer_new_mail first_mail, last, mail, *link = NULL;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	725
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	726 if (*newmail->guid != '\0') {
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	727 first_mail = hash_table_lookup(importer->import_guids,
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	728 newmail->guid);
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	729 if (first_mail == NULL) {
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	730 /* first mail for this GUID */
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	731 hash_table_insert(importer->import_guids,
14918 8eae4e205c82 Hash table API is now (mostly) type safe. Timo Sirainen <tss@iki.fi> parents: 14917 diff changeset	732 newmail->guid, newmail);
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	733 return;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	734 }
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	735 } else {
15766 4260244e57e2 dsync: Various importer fixes. Handle unexpectedly wrong mailbox state better. Timo Sirainen <tss@iki.fi> parents: 15754 diff changeset	736 if (remote_uid == 0) {
4260244e57e2 dsync: Various importer fixes. Handle unexpectedly wrong mailbox state better. Timo Sirainen <tss@iki.fi> parents: 15754 diff changeset	737 /* mail exists only locally. we don't want to request
4260244e57e2 dsync: Various importer fixes. Handle unexpectedly wrong mailbox state better. Timo Sirainen <tss@iki.fi> parents: 15754 diff changeset	738 it, and we'll assume it has no duplicate
4260244e57e2 dsync: Various importer fixes. Handle unexpectedly wrong mailbox state better. Timo Sirainen <tss@iki.fi> parents: 15754 diff changeset	739 instances. */
4260244e57e2 dsync: Various importer fixes. Handle unexpectedly wrong mailbox state better. Timo Sirainen <tss@iki.fi> parents: 15754 diff changeset	740 return;
4260244e57e2 dsync: Various importer fixes. Handle unexpectedly wrong mailbox state better. Timo Sirainen <tss@iki.fi> parents: 15754 diff changeset	741 }
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	742 first_mail = hash_table_lookup(importer->import_uids,
15753 f58e7b386c6e dsync: Fixes for syncing without GUIDs. Timo Sirainen <tss@iki.fi> parents: 15743 diff changeset	743 POINTER_CAST(remote_uid));
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	744 if (first_mail == NULL) {
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	745 /* first mail for this UID */
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	746 hash_table_insert(importer->import_uids,
15753 f58e7b386c6e dsync: Fixes for syncing without GUIDs. Timo Sirainen <tss@iki.fi> parents: 15743 diff changeset	747 POINTER_CAST(remote_uid), newmail);
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	748 return;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	749 }
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	750 }
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	751 /* 1) add the newmail to the end of the linked list
17174 ea737947fac8 dsync: Optimization for a large number of mails with the same GUID. Timo Sirainen <tss@iki.fi> parents: 17130 diff changeset	752 2) find our link
ea737947fac8 dsync: Optimization for a large number of mails with the same GUID. Timo Sirainen <tss@iki.fi> parents: 17130 diff changeset	753
ea737947fac8 dsync: Optimization for a large number of mails with the same GUID. Timo Sirainen <tss@iki.fi> parents: 17130 diff changeset	754 FIXME: this loop is slow if the same GUID has a ton of instances.
ea737947fac8 dsync: Optimization for a large number of mails with the same GUID. Timo Sirainen <tss@iki.fi> parents: 17130 diff changeset	755 Could it be improved in some way? */
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	756 last = &first_mail->next;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	757 for (mail = first_mail; mail != NULL; mail = mail->next) {
15766 4260244e57e2 dsync: Various importer fixes. Handle unexpectedly wrong mailbox state better. Timo Sirainen <tss@iki.fi> parents: 15754 diff changeset	758 if (mail->final_uid == newmail->final_uid)
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	759 mail->uid_is_usable = TRUE;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	760 if (link == NULL && mail->link == NULL &&
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	761 mail->uid_in_local != newmail->uid_in_local)
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	762 link = mail;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	763 last = &mail->next;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	764 }
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	765 *last = newmail;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	766 if (link != NULL && newmail->link == NULL) {
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	767 link->link = newmail;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	768 newmail->link = link;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	769 }
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	770 }
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	771
19660 7a3a138a4986 doveadm backup: Detect and handle conflicts earlier. Timo Sirainen <timo.sirainen@dovecot.fi> parents: 19646 diff changeset	772 static void
7a3a138a4986 doveadm backup: Detect and handle conflicts earlier. Timo Sirainen <timo.sirainen@dovecot.fi> parents: 19646 diff changeset	773 dsync_mailbox_revert_existing_uid(struct dsync_mailbox_importer *importer,
7a3a138a4986 doveadm backup: Detect and handle conflicts earlier. Timo Sirainen <timo.sirainen@dovecot.fi> parents: 19646 diff changeset	774 uint32_t uid, const char *reason)
7a3a138a4986 doveadm backup: Detect and handle conflicts earlier. Timo Sirainen <timo.sirainen@dovecot.fi> parents: 19646 diff changeset	775 {
7a3a138a4986 doveadm backup: Detect and handle conflicts earlier. Timo Sirainen <timo.sirainen@dovecot.fi> parents: 19646 diff changeset	776 i_assert(importer->revert_local_changes);
7a3a138a4986 doveadm backup: Detect and handle conflicts earlier. Timo Sirainen <timo.sirainen@dovecot.fi> parents: 19646 diff changeset	777
7a3a138a4986 doveadm backup: Detect and handle conflicts earlier. Timo Sirainen <timo.sirainen@dovecot.fi> parents: 19646 diff changeset	778 /* UID either already exists or UIDNEXT is too high. we can't set the
7a3a138a4986 doveadm backup: Detect and handle conflicts earlier. Timo Sirainen <timo.sirainen@dovecot.fi> parents: 19646 diff changeset	779 wanted UID, so we'll need to delete the whole mailbox and resync */
7a3a138a4986 doveadm backup: Detect and handle conflicts earlier. Timo Sirainen <timo.sirainen@dovecot.fi> parents: 19646 diff changeset	780 i_warning("Deleting mailbox '%s': UID=%u already exists locally for a different mail: %s",
7a3a138a4986 doveadm backup: Detect and handle conflicts earlier. Timo Sirainen <timo.sirainen@dovecot.fi> parents: 19646 diff changeset	781 mailbox_get_vname(importer->box), uid, reason);
7a3a138a4986 doveadm backup: Detect and handle conflicts earlier. Timo Sirainen <timo.sirainen@dovecot.fi> parents: 19646 diff changeset	782 importer->delete_mailbox = TRUE;
7a3a138a4986 doveadm backup: Detect and handle conflicts earlier. Timo Sirainen <timo.sirainen@dovecot.fi> parents: 19646 diff changeset	783 importer->mail_error = MAIL_ERROR_TEMP;
7a3a138a4986 doveadm backup: Detect and handle conflicts earlier. Timo Sirainen <timo.sirainen@dovecot.fi> parents: 19646 diff changeset	784 importer->failed = TRUE;
7a3a138a4986 doveadm backup: Detect and handle conflicts earlier. Timo Sirainen <timo.sirainen@dovecot.fi> parents: 19646 diff changeset	785 }
7a3a138a4986 doveadm backup: Detect and handle conflicts earlier. Timo Sirainen <timo.sirainen@dovecot.fi> parents: 19646 diff changeset	786
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	787 static bool dsync_mailbox_try_save_cur(struct dsync_mailbox_importer *importer,
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	788 struct dsync_mail_change *save_change)
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	789 {
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	790 struct importer_mail m1, m2;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	791 struct importer_new_mail *newmail;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	792 int diff;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	793 bool remote_saved;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	794
21389 59437f8764c6 global: Replaced all instances of memset(p, 0, sizeof(p)) with the new i_zero() macro. Stephan Bosch <stephan.bosch@dovecot.fi>* parents: 21267 diff changeset	795 i_zero(&m1);
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	796 if (importer->cur_mail != NULL) {
15754 f6a8f0522634 dsync: Minor fix to deciding when to use GUIDs vs. header hashes. Timo Sirainen <tss@iki.fi> parents: 15753 diff changeset	797 m1.guid = importer->mails_have_guids ?
15753 f58e7b386c6e dsync: Fixes for syncing without GUIDs. Timo Sirainen <tss@iki.fi> parents: 15743 diff changeset	798 importer->cur_guid : importer->cur_hdr_hash;
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	799 m1.uid = importer->cur_mail->uid;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	800 }
21389 59437f8764c6 global: Replaced all instances of memset(p, 0, sizeof(p)) with the new i_zero() macro. Stephan Bosch <stephan.bosch@dovecot.fi>* parents: 21267 diff changeset	801 i_zero(&m2);
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	802 if (save_change != NULL) {
15754 f6a8f0522634 dsync: Minor fix to deciding when to use GUIDs vs. header hashes. Timo Sirainen <tss@iki.fi> parents: 15753 diff changeset	803 m2.guid = importer->mails_have_guids ?
f6a8f0522634 dsync: Minor fix to deciding when to use GUIDs vs. header hashes. Timo Sirainen <tss@iki.fi> parents: 15753 diff changeset	804 save_change->guid : save_change->hdr_hash;
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	805 m2.uid = save_change->uid;
15667 1b18ff11effc dsync: Fixes to handling storage formats that don't use 128bit GUIDs. Timo Sirainen <tss@iki.fi> parents: 15664 diff changeset	806 i_assert(save_change->type != DSYNC_MAIL_CHANGE_TYPE_EXPUNGE);
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	807 }
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	808
20595 40ce04c672a4 dsync: Add support for features Aki Tuomi <aki.tuomi@dovecot.fi> parents: 20262 diff changeset	809 if (importer->empty_hdr_workaround && !importer->mails_have_guids &&
40ce04c672a4 dsync: Add support for features Aki Tuomi <aki.tuomi@dovecot.fi> parents: 20262 diff changeset	810 importer->cur_mail != NULL && save_change != NULL &&
40ce04c672a4 dsync: Add support for features Aki Tuomi <aki.tuomi@dovecot.fi> parents: 20262 diff changeset	811 (dsync_mail_hdr_hash_is_empty(m1.guid) \|\|
40ce04c672a4 dsync: Add support for features Aki Tuomi <aki.tuomi@dovecot.fi> parents: 20262 diff changeset	812 dsync_mail_hdr_hash_is_empty(m2.guid))) {
40ce04c672a4 dsync: Add support for features Aki Tuomi <aki.tuomi@dovecot.fi> parents: 20262 diff changeset	813 /* one of the headers is empty. assume it's broken and that
40ce04c672a4 dsync: Add support for features Aki Tuomi <aki.tuomi@dovecot.fi> parents: 20262 diff changeset	814 the header matches what we have currently. */
40ce04c672a4 dsync: Add support for features Aki Tuomi <aki.tuomi@dovecot.fi> parents: 20262 diff changeset	815 diff = 0;
40ce04c672a4 dsync: Add support for features Aki Tuomi <aki.tuomi@dovecot.fi> parents: 20262 diff changeset	816 } else {
40ce04c672a4 dsync: Add support for features Aki Tuomi <aki.tuomi@dovecot.fi> parents: 20262 diff changeset	817 diff = importer_mail_cmp(&m1, &m2);
40ce04c672a4 dsync: Add support for features Aki Tuomi <aki.tuomi@dovecot.fi> parents: 20262 diff changeset	818 }
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	819 if (diff < 0) {
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	820 /* add a record for local mail */
14675 f07ba5e7d97d Make static analyzer happier. Timo Sirainen <tss@iki.fi> parents: 14629 diff changeset	821 i_assert(importer->cur_mail != NULL);
15661 940414cea18d dsync backup: Old unwanted messages weren't deleted as they should have. Timo Sirainen <tss@iki.fi> parents: 15658 diff changeset	822 if (importer->revert_local_changes) {
20191 8e6442a0888f doveadm backup: Fixed unnecessary mailbox deletion. Timo Sirainen <timo.sirainen@dovecot.fi> parents: 20127 diff changeset	823 if (save_change == NULL &&
8e6442a0888f doveadm backup: Fixed unnecessary mailbox deletion. Timo Sirainen <timo.sirainen@dovecot.fi> parents: 20127 diff changeset	824 importer->cur_mail->uid >= importer->remote_uid_next) {
19670 b467f6d24fd5 doveadm backup: If local mailbox has more mails than remote, detect and delete it earlier. Timo Sirainen <timo.sirainen@dovecot.fi> parents: 19660 diff changeset	825 dsync_mailbox_revert_existing_uid(importer, importer->cur_mail->uid,
20191 8e6442a0888f doveadm backup: Fixed unnecessary mailbox deletion. Timo Sirainen <timo.sirainen@dovecot.fi> parents: 20127 diff changeset	826 t_strdup_printf("higher than remote's UIDs (remote UIDNEXT=%u)", importer->remote_uid_next));
19670 b467f6d24fd5 doveadm backup: If local mailbox has more mails than remote, detect and delete it earlier. Timo Sirainen <timo.sirainen@dovecot.fi> parents: 19660 diff changeset	827 return TRUE;
b467f6d24fd5 doveadm backup: If local mailbox has more mails than remote, detect and delete it earlier. Timo Sirainen <timo.sirainen@dovecot.fi> parents: 19660 diff changeset	828 }
15661 940414cea18d dsync backup: Old unwanted messages weren't deleted as they should have. Timo Sirainen <tss@iki.fi> parents: 15658 diff changeset	829 mail_expunge(importer->cur_mail);
940414cea18d dsync backup: Old unwanted messages weren't deleted as they should have. Timo Sirainen <tss@iki.fi> parents: 15658 diff changeset	830 importer->cur_mail_skip = TRUE;
940414cea18d dsync backup: Old unwanted messages weren't deleted as they should have. Timo Sirainen <tss@iki.fi> parents: 15658 diff changeset	831 importer->next_local_seq++;
940414cea18d dsync backup: Old unwanted messages weren't deleted as they should have. Timo Sirainen <tss@iki.fi> parents: 15658 diff changeset	832 return FALSE;
940414cea18d dsync backup: Old unwanted messages weren't deleted as they should have. Timo Sirainen <tss@iki.fi> parents: 15658 diff changeset	833 }
940414cea18d dsync backup: Old unwanted messages weren't deleted as they should have. Timo Sirainen <tss@iki.fi> parents: 15658 diff changeset	834 newmail = p_new(importer->pool, struct importer_new_mail, 1);
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	835 newmail->guid = p_strdup(importer->pool, importer->cur_guid);
15766 4260244e57e2 dsync: Various importer fixes. Handle unexpectedly wrong mailbox state better. Timo Sirainen <tss@iki.fi> parents: 15754 diff changeset	836 newmail->final_uid = importer->cur_mail->uid;
4260244e57e2 dsync: Various importer fixes. Handle unexpectedly wrong mailbox state better. Timo Sirainen <tss@iki.fi> parents: 15754 diff changeset	837 newmail->local_uid = importer->cur_mail->uid;
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	838 newmail->uid_in_local = TRUE;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	839 newmail->uid_is_usable =
15766 4260244e57e2 dsync: Various importer fixes. Handle unexpectedly wrong mailbox state better. Timo Sirainen <tss@iki.fi> parents: 15754 diff changeset	840 newmail->final_uid >= importer->remote_uid_next;
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	841 remote_saved = FALSE;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	842 } else if (diff > 0) {
14675 f07ba5e7d97d Make static analyzer happier. Timo Sirainen <tss@iki.fi> parents: 14629 diff changeset	843 i_assert(save_change != NULL);
15661 940414cea18d dsync backup: Old unwanted messages weren't deleted as they should have. Timo Sirainen <tss@iki.fi> parents: 15658 diff changeset	844 newmail = p_new(importer->pool, struct importer_new_mail, 1);
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	845 newmail->guid = save_change->guid;
15766 4260244e57e2 dsync: Various importer fixes. Handle unexpectedly wrong mailbox state better. Timo Sirainen <tss@iki.fi> parents: 15754 diff changeset	846 newmail->final_uid = save_change->uid;
4260244e57e2 dsync: Various importer fixes. Handle unexpectedly wrong mailbox state better. Timo Sirainen <tss@iki.fi> parents: 15754 diff changeset	847 newmail->remote_uid = save_change->uid;
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	848 newmail->uid_in_local = FALSE;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	849 newmail->uid_is_usable =
15766 4260244e57e2 dsync: Various importer fixes. Handle unexpectedly wrong mailbox state better. Timo Sirainen <tss@iki.fi> parents: 15754 diff changeset	850 newmail->final_uid >= importer->local_uid_next;
19660 7a3a138a4986 doveadm backup: Detect and handle conflicts earlier. Timo Sirainen <timo.sirainen@dovecot.fi> parents: 19646 diff changeset	851 if (!newmail->uid_is_usable && importer->revert_local_changes) {
7a3a138a4986 doveadm backup: Detect and handle conflicts earlier. Timo Sirainen <timo.sirainen@dovecot.fi> parents: 19646 diff changeset	852 dsync_mailbox_revert_existing_uid(importer, newmail->final_uid,
7a3a138a4986 doveadm backup: Detect and handle conflicts earlier. Timo Sirainen <timo.sirainen@dovecot.fi> parents: 19646 diff changeset	853 t_strdup_printf("UID >= local UIDNEXT=%u", importer->local_uid_next));
7a3a138a4986 doveadm backup: Detect and handle conflicts earlier. Timo Sirainen <timo.sirainen@dovecot.fi> parents: 19646 diff changeset	854 return TRUE;
7a3a138a4986 doveadm backup: Detect and handle conflicts earlier. Timo Sirainen <timo.sirainen@dovecot.fi> parents: 19646 diff changeset	855 }
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	856 remote_saved = TRUE;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	857 } else {
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	858 /* identical */
14675 f07ba5e7d97d Make static analyzer happier. Timo Sirainen <tss@iki.fi> parents: 14629 diff changeset	859 i_assert(importer->cur_mail != NULL);
f07ba5e7d97d Make static analyzer happier. Timo Sirainen <tss@iki.fi> parents: 14629 diff changeset	860 i_assert(save_change != NULL);
15661 940414cea18d dsync backup: Old unwanted messages weren't deleted as they should have. Timo Sirainen <tss@iki.fi> parents: 15658 diff changeset	861 newmail = p_new(importer->pool, struct importer_new_mail, 1);
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	862 newmail->guid = save_change->guid;
15766 4260244e57e2 dsync: Various importer fixes. Handle unexpectedly wrong mailbox state better. Timo Sirainen <tss@iki.fi> parents: 15754 diff changeset	863 newmail->final_uid = importer->cur_mail->uid;
4260244e57e2 dsync: Various importer fixes. Handle unexpectedly wrong mailbox state better. Timo Sirainen <tss@iki.fi> parents: 15754 diff changeset	864 newmail->local_uid = importer->cur_mail->uid;
4260244e57e2 dsync: Various importer fixes. Handle unexpectedly wrong mailbox state better. Timo Sirainen <tss@iki.fi> parents: 15754 diff changeset	865 newmail->remote_uid = save_change->uid;
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	866 newmail->uid_in_local = TRUE;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	867 newmail->uid_is_usable = TRUE;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	868 newmail->link = newmail;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	869 remote_saved = TRUE;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	870 }
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	871
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	872 if (newmail->uid_in_local) {
15661 940414cea18d dsync backup: Old unwanted messages weren't deleted as they should have. Timo Sirainen <tss@iki.fi> parents: 15658 diff changeset	873 importer->cur_mail_skip = TRUE;
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	874 importer->next_local_seq++;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	875 }
15772 360661d99c42 dsync: Various importer fixes. Timo Sirainen <tss@iki.fi> parents: 15770 diff changeset	876 /* NOTE: assumes save_change is allocated from importer pool */
360661d99c42 dsync: Various importer fixes. Timo Sirainen <tss@iki.fi> parents: 15770 diff changeset	877 newmail->change = save_change;
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	878
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	879 array_append(&importer->newmails, &newmail, 1);
15766 4260244e57e2 dsync: Various importer fixes. Handle unexpectedly wrong mailbox state better. Timo Sirainen <tss@iki.fi> parents: 15754 diff changeset	880 newmail_link(importer, newmail,
4260244e57e2 dsync: Various importer fixes. Handle unexpectedly wrong mailbox state better. Timo Sirainen <tss@iki.fi> parents: 15754 diff changeset	881 save_change == NULL ? 0 : save_change->uid);
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	882 return remote_saved;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	883 }
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	884
14629 c93ca5e46a8a Marked functions parameters that are allowed to be NULL. Some APIs were also changed. Timo Sirainen <tss@iki.fi> parents: 14584 diff changeset	885 static bool ATTR_NULL(2)
c93ca5e46a8a Marked functions parameters that are allowed to be NULL. Some APIs were also changed. Timo Sirainen <tss@iki.fi> parents: 14584 diff changeset	886 dsync_mailbox_try_save(struct dsync_mailbox_importer *importer,
c93ca5e46a8a Marked functions parameters that are allowed to be NULL. Some APIs were also changed. Timo Sirainen <tss@iki.fi> parents: 14584 diff changeset	887 struct dsync_mail_change *save_change)
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	888 {
15661 940414cea18d dsync backup: Old unwanted messages weren't deleted as they should have. Timo Sirainen <tss@iki.fi> parents: 15658 diff changeset	889 if (importer->cur_mail_skip) {
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	890 if (!importer_next_mail(importer, 0) && save_change == NULL)
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	891 return FALSE;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	892 }
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	893 return dsync_mailbox_try_save_cur(importer, save_change);
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	894 }
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	895
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	896 static void dsync_mailbox_save(struct dsync_mailbox_importer *importer,
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	897 struct dsync_mail_change *save_change)
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	898 {
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	899 while (!dsync_mailbox_try_save(importer, save_change)) ;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	900 }
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	901
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	902 static bool
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	903 dsync_import_set_mail(struct dsync_mailbox_importer *importer,
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	904 const struct dsync_mail_change *change)
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	905 {
15819 9c095a50ba3e dsync: Improved unexpected error logging. Timo Sirainen <tss@iki.fi> parents: 15781 diff changeset	906 const char guid, cmp_guid;
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	907
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	908 if (!mail_set_uid(importer->mail, change->uid))
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	909 return FALSE;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	910 if (change->guid == NULL) {
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	911 /* GUID is unknown */
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	912 return TRUE;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	913 }
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	914 if (*change->guid == '\0') {
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	915 /* backend doesn't support GUIDs. if hdr_hash is set, we could
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	916 verify it, but since this message really is supposed to
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	917 match, it's probably too much trouble. */
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	918 return TRUE;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	919 }
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	920
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	921 /* verify that GUID matches, just in case */
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	922 if (mail_get_special(importer->mail, MAIL_FETCH_GUID, &guid) < 0) {
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	923 dsync_mail_error(importer, importer->mail, "GUID");
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	924 return FALSE;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	925 }
16977 8a7ecddca4ad dsync: Support syncing storages with 128bit GUIDs <-> string GUIDs. Timo Sirainen <tss@iki.fi> parents: 16940 diff changeset	926 if (!dsync_mail_change_guid_equals(importer, change, guid, &cmp_guid)) {
15766 4260244e57e2 dsync: Various importer fixes. Handle unexpectedly wrong mailbox state better. Timo Sirainen <tss@iki.fi> parents: 15754 diff changeset	927 dsync_import_unexpected_state(importer, t_strdup_printf(
4260244e57e2 dsync: Various importer fixes. Handle unexpectedly wrong mailbox state better. Timo Sirainen <tss@iki.fi> parents: 15754 diff changeset	928 "Unexpected GUID mismatch for UID=%u: %s != %s",
15819 9c095a50ba3e dsync: Improved unexpected error logging. Timo Sirainen <tss@iki.fi> parents: 15781 diff changeset	929 change->uid, guid, cmp_guid));
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	930 return FALSE;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	931 }
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	932 return TRUE;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	933 }
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	934
15657 f0dbe8fc8905 dsync: Verify messages' GUIDs better. Timo Sirainen <tss@iki.fi> parents: 15619 diff changeset	935 static bool dsync_check_cur_guid(struct dsync_mailbox_importer *importer,
f0dbe8fc8905 dsync: Verify messages' GUIDs better. Timo Sirainen <tss@iki.fi> parents: 15619 diff changeset	936 const struct dsync_mail_change *change)
f0dbe8fc8905 dsync: Verify messages' GUIDs better. Timo Sirainen <tss@iki.fi> parents: 15619 diff changeset	937 {
15819 9c095a50ba3e dsync: Improved unexpected error logging. Timo Sirainen <tss@iki.fi> parents: 15781 diff changeset	938 const char *cmp_guid;
9c095a50ba3e dsync: Improved unexpected error logging. Timo Sirainen <tss@iki.fi> parents: 15781 diff changeset	939
16558 f3b9325509fd dsync: Fixed syncing when one of the backends supported GUIDs. Timo Sirainen <tss@iki.fi> parents: 16556 diff changeset	940 if (change->guid == NULL \|\| change->guid[0] == '\0' \|\|
f3b9325509fd dsync: Fixed syncing when one of the backends supported GUIDs. Timo Sirainen <tss@iki.fi> parents: 16556 diff changeset	941 importer->cur_guid[0] == '\0')
15657 f0dbe8fc8905 dsync: Verify messages' GUIDs better. Timo Sirainen <tss@iki.fi> parents: 15619 diff changeset	942 return TRUE;
16977 8a7ecddca4ad dsync: Support syncing storages with 128bit GUIDs <-> string GUIDs. Timo Sirainen <tss@iki.fi> parents: 16940 diff changeset	943 if (!dsync_mail_change_guid_equals(importer, change,
8a7ecddca4ad dsync: Support syncing storages with 128bit GUIDs <-> string GUIDs. Timo Sirainen <tss@iki.fi> parents: 16940 diff changeset	944 importer->cur_guid, &cmp_guid)) {
15766 4260244e57e2 dsync: Various importer fixes. Handle unexpectedly wrong mailbox state better. Timo Sirainen <tss@iki.fi> parents: 15754 diff changeset	945 dsync_import_unexpected_state(importer, t_strdup_printf(
4260244e57e2 dsync: Various importer fixes. Handle unexpectedly wrong mailbox state better. Timo Sirainen <tss@iki.fi> parents: 15754 diff changeset	946 "Unexpected GUID mismatch (2) for UID=%u: %s != %s",
15819 9c095a50ba3e dsync: Improved unexpected error logging. Timo Sirainen <tss@iki.fi> parents: 15781 diff changeset	947 change->uid, importer->cur_guid, cmp_guid));
15657 f0dbe8fc8905 dsync: Verify messages' GUIDs better. Timo Sirainen <tss@iki.fi> parents: 15619 diff changeset	948 return FALSE;
f0dbe8fc8905 dsync: Verify messages' GUIDs better. Timo Sirainen <tss@iki.fi> parents: 15619 diff changeset	949 }
f0dbe8fc8905 dsync: Verify messages' GUIDs better. Timo Sirainen <tss@iki.fi> parents: 15619 diff changeset	950 return TRUE;
f0dbe8fc8905 dsync: Verify messages' GUIDs better. Timo Sirainen <tss@iki.fi> parents: 15619 diff changeset	951 }
f0dbe8fc8905 dsync: Verify messages' GUIDs better. Timo Sirainen <tss@iki.fi> parents: 15619 diff changeset	952
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	953 static void
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	954 merge_flags(uint32_t local_final, uint32_t local_add, uint32_t local_remove,
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	955 uint32_t remote_final, uint32_t remote_add, uint32_t remote_remove,
15496 5d90e2aa1ba8 dsync: Use private modseqs to support syncing private flags in shared mailboxes. Timo Sirainen <tss@iki.fi> parents: 15257 diff changeset	956 uint32_t pvt_mask, bool prefer_remote, bool prefer_pvt_remote,
15698 798a335d99b0 dsync: When we know that remote does a flag update, try to guess the remote's new modseq and set it Timo Sirainen <tss@iki.fi> parents: 15696 diff changeset	957 uint32_t change_add_r, uint32_t change_remove_r,
15702 85e464d088da dsync: More fixes to merging flag/keyword changes. Timo Sirainen <tss@iki.fi> parents: 15700 diff changeset	958 bool remote_changed, bool remote_pvt_changed)
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	959 {
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	960 uint32_t combined_add, combined_remove, conflict_flags;
15496 5d90e2aa1ba8 dsync: Use private modseqs to support syncing private flags in shared mailboxes. Timo Sirainen <tss@iki.fi> parents: 15257 diff changeset	961 uint32_t local_wanted, remote_wanted, conflict_pvt_flags;
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	962
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	963 /* resolve conflicts */
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	964 conflict_flags = local_add & remote_remove;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	965 if (conflict_flags != 0) {
15496 5d90e2aa1ba8 dsync: Use private modseqs to support syncing private flags in shared mailboxes. Timo Sirainen <tss@iki.fi> parents: 15257 diff changeset	966 conflict_pvt_flags = conflict_flags & pvt_mask;
5d90e2aa1ba8 dsync: Use private modseqs to support syncing private flags in shared mailboxes. Timo Sirainen <tss@iki.fi> parents: 15257 diff changeset	967 conflict_flags &= ~pvt_mask;
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	968 if (prefer_remote)
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	969 local_add &= ~conflict_flags;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	970 else
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	971 remote_remove &= ~conflict_flags;
15496 5d90e2aa1ba8 dsync: Use private modseqs to support syncing private flags in shared mailboxes. Timo Sirainen <tss@iki.fi> parents: 15257 diff changeset	972 if (prefer_pvt_remote)
5d90e2aa1ba8 dsync: Use private modseqs to support syncing private flags in shared mailboxes. Timo Sirainen <tss@iki.fi> parents: 15257 diff changeset	973 local_add &= ~conflict_pvt_flags;
5d90e2aa1ba8 dsync: Use private modseqs to support syncing private flags in shared mailboxes. Timo Sirainen <tss@iki.fi> parents: 15257 diff changeset	974 else
5d90e2aa1ba8 dsync: Use private modseqs to support syncing private flags in shared mailboxes. Timo Sirainen <tss@iki.fi> parents: 15257 diff changeset	975 remote_remove &= ~conflict_pvt_flags;
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	976 }
15841 7f5c29a81f03 dsync: Assert-crashfix for handling conflicting private flags. Timo Sirainen <tss@iki.fi> parents: 15824 diff changeset	977 conflict_flags = local_remove & remote_add;
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	978 if (conflict_flags != 0) {
15496 5d90e2aa1ba8 dsync: Use private modseqs to support syncing private flags in shared mailboxes. Timo Sirainen <tss@iki.fi> parents: 15257 diff changeset	979 conflict_pvt_flags = conflict_flags & pvt_mask;
5d90e2aa1ba8 dsync: Use private modseqs to support syncing private flags in shared mailboxes. Timo Sirainen <tss@iki.fi> parents: 15257 diff changeset	980 conflict_flags &= ~pvt_mask;
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	981 if (prefer_remote)
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	982 local_remove &= ~conflict_flags;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	983 else
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	984 remote_add &= ~conflict_flags;
15496 5d90e2aa1ba8 dsync: Use private modseqs to support syncing private flags in shared mailboxes. Timo Sirainen <tss@iki.fi> parents: 15257 diff changeset	985 if (prefer_pvt_remote)
5d90e2aa1ba8 dsync: Use private modseqs to support syncing private flags in shared mailboxes. Timo Sirainen <tss@iki.fi> parents: 15257 diff changeset	986 local_remove &= ~conflict_pvt_flags;
5d90e2aa1ba8 dsync: Use private modseqs to support syncing private flags in shared mailboxes. Timo Sirainen <tss@iki.fi> parents: 15257 diff changeset	987 else
5d90e2aa1ba8 dsync: Use private modseqs to support syncing private flags in shared mailboxes. Timo Sirainen <tss@iki.fi> parents: 15257 diff changeset	988 remote_add &= ~conflict_pvt_flags;
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	989 }
15496 5d90e2aa1ba8 dsync: Use private modseqs to support syncing private flags in shared mailboxes. Timo Sirainen <tss@iki.fi> parents: 15257 diff changeset	990
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	991 combined_add = local_add\|remote_add;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	992 combined_remove = local_remove\|remote_remove;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	993 i_assert((combined_add & combined_remove) == 0);
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	994
15699 9b7d80a8db44 dsync: If both local and remote have the same message flags, don't go changing them. Timo Sirainen <tss@iki.fi> parents: 15698 diff changeset	995 /* don't change flags that are currently identical in both sides */
9b7d80a8db44 dsync: If both local and remote have the same message flags, don't go changing them. Timo Sirainen <tss@iki.fi> parents: 15698 diff changeset	996 conflict_flags = local_final ^ remote_final;
9b7d80a8db44 dsync: If both local and remote have the same message flags, don't go changing them. Timo Sirainen <tss@iki.fi> parents: 15698 diff changeset	997 combined_add &= conflict_flags;
9b7d80a8db44 dsync: If both local and remote have the same message flags, don't go changing them. Timo Sirainen <tss@iki.fi> parents: 15698 diff changeset	998 combined_remove &= conflict_flags;
9b7d80a8db44 dsync: If both local and remote have the same message flags, don't go changing them. Timo Sirainen <tss@iki.fi> parents: 15698 diff changeset	999
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1000 /* see if there are conflicting final flags */
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1001 local_wanted = (local_final\|combined_add) & ~combined_remove;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1002 remote_wanted = (remote_final\|combined_add) & ~combined_remove;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1003
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1004 conflict_flags = local_wanted ^ remote_wanted;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1005 if (conflict_flags != 0) {
15496 5d90e2aa1ba8 dsync: Use private modseqs to support syncing private flags in shared mailboxes. Timo Sirainen <tss@iki.fi> parents: 15257 diff changeset	1006 if (prefer_remote && prefer_pvt_remote)
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1007 local_wanted = remote_wanted;
15496 5d90e2aa1ba8 dsync: Use private modseqs to support syncing private flags in shared mailboxes. Timo Sirainen <tss@iki.fi> parents: 15257 diff changeset	1008 else if (prefer_remote && !prefer_pvt_remote) {
5d90e2aa1ba8 dsync: Use private modseqs to support syncing private flags in shared mailboxes. Timo Sirainen <tss@iki.fi> parents: 15257 diff changeset	1009 local_wanted = (local_wanted & pvt_mask) \|
5d90e2aa1ba8 dsync: Use private modseqs to support syncing private flags in shared mailboxes. Timo Sirainen <tss@iki.fi> parents: 15257 diff changeset	1010 (remote_wanted & ~pvt_mask);
5d90e2aa1ba8 dsync: Use private modseqs to support syncing private flags in shared mailboxes. Timo Sirainen <tss@iki.fi> parents: 15257 diff changeset	1011 } else if (!prefer_remote && prefer_pvt_remote) {
5d90e2aa1ba8 dsync: Use private modseqs to support syncing private flags in shared mailboxes. Timo Sirainen <tss@iki.fi> parents: 15257 diff changeset	1012 local_wanted = (local_wanted & ~pvt_mask) \|
5d90e2aa1ba8 dsync: Use private modseqs to support syncing private flags in shared mailboxes. Timo Sirainen <tss@iki.fi> parents: 15257 diff changeset	1013 (remote_wanted & pvt_mask);
5d90e2aa1ba8 dsync: Use private modseqs to support syncing private flags in shared mailboxes. Timo Sirainen <tss@iki.fi> parents: 15257 diff changeset	1014 }
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1015 }
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1016
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1017 *change_add_r = local_wanted & ~local_final;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1018 *change_remove_r = local_final & ~local_wanted;
15702 85e464d088da dsync: More fixes to merging flag/keyword changes. Timo Sirainen <tss@iki.fi> parents: 15700 diff changeset	1019 if ((local_wanted & ~pvt_mask) != (remote_final & ~pvt_mask))
15698 798a335d99b0 dsync: When we know that remote does a flag update, try to guess the remote's new modseq and set it Timo Sirainen <tss@iki.fi> parents: 15696 diff changeset	1020 *remote_changed = TRUE;
15702 85e464d088da dsync: More fixes to merging flag/keyword changes. Timo Sirainen <tss@iki.fi> parents: 15700 diff changeset	1021 if ((local_wanted & pvt_mask) != (remote_final & pvt_mask))
85e464d088da dsync: More fixes to merging flag/keyword changes. Timo Sirainen <tss@iki.fi> parents: 15700 diff changeset	1022 *remote_pvt_changed = TRUE;
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1023 }
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1024
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1025 static bool
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1026 keyword_find(ARRAY_TYPE(const_string) keywords, const char name,
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1027 unsigned int *idx_r)
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1028 {
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1029 const char const names;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1030 unsigned int i, count;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1031
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1032 names = array_get(keywords, &count);
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1033 for (i = 0; i < count; i++) {
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1034 if (strcmp(names[i], name) == 0) {
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1035 *idx_r = i;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1036 return TRUE;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1037 }
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1038 }
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1039 return FALSE;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1040 }
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1041
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1042 static void keywords_append(ARRAY_TYPE(const_string) *dest,
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1043 const ARRAY_TYPE(const_string) *keywords,
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1044 uint32_t bits, unsigned int start_idx)
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1045 {
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1046 const char const namep;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1047 unsigned int i;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1048
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1049 for (i = 0; i < 32; i++) {
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1050 if ((bits & (1U << i)) == 0)
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1051 continue;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1052
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1053 namep = array_idx(keywords, start_idx+i);
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1054 array_append(dest, namep, 1);
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1055 }
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1056 }
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1057
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1058 static void
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1059 merge_keywords(struct mail mail, const ARRAY_TYPE(const_string) local_changes,
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1060 const ARRAY_TYPE(const_string) *remote_changes,
15702 85e464d088da dsync: More fixes to merging flag/keyword changes. Timo Sirainen <tss@iki.fi> parents: 15700 diff changeset	1061 bool prefer_remote,
85e464d088da dsync: More fixes to merging flag/keyword changes. Timo Sirainen <tss@iki.fi> parents: 15700 diff changeset	1062 bool remote_changed, bool remote_pvt_changed)
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1063 {
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1064 /* local_changes and remote_changes are assumed to have no
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1065 duplicates names */
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1066 uint32_t local_add, local_remove, *local_final;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1067 uint32_t remote_add, remote_remove, *remote_final;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1068 uint32_t change_add, change_remove;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1069 ARRAY_TYPE(const_string) all_keywords, add_keywords, remove_keywords;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1070 const char const changes, name, const *local_keywords;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1071 struct mail_keywords *kw;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1072 unsigned int i, count, name_idx, array_size;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1073
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1074 local_keywords = mail_get_keywords(mail);
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1075
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1076 /* we'll assign a common index for each keyword name and place
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1077 the changes to separate bit arrays. */
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1078 if (array_is_created(remote_changes))
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1079 changes = array_get(remote_changes, &count);
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1080 else {
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1081 changes = NULL;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1082 count = 0;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1083 }
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1084
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1085 array_size = str_array_length(local_keywords) + count;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1086 if (array_is_created(local_changes))
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1087 array_size += array_count(local_changes);
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1088 if (array_size == 0) {
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1089 /* this message has no keywords */
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1090 return;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1091 }
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1092 t_array_init(&all_keywords, array_size);
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1093 t_array_init(&add_keywords, array_size);
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1094 t_array_init(&remove_keywords, array_size);
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1095
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1096 /* @UNSAFE: create large enough arrays to fit all keyword indexes. */
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1097 array_size = (array_size+31)/32;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1098 local_add = t_new(uint32_t, array_size);
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1099 local_remove = t_new(uint32_t, array_size);
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1100 local_final = t_new(uint32_t, array_size);
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1101 remote_add = t_new(uint32_t, array_size);
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1102 remote_remove = t_new(uint32_t, array_size);
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1103 remote_final = t_new(uint32_t, array_size);
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1104 change_add = t_new(uint32_t, array_size);
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1105 change_remove = t_new(uint32_t, array_size);
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1106
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1107 /* get remote changes */
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1108 for (i = 0; i < count; i++) {
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1109 name = changes[i]+1;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1110 name_idx = array_count(&all_keywords);
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1111 array_append(&all_keywords, &name, 1);
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1112
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1113 switch (changes[i][0]) {
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1114 case KEYWORD_CHANGE_ADD:
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1115 remote_add[name_idx/32] \|= 1U << (name_idx%32);
15700 067179cbabc2 dsync: Fixed syncing message keywords. Timo Sirainen <tss@iki.fi> parents: 15699 diff changeset	1116 break;
067179cbabc2 dsync: Fixed syncing message keywords. Timo Sirainen <tss@iki.fi> parents: 15699 diff changeset	1117 case KEYWORD_CHANGE_REMOVE:
067179cbabc2 dsync: Fixed syncing message keywords. Timo Sirainen <tss@iki.fi> parents: 15699 diff changeset	1118 remote_remove[name_idx/32] \|= 1U << (name_idx%32);
067179cbabc2 dsync: Fixed syncing message keywords. Timo Sirainen <tss@iki.fi> parents: 15699 diff changeset	1119 break;
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1120 case KEYWORD_CHANGE_FINAL:
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1121 remote_final[name_idx/32] \|= 1U << (name_idx%32);
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1122 break;
15700 067179cbabc2 dsync: Fixed syncing message keywords. Timo Sirainen <tss@iki.fi> parents: 15699 diff changeset	1123 case KEYWORD_CHANGE_ADD_AND_FINAL:
067179cbabc2 dsync: Fixed syncing message keywords. Timo Sirainen <tss@iki.fi> parents: 15699 diff changeset	1124 remote_add[name_idx/32] \|= 1U << (name_idx%32);
067179cbabc2 dsync: Fixed syncing message keywords. Timo Sirainen <tss@iki.fi> parents: 15699 diff changeset	1125 remote_final[name_idx/32] \|= 1U << (name_idx%32);
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1126 break;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1127 }
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1128 }
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1129
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1130 /* get local changes. use existing indexes for names when they exist. */
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1131 if (array_is_created(local_changes))
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1132 changes = array_get(local_changes, &count);
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1133 else {
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1134 changes = NULL;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1135 count = 0;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1136 }
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1137 for (i = 0; i < count; i++) {
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1138 name = changes[i]+1;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1139 if (!keyword_find(&all_keywords, name, &name_idx)) {
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1140 name_idx = array_count(&all_keywords);
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1141 array_append(&all_keywords, &name, 1);
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1142 }
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1143
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1144 switch (changes[i][0]) {
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1145 case KEYWORD_CHANGE_ADD:
15700 067179cbabc2 dsync: Fixed syncing message keywords. Timo Sirainen <tss@iki.fi> parents: 15699 diff changeset	1146 case KEYWORD_CHANGE_ADD_AND_FINAL:
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1147 local_add[name_idx/32] \|= 1U << (name_idx%32);
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1148 break;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1149 case KEYWORD_CHANGE_REMOVE:
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1150 local_remove[name_idx/32] \|= 1U << (name_idx%32);
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1151 break;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1152 case KEYWORD_CHANGE_FINAL:
15702 85e464d088da dsync: More fixes to merging flag/keyword changes. Timo Sirainen <tss@iki.fi> parents: 15700 diff changeset	1153 break;
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1154 }
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1155 }
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1156 for (i = 0; local_keywords[i] != NULL; i++) {
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1157 name = local_keywords[i];
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1158 if (!keyword_find(&all_keywords, name, &name_idx)) {
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1159 name_idx = array_count(&all_keywords);
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1160 array_append(&all_keywords, &name, 1);
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1161 }
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1162 local_final[name_idx/32] \|= 1U << (name_idx%32);
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1163 }
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1164 i_assert(array_count(&all_keywords) <= array_size*32);
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1165 array_size = (array_count(&all_keywords)+31) / 32;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1166
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1167 /* merge keywords */
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1168 for (i = 0; i < array_size; i++) {
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1169 merge_flags(local_final[i], local_add[i], local_remove[i],
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1170 remote_final[i], remote_add[i], remote_remove[i],
15496 5d90e2aa1ba8 dsync: Use private modseqs to support syncing private flags in shared mailboxes. Timo Sirainen <tss@iki.fi> parents: 15257 diff changeset	1171 0, prefer_remote, prefer_remote,
15702 85e464d088da dsync: More fixes to merging flag/keyword changes. Timo Sirainen <tss@iki.fi> parents: 15700 diff changeset	1172 &change_add[i], &change_remove[i],
85e464d088da dsync: More fixes to merging flag/keyword changes. Timo Sirainen <tss@iki.fi> parents: 15700 diff changeset	1173 remote_changed, remote_pvt_changed);
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1174 if (change_add[i] != 0) {
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1175 keywords_append(&add_keywords, &all_keywords,
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1176 change_add[i], i*32);
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1177 }
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1178 if (change_remove[i] != 0) {
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1179 keywords_append(&remove_keywords, &all_keywords,
15696 2801de211b78 dsync: Fixed syncing keyword removals. Timo Sirainen <tss@iki.fi> parents: 15694 diff changeset	1180 change_remove[i], i*32);
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1181 }
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1182 }
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1183
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1184 /* apply changes */
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1185 if (array_count(&add_keywords) > 0) {
14686 9ff19c1d5f69 Added array_append_zero() to write a zero-filled record to an array. Timo Sirainen <tss@iki.fi> parents: 14682 diff changeset	1186 array_append_zero(&add_keywords);
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1187 kw = mailbox_keywords_create_valid(mail->box,
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1188 array_idx(&add_keywords, 0));
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1189 mail_update_keywords(mail, MODIFY_ADD, kw);
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1190 mailbox_keywords_unref(&kw);
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1191 }
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1192 if (array_count(&remove_keywords) > 0) {
14686 9ff19c1d5f69 Added array_append_zero() to write a zero-filled record to an array. Timo Sirainen <tss@iki.fi> parents: 14682 diff changeset	1193 array_append_zero(&remove_keywords);
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1194 kw = mailbox_keywords_create_valid(mail->box,
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1195 array_idx(&remove_keywords, 0));
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1196 mail_update_keywords(mail, MODIFY_REMOVE, kw);
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1197 mailbox_keywords_unref(&kw);
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1198 }
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1199 }
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1200
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1201 static void
15257 60f0cb48fdb2 doveadm backup: Revert all local changes. Timo Sirainen <tss@iki.fi> parents: 15233 diff changeset	1202 dsync_mailbox_import_replace_flags(struct mail *mail,
60f0cb48fdb2 doveadm backup: Revert all local changes. Timo Sirainen <tss@iki.fi> parents: 15233 diff changeset	1203 const struct dsync_mail_change *change)
60f0cb48fdb2 doveadm backup: Revert all local changes. Timo Sirainen <tss@iki.fi> parents: 15233 diff changeset	1204 {
60f0cb48fdb2 doveadm backup: Revert all local changes. Timo Sirainen <tss@iki.fi> parents: 15233 diff changeset	1205 ARRAY_TYPE(const_string) keywords;
60f0cb48fdb2 doveadm backup: Revert all local changes. Timo Sirainen <tss@iki.fi> parents: 15233 diff changeset	1206 struct mail_keywords *kw;
60f0cb48fdb2 doveadm backup: Revert all local changes. Timo Sirainen <tss@iki.fi> parents: 15233 diff changeset	1207 const char const changes, *name;
60f0cb48fdb2 doveadm backup: Revert all local changes. Timo Sirainen <tss@iki.fi> parents: 15233 diff changeset	1208 unsigned int i, count;
60f0cb48fdb2 doveadm backup: Revert all local changes. Timo Sirainen <tss@iki.fi> parents: 15233 diff changeset	1209
60f0cb48fdb2 doveadm backup: Revert all local changes. Timo Sirainen <tss@iki.fi> parents: 15233 diff changeset	1210 if (array_is_created(&change->keyword_changes))
60f0cb48fdb2 doveadm backup: Revert all local changes. Timo Sirainen <tss@iki.fi> parents: 15233 diff changeset	1211 changes = array_get(&change->keyword_changes, &count);
60f0cb48fdb2 doveadm backup: Revert all local changes. Timo Sirainen <tss@iki.fi> parents: 15233 diff changeset	1212 else {
60f0cb48fdb2 doveadm backup: Revert all local changes. Timo Sirainen <tss@iki.fi> parents: 15233 diff changeset	1213 changes = NULL;
60f0cb48fdb2 doveadm backup: Revert all local changes. Timo Sirainen <tss@iki.fi> parents: 15233 diff changeset	1214 count = 0;
60f0cb48fdb2 doveadm backup: Revert all local changes. Timo Sirainen <tss@iki.fi> parents: 15233 diff changeset	1215 }
60f0cb48fdb2 doveadm backup: Revert all local changes. Timo Sirainen <tss@iki.fi> parents: 15233 diff changeset	1216 t_array_init(&keywords, count+1);
60f0cb48fdb2 doveadm backup: Revert all local changes. Timo Sirainen <tss@iki.fi> parents: 15233 diff changeset	1217 for (i = 0; i < count; i++) {
60f0cb48fdb2 doveadm backup: Revert all local changes. Timo Sirainen <tss@iki.fi> parents: 15233 diff changeset	1218 switch (changes[i][0]) {
60f0cb48fdb2 doveadm backup: Revert all local changes. Timo Sirainen <tss@iki.fi> parents: 15233 diff changeset	1219 case KEYWORD_CHANGE_ADD:
60f0cb48fdb2 doveadm backup: Revert all local changes. Timo Sirainen <tss@iki.fi> parents: 15233 diff changeset	1220 case KEYWORD_CHANGE_FINAL:
15700 067179cbabc2 dsync: Fixed syncing message keywords. Timo Sirainen <tss@iki.fi> parents: 15699 diff changeset	1221 case KEYWORD_CHANGE_ADD_AND_FINAL:
15257 60f0cb48fdb2 doveadm backup: Revert all local changes. Timo Sirainen <tss@iki.fi> parents: 15233 diff changeset	1222 name = changes[i]+1;
60f0cb48fdb2 doveadm backup: Revert all local changes. Timo Sirainen <tss@iki.fi> parents: 15233 diff changeset	1223 array_append(&keywords, &name, 1);
60f0cb48fdb2 doveadm backup: Revert all local changes. Timo Sirainen <tss@iki.fi> parents: 15233 diff changeset	1224 break;
60f0cb48fdb2 doveadm backup: Revert all local changes. Timo Sirainen <tss@iki.fi> parents: 15233 diff changeset	1225 case KEYWORD_CHANGE_REMOVE:
60f0cb48fdb2 doveadm backup: Revert all local changes. Timo Sirainen <tss@iki.fi> parents: 15233 diff changeset	1226 break;
60f0cb48fdb2 doveadm backup: Revert all local changes. Timo Sirainen <tss@iki.fi> parents: 15233 diff changeset	1227 }
60f0cb48fdb2 doveadm backup: Revert all local changes. Timo Sirainen <tss@iki.fi> parents: 15233 diff changeset	1228 }
60f0cb48fdb2 doveadm backup: Revert all local changes. Timo Sirainen <tss@iki.fi> parents: 15233 diff changeset	1229 array_append_zero(&keywords);
60f0cb48fdb2 doveadm backup: Revert all local changes. Timo Sirainen <tss@iki.fi> parents: 15233 diff changeset	1230
60f0cb48fdb2 doveadm backup: Revert all local changes. Timo Sirainen <tss@iki.fi> parents: 15233 diff changeset	1231 kw = mailbox_keywords_create_valid(mail->box, array_idx(&keywords, 0));
60f0cb48fdb2 doveadm backup: Revert all local changes. Timo Sirainen <tss@iki.fi> parents: 15233 diff changeset	1232 mail_update_keywords(mail, MODIFY_REPLACE, kw);
60f0cb48fdb2 doveadm backup: Revert all local changes. Timo Sirainen <tss@iki.fi> parents: 15233 diff changeset	1233 mailbox_keywords_unref(&kw);
60f0cb48fdb2 doveadm backup: Revert all local changes. Timo Sirainen <tss@iki.fi> parents: 15233 diff changeset	1234
60f0cb48fdb2 doveadm backup: Revert all local changes. Timo Sirainen <tss@iki.fi> parents: 15233 diff changeset	1235 mail_update_flags(mail, MODIFY_REPLACE,
60f0cb48fdb2 doveadm backup: Revert all local changes. Timo Sirainen <tss@iki.fi> parents: 15233 diff changeset	1236 change->add_flags \| change->final_flags);
15589 df5b25b11592 dsync: Avoid updating modseqs unnecessarily. Timo Sirainen <tss@iki.fi> parents: 15496 diff changeset	1237 if (mail_get_modseq(mail) < change->modseq)
df5b25b11592 dsync: Avoid updating modseqs unnecessarily. Timo Sirainen <tss@iki.fi> parents: 15496 diff changeset	1238 mail_update_modseq(mail, change->modseq);
df5b25b11592 dsync: Avoid updating modseqs unnecessarily. Timo Sirainen <tss@iki.fi> parents: 15496 diff changeset	1239 if (mail_get_pvt_modseq(mail) < change->pvt_modseq)
df5b25b11592 dsync: Avoid updating modseqs unnecessarily. Timo Sirainen <tss@iki.fi> parents: 15496 diff changeset	1240 mail_update_pvt_modseq(mail, change->pvt_modseq);
15257 60f0cb48fdb2 doveadm backup: Revert all local changes. Timo Sirainen <tss@iki.fi> parents: 15233 diff changeset	1241 }
60f0cb48fdb2 doveadm backup: Revert all local changes. Timo Sirainen <tss@iki.fi> parents: 15233 diff changeset	1242
60f0cb48fdb2 doveadm backup: Revert all local changes. Timo Sirainen <tss@iki.fi> parents: 15233 diff changeset	1243 static void
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1244 dsync_mailbox_import_flag_change(struct dsync_mailbox_importer *importer,
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1245 const struct dsync_mail_change *change)
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1246 {
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1247 const struct dsync_mail_change *local_change;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1248 enum mail_flags local_add, local_remove;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1249 uint32_t change_add, change_remove;
15698 798a335d99b0 dsync: When we know that remote does a flag update, try to guess the remote's new modseq and set it Timo Sirainen <tss@iki.fi> parents: 15696 diff changeset	1250 uint64_t new_modseq;
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1251 ARRAY_TYPE(const_string) local_keyword_changes = ARRAY_INIT;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1252 struct mail *mail;
15702 85e464d088da dsync: More fixes to merging flag/keyword changes. Timo Sirainen <tss@iki.fi> parents: 15700 diff changeset	1253 bool prefer_remote, prefer_pvt_remote;
85e464d088da dsync: More fixes to merging flag/keyword changes. Timo Sirainen <tss@iki.fi> parents: 15700 diff changeset	1254 bool remote_changed = FALSE, remote_pvt_changed = FALSE;
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1255
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1256 i_assert((change->add_flags & change->remove_flags) == 0);
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1257
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1258 if (importer->cur_mail != NULL &&
15657 f0dbe8fc8905 dsync: Verify messages' GUIDs better. Timo Sirainen <tss@iki.fi> parents: 15619 diff changeset	1259 importer->cur_mail->uid == change->uid) {
f0dbe8fc8905 dsync: Verify messages' GUIDs better. Timo Sirainen <tss@iki.fi> parents: 15619 diff changeset	1260 if (!dsync_check_cur_guid(importer, change))
f0dbe8fc8905 dsync: Verify messages' GUIDs better. Timo Sirainen <tss@iki.fi> parents: 15619 diff changeset	1261 return;
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1262 mail = importer->cur_mail;
15657 f0dbe8fc8905 dsync: Verify messages' GUIDs better. Timo Sirainen <tss@iki.fi> parents: 15619 diff changeset	1263 } else {
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1264 if (!dsync_import_set_mail(importer, change))
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1265 return;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1266 mail = importer->mail;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1267 }
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1268
15257 60f0cb48fdb2 doveadm backup: Revert all local changes. Timo Sirainen <tss@iki.fi> parents: 15233 diff changeset	1269 if (importer->revert_local_changes) {
60f0cb48fdb2 doveadm backup: Revert all local changes. Timo Sirainen <tss@iki.fi> parents: 15233 diff changeset	1270 /* dsync backup: just make the local look like remote. */
60f0cb48fdb2 doveadm backup: Revert all local changes. Timo Sirainen <tss@iki.fi> parents: 15233 diff changeset	1271 dsync_mailbox_import_replace_flags(mail, change);
60f0cb48fdb2 doveadm backup: Revert all local changes. Timo Sirainen <tss@iki.fi> parents: 15233 diff changeset	1272 return;
60f0cb48fdb2 doveadm backup: Revert all local changes. Timo Sirainen <tss@iki.fi> parents: 15233 diff changeset	1273 }
60f0cb48fdb2 doveadm backup: Revert all local changes. Timo Sirainen <tss@iki.fi> parents: 15233 diff changeset	1274
14923 96fd2c3bf932 Reverted "support for non-pointers" part of the hash table API changes. Timo Sirainen <tss@iki.fi> parents: 14922 diff changeset	1275 local_change = hash_table_lookup(importer->local_changes,
96fd2c3bf932 Reverted "support for non-pointers" part of the hash table API changes. Timo Sirainen <tss@iki.fi> parents: 14922 diff changeset	1276 POINTER_CAST(change->uid));
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1277 if (local_change == NULL) {
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1278 local_add = local_remove = 0;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1279 } else {
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1280 local_add = local_change->add_flags;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1281 local_remove = local_change->remove_flags;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1282 local_keyword_changes = local_change->keyword_changes;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1283 }
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1284
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1285 if (mail_get_modseq(mail) < change->modseq)
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1286 prefer_remote = TRUE;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1287 else if (mail_get_modseq(mail) > change->modseq)
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1288 prefer_remote = FALSE;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1289 else {
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1290 /* identical modseq, we'll just have to pick one.
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1291 Note that both brains need to pick the same one, otherwise
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1292 they become unsynced. */
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1293 prefer_remote = !importer->master_brain;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1294 }
15496 5d90e2aa1ba8 dsync: Use private modseqs to support syncing private flags in shared mailboxes. Timo Sirainen <tss@iki.fi> parents: 15257 diff changeset	1295 if (mail_get_pvt_modseq(mail) < change->pvt_modseq)
5d90e2aa1ba8 dsync: Use private modseqs to support syncing private flags in shared mailboxes. Timo Sirainen <tss@iki.fi> parents: 15257 diff changeset	1296 prefer_pvt_remote = TRUE;
5d90e2aa1ba8 dsync: Use private modseqs to support syncing private flags in shared mailboxes. Timo Sirainen <tss@iki.fi> parents: 15257 diff changeset	1297 else if (mail_get_pvt_modseq(mail) > change->pvt_modseq)
5d90e2aa1ba8 dsync: Use private modseqs to support syncing private flags in shared mailboxes. Timo Sirainen <tss@iki.fi> parents: 15257 diff changeset	1298 prefer_pvt_remote = FALSE;
5d90e2aa1ba8 dsync: Use private modseqs to support syncing private flags in shared mailboxes. Timo Sirainen <tss@iki.fi> parents: 15257 diff changeset	1299 else
5d90e2aa1ba8 dsync: Use private modseqs to support syncing private flags in shared mailboxes. Timo Sirainen <tss@iki.fi> parents: 15257 diff changeset	1300 prefer_pvt_remote = !importer->master_brain;
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1301
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1302 /* merge flags */
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1303 merge_flags(mail_get_flags(mail), local_add, local_remove,
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1304 change->final_flags, change->add_flags, change->remove_flags,
15496 5d90e2aa1ba8 dsync: Use private modseqs to support syncing private flags in shared mailboxes. Timo Sirainen <tss@iki.fi> parents: 15257 diff changeset	1305 mailbox_get_private_flags_mask(mail->box),
5d90e2aa1ba8 dsync: Use private modseqs to support syncing private flags in shared mailboxes. Timo Sirainen <tss@iki.fi> parents: 15257 diff changeset	1306 prefer_remote, prefer_pvt_remote,
15702 85e464d088da dsync: More fixes to merging flag/keyword changes. Timo Sirainen <tss@iki.fi> parents: 15700 diff changeset	1307 &change_add, &change_remove,
85e464d088da dsync: More fixes to merging flag/keyword changes. Timo Sirainen <tss@iki.fi> parents: 15700 diff changeset	1308 &remote_changed, &remote_pvt_changed);
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1309
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1310 if (change_add != 0)
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1311 mail_update_flags(mail, MODIFY_ADD, change_add);
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1312 if (change_remove != 0)
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1313 mail_update_flags(mail, MODIFY_REMOVE, change_remove);
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1314
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1315 /* merge keywords */
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1316 merge_keywords(mail, &local_keyword_changes, &change->keyword_changes,
15702 85e464d088da dsync: More fixes to merging flag/keyword changes. Timo Sirainen <tss@iki.fi> parents: 15700 diff changeset	1317 prefer_remote, &remote_changed, &remote_pvt_changed);
15698 798a335d99b0 dsync: When we know that remote does a flag update, try to guess the remote's new modseq and set it Timo Sirainen <tss@iki.fi> parents: 15696 diff changeset	1318
798a335d99b0 dsync: When we know that remote does a flag update, try to guess the remote's new modseq and set it Timo Sirainen <tss@iki.fi> parents: 15696 diff changeset	1319 /* update modseqs. try to anticipate when we have to increase modseq
798a335d99b0 dsync: When we know that remote does a flag update, try to guess the remote's new modseq and set it Timo Sirainen <tss@iki.fi> parents: 15696 diff changeset	1320 to get it closer to what remote has (although we can't guess it
798a335d99b0 dsync: When we know that remote does a flag update, try to guess the remote's new modseq and set it Timo Sirainen <tss@iki.fi> parents: 15696 diff changeset	1321 exactly correctly) */
798a335d99b0 dsync: When we know that remote does a flag update, try to guess the remote's new modseq and set it Timo Sirainen <tss@iki.fi> parents: 15696 diff changeset	1322 new_modseq = change->modseq;
798a335d99b0 dsync: When we know that remote does a flag update, try to guess the remote's new modseq and set it Timo Sirainen <tss@iki.fi> parents: 15696 diff changeset	1323 if (remote_changed && new_modseq <= importer->remote_highest_modseq)
798a335d99b0 dsync: When we know that remote does a flag update, try to guess the remote's new modseq and set it Timo Sirainen <tss@iki.fi> parents: 15696 diff changeset	1324 new_modseq = importer->remote_highest_modseq+1;
798a335d99b0 dsync: When we know that remote does a flag update, try to guess the remote's new modseq and set it Timo Sirainen <tss@iki.fi> parents: 15696 diff changeset	1325 if (mail_get_modseq(mail) < new_modseq)
798a335d99b0 dsync: When we know that remote does a flag update, try to guess the remote's new modseq and set it Timo Sirainen <tss@iki.fi> parents: 15696 diff changeset	1326 mail_update_modseq(mail, new_modseq);
798a335d99b0 dsync: When we know that remote does a flag update, try to guess the remote's new modseq and set it Timo Sirainen <tss@iki.fi> parents: 15696 diff changeset	1327
798a335d99b0 dsync: When we know that remote does a flag update, try to guess the remote's new modseq and set it Timo Sirainen <tss@iki.fi> parents: 15696 diff changeset	1328 new_modseq = change->pvt_modseq;
15702 85e464d088da dsync: More fixes to merging flag/keyword changes. Timo Sirainen <tss@iki.fi> parents: 15700 diff changeset	1329 if (remote_pvt_changed && new_modseq <= importer->remote_highest_pvt_modseq)
15698 798a335d99b0 dsync: When we know that remote does a flag update, try to guess the remote's new modseq and set it Timo Sirainen <tss@iki.fi> parents: 15696 diff changeset	1330 new_modseq = importer->remote_highest_pvt_modseq+1;
798a335d99b0 dsync: When we know that remote does a flag update, try to guess the remote's new modseq and set it Timo Sirainen <tss@iki.fi> parents: 15696 diff changeset	1331 if (mail_get_pvt_modseq(mail) < new_modseq)
798a335d99b0 dsync: When we know that remote does a flag update, try to guess the remote's new modseq and set it Timo Sirainen <tss@iki.fi> parents: 15696 diff changeset	1332 mail_update_pvt_modseq(mail, new_modseq);
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1333 }
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1334
18181 35e4a6ae8d85 dsync: Added -F parameter to sync only mails with[out] specific flag. Timo Sirainen <tss@iki.fi> parents: 18180 diff changeset	1335 static bool
35e4a6ae8d85 dsync: Added -F parameter to sync only mails with[out] specific flag. Timo Sirainen <tss@iki.fi> parents: 18180 diff changeset	1336 dsync_mail_change_have_keyword(const struct dsync_mail_change *change,
35e4a6ae8d85 dsync: Added -F parameter to sync only mails with[out] specific flag. Timo Sirainen <tss@iki.fi> parents: 18180 diff changeset	1337 const char *keyword)
35e4a6ae8d85 dsync: Added -F parameter to sync only mails with[out] specific flag. Timo Sirainen <tss@iki.fi> parents: 18180 diff changeset	1338 {
35e4a6ae8d85 dsync: Added -F parameter to sync only mails with[out] specific flag. Timo Sirainen <tss@iki.fi> parents: 18180 diff changeset	1339 const char const strp;
35e4a6ae8d85 dsync: Added -F parameter to sync only mails with[out] specific flag. Timo Sirainen <tss@iki.fi> parents: 18180 diff changeset	1340
18183 b505739970e7 dsync: Crashfix to -F parameter handling. Timo Sirainen <tss@iki.fi> parents: 18181 diff changeset	1341 if (!array_is_created(&change->keyword_changes))
b505739970e7 dsync: Crashfix to -F parameter handling. Timo Sirainen <tss@iki.fi> parents: 18181 diff changeset	1342 return FALSE;
b505739970e7 dsync: Crashfix to -F parameter handling. Timo Sirainen <tss@iki.fi> parents: 18181 diff changeset	1343
18181 35e4a6ae8d85 dsync: Added -F parameter to sync only mails with[out] specific flag. Timo Sirainen <tss@iki.fi> parents: 18180 diff changeset	1344 array_foreach(&change->keyword_changes, strp) {
35e4a6ae8d85 dsync: Added -F parameter to sync only mails with[out] specific flag. Timo Sirainen <tss@iki.fi> parents: 18180 diff changeset	1345 switch ((*strp)[0]) {
35e4a6ae8d85 dsync: Added -F parameter to sync only mails with[out] specific flag. Timo Sirainen <tss@iki.fi> parents: 18180 diff changeset	1346 case KEYWORD_CHANGE_FINAL:
35e4a6ae8d85 dsync: Added -F parameter to sync only mails with[out] specific flag. Timo Sirainen <tss@iki.fi> parents: 18180 diff changeset	1347 case KEYWORD_CHANGE_ADD_AND_FINAL:
35e4a6ae8d85 dsync: Added -F parameter to sync only mails with[out] specific flag. Timo Sirainen <tss@iki.fi> parents: 18180 diff changeset	1348 if (strcasecmp((*strp)+1, keyword) == 0)
35e4a6ae8d85 dsync: Added -F parameter to sync only mails with[out] specific flag. Timo Sirainen <tss@iki.fi> parents: 18180 diff changeset	1349 return TRUE;
35e4a6ae8d85 dsync: Added -F parameter to sync only mails with[out] specific flag. Timo Sirainen <tss@iki.fi> parents: 18180 diff changeset	1350 break;
35e4a6ae8d85 dsync: Added -F parameter to sync only mails with[out] specific flag. Timo Sirainen <tss@iki.fi> parents: 18180 diff changeset	1351 default:
35e4a6ae8d85 dsync: Added -F parameter to sync only mails with[out] specific flag. Timo Sirainen <tss@iki.fi> parents: 18180 diff changeset	1352 break;
35e4a6ae8d85 dsync: Added -F parameter to sync only mails with[out] specific flag. Timo Sirainen <tss@iki.fi> parents: 18180 diff changeset	1353 }
35e4a6ae8d85 dsync: Added -F parameter to sync only mails with[out] specific flag. Timo Sirainen <tss@iki.fi> parents: 18180 diff changeset	1354 }
35e4a6ae8d85 dsync: Added -F parameter to sync only mails with[out] specific flag. Timo Sirainen <tss@iki.fi> parents: 18180 diff changeset	1355 return FALSE;
35e4a6ae8d85 dsync: Added -F parameter to sync only mails with[out] specific flag. Timo Sirainen <tss@iki.fi> parents: 18180 diff changeset	1356 }
35e4a6ae8d85 dsync: Added -F parameter to sync only mails with[out] specific flag. Timo Sirainen <tss@iki.fi> parents: 18180 diff changeset	1357
35e4a6ae8d85 dsync: Added -F parameter to sync only mails with[out] specific flag. Timo Sirainen <tss@iki.fi> parents: 18180 diff changeset	1358 static bool
35e4a6ae8d85 dsync: Added -F parameter to sync only mails with[out] specific flag. Timo Sirainen <tss@iki.fi> parents: 18180 diff changeset	1359 dsync_mailbox_import_want_change(struct dsync_mailbox_importer *importer,
35e4a6ae8d85 dsync: Added -F parameter to sync only mails with[out] specific flag. Timo Sirainen <tss@iki.fi> parents: 18180 diff changeset	1360 const struct dsync_mail_change *change,
35e4a6ae8d85 dsync: Added -F parameter to sync only mails with[out] specific flag. Timo Sirainen <tss@iki.fi> parents: 18180 diff changeset	1361 const char **result_r)
35e4a6ae8d85 dsync: Added -F parameter to sync only mails with[out] specific flag. Timo Sirainen <tss@iki.fi> parents: 18180 diff changeset	1362 {
35e4a6ae8d85 dsync: Added -F parameter to sync only mails with[out] specific flag. Timo Sirainen <tss@iki.fi> parents: 18180 diff changeset	1363 if (importer->sync_since_timestamp > 0) {
35e4a6ae8d85 dsync: Added -F parameter to sync only mails with[out] specific flag. Timo Sirainen <tss@iki.fi> parents: 18180 diff changeset	1364 i_assert(change->received_timestamp > 0);
35e4a6ae8d85 dsync: Added -F parameter to sync only mails with[out] specific flag. Timo Sirainen <tss@iki.fi> parents: 18180 diff changeset	1365 if (change->received_timestamp < importer->sync_since_timestamp) {
35e4a6ae8d85 dsync: Added -F parameter to sync only mails with[out] specific flag. Timo Sirainen <tss@iki.fi> parents: 18180 diff changeset	1366 /* mail has too old timestamp - skip it */
35e4a6ae8d85 dsync: Added -F parameter to sync only mails with[out] specific flag. Timo Sirainen <tss@iki.fi> parents: 18180 diff changeset	1367 *result_r = "Ignoring missing local mail with too old timestamp";
35e4a6ae8d85 dsync: Added -F parameter to sync only mails with[out] specific flag. Timo Sirainen <tss@iki.fi> parents: 18180 diff changeset	1368 return FALSE;
35e4a6ae8d85 dsync: Added -F parameter to sync only mails with[out] specific flag. Timo Sirainen <tss@iki.fi> parents: 18180 diff changeset	1369 }
35e4a6ae8d85 dsync: Added -F parameter to sync only mails with[out] specific flag. Timo Sirainen <tss@iki.fi> parents: 18180 diff changeset	1370 }
20831 0c4e5c2725a3 doveadm-sync: Add end-date support Aki Tuomi <aki.tuomi@dovecot.fi> parents: 20633 diff changeset	1371 if (importer->sync_until_timestamp > 0) {
0c4e5c2725a3 doveadm-sync: Add end-date support Aki Tuomi <aki.tuomi@dovecot.fi> parents: 20633 diff changeset	1372 i_assert(change->received_timestamp > 0);
0c4e5c2725a3 doveadm-sync: Add end-date support Aki Tuomi <aki.tuomi@dovecot.fi> parents: 20633 diff changeset	1373 if (change->received_timestamp > importer->sync_until_timestamp) {
0c4e5c2725a3 doveadm-sync: Add end-date support Aki Tuomi <aki.tuomi@dovecot.fi> parents: 20633 diff changeset	1374 /* mail has too new timestamp - skip it */
0c4e5c2725a3 doveadm-sync: Add end-date support Aki Tuomi <aki.tuomi@dovecot.fi> parents: 20633 diff changeset	1375 *result_r = "Ignoring missing local mail with too new timestamp";
0c4e5c2725a3 doveadm-sync: Add end-date support Aki Tuomi <aki.tuomi@dovecot.fi> parents: 20633 diff changeset	1376 return FALSE;
0c4e5c2725a3 doveadm-sync: Add end-date support Aki Tuomi <aki.tuomi@dovecot.fi> parents: 20633 diff changeset	1377 }
0c4e5c2725a3 doveadm-sync: Add end-date support Aki Tuomi <aki.tuomi@dovecot.fi> parents: 20633 diff changeset	1378 }
20633 d54651ba988a doveadm sync/backup: Added -S <max size> parameter to skip too large mails. Timo Sirainen <timo.sirainen@dovecot.fi> parents: 20605 diff changeset	1379 if (importer->sync_max_size > 0) {
d54651ba988a doveadm sync/backup: Added -S <max size> parameter to skip too large mails. Timo Sirainen <timo.sirainen@dovecot.fi> parents: 20605 diff changeset	1380 i_assert(change->virtual_size != (uoff_t)-1);
d54651ba988a doveadm sync/backup: Added -S <max size> parameter to skip too large mails. Timo Sirainen <timo.sirainen@dovecot.fi> parents: 20605 diff changeset	1381 if (change->virtual_size < importer->sync_max_size) {
d54651ba988a doveadm sync/backup: Added -S <max size> parameter to skip too large mails. Timo Sirainen <timo.sirainen@dovecot.fi> parents: 20605 diff changeset	1382 /* mail is too large - skip it */
d54651ba988a doveadm sync/backup: Added -S <max size> parameter to skip too large mails. Timo Sirainen <timo.sirainen@dovecot.fi> parents: 20605 diff changeset	1383 *result_r = "Ignoring missing local mail with too large size";
d54651ba988a doveadm sync/backup: Added -S <max size> parameter to skip too large mails. Timo Sirainen <timo.sirainen@dovecot.fi> parents: 20605 diff changeset	1384 return FALSE;
d54651ba988a doveadm sync/backup: Added -S <max size> parameter to skip too large mails. Timo Sirainen <timo.sirainen@dovecot.fi> parents: 20605 diff changeset	1385 }
d54651ba988a doveadm sync/backup: Added -S <max size> parameter to skip too large mails. Timo Sirainen <timo.sirainen@dovecot.fi> parents: 20605 diff changeset	1386 }
18181 35e4a6ae8d85 dsync: Added -F parameter to sync only mails with[out] specific flag. Timo Sirainen <tss@iki.fi> parents: 18180 diff changeset	1387 if (importer->sync_flag != 0) {
35e4a6ae8d85 dsync: Added -F parameter to sync only mails with[out] specific flag. Timo Sirainen <tss@iki.fi> parents: 18180 diff changeset	1388 bool have_flag = (change->final_flags & importer->sync_flag) != 0;
35e4a6ae8d85 dsync: Added -F parameter to sync only mails with[out] specific flag. Timo Sirainen <tss@iki.fi> parents: 18180 diff changeset	1389
35e4a6ae8d85 dsync: Added -F parameter to sync only mails with[out] specific flag. Timo Sirainen <tss@iki.fi> parents: 18180 diff changeset	1390 if (have_flag && importer->sync_flag_dontwant) {
35e4a6ae8d85 dsync: Added -F parameter to sync only mails with[out] specific flag. Timo Sirainen <tss@iki.fi> parents: 18180 diff changeset	1391 *result_r = "Ignoring missing local mail that doesn't have wanted flags";
35e4a6ae8d85 dsync: Added -F parameter to sync only mails with[out] specific flag. Timo Sirainen <tss@iki.fi> parents: 18180 diff changeset	1392 return FALSE;
35e4a6ae8d85 dsync: Added -F parameter to sync only mails with[out] specific flag. Timo Sirainen <tss@iki.fi> parents: 18180 diff changeset	1393 }
35e4a6ae8d85 dsync: Added -F parameter to sync only mails with[out] specific flag. Timo Sirainen <tss@iki.fi> parents: 18180 diff changeset	1394 if (!have_flag && !importer->sync_flag_dontwant) {
35e4a6ae8d85 dsync: Added -F parameter to sync only mails with[out] specific flag. Timo Sirainen <tss@iki.fi> parents: 18180 diff changeset	1395 *result_r = "Ignoring missing local mail that has unwanted flags";
35e4a6ae8d85 dsync: Added -F parameter to sync only mails with[out] specific flag. Timo Sirainen <tss@iki.fi> parents: 18180 diff changeset	1396 return FALSE;
35e4a6ae8d85 dsync: Added -F parameter to sync only mails with[out] specific flag. Timo Sirainen <tss@iki.fi> parents: 18180 diff changeset	1397 }
35e4a6ae8d85 dsync: Added -F parameter to sync only mails with[out] specific flag. Timo Sirainen <tss@iki.fi> parents: 18180 diff changeset	1398 }
35e4a6ae8d85 dsync: Added -F parameter to sync only mails with[out] specific flag. Timo Sirainen <tss@iki.fi> parents: 18180 diff changeset	1399 if (importer->sync_keyword != NULL) {
35e4a6ae8d85 dsync: Added -F parameter to sync only mails with[out] specific flag. Timo Sirainen <tss@iki.fi> parents: 18180 diff changeset	1400 bool have_kw = dsync_mail_change_have_keyword(change, importer->sync_keyword);
35e4a6ae8d85 dsync: Added -F parameter to sync only mails with[out] specific flag. Timo Sirainen <tss@iki.fi> parents: 18180 diff changeset	1401
35e4a6ae8d85 dsync: Added -F parameter to sync only mails with[out] specific flag. Timo Sirainen <tss@iki.fi> parents: 18180 diff changeset	1402 if (have_kw && importer->sync_flag_dontwant) {
35e4a6ae8d85 dsync: Added -F parameter to sync only mails with[out] specific flag. Timo Sirainen <tss@iki.fi> parents: 18180 diff changeset	1403 *result_r = "Ignoring missing local mail that doesn't have wanted keywords";
35e4a6ae8d85 dsync: Added -F parameter to sync only mails with[out] specific flag. Timo Sirainen <tss@iki.fi> parents: 18180 diff changeset	1404 return FALSE;
35e4a6ae8d85 dsync: Added -F parameter to sync only mails with[out] specific flag. Timo Sirainen <tss@iki.fi> parents: 18180 diff changeset	1405 }
35e4a6ae8d85 dsync: Added -F parameter to sync only mails with[out] specific flag. Timo Sirainen <tss@iki.fi> parents: 18180 diff changeset	1406 if (!have_kw && !importer->sync_flag_dontwant) {
35e4a6ae8d85 dsync: Added -F parameter to sync only mails with[out] specific flag. Timo Sirainen <tss@iki.fi> parents: 18180 diff changeset	1407 *result_r = "Ignoring missing local mail that has unwanted keywords";
35e4a6ae8d85 dsync: Added -F parameter to sync only mails with[out] specific flag. Timo Sirainen <tss@iki.fi> parents: 18180 diff changeset	1408 return FALSE;
35e4a6ae8d85 dsync: Added -F parameter to sync only mails with[out] specific flag. Timo Sirainen <tss@iki.fi> parents: 18180 diff changeset	1409 }
35e4a6ae8d85 dsync: Added -F parameter to sync only mails with[out] specific flag. Timo Sirainen <tss@iki.fi> parents: 18180 diff changeset	1410 }
35e4a6ae8d85 dsync: Added -F parameter to sync only mails with[out] specific flag. Timo Sirainen <tss@iki.fi> parents: 18180 diff changeset	1411 return TRUE;
35e4a6ae8d85 dsync: Added -F parameter to sync only mails with[out] specific flag. Timo Sirainen <tss@iki.fi> parents: 18180 diff changeset	1412 }
35e4a6ae8d85 dsync: Added -F parameter to sync only mails with[out] specific flag. Timo Sirainen <tss@iki.fi> parents: 18180 diff changeset	1413
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1414 static void
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1415 dsync_mailbox_import_save(struct dsync_mailbox_importer *importer,
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1416 const struct dsync_mail_change *change)
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1417 {
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1418 struct dsync_mail_change *save;
18181 35e4a6ae8d85 dsync: Added -F parameter to sync only mails with[out] specific flag. Timo Sirainen <tss@iki.fi> parents: 18180 diff changeset	1419 const char *result;
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1420
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1421 i_assert(change->guid != NULL);
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1422
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1423 if (change->uid == importer->last_common_uid) {
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1424 /* we've already verified that the GUID matches.
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1425 apply flag changes if there are any. */
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1426 i_assert(!importer->last_common_uid_found);
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1427 dsync_mailbox_import_flag_change(importer, change);
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1428 return;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1429 }
18181 35e4a6ae8d85 dsync: Added -F parameter to sync only mails with[out] specific flag. Timo Sirainen <tss@iki.fi> parents: 18180 diff changeset	1430 if (!dsync_mailbox_import_want_change(importer, change, &result))
35e4a6ae8d85 dsync: Added -F parameter to sync only mails with[out] specific flag. Timo Sirainen <tss@iki.fi> parents: 18180 diff changeset	1431 return;
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1432
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1433 save = p_new(importer->pool, struct dsync_mail_change, 1);
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1434 dsync_mail_change_dup(importer->pool, change, save);
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1435
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1436 if (importer->last_common_uid_found) {
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1437 /* this is a new mail. its UID may or may not conflict with
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1438 an existing local mail, we'll figure it out later. */
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1439 i_assert(change->uid > importer->last_common_uid);
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1440 dsync_mailbox_save(importer, save);
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1441 } else {
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1442 /* the local mail is expunged. we'll decide later if we want
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1443 to save this mail locally or expunge it form remote. */
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1444 i_assert(change->uid > importer->last_common_uid);
15772 360661d99c42 dsync: Various importer fixes. Timo Sirainen <tss@iki.fi> parents: 15770 diff changeset	1445 i_assert(importer->cur_mail == NULL \|\|
360661d99c42 dsync: Various importer fixes. Timo Sirainen <tss@iki.fi> parents: 15770 diff changeset	1446 change->uid < importer->cur_mail->uid);
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1447 array_append(&importer->maybe_saves, &save, 1);
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1448 }
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1449 }
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1450
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1451 static void
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1452 dsync_mailbox_import_expunge(struct dsync_mailbox_importer *importer,
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1453 const struct dsync_mail_change *change)
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1454 {
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1455
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1456 if (importer->last_common_uid_found) {
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1457 /* expunge the message, unless its GUID unexpectedly doesn't
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1458 match */
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1459 i_assert(change->uid <= importer->last_common_uid);
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1460 if (dsync_import_set_mail(importer, change))
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1461 mail_expunge(importer->mail);
15591 a7be0c9a00df dsync: Crashfix for handling expunges from end of mailbox. Timo Sirainen <tss@iki.fi> parents: 15590 diff changeset	1462 } else if (importer->cur_mail == NULL \|\|
a7be0c9a00df dsync: Crashfix for handling expunges from end of mailbox. Timo Sirainen <tss@iki.fi> parents: 15590 diff changeset	1463 change->uid < importer->cur_mail->uid) {
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1464 /* already expunged locally, we can ignore this.
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1465 uid=last_common_uid if we managed to verify from
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1466 transaction log that the GUIDs match */
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1467 i_assert(change->uid >= importer->last_common_uid);
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1468 } else if (change->uid == importer->last_common_uid) {
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1469 /* already verified that the GUID matches */
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1470 i_assert(importer->cur_mail->uid == change->uid);
15657 f0dbe8fc8905 dsync: Verify messages' GUIDs better. Timo Sirainen <tss@iki.fi> parents: 15619 diff changeset	1471 if (dsync_check_cur_guid(importer, change))
f0dbe8fc8905 dsync: Verify messages' GUIDs better. Timo Sirainen <tss@iki.fi> parents: 15619 diff changeset	1472 mail_expunge(importer->cur_mail);
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1473 } else {
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1474 /* we don't know yet if we should expunge this
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1475 message or not. queue it until we do. */
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1476 i_assert(change->uid > importer->last_common_uid);
14676 69ba6977bed8 seq_range_array_add() API changed. Added other functions to provide the less common use cases. Timo Sirainen <tss@iki.fi> parents: 14675 diff changeset	1477 seq_range_array_add(&importer->maybe_expunge_uids, change->uid);
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1478 }
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1479 }
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1480
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1481 static void
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1482 dsync_mailbox_rewind_search(struct dsync_mailbox_importer *importer)
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1483 {
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1484 /* If there are local mails after last_common_uid which we skipped
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1485 while trying to match the next message, we need to now go back */
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1486 if (importer->cur_mail != NULL &&
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1487 importer->cur_mail->uid <= importer->last_common_uid+1)
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1488 return;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1489
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1490 importer->cur_mail = NULL;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1491 importer->cur_guid = NULL;
15753 f58e7b386c6e dsync: Fixes for syncing without GUIDs. Timo Sirainen <tss@iki.fi> parents: 15743 diff changeset	1492 importer->cur_hdr_hash = NULL;
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1493 importer->next_local_seq = 0;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1494
14682 d0d7b810646b Make sure we check all the functions' return values. Minor API changes to simplify this. Timo Sirainen <tss@iki.fi> parents: 14676 diff changeset	1495 (void)mailbox_search_deinit(&importer->search_ctx);
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1496 dsync_mailbox_import_search_init(importer);
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1497 }
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1498
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1499 static void
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1500 dsync_mailbox_common_uid_found(struct dsync_mailbox_importer *importer)
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1501 {
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1502 struct dsync_mail_change const saves;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1503 struct seq_range_iter iter;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1504 unsigned int n, i, count;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1505 uint32_t uid;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1506
17861 0b4352d6dfdb dsync: Added more debug logging. Timo Sirainen <tss@iki.fi> parents: 17860 diff changeset	1507 if (importer->debug) T_BEGIN {
0b4352d6dfdb dsync: Added more debug logging. Timo Sirainen <tss@iki.fi> parents: 17860 diff changeset	1508 string_t *expunges = t_str_new(64);
0b4352d6dfdb dsync: Added more debug logging. Timo Sirainen <tss@iki.fi> parents: 17860 diff changeset	1509
0b4352d6dfdb dsync: Added more debug logging. Timo Sirainen <tss@iki.fi> parents: 17860 diff changeset	1510 imap_write_seq_range(expunges, &importer->maybe_expunge_uids);
0b4352d6dfdb dsync: Added more debug logging. Timo Sirainen <tss@iki.fi> parents: 17860 diff changeset	1511 imp_debug(importer, "Last common UID=%u. Delayed expunges=%s",
0b4352d6dfdb dsync: Added more debug logging. Timo Sirainen <tss@iki.fi> parents: 17860 diff changeset	1512 importer->last_common_uid, str_c(expunges));
0b4352d6dfdb dsync: Added more debug logging. Timo Sirainen <tss@iki.fi> parents: 17860 diff changeset	1513 } T_END;
16792 4a969a2e9399 dsync: Added some more debug logging. Timo Sirainen <tss@iki.fi> parents: 16762 diff changeset	1514
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1515 importer->last_common_uid_found = TRUE;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1516 dsync_mailbox_rewind_search(importer);
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1517
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1518 /* expunge the messages whose expunge-decision we delayed previously */
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1519 seq_range_array_iter_init(&iter, &importer->maybe_expunge_uids); n = 0;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1520 while (seq_range_array_iter_nth(&iter, n++, &uid)) {
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1521 if (uid > importer->last_common_uid) {
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1522 /* we expunge messages only up to last_common_uid,
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1523 ignore the rest */
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1524 break;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1525 }
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1526
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1527 if (mail_set_uid(importer->mail, uid))
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1528 mail_expunge(importer->mail);
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1529 }
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1530
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1531 /* handle pending saves */
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1532 saves = array_get(&importer->maybe_saves, &count);
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1533 for (i = 0; i < count; i++) {
17861 0b4352d6dfdb dsync: Added more debug logging. Timo Sirainen <tss@iki.fi> parents: 17860 diff changeset	1534 if (saves[i]->uid > importer->last_common_uid) {
0b4352d6dfdb dsync: Added more debug logging. Timo Sirainen <tss@iki.fi> parents: 17860 diff changeset	1535 imp_debug(importer, "Delayed save UID=%u: Save",
0b4352d6dfdb dsync: Added more debug logging. Timo Sirainen <tss@iki.fi> parents: 17860 diff changeset	1536 saves[i]->uid);
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1537 dsync_mailbox_save(importer, saves[i]);
17861 0b4352d6dfdb dsync: Added more debug logging. Timo Sirainen <tss@iki.fi> parents: 17860 diff changeset	1538 } else {
0b4352d6dfdb dsync: Added more debug logging. Timo Sirainen <tss@iki.fi> parents: 17860 diff changeset	1539 imp_debug(importer, "Delayed save UID=%u: Ignore",
0b4352d6dfdb dsync: Added more debug logging. Timo Sirainen <tss@iki.fi> parents: 17860 diff changeset	1540 saves[i]->uid);
0b4352d6dfdb dsync: Added more debug logging. Timo Sirainen <tss@iki.fi> parents: 17860 diff changeset	1541 }
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1542 }
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1543 }
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1544
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1545 static int
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1546 dsync_mailbox_import_match_msg(struct dsync_mailbox_importer *importer,
17862 c3ef6f19d518 dsync: Improved debug logging. Timo Sirainen <tss@iki.fi> parents: 17861 diff changeset	1547 const struct dsync_mail_change *change,
c3ef6f19d518 dsync: Improved debug logging. Timo Sirainen <tss@iki.fi> parents: 17861 diff changeset	1548 const char **result_r)
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1549 {
17862 c3ef6f19d518 dsync: Improved debug logging. Timo Sirainen <tss@iki.fi> parents: 17861 diff changeset	1550 const char hdr_hash, cmp_guid;
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1551
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1552 if (change->guid != '\0' && importer->cur_guid != '\0') {
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1553 /* we have GUIDs, verify them */
16977 8a7ecddca4ad dsync: Support syncing storages with 128bit GUIDs <-> string GUIDs. Timo Sirainen <tss@iki.fi> parents: 16940 diff changeset	1554 if (dsync_mail_change_guid_equals(importer, change,
17862 c3ef6f19d518 dsync: Improved debug logging. Timo Sirainen <tss@iki.fi> parents: 17861 diff changeset	1555 importer->cur_guid, &cmp_guid)) {
c3ef6f19d518 dsync: Improved debug logging. Timo Sirainen <tss@iki.fi> parents: 17861 diff changeset	1556 *result_r = "GUIDs match";
15819 9c095a50ba3e dsync: Improved unexpected error logging. Timo Sirainen <tss@iki.fi> parents: 15781 diff changeset	1557 return 1;
17862 c3ef6f19d518 dsync: Improved debug logging. Timo Sirainen <tss@iki.fi> parents: 17861 diff changeset	1558 } else {
c3ef6f19d518 dsync: Improved debug logging. Timo Sirainen <tss@iki.fi> parents: 17861 diff changeset	1559 *result_r = t_strdup_printf("GUIDs don't match (%s vs %s)",
17864 01ba94a2d4c5 dsync: Debug message fix Timo Sirainen <tss@iki.fi> parents: 17862 diff changeset	1560 importer->cur_guid, cmp_guid);
15819 9c095a50ba3e dsync: Improved unexpected error logging. Timo Sirainen <tss@iki.fi> parents: 15781 diff changeset	1561 return 0;
17862 c3ef6f19d518 dsync: Improved debug logging. Timo Sirainen <tss@iki.fi> parents: 17861 diff changeset	1562 }
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1563 }
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1564
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1565 /* verify hdr_hash if it exists */
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1566 if (change->hdr_hash == NULL) {
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1567 i_assert(*importer->cur_guid == '\0');
16556 af8ce0a84bb5 dsync: Fixed handling expunges when GUIDs aren't supported by the backend(s). Timo Sirainen <tss@iki.fi> parents: 16539 diff changeset	1568 if (change->type == DSYNC_MAIL_CHANGE_TYPE_EXPUNGE) {
af8ce0a84bb5 dsync: Fixed handling expunges when GUIDs aren't supported by the backend(s). Timo Sirainen <tss@iki.fi> parents: 16539 diff changeset	1569 /* the message was already expunged, so we don't know
af8ce0a84bb5 dsync: Fixed handling expunges when GUIDs aren't supported by the backend(s). Timo Sirainen <tss@iki.fi> parents: 16539 diff changeset	1570 its header. return "unknown". */
17862 c3ef6f19d518 dsync: Improved debug logging. Timo Sirainen <tss@iki.fi> parents: 17861 diff changeset	1571 *result_r = "Unknown match for expunge";
16556 af8ce0a84bb5 dsync: Fixed handling expunges when GUIDs aren't supported by the backend(s). Timo Sirainen <tss@iki.fi> parents: 16539 diff changeset	1572 return -1;
af8ce0a84bb5 dsync: Fixed handling expunges when GUIDs aren't supported by the backend(s). Timo Sirainen <tss@iki.fi> parents: 16539 diff changeset	1573 }
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1574 i_error("Mailbox %s: GUIDs not supported, "
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1575 "sync with header hashes instead",
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1576 mailbox_get_vname(importer->box));
18371 b900b50085fc dsync: Use storage's mail_error to choose the doveadm exit code. Timo Sirainen <tss@iki.fi> parents: 18183 diff changeset	1577 importer->mail_error = MAIL_ERROR_TEMP;
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1578 importer->failed = TRUE;
17862 c3ef6f19d518 dsync: Improved debug logging. Timo Sirainen <tss@iki.fi> parents: 17861 diff changeset	1579 *result_r = "Error, invalid parameters";
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1580 return -1;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1581 }
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1582
19646 25f06710e671 dsync: When comparing headers' hashes to match messages, try to normalize the input. Timo Sirainen <timo.sirainen@dovecot.fi> parents: 19552 diff changeset	1583 if (dsync_mail_get_hdr_hash(importer->cur_mail,
22549 400ff84f109d dsync: Add hashed_headers setting Aki Tuomi <aki.tuomi@dovecot.fi> parents: 22264 diff changeset	1584 importer->hdr_hash_version,
400ff84f109d dsync: Add hashed_headers setting Aki Tuomi <aki.tuomi@dovecot.fi> parents: 22264 diff changeset	1585 importer->hashed_headers, &hdr_hash) < 0) {
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1586 dsync_mail_error(importer, importer->cur_mail, "hdr-stream");
17862 c3ef6f19d518 dsync: Improved debug logging. Timo Sirainen <tss@iki.fi> parents: 17861 diff changeset	1587 *result_r = "Error fetching header stream";
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1588 return -1;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1589 }
20605 b7616318f4d0 dsync: Fixed empty-header-workaround Timo Sirainen <timo.sirainen@dovecot.fi> parents: 20595 diff changeset	1590 if (importer->empty_hdr_workaround &&
b7616318f4d0 dsync: Fixed empty-header-workaround Timo Sirainen <timo.sirainen@dovecot.fi> parents: 20595 diff changeset	1591 (dsync_mail_hdr_hash_is_empty(change->hdr_hash) \|\|
b7616318f4d0 dsync: Fixed empty-header-workaround Timo Sirainen <timo.sirainen@dovecot.fi> parents: 20595 diff changeset	1592 dsync_mail_hdr_hash_is_empty(hdr_hash))) {
b7616318f4d0 dsync: Fixed empty-header-workaround Timo Sirainen <timo.sirainen@dovecot.fi> parents: 20595 diff changeset	1593 *result_r = "Empty headers found with workaround enabled - assuming a match";
b7616318f4d0 dsync: Fixed empty-header-workaround Timo Sirainen <timo.sirainen@dovecot.fi> parents: 20595 diff changeset	1594 return 1;
b7616318f4d0 dsync: Fixed empty-header-workaround Timo Sirainen <timo.sirainen@dovecot.fi> parents: 20595 diff changeset	1595 } else if (strcmp(change->hdr_hash, hdr_hash) == 0) {
17862 c3ef6f19d518 dsync: Improved debug logging. Timo Sirainen <tss@iki.fi> parents: 17861 diff changeset	1596 *result_r = "Headers hashes match";
c3ef6f19d518 dsync: Improved debug logging. Timo Sirainen <tss@iki.fi> parents: 17861 diff changeset	1597 return 1;
c3ef6f19d518 dsync: Improved debug logging. Timo Sirainen <tss@iki.fi> parents: 17861 diff changeset	1598 } else {
c3ef6f19d518 dsync: Improved debug logging. Timo Sirainen <tss@iki.fi> parents: 17861 diff changeset	1599 *result_r = t_strdup_printf("Headers hashes don't match (%s vs %s)",
c3ef6f19d518 dsync: Improved debug logging. Timo Sirainen <tss@iki.fi> parents: 17861 diff changeset	1600 change->hdr_hash, hdr_hash);
c3ef6f19d518 dsync: Improved debug logging. Timo Sirainen <tss@iki.fi> parents: 17861 diff changeset	1601 return 0;
c3ef6f19d518 dsync: Improved debug logging. Timo Sirainen <tss@iki.fi> parents: 17861 diff changeset	1602 }
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1603 }
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1604
15590 ea3c1bd6d434 dsync: Fixed syncing expunge for last mail in mailbox. Timo Sirainen <tss@iki.fi> parents: 15589 diff changeset	1605 static bool
ea3c1bd6d434 dsync: Fixed syncing expunge for last mail in mailbox. Timo Sirainen <tss@iki.fi> parents: 15589 diff changeset	1606 dsync_mailbox_find_common_expunged_uid(struct dsync_mailbox_importer *importer,
17890 bf086c6f6e4a dsync: Debug logging improvements and comment clarifications. Timo Sirainen <tss@iki.fi> parents: 17888 diff changeset	1607 const struct dsync_mail_change *change,
bf086c6f6e4a dsync: Debug logging improvements and comment clarifications. Timo Sirainen <tss@iki.fi> parents: 17888 diff changeset	1608 const char **result_r)
15590 ea3c1bd6d434 dsync: Fixed syncing expunge for last mail in mailbox. Timo Sirainen <tss@iki.fi> parents: 15589 diff changeset	1609 {
ea3c1bd6d434 dsync: Fixed syncing expunge for last mail in mailbox. Timo Sirainen <tss@iki.fi> parents: 15589 diff changeset	1610 const struct dsync_mail_change *local_change;
ea3c1bd6d434 dsync: Fixed syncing expunge for last mail in mailbox. Timo Sirainen <tss@iki.fi> parents: 15589 diff changeset	1611
ea3c1bd6d434 dsync: Fixed syncing expunge for last mail in mailbox. Timo Sirainen <tss@iki.fi> parents: 15589 diff changeset	1612 if (*change->guid == '\0') {
ea3c1bd6d434 dsync: Fixed syncing expunge for last mail in mailbox. Timo Sirainen <tss@iki.fi> parents: 15589 diff changeset	1613 /* remote doesn't support GUIDs, can't verify expunge */
17890 bf086c6f6e4a dsync: Debug logging improvements and comment clarifications. Timo Sirainen <tss@iki.fi> parents: 17888 diff changeset	1614 *result_r = "GUIDs not supported, can't verify expunge";
15590 ea3c1bd6d434 dsync: Fixed syncing expunge for last mail in mailbox. Timo Sirainen <tss@iki.fi> parents: 15589 diff changeset	1615 return FALSE;
ea3c1bd6d434 dsync: Fixed syncing expunge for last mail in mailbox. Timo Sirainen <tss@iki.fi> parents: 15589 diff changeset	1616 }
ea3c1bd6d434 dsync: Fixed syncing expunge for last mail in mailbox. Timo Sirainen <tss@iki.fi> parents: 15589 diff changeset	1617
ea3c1bd6d434 dsync: Fixed syncing expunge for last mail in mailbox. Timo Sirainen <tss@iki.fi> parents: 15589 diff changeset	1618 /* local message is expunged. see if we can find its GUID from
ea3c1bd6d434 dsync: Fixed syncing expunge for last mail in mailbox. Timo Sirainen <tss@iki.fi> parents: 15589 diff changeset	1619 transaction log and check if the GUIDs match. The GUID in
ea3c1bd6d434 dsync: Fixed syncing expunge for last mail in mailbox. Timo Sirainen <tss@iki.fi> parents: 15589 diff changeset	1620 log is a 128bit GUID, so we may need to convert the remote's
ea3c1bd6d434 dsync: Fixed syncing expunge for last mail in mailbox. Timo Sirainen <tss@iki.fi> parents: 15589 diff changeset	1621 GUID string to 128bit GUID first. */
ea3c1bd6d434 dsync: Fixed syncing expunge for last mail in mailbox. Timo Sirainen <tss@iki.fi> parents: 15589 diff changeset	1622 local_change = hash_table_lookup(importer->local_changes,
ea3c1bd6d434 dsync: Fixed syncing expunge for last mail in mailbox. Timo Sirainen <tss@iki.fi> parents: 15589 diff changeset	1623 POINTER_CAST(change->uid));
17890 bf086c6f6e4a dsync: Debug logging improvements and comment clarifications. Timo Sirainen <tss@iki.fi> parents: 17888 diff changeset	1624 if (local_change == NULL \|\| local_change->guid == NULL) {
bf086c6f6e4a dsync: Debug logging improvements and comment clarifications. Timo Sirainen <tss@iki.fi> parents: 17888 diff changeset	1625 *result_r = "Expunged local mail's GUID not found";
15590 ea3c1bd6d434 dsync: Fixed syncing expunge for last mail in mailbox. Timo Sirainen <tss@iki.fi> parents: 15589 diff changeset	1626 return FALSE;
17890 bf086c6f6e4a dsync: Debug logging improvements and comment clarifications. Timo Sirainen <tss@iki.fi> parents: 17888 diff changeset	1627 }
15590 ea3c1bd6d434 dsync: Fixed syncing expunge for last mail in mailbox. Timo Sirainen <tss@iki.fi> parents: 15589 diff changeset	1628
15667 1b18ff11effc dsync: Fixes to handling storage formats that don't use 128bit GUIDs. Timo Sirainen <tss@iki.fi> parents: 15664 diff changeset	1629 i_assert(local_change->type == DSYNC_MAIL_CHANGE_TYPE_EXPUNGE);
16977 8a7ecddca4ad dsync: Support syncing storages with 128bit GUIDs <-> string GUIDs. Timo Sirainen <tss@iki.fi> parents: 16940 diff changeset	1630 if (dsync_mail_change_guid_equals(importer, local_change,
17890 bf086c6f6e4a dsync: Debug logging improvements and comment clarifications. Timo Sirainen <tss@iki.fi> parents: 17888 diff changeset	1631 change->guid, NULL)) {
15779 4e90a6911b26 dsync: Fixed last-common-uid lookup when local UID was expunged but remote wasn't. Timo Sirainen <tss@iki.fi> parents: 15778 diff changeset	1632 importer->last_common_uid = change->uid;
17890 bf086c6f6e4a dsync: Debug logging improvements and comment clarifications. Timo Sirainen <tss@iki.fi> parents: 17888 diff changeset	1633 *result_r = "Expunged local mail's GUID matches remote";
bf086c6f6e4a dsync: Debug logging improvements and comment clarifications. Timo Sirainen <tss@iki.fi> parents: 17888 diff changeset	1634 } else if (change->type != DSYNC_MAIL_CHANGE_TYPE_EXPUNGE) {
15779 4e90a6911b26 dsync: Fixed last-common-uid lookup when local UID was expunged but remote wasn't. Timo Sirainen <tss@iki.fi> parents: 15778 diff changeset	1635 dsync_mailbox_common_uid_found(importer);
17890 bf086c6f6e4a dsync: Debug logging improvements and comment clarifications. Timo Sirainen <tss@iki.fi> parents: 17888 diff changeset	1636 *result_r = "Expunged local mail's GUID doesn't match remote GUID";
bf086c6f6e4a dsync: Debug logging improvements and comment clarifications. Timo Sirainen <tss@iki.fi> parents: 17888 diff changeset	1637 } else {
15694 1d88f01ba2aa dsync: Ignore GUID mismatch on expunged message pair. Timo Sirainen <tss@iki.fi> parents: 15676 diff changeset	1638 /* GUID mismatch for two expunged mails. dsync can't update
1d88f01ba2aa dsync: Ignore GUID mismatch on expunged message pair. Timo Sirainen <tss@iki.fi> parents: 15676 diff changeset	1639 GUIDs for already expunged messages, so we can't immediately
1d88f01ba2aa dsync: Ignore GUID mismatch on expunged message pair. Timo Sirainen <tss@iki.fi> parents: 15676 diff changeset	1640 determine that the rest of the messages are a mismatch. so
1d88f01ba2aa dsync: Ignore GUID mismatch on expunged message pair. Timo Sirainen <tss@iki.fi> parents: 15676 diff changeset	1641 for now we'll just skip over this pair. */
17890 bf086c6f6e4a dsync: Debug logging improvements and comment clarifications. Timo Sirainen <tss@iki.fi> parents: 17888 diff changeset	1642 *result_r = "Expunged mails' GUIDs don't match - delaying decision";
bf086c6f6e4a dsync: Debug logging improvements and comment clarifications. Timo Sirainen <tss@iki.fi> parents: 17888 diff changeset	1643 /* NOTE: the return value here doesn't matter, because the only
bf086c6f6e4a dsync: Debug logging improvements and comment clarifications. Timo Sirainen <tss@iki.fi> parents: 17888 diff changeset	1644 caller that checks for it never reaches this code path */
15590 ea3c1bd6d434 dsync: Fixed syncing expunge for last mail in mailbox. Timo Sirainen <tss@iki.fi> parents: 15589 diff changeset	1645 }
ea3c1bd6d434 dsync: Fixed syncing expunge for last mail in mailbox. Timo Sirainen <tss@iki.fi> parents: 15589 diff changeset	1646 return TRUE;
ea3c1bd6d434 dsync: Fixed syncing expunge for last mail in mailbox. Timo Sirainen <tss@iki.fi> parents: 15589 diff changeset	1647 }
ea3c1bd6d434 dsync: Fixed syncing expunge for last mail in mailbox. Timo Sirainen <tss@iki.fi> parents: 15589 diff changeset	1648
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1649 static void
17859 c02918434721 doveadm backup: Locally expunged mails weren't always added back. Timo Sirainen <tss@iki.fi> parents: 17845 diff changeset	1650 dsync_mailbox_revert_missing(struct dsync_mailbox_importer *importer,
c02918434721 doveadm backup: Locally expunged mails weren't always added back. Timo Sirainen <tss@iki.fi> parents: 17845 diff changeset	1651 const struct dsync_mail_change *change)
c02918434721 doveadm backup: Locally expunged mails weren't always added back. Timo Sirainen <tss@iki.fi> parents: 17845 diff changeset	1652 {
c02918434721 doveadm backup: Locally expunged mails weren't always added back. Timo Sirainen <tss@iki.fi> parents: 17845 diff changeset	1653 i_assert(importer->revert_local_changes);
c02918434721 doveadm backup: Locally expunged mails weren't always added back. Timo Sirainen <tss@iki.fi> parents: 17845 diff changeset	1654
c02918434721 doveadm backup: Locally expunged mails weren't always added back. Timo Sirainen <tss@iki.fi> parents: 17845 diff changeset	1655 /* mail exists on remote, but not locally. we'll need to
c02918434721 doveadm backup: Locally expunged mails weren't always added back. Timo Sirainen <tss@iki.fi> parents: 17845 diff changeset	1656 insert this mail back, which means deleting the whole
c02918434721 doveadm backup: Locally expunged mails weren't always added back. Timo Sirainen <tss@iki.fi> parents: 17845 diff changeset	1657 mailbox and resyncing. */
17886 d7ac3056ed7c doveadm backup: When deleting a mailbox, log a warning, not just a debug message. Timo Sirainen <tss@iki.fi> parents: 17864 diff changeset	1658 i_warning("Deleting mailbox '%s': UID=%u GUID=%s is missing locally",
17859 c02918434721 doveadm backup: Locally expunged mails weren't always added back. Timo Sirainen <tss@iki.fi> parents: 17845 diff changeset	1659 mailbox_get_vname(importer->box),
c02918434721 doveadm backup: Locally expunged mails weren't always added back. Timo Sirainen <tss@iki.fi> parents: 17845 diff changeset	1660 change->uid, change->guid);
c02918434721 doveadm backup: Locally expunged mails weren't always added back. Timo Sirainen <tss@iki.fi> parents: 17845 diff changeset	1661 importer->delete_mailbox = TRUE;
18371 b900b50085fc dsync: Use storage's mail_error to choose the doveadm exit code. Timo Sirainen <tss@iki.fi> parents: 18183 diff changeset	1662 importer->mail_error = MAIL_ERROR_TEMP;
17859 c02918434721 doveadm backup: Locally expunged mails weren't always added back. Timo Sirainen <tss@iki.fi> parents: 17845 diff changeset	1663 importer->failed = TRUE;
c02918434721 doveadm backup: Locally expunged mails weren't always added back. Timo Sirainen <tss@iki.fi> parents: 17845 diff changeset	1664 }
c02918434721 doveadm backup: Locally expunged mails weren't always added back. Timo Sirainen <tss@iki.fi> parents: 17845 diff changeset	1665
c02918434721 doveadm backup: Locally expunged mails weren't always added back. Timo Sirainen <tss@iki.fi> parents: 17845 diff changeset	1666 static void
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1667 dsync_mailbox_find_common_uid(struct dsync_mailbox_importer *importer,
17861 0b4352d6dfdb dsync: Added more debug logging. Timo Sirainen <tss@iki.fi> parents: 17860 diff changeset	1668 const struct dsync_mail_change *change,
0b4352d6dfdb dsync: Added more debug logging. Timo Sirainen <tss@iki.fi> parents: 17860 diff changeset	1669 const char **result_r)
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1670 {
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1671 int ret;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1672
20633 d54651ba988a doveadm sync/backup: Added -S <max size> parameter to skip too large mails. Timo Sirainen <timo.sirainen@dovecot.fi> parents: 20605 diff changeset	1673 i_assert(change->type == DSYNC_MAIL_CHANGE_TYPE_EXPUNGE \|\|
20831 0c4e5c2725a3 doveadm-sync: Add end-date support Aki Tuomi <aki.tuomi@dovecot.fi> parents: 20633 diff changeset	1674 ((change->received_timestamp > 0 \|\|
0c4e5c2725a3 doveadm-sync: Add end-date support Aki Tuomi <aki.tuomi@dovecot.fi> parents: 20633 diff changeset	1675 (importer->sync_since_timestamp == 0 &&
0c4e5c2725a3 doveadm-sync: Add end-date support Aki Tuomi <aki.tuomi@dovecot.fi> parents: 20633 diff changeset	1676 importer->sync_until_timestamp == 0)) &&
20633 d54651ba988a doveadm sync/backup: Added -S <max size> parameter to skip too large mails. Timo Sirainen <timo.sirainen@dovecot.fi> parents: 20605 diff changeset	1677 (change->virtual_size != (uoff_t)-1 \|\| importer->sync_max_size == 0)));
18177 f393f63764e0 dsync: Added -t <timestamp> parameter to save only mails newer than <timestamp> Timo Sirainen <tss@iki.fi> parents: 18137 diff changeset	1678
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1679 /* try to find the matching local mail */
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1680 if (!importer_next_mail(importer, change->uid)) {
15590 ea3c1bd6d434 dsync: Fixed syncing expunge for last mail in mailbox. Timo Sirainen <tss@iki.fi> parents: 15589 diff changeset	1681 /* no more local mails. we can still try to match
ea3c1bd6d434 dsync: Fixed syncing expunge for last mail in mailbox. Timo Sirainen <tss@iki.fi> parents: 15589 diff changeset	1682 expunged mails though. */
15619 b27245a6adde dsync: Another fix to handling expunges from the end of mailbox. Timo Sirainen <tss@iki.fi> parents: 15618 diff changeset	1683 if (change->type == DSYNC_MAIL_CHANGE_TYPE_EXPUNGE) {
b27245a6adde dsync: Another fix to handling expunges from the end of mailbox. Timo Sirainen <tss@iki.fi> parents: 15618 diff changeset	1684 /* mail doesn't exist remotely either, don't bother
b27245a6adde dsync: Another fix to handling expunges from the end of mailbox. Timo Sirainen <tss@iki.fi> parents: 15618 diff changeset	1685 looking it up locally. */
17861 0b4352d6dfdb dsync: Added more debug logging. Timo Sirainen <tss@iki.fi> parents: 17860 diff changeset	1686 *result_r = "Expunged mail not found locally";
15619 b27245a6adde dsync: Another fix to handling expunges from the end of mailbox. Timo Sirainen <tss@iki.fi> parents: 15618 diff changeset	1687 return;
b27245a6adde dsync: Another fix to handling expunges from the end of mailbox. Timo Sirainen <tss@iki.fi> parents: 15618 diff changeset	1688 }
17890 bf086c6f6e4a dsync: Debug logging improvements and comment clarifications. Timo Sirainen <tss@iki.fi> parents: 17888 diff changeset	1689 i_assert(change->guid != NULL);
18181 35e4a6ae8d85 dsync: Added -F parameter to sync only mails with[out] specific flag. Timo Sirainen <tss@iki.fi> parents: 18180 diff changeset	1690 if (!dsync_mailbox_import_want_change(importer, change, result_r))
35e4a6ae8d85 dsync: Added -F parameter to sync only mails with[out] specific flag. Timo Sirainen <tss@iki.fi> parents: 18180 diff changeset	1691 ;
35e4a6ae8d85 dsync: Added -F parameter to sync only mails with[out] specific flag. Timo Sirainen <tss@iki.fi> parents: 18180 diff changeset	1692 else if (importer->local_uid_next <= change->uid) {
17890 bf086c6f6e4a dsync: Debug logging improvements and comment clarifications. Timo Sirainen <tss@iki.fi> parents: 17888 diff changeset	1693 dsync_mailbox_common_uid_found(importer);
bf086c6f6e4a dsync: Debug logging improvements and comment clarifications. Timo Sirainen <tss@iki.fi> parents: 17888 diff changeset	1694 *result_r = "Mail's UID is above local UIDNEXT";
bf086c6f6e4a dsync: Debug logging improvements and comment clarifications. Timo Sirainen <tss@iki.fi> parents: 17888 diff changeset	1695 } else if (importer->revert_local_changes) {
17859 c02918434721 doveadm backup: Locally expunged mails weren't always added back. Timo Sirainen <tss@iki.fi> parents: 17845 diff changeset	1696 dsync_mailbox_revert_missing(importer, change);
17861 0b4352d6dfdb dsync: Added more debug logging. Timo Sirainen <tss@iki.fi> parents: 17860 diff changeset	1697 *result_r = "Reverting local change by deleting mailbox";
17890 bf086c6f6e4a dsync: Debug logging improvements and comment clarifications. Timo Sirainen <tss@iki.fi> parents: 17888 diff changeset	1698 } else if (!dsync_mailbox_find_common_expunged_uid(importer, change, result_r)) {
bf086c6f6e4a dsync: Debug logging improvements and comment clarifications. Timo Sirainen <tss@iki.fi> parents: 17888 diff changeset	1699 /* it's unknown if this mail existed locally and was
bf086c6f6e4a dsync: Debug logging improvements and comment clarifications. Timo Sirainen <tss@iki.fi> parents: 17888 diff changeset	1700 expunged. since we don't want to lose any mails,
bf086c6f6e4a dsync: Debug logging improvements and comment clarifications. Timo Sirainen <tss@iki.fi> parents: 17888 diff changeset	1701 assume that we need to preserve the mail. use the
bf086c6f6e4a dsync: Debug logging improvements and comment clarifications. Timo Sirainen <tss@iki.fi> parents: 17888 diff changeset	1702 last message with a matching GUID as the last common
15590 ea3c1bd6d434 dsync: Fixed syncing expunge for last mail in mailbox. Timo Sirainen <tss@iki.fi> parents: 15589 diff changeset	1703 UID. */
ea3c1bd6d434 dsync: Fixed syncing expunge for last mail in mailbox. Timo Sirainen <tss@iki.fi> parents: 15589 diff changeset	1704 dsync_mailbox_common_uid_found(importer);
ea3c1bd6d434 dsync: Fixed syncing expunge for last mail in mailbox. Timo Sirainen <tss@iki.fi> parents: 15589 diff changeset	1705 }
17890 bf086c6f6e4a dsync: Debug logging improvements and comment clarifications. Timo Sirainen <tss@iki.fi> parents: 17888 diff changeset	1706 result_r = t_strdup_printf("%s - No more local mails found", result_r);
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1707 return;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1708 }
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1709
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1710 if (change->guid == NULL) {
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1711 /* we can't know if this UID matches */
17861 0b4352d6dfdb dsync: Added more debug logging. Timo Sirainen <tss@iki.fi> parents: 17860 diff changeset	1712 i_assert(change->type == DSYNC_MAIL_CHANGE_TYPE_EXPUNGE);
0b4352d6dfdb dsync: Added more debug logging. Timo Sirainen <tss@iki.fi> parents: 17860 diff changeset	1713 *result_r = "Expunged mail has no GUID, can't verify it";
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1714 return;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1715 }
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1716 if (importer->cur_mail->uid == change->uid) {
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1717 /* we have a matching local UID. check GUID to see if it's
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1718 really the same mail or not */
17862 c3ef6f19d518 dsync: Improved debug logging. Timo Sirainen <tss@iki.fi> parents: 17861 diff changeset	1719 if ((ret = dsync_mailbox_import_match_msg(importer, change, result_r)) < 0) {
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1720 /* unknown */
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1721 return;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1722 }
19660 7a3a138a4986 doveadm backup: Detect and handle conflicts earlier. Timo Sirainen <timo.sirainen@dovecot.fi> parents: 19646 diff changeset	1723 if (ret > 0) {
7a3a138a4986 doveadm backup: Detect and handle conflicts earlier. Timo Sirainen <timo.sirainen@dovecot.fi> parents: 19646 diff changeset	1724 importer->last_common_uid = change->uid;
7a3a138a4986 doveadm backup: Detect and handle conflicts earlier. Timo Sirainen <timo.sirainen@dovecot.fi> parents: 19646 diff changeset	1725 } else if (!importer->revert_local_changes) {
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1726 /* mismatch - found the first non-common UID */
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1727 dsync_mailbox_common_uid_found(importer);
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1728 } else {
19660 7a3a138a4986 doveadm backup: Detect and handle conflicts earlier. Timo Sirainen <timo.sirainen@dovecot.fi> parents: 19646 diff changeset	1729 /* mismatch and we want to revert local changes -
7a3a138a4986 doveadm backup: Detect and handle conflicts earlier. Timo Sirainen <timo.sirainen@dovecot.fi> parents: 19646 diff changeset	1730 need to delete the mailbox. */
7a3a138a4986 doveadm backup: Detect and handle conflicts earlier. Timo Sirainen <timo.sirainen@dovecot.fi> parents: 19646 diff changeset	1731 dsync_mailbox_revert_existing_uid(importer, change->uid, *result_r);
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1732 }
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1733 return;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1734 }
18177 f393f63764e0 dsync: Added -t <timestamp> parameter to save only mails newer than <timestamp> Timo Sirainen <tss@iki.fi> parents: 18137 diff changeset	1735 /* mail exists remotely, but doesn't exist locally. */
18181 35e4a6ae8d85 dsync: Added -F parameter to sync only mails with[out] specific flag. Timo Sirainen <tss@iki.fi> parents: 18180 diff changeset	1736 if (!dsync_mailbox_import_want_change(importer, change, result_r))
18177 f393f63764e0 dsync: Added -t <timestamp> parameter to save only mails newer than <timestamp> Timo Sirainen <tss@iki.fi> parents: 18137 diff changeset	1737 return;
17888 68e5eaecc8b4 doveadm backup: Don't unnecessarily delete mailbox when handling expunges Timo Sirainen <tss@iki.fi> parents: 17887 diff changeset	1738 if (importer->revert_local_changes &&
68e5eaecc8b4 doveadm backup: Don't unnecessarily delete mailbox when handling expunges Timo Sirainen <tss@iki.fi> parents: 17887 diff changeset	1739 change->type != DSYNC_MAIL_CHANGE_TYPE_EXPUNGE) {
17859 c02918434721 doveadm backup: Locally expunged mails weren't always added back. Timo Sirainen <tss@iki.fi> parents: 17845 diff changeset	1740 dsync_mailbox_revert_missing(importer, change);
17861 0b4352d6dfdb dsync: Added more debug logging. Timo Sirainen <tss@iki.fi> parents: 17860 diff changeset	1741 *result_r = "Reverting local change by deleting mailbox";
0b4352d6dfdb dsync: Added more debug logging. Timo Sirainen <tss@iki.fi> parents: 17860 diff changeset	1742 } else {
17890 bf086c6f6e4a dsync: Debug logging improvements and comment clarifications. Timo Sirainen <tss@iki.fi> parents: 17888 diff changeset	1743 (void)dsync_mailbox_find_common_expunged_uid(importer, change, result_r);
17861 0b4352d6dfdb dsync: Added more debug logging. Timo Sirainen <tss@iki.fi> parents: 17860 diff changeset	1744 }
19228 83ccad730ea3 dsync: Removed unnecessary code - cur_mail is never NULL at this point. Timo Sirainen <tss@iki.fi> parents: 19084 diff changeset	1745 *result_r = t_strdup_printf("%s (next local mail UID=%u)",
20262 1f4e2c3a8512 dsync: Fixed potential crash Timo Sirainen <timo.sirainen@dovecot.fi> parents: 20208 diff changeset	1746 *result_r, importer->cur_mail == NULL ? 0 : importer->cur_mail->uid);
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1747 }
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1748
15772 360661d99c42 dsync: Various importer fixes. Timo Sirainen <tss@iki.fi> parents: 15770 diff changeset	1749 int dsync_mailbox_import_change(struct dsync_mailbox_importer *importer,
360661d99c42 dsync: Various importer fixes. Timo Sirainen <tss@iki.fi> parents: 15770 diff changeset	1750 const struct dsync_mail_change *change)
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1751 {
17861 0b4352d6dfdb dsync: Added more debug logging. Timo Sirainen <tss@iki.fi> parents: 17860 diff changeset	1752 const char *result;
0b4352d6dfdb dsync: Added more debug logging. Timo Sirainen <tss@iki.fi> parents: 17860 diff changeset	1753
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1754 i_assert(!importer->new_uids_assigned);
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1755 i_assert(importer->prev_uid < change->uid);
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1756
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1757 importer->prev_uid = change->uid;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1758
15772 360661d99c42 dsync: Various importer fixes. Timo Sirainen <tss@iki.fi> parents: 15770 diff changeset	1759 if (importer->failed)
360661d99c42 dsync: Various importer fixes. Timo Sirainen <tss@iki.fi> parents: 15770 diff changeset	1760 return -1;
20074 7a29a0930b63 dsync: If state is invalid, exit with code 2 instead of tempfail. Timo Sirainen <timo.sirainen@dovecot.fi> parents: 20072 diff changeset	1761 if (importer->require_full_resync)
7a29a0930b63 dsync: If state is invalid, exit with code 2 instead of tempfail. Timo Sirainen <timo.sirainen@dovecot.fi> parents: 20072 diff changeset	1762 return 0;
15772 360661d99c42 dsync: Various importer fixes. Timo Sirainen <tss@iki.fi> parents: 15770 diff changeset	1763
17861 0b4352d6dfdb dsync: Added more debug logging. Timo Sirainen <tss@iki.fi> parents: 17860 diff changeset	1764 if (!importer->last_common_uid_found) {
0b4352d6dfdb dsync: Added more debug logging. Timo Sirainen <tss@iki.fi> parents: 17860 diff changeset	1765 result = NULL;
0b4352d6dfdb dsync: Added more debug logging. Timo Sirainen <tss@iki.fi> parents: 17860 diff changeset	1766 dsync_mailbox_find_common_uid(importer, change, &result);
0b4352d6dfdb dsync: Added more debug logging. Timo Sirainen <tss@iki.fi> parents: 17860 diff changeset	1767 i_assert(result != NULL);
0b4352d6dfdb dsync: Added more debug logging. Timo Sirainen <tss@iki.fi> parents: 17860 diff changeset	1768 } else {
0b4352d6dfdb dsync: Added more debug logging. Timo Sirainen <tss@iki.fi> parents: 17860 diff changeset	1769 result = "New mail";
0b4352d6dfdb dsync: Added more debug logging. Timo Sirainen <tss@iki.fi> parents: 17860 diff changeset	1770 }
16792 4a969a2e9399 dsync: Added some more debug logging. Timo Sirainen <tss@iki.fi> parents: 16762 diff changeset	1771
20208 f1a5362b8a85 dsync: Debug logging improvement. Timo Sirainen <timo.sirainen@dovecot.fi> parents: 20191 diff changeset	1772 imp_debug(importer, "Import change type=%s GUID=%s UID=%u hdr_hash=%s result=%s",
f1a5362b8a85 dsync: Debug logging improvement. Timo Sirainen <timo.sirainen@dovecot.fi> parents: 20191 diff changeset	1773 dsync_mail_change_type_names[change->type],
17861 0b4352d6dfdb dsync: Added more debug logging. Timo Sirainen <tss@iki.fi> parents: 17860 diff changeset	1774 change->guid != NULL ? change->guid : "<unknown>", change->uid,
0b4352d6dfdb dsync: Added more debug logging. Timo Sirainen <tss@iki.fi> parents: 17860 diff changeset	1775 change->hdr_hash != NULL ? change->hdr_hash : "", result);
0b4352d6dfdb dsync: Added more debug logging. Timo Sirainen <tss@iki.fi> parents: 17860 diff changeset	1776
0b4352d6dfdb dsync: Added more debug logging. Timo Sirainen <tss@iki.fi> parents: 17860 diff changeset	1777 if (importer->failed)
0b4352d6dfdb dsync: Added more debug logging. Timo Sirainen <tss@iki.fi> parents: 17860 diff changeset	1778 return -1;
20074 7a29a0930b63 dsync: If state is invalid, exit with code 2 instead of tempfail. Timo Sirainen <timo.sirainen@dovecot.fi> parents: 20072 diff changeset	1779 if (importer->require_full_resync)
7a29a0930b63 dsync: If state is invalid, exit with code 2 instead of tempfail. Timo Sirainen <timo.sirainen@dovecot.fi> parents: 20072 diff changeset	1780 return 0;
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1781
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1782 if (importer->last_common_uid_found) {
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1783 /* a) uid <= last_common_uid for flag changes and expunges.
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1784 this happens only when last_common_uid was originally given
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1785 as parameter to importer.
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1786
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1787 when we're finding the last_common_uid ourself,
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1788 uid>last_common_uid always in here, because
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1789 last_common_uid_found=TRUE only after we find the first
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1790 mismatch.
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1791
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1792 b) uid > last_common_uid for i) new messages, ii) expunges
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1793 that were sent "just in case" */
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1794 if (change->uid <= importer->last_common_uid) {
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1795 i_assert(change->type != DSYNC_MAIL_CHANGE_TYPE_SAVE);
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1796 } else if (change->type == DSYNC_MAIL_CHANGE_TYPE_EXPUNGE) {
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1797 /* ignore */
15772 360661d99c42 dsync: Various importer fixes. Timo Sirainen <tss@iki.fi> parents: 15770 diff changeset	1798 return 0;
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1799 } else {
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1800 i_assert(change->type == DSYNC_MAIL_CHANGE_TYPE_SAVE);
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1801 }
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1802 } else {
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1803 /* a) uid < last_common_uid can never happen */
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1804 i_assert(change->uid >= importer->last_common_uid);
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1805 /* b) uid = last_common_uid if we've verified that the
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1806 messages' GUIDs match so far.
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1807
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1808 c) uid > last_common_uid: i) TYPE_EXPUNGE change has
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1809 GUID=NULL, so we couldn't verify yet if it matches our
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1810 local message, ii) local message is expunged and we couldn't
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1811 find its GUID */
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1812 if (change->uid > importer->last_common_uid) {
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1813 i_assert(change->type == DSYNC_MAIL_CHANGE_TYPE_EXPUNGE \|\|
15772 360661d99c42 dsync: Various importer fixes. Timo Sirainen <tss@iki.fi> parents: 15770 diff changeset	1814 importer->cur_mail == NULL \|\|
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1815 change->uid < importer->cur_mail->uid);
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1816 }
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1817 }
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1818
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1819 switch (change->type) {
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1820 case DSYNC_MAIL_CHANGE_TYPE_SAVE:
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1821 dsync_mailbox_import_save(importer, change);
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1822 break;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1823 case DSYNC_MAIL_CHANGE_TYPE_EXPUNGE:
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1824 dsync_mailbox_import_expunge(importer, change);
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1825 break;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1826 case DSYNC_MAIL_CHANGE_TYPE_FLAG_CHANGE:
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1827 i_assert(importer->last_common_uid_found);
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1828 dsync_mailbox_import_flag_change(importer, change);
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1829 break;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1830 }
15772 360661d99c42 dsync: Various importer fixes. Timo Sirainen <tss@iki.fi> parents: 15770 diff changeset	1831 return importer->failed ? -1 : 0;
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1832 }
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1833
22043 81e013b3207d dsync: Try to commit transactions every dsync_commit_msgs_interval messages Timo Sirainen <timo.sirainen@dovecot.fi> parents: 21814 diff changeset	1834 static int
81e013b3207d dsync: Try to commit transactions every dsync_commit_msgs_interval messages Timo Sirainen <timo.sirainen@dovecot.fi> parents: 21814 diff changeset	1835 importer_new_mail_final_uid_cmp(struct importer_new_mail const newmail1,
81e013b3207d dsync: Try to commit transactions every dsync_commit_msgs_interval messages Timo Sirainen <timo.sirainen@dovecot.fi> parents: 21814 diff changeset	1836 struct importer_new_mail const newmail2)
81e013b3207d dsync: Try to commit transactions every dsync_commit_msgs_interval messages Timo Sirainen <timo.sirainen@dovecot.fi> parents: 21814 diff changeset	1837 {
81e013b3207d dsync: Try to commit transactions every dsync_commit_msgs_interval messages Timo Sirainen <timo.sirainen@dovecot.fi> parents: 21814 diff changeset	1838 if ((newmail1)->final_uid < (newmail2)->final_uid)
81e013b3207d dsync: Try to commit transactions every dsync_commit_msgs_interval messages Timo Sirainen <timo.sirainen@dovecot.fi> parents: 21814 diff changeset	1839 return -1;
81e013b3207d dsync: Try to commit transactions every dsync_commit_msgs_interval messages Timo Sirainen <timo.sirainen@dovecot.fi> parents: 21814 diff changeset	1840 if ((newmail1)->final_uid > (newmail2)->final_uid)
81e013b3207d dsync: Try to commit transactions every dsync_commit_msgs_interval messages Timo Sirainen <timo.sirainen@dovecot.fi> parents: 21814 diff changeset	1841 return 1;
81e013b3207d dsync: Try to commit transactions every dsync_commit_msgs_interval messages Timo Sirainen <timo.sirainen@dovecot.fi> parents: 21814 diff changeset	1842 return 0;
81e013b3207d dsync: Try to commit transactions every dsync_commit_msgs_interval messages Timo Sirainen <timo.sirainen@dovecot.fi> parents: 21814 diff changeset	1843 }
81e013b3207d dsync: Try to commit transactions every dsync_commit_msgs_interval messages Timo Sirainen <timo.sirainen@dovecot.fi> parents: 21814 diff changeset	1844
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1845 static void
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1846 dsync_mailbox_import_assign_new_uids(struct dsync_mailbox_importer *importer)
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1847 {
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1848 struct importer_new_mail newmail, const *newmailp;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1849 uint32_t common_uid_next, new_uid;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1850
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1851 common_uid_next = I_MAX(importer->local_uid_next,
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1852 importer->remote_uid_next);
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1853 array_foreach_modifiable(&importer->newmails, newmailp) {
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1854 newmail = *newmailp;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1855 if (newmail->skip) {
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1856 /* already assigned */
22043 81e013b3207d dsync: Try to commit transactions every dsync_commit_msgs_interval messages Timo Sirainen <timo.sirainen@dovecot.fi> parents: 21814 diff changeset	1857 i_assert(newmail->final_uid != 0);
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1858 continue;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1859 }
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1860
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1861 /* figure out what UID to use for the mail */
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1862 if (newmail->uid_is_usable) {
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1863 /* keep the UID */
15766 4260244e57e2 dsync: Various importer fixes. Handle unexpectedly wrong mailbox state better. Timo Sirainen <tss@iki.fi> parents: 15754 diff changeset	1864 new_uid = newmail->final_uid;
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1865 } else if (newmail->link != NULL &&
15618 b338c4a08f42 dsync: Mailbox syncing fix Timo Sirainen <tss@iki.fi> parents: 15617 diff changeset	1866 newmail->link->uid_is_usable) {
15772 360661d99c42 dsync: Various importer fixes. Timo Sirainen <tss@iki.fi> parents: 15770 diff changeset	1867 /* we can use the linked message's UID and expunge
360661d99c42 dsync: Various importer fixes. Timo Sirainen <tss@iki.fi> parents: 15770 diff changeset	1868 this mail */
15766 4260244e57e2 dsync: Various importer fixes. Handle unexpectedly wrong mailbox state better. Timo Sirainen <tss@iki.fi> parents: 15754 diff changeset	1869 new_uid = newmail->link->final_uid;
15618 b338c4a08f42 dsync: Mailbox syncing fix Timo Sirainen <tss@iki.fi> parents: 15617 diff changeset	1870 } else {
19660 7a3a138a4986 doveadm backup: Detect and handle conflicts earlier. Timo Sirainen <timo.sirainen@dovecot.fi> parents: 19646 diff changeset	1871 i_assert(!importer->revert_local_changes);
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1872 new_uid = common_uid_next++;
18178 dbf3a0701810 dsync: Added more debug logging. Timo Sirainen <tss@iki.fi> parents: 18177 diff changeset	1873 imp_debug(importer, "UID %u isn't usable, assigning new UID %u",
dbf3a0701810 dsync: Added more debug logging. Timo Sirainen <tss@iki.fi> parents: 18177 diff changeset	1874 newmail->final_uid, new_uid);
15618 b338c4a08f42 dsync: Mailbox syncing fix Timo Sirainen <tss@iki.fi> parents: 15617 diff changeset	1875 }
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1876
15766 4260244e57e2 dsync: Various importer fixes. Handle unexpectedly wrong mailbox state better. Timo Sirainen <tss@iki.fi> parents: 15754 diff changeset	1877 newmail->final_uid = new_uid;
15778 e49424a3b13e dsync: Assert-crashfix for previous importer changes. Timo Sirainen <tss@iki.fi> parents: 15772 diff changeset	1878 if (newmail->link != NULL && newmail->link != newmail) {
15772 360661d99c42 dsync: Various importer fixes. Timo Sirainen <tss@iki.fi> parents: 15770 diff changeset	1879 /* skip processing the linked mail */
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1880 newmail->link->skip = TRUE;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1881 }
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1882 }
15232 6c850258002f doveadm sync/backup: Added -s <state> parameter to do a fast stateful sync. Timo Sirainen <tss@iki.fi> parents: 14926 diff changeset	1883 importer->last_common_uid = common_uid_next-1;
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1884 importer->new_uids_assigned = TRUE;
22043 81e013b3207d dsync: Try to commit transactions every dsync_commit_msgs_interval messages Timo Sirainen <timo.sirainen@dovecot.fi> parents: 21814 diff changeset	1885 /* Sort the newmails by their final_uid. This is used for tracking
81e013b3207d dsync: Try to commit transactions every dsync_commit_msgs_interval messages Timo Sirainen <timo.sirainen@dovecot.fi> parents: 21814 diff changeset	1886 whether an intermediate commit is allowed. */
81e013b3207d dsync: Try to commit transactions every dsync_commit_msgs_interval messages Timo Sirainen <timo.sirainen@dovecot.fi> parents: 21814 diff changeset	1887 array_sort(&importer->newmails, importer_new_mail_final_uid_cmp);
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1888 }
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	1889
15766 4260244e57e2 dsync: Various importer fixes. Handle unexpectedly wrong mailbox state better. Timo Sirainen <tss@iki.fi> parents: 15754 diff changeset	1890 static int
4260244e57e2 dsync: Various importer fixes. Handle unexpectedly wrong mailbox state better. Timo Sirainen <tss@iki.fi> parents: 15754 diff changeset	1891 dsync_mailbox_import_local_uid(struct dsync_mailbox_importer *importer,
18180 39d00448490f dsync: If same GUID already exists in storage, try to copy it instead of recreating the mail. Timo Sirainen <tss@iki.fi> parents: 18178 diff changeset	1892 struct mail mail, uint32_t uid, const char guid,
15766 4260244e57e2 dsync: Various importer fixes. Handle unexpectedly wrong mailbox state better. Timo Sirainen <tss@iki.fi> parents: 15754 diff changeset	1893 struct dsync_mail *dmail_r)
4260244e57e2 dsync: Various importer fixes. Handle unexpectedly wrong mailbox state better. Timo Sirainen <tss@iki.fi> parents: 15754 diff changeset	1894 {
4260244e57e2 dsync: Various importer fixes. Handle unexpectedly wrong mailbox state better. Timo Sirainen <tss@iki.fi> parents: 15754 diff changeset	1895 const char error_field, errstr;
4260244e57e2 dsync: Various importer fixes. Handle unexpectedly wrong mailbox state better. Timo Sirainen <tss@iki.fi> parents: 15754 diff changeset	1896 enum mail_error error;
4260244e57e2 dsync: Various importer fixes. Handle unexpectedly wrong mailbox state better. Timo Sirainen <tss@iki.fi> parents: 15754 diff changeset	1897
18180 39d00448490f dsync: If same GUID already exists in storage, try to copy it instead of recreating the mail. Timo Sirainen <tss@iki.fi> parents: 18178 diff changeset	1898 if (!mail_set_uid(mail, uid))
15766 4260244e57e2 dsync: Various importer fixes. Handle unexpectedly wrong mailbox state better. Timo Sirainen <tss@iki.fi> parents: 15754 diff changeset	1899 return 0;
4260244e57e2 dsync: Various importer fixes. Handle unexpectedly wrong mailbox state better. Timo Sirainen <tss@iki.fi> parents: 15754 diff changeset	1900
18371 b900b50085fc dsync: Use storage's mail_error to choose the doveadm exit code. Timo Sirainen <tss@iki.fi> parents: 18183 diff changeset	1901 /* NOTE: Errors are logged, but they don't cause the entire import
b900b50085fc dsync: Use storage's mail_error to choose the doveadm exit code. Timo Sirainen <tss@iki.fi> parents: 18183 diff changeset	1902 to fail. */
18180 39d00448490f dsync: If same GUID already exists in storage, try to copy it instead of recreating the mail. Timo Sirainen <tss@iki.fi> parents: 18178 diff changeset	1903 if (dsync_mail_fill(mail, TRUE, dmail_r, &error_field) < 0) {
21814 759962e70148 global: Log internal storage error on failure Martti Rannanjärvi <martti.rannanjarvi@dovecot.fi> parents: 21390 diff changeset	1904 errstr = mailbox_get_last_internal_error(mail->box, &error);
15766 4260244e57e2 dsync: Various importer fixes. Handle unexpectedly wrong mailbox state better. Timo Sirainen <tss@iki.fi> parents: 15754 diff changeset	1905 if (error == MAIL_ERROR_EXPUNGED)
4260244e57e2 dsync: Various importer fixes. Handle unexpectedly wrong mailbox state better. Timo Sirainen <tss@iki.fi> parents: 15754 diff changeset	1906 return 0;
4260244e57e2 dsync: Various importer fixes. Handle unexpectedly wrong mailbox state better. Timo Sirainen <tss@iki.fi> parents: 15754 diff changeset	1907
4260244e57e2 dsync: Various importer fixes. Handle unexpectedly wrong mailbox state better. Timo Sirainen <tss@iki.fi> parents: 15754 diff changeset	1908 i_error("Mailbox %s: Can't lookup %s for UID=%u: %s",
4260244e57e2 dsync: Various importer fixes. Handle unexpectedly wrong mailbox state better. Timo Sirainen <tss@iki.fi> parents: 15754 diff changeset	1909 mailbox_get_vname(importer->box),
15772 360661d99c42 dsync: Various importer fixes. Timo Sirainen <tss@iki.fi> parents: 15770 diff changeset	1910 error_field, uid, errstr);
15766 4260244e57e2 dsync: Various importer fixes. Handle unexpectedly wrong mailbox state better. Timo Sirainen <tss@iki.fi> parents: 15754 diff changeset	1911 return -1;
4260244e57e2 dsync: Various importer fixes. Handle unexpectedly wrong mailbox state better. Timo Sirainen <tss@iki.fi> parents: 15754 diff changeset	1912 }
15772 360661d99c42 dsync: Various importer fixes. Timo Sirainen <tss@iki.fi> parents: 15770 diff changeset	1913 if (*guid != '\0' && strcmp(guid, dmail_r->guid) != 0) {
15766 4260244e57e2 dsync: Various importer fixes. Handle unexpectedly wrong mailbox state better. Timo Sirainen <tss@iki.fi> parents: 15754 diff changeset	1914 dsync_import_unexpected_state(importer, t_strdup_printf(
4260244e57e2 dsync: Various importer fixes. Handle unexpectedly wrong mailbox state better. Timo Sirainen <tss@iki.fi> parents: 15754 diff changeset	1915 "Unexpected GUID mismatch (3) for UID=%u: %s != %s",
15772 360661d99c42 dsync: Various importer fixes. Timo Sirainen <tss@iki.fi> parents: 15770 diff changeset	1916 uid, dmail_r->guid, guid));
15766 4260244e57e2 dsync: Various importer fixes. Handle unexpectedly wrong mailbox state better. Timo Sirainen <tss@iki.fi> parents: 15754 diff changeset	1917 return -1;
4260244e57e2 dsync: Various importer fixes. Handle unexpectedly wrong mailbox state better. Timo Sirainen <tss@iki.fi> parents: 15754 diff changeset	1918 }
4260244e57e2 dsync: Various importer fixes. Handle unexpectedly wrong mailbox state better. Timo Sirainen <tss@iki.fi> parents: 15754 diff changeset	1919 return 1;
4260244e57e2 dsync: Various importer fixes. Handle unexpectedly wrong mailbox state better. Timo Sirainen <tss@iki.fi> parents: 15754 diff changeset	1920 }
4260244e57e2 dsync: Various importer fixes. Handle unexpectedly wrong mailbox state better. Timo Sirainen <tss@iki.fi> parents: 15754 diff changeset	1921
15824 953594b35bb3 dsync: Fixes to handling local changes during dsync. Timo Sirainen <tss@iki.fi> parents: 15820 diff changeset	1922 static void
16255 548e59794f2e dsync: Commit large transactions every 100 new messages. Timo Sirainen <tss@iki.fi> parents: 16254 diff changeset	1923 dsync_mailbox_import_saved_uid(struct dsync_mailbox_importer *importer,
548e59794f2e dsync: Commit large transactions every 100 new messages. Timo Sirainen <tss@iki.fi> parents: 16254 diff changeset	1924 uint32_t uid)
15824 953594b35bb3 dsync: Fixes to handling local changes during dsync. Timo Sirainen <tss@iki.fi> parents: 15820 diff changeset	1925 {
16255 548e59794f2e dsync: Commit large transactions every 100 new messages. Timo Sirainen <tss@iki.fi> parents: 16254 diff changeset	1926 i_assert(importer->search_ctx == NULL);
548e59794f2e dsync: Commit large transactions every 100 new messages. Timo Sirainen <tss@iki.fi> parents: 16254 diff changeset	1927
15824 953594b35bb3 dsync: Fixes to handling local changes during dsync. Timo Sirainen <tss@iki.fi> parents: 15820 diff changeset	1928 if (importer->highest_wanted_uid < uid)
953594b35bb3 dsync: Fixes to handling local changes during dsync. Timo Sirainen <tss@iki.fi> parents: 15820 diff changeset	1929 importer->highest_wanted_uid = uid;
953594b35bb3 dsync: Fixes to handling local changes during dsync. Timo Sirainen <tss@iki.fi> parents: 15820 diff changeset	1930 array_append(&importer->wanted_uids, &uid, 1);
953594b35bb3 dsync: Fixes to handling local changes during dsync. Timo Sirainen <tss@iki.fi> parents: 15820 diff changeset	1931 }
953594b35bb3 dsync: Fixes to handling local changes during dsync. Timo Sirainen <tss@iki.fi> parents: 15820 diff changeset	1932
22043 81e013b3207d dsync: Try to commit transactions every dsync_commit_msgs_interval messages Timo Sirainen <timo.sirainen@dovecot.fi> parents: 21814 diff changeset	1933 static void
81e013b3207d dsync: Try to commit transactions every dsync_commit_msgs_interval messages Timo Sirainen <timo.sirainen@dovecot.fi> parents: 21814 diff changeset	1934 dsync_mailbox_import_update_first_saved(struct dsync_mailbox_importer *importer)
81e013b3207d dsync: Try to commit transactions every dsync_commit_msgs_interval messages Timo Sirainen <timo.sirainen@dovecot.fi> parents: 21814 diff changeset	1935 {
81e013b3207d dsync: Try to commit transactions every dsync_commit_msgs_interval messages Timo Sirainen <timo.sirainen@dovecot.fi> parents: 21814 diff changeset	1936 struct importer_new_mail const newmails;
81e013b3207d dsync: Try to commit transactions every dsync_commit_msgs_interval messages Timo Sirainen <timo.sirainen@dovecot.fi> parents: 21814 diff changeset	1937 unsigned int count;
81e013b3207d dsync: Try to commit transactions every dsync_commit_msgs_interval messages Timo Sirainen <timo.sirainen@dovecot.fi> parents: 21814 diff changeset	1938
81e013b3207d dsync: Try to commit transactions every dsync_commit_msgs_interval messages Timo Sirainen <timo.sirainen@dovecot.fi> parents: 21814 diff changeset	1939 newmails = array_get(&importer->newmails, &count);
81e013b3207d dsync: Try to commit transactions every dsync_commit_msgs_interval messages Timo Sirainen <timo.sirainen@dovecot.fi> parents: 21814 diff changeset	1940 while (importer->first_unsaved_idx < count) {
81e013b3207d dsync: Try to commit transactions every dsync_commit_msgs_interval messages Timo Sirainen <timo.sirainen@dovecot.fi> parents: 21814 diff changeset	1941 if (!newmails[importer->first_unsaved_idx]->saved)
81e013b3207d dsync: Try to commit transactions every dsync_commit_msgs_interval messages Timo Sirainen <timo.sirainen@dovecot.fi> parents: 21814 diff changeset	1942 break;
81e013b3207d dsync: Try to commit transactions every dsync_commit_msgs_interval messages Timo Sirainen <timo.sirainen@dovecot.fi> parents: 21814 diff changeset	1943 importer->first_unsaved_idx++;
81e013b3207d dsync: Try to commit transactions every dsync_commit_msgs_interval messages Timo Sirainen <timo.sirainen@dovecot.fi> parents: 21814 diff changeset	1944 }
81e013b3207d dsync: Try to commit transactions every dsync_commit_msgs_interval messages Timo Sirainen <timo.sirainen@dovecot.fi> parents: 21814 diff changeset	1945 }
81e013b3207d dsync: Try to commit transactions every dsync_commit_msgs_interval messages Timo Sirainen <timo.sirainen@dovecot.fi> parents: 21814 diff changeset	1946
81e013b3207d dsync: Try to commit transactions every dsync_commit_msgs_interval messages Timo Sirainen <timo.sirainen@dovecot.fi> parents: 21814 diff changeset	1947 static void
81e013b3207d dsync: Try to commit transactions every dsync_commit_msgs_interval messages Timo Sirainen <timo.sirainen@dovecot.fi> parents: 21814 diff changeset	1948 dsync_mailbox_import_saved_newmail(struct dsync_mailbox_importer *importer,
81e013b3207d dsync: Try to commit transactions every dsync_commit_msgs_interval messages Timo Sirainen <timo.sirainen@dovecot.fi> parents: 21814 diff changeset	1949 struct importer_new_mail *newmail)
81e013b3207d dsync: Try to commit transactions every dsync_commit_msgs_interval messages Timo Sirainen <timo.sirainen@dovecot.fi> parents: 21814 diff changeset	1950 {
81e013b3207d dsync: Try to commit transactions every dsync_commit_msgs_interval messages Timo Sirainen <timo.sirainen@dovecot.fi> parents: 21814 diff changeset	1951 dsync_mailbox_import_saved_uid(importer, newmail->final_uid);
81e013b3207d dsync: Try to commit transactions every dsync_commit_msgs_interval messages Timo Sirainen <timo.sirainen@dovecot.fi> parents: 21814 diff changeset	1952 newmail->saved = TRUE;
81e013b3207d dsync: Try to commit transactions every dsync_commit_msgs_interval messages Timo Sirainen <timo.sirainen@dovecot.fi> parents: 21814 diff changeset	1953
81e013b3207d dsync: Try to commit transactions every dsync_commit_msgs_interval messages Timo Sirainen <timo.sirainen@dovecot.fi> parents: 21814 diff changeset	1954 dsync_mailbox_import_update_first_saved(importer);
81e013b3207d dsync: Try to commit transactions every dsync_commit_msgs_interval messages Timo Sirainen <timo.sirainen@dovecot.fi> parents: 21814 diff changeset	1955 importer->saves_since_commit++;
81e013b3207d dsync: Try to commit transactions every dsync_commit_msgs_interval messages Timo Sirainen <timo.sirainen@dovecot.fi> parents: 21814 diff changeset	1956 /* we can commit only if all the upcoming mails will have UIDs that
81e013b3207d dsync: Try to commit transactions every dsync_commit_msgs_interval messages Timo Sirainen <timo.sirainen@dovecot.fi> parents: 21814 diff changeset	1957 are larger than we're committing.
81e013b3207d dsync: Try to commit transactions every dsync_commit_msgs_interval messages Timo Sirainen <timo.sirainen@dovecot.fi> parents: 21814 diff changeset	1958
81e013b3207d dsync: Try to commit transactions every dsync_commit_msgs_interval messages Timo Sirainen <timo.sirainen@dovecot.fi> parents: 21814 diff changeset	1959 Note that if any existing UIDs have been changed, the new UID is
81e013b3207d dsync: Try to commit transactions every dsync_commit_msgs_interval messages Timo Sirainen <timo.sirainen@dovecot.fi> parents: 21814 diff changeset	1960 usually higher than anything that is being saved so we can't do
81e013b3207d dsync: Try to commit transactions every dsync_commit_msgs_interval messages Timo Sirainen <timo.sirainen@dovecot.fi> parents: 21814 diff changeset	1961 an intermediate commit. It's too much extra work to try to handle
81e013b3207d dsync: Try to commit transactions every dsync_commit_msgs_interval messages Timo Sirainen <timo.sirainen@dovecot.fi> parents: 21814 diff changeset	1962 that situation. So here this never happens, because then
81e013b3207d dsync: Try to commit transactions every dsync_commit_msgs_interval messages Timo Sirainen <timo.sirainen@dovecot.fi> parents: 21814 diff changeset	1963 array_count(wanted_uids) is always higher than first_unsaved_idx. */
81e013b3207d dsync: Try to commit transactions every dsync_commit_msgs_interval messages Timo Sirainen <timo.sirainen@dovecot.fi> parents: 21814 diff changeset	1964 if (importer->saves_since_commit >= importer->commit_msgs_interval &&
81e013b3207d dsync: Try to commit transactions every dsync_commit_msgs_interval messages Timo Sirainen <timo.sirainen@dovecot.fi> parents: 21814 diff changeset	1965 importer->first_unsaved_idx == array_count(&importer->wanted_uids)) {
81e013b3207d dsync: Try to commit transactions every dsync_commit_msgs_interval messages Timo Sirainen <timo.sirainen@dovecot.fi> parents: 21814 diff changeset	1966 if (dsync_mailbox_import_commit(importer, FALSE) < 0)
81e013b3207d dsync: Try to commit transactions every dsync_commit_msgs_interval messages Timo Sirainen <timo.sirainen@dovecot.fi> parents: 21814 diff changeset	1967 importer->failed = TRUE;
81e013b3207d dsync: Try to commit transactions every dsync_commit_msgs_interval messages Timo Sirainen <timo.sirainen@dovecot.fi> parents: 21814 diff changeset	1968 importer->saves_since_commit = 0;
81e013b3207d dsync: Try to commit transactions every dsync_commit_msgs_interval messages Timo Sirainen <timo.sirainen@dovecot.fi> parents: 21814 diff changeset	1969 }
81e013b3207d dsync: Try to commit transactions every dsync_commit_msgs_interval messages Timo Sirainen <timo.sirainen@dovecot.fi> parents: 21814 diff changeset	1970 }
81e013b3207d dsync: Try to commit transactions every dsync_commit_msgs_interval messages Timo Sirainen <timo.sirainen@dovecot.fi> parents: 21814 diff changeset	1971
15772 360661d99c42 dsync: Various importer fixes. Timo Sirainen <tss@iki.fi> parents: 15770 diff changeset	1972 static bool
360661d99c42 dsync: Various importer fixes. Timo Sirainen <tss@iki.fi> parents: 15770 diff changeset	1973 dsync_msg_change_uid(struct dsync_mailbox_importer *importer,
360661d99c42 dsync: Various importer fixes. Timo Sirainen <tss@iki.fi> parents: 15770 diff changeset	1974 uint32_t old_uid, uint32_t new_uid)
360661d99c42 dsync: Various importer fixes. Timo Sirainen <tss@iki.fi> parents: 15770 diff changeset	1975 {
360661d99c42 dsync: Various importer fixes. Timo Sirainen <tss@iki.fi> parents: 15770 diff changeset	1976 struct mail_save_context *save_ctx;
360661d99c42 dsync: Various importer fixes. Timo Sirainen <tss@iki.fi> parents: 15770 diff changeset	1977
360661d99c42 dsync: Various importer fixes. Timo Sirainen <tss@iki.fi> parents: 15770 diff changeset	1978 IMPORTER_DEBUG_CHANGE(importer);
360661d99c42 dsync: Various importer fixes. Timo Sirainen <tss@iki.fi> parents: 15770 diff changeset	1979
360661d99c42 dsync: Various importer fixes. Timo Sirainen <tss@iki.fi> parents: 15770 diff changeset	1980 if (!mail_set_uid(importer->mail, old_uid))
360661d99c42 dsync: Various importer fixes. Timo Sirainen <tss@iki.fi> parents: 15770 diff changeset	1981 return FALSE;
360661d99c42 dsync: Various importer fixes. Timo Sirainen <tss@iki.fi> parents: 15770 diff changeset	1982
360661d99c42 dsync: Various importer fixes. Timo Sirainen <tss@iki.fi> parents: 15770 diff changeset	1983 save_ctx = mailbox_save_alloc(importer->ext_trans);
360661d99c42 dsync: Various importer fixes. Timo Sirainen <tss@iki.fi> parents: 15770 diff changeset	1984 mailbox_save_copy_flags(save_ctx, importer->mail);
360661d99c42 dsync: Various importer fixes. Timo Sirainen <tss@iki.fi> parents: 15770 diff changeset	1985 mailbox_save_set_uid(save_ctx, new_uid);
360661d99c42 dsync: Various importer fixes. Timo Sirainen <tss@iki.fi> parents: 15770 diff changeset	1986 if (mailbox_move(&save_ctx, importer->mail) < 0)
360661d99c42 dsync: Various importer fixes. Timo Sirainen <tss@iki.fi> parents: 15770 diff changeset	1987 return FALSE;
16255 548e59794f2e dsync: Commit large transactions every 100 new messages. Timo Sirainen <tss@iki.fi> parents: 16254 diff changeset	1988 dsync_mailbox_import_saved_uid(importer, new_uid);
15772 360661d99c42 dsync: Various importer fixes. Timo Sirainen <tss@iki.fi> parents: 15770 diff changeset	1989 return TRUE;
360661d99c42 dsync: Various importer fixes. Timo Sirainen <tss@iki.fi> parents: 15770 diff changeset	1990 }
360661d99c42 dsync: Various importer fixes. Timo Sirainen <tss@iki.fi> parents: 15770 diff changeset	1991
360661d99c42 dsync: Various importer fixes. Timo Sirainen <tss@iki.fi> parents: 15770 diff changeset	1992 static bool
360661d99c42 dsync: Various importer fixes. Timo Sirainen <tss@iki.fi> parents: 15770 diff changeset	1993 dsync_mailbox_import_change_uid(struct dsync_mailbox_importer *importer,
360661d99c42 dsync: Various importer fixes. Timo Sirainen <tss@iki.fi> parents: 15770 diff changeset	1994 ARRAY_TYPE(seq_range) *unwanted_uids,
360661d99c42 dsync: Various importer fixes. Timo Sirainen <tss@iki.fi> parents: 15770 diff changeset	1995 uint32_t wanted_uid)
15766 4260244e57e2 dsync: Various importer fixes. Handle unexpectedly wrong mailbox state better. Timo Sirainen <tss@iki.fi> parents: 15754 diff changeset	1996 {
15772 360661d99c42 dsync: Various importer fixes. Timo Sirainen <tss@iki.fi> parents: 15770 diff changeset	1997 const struct seq_range *range;
15820 e7396c0f7f66 dsync: Fixed infinite loop in importer. Timo Sirainen <tss@iki.fi> parents: 15819 diff changeset	1998 unsigned int count, n;
e7396c0f7f66 dsync: Fixed infinite loop in importer. Timo Sirainen <tss@iki.fi> parents: 15819 diff changeset	1999 struct seq_range_iter iter;
e7396c0f7f66 dsync: Fixed infinite loop in importer. Timo Sirainen <tss@iki.fi> parents: 15819 diff changeset	2000 uint32_t uid;
15772 360661d99c42 dsync: Various importer fixes. Timo Sirainen <tss@iki.fi> parents: 15770 diff changeset	2001
15820 e7396c0f7f66 dsync: Fixed infinite loop in importer. Timo Sirainen <tss@iki.fi> parents: 15819 diff changeset	2002 /* optimize by first trying to use the latest UID */
e7396c0f7f66 dsync: Fixed infinite loop in importer. Timo Sirainen <tss@iki.fi> parents: 15819 diff changeset	2003 range = array_get(unwanted_uids, &count);
e7396c0f7f66 dsync: Fixed infinite loop in importer. Timo Sirainen <tss@iki.fi> parents: 15819 diff changeset	2004 if (count == 0)
e7396c0f7f66 dsync: Fixed infinite loop in importer. Timo Sirainen <tss@iki.fi> parents: 15819 diff changeset	2005 return FALSE;
e7396c0f7f66 dsync: Fixed infinite loop in importer. Timo Sirainen <tss@iki.fi> parents: 15819 diff changeset	2006 if (dsync_msg_change_uid(importer, range[count-1].seq2, wanted_uid)) {
e7396c0f7f66 dsync: Fixed infinite loop in importer. Timo Sirainen <tss@iki.fi> parents: 15819 diff changeset	2007 seq_range_array_remove(unwanted_uids, range[count-1].seq2);
e7396c0f7f66 dsync: Fixed infinite loop in importer. Timo Sirainen <tss@iki.fi> parents: 15819 diff changeset	2008 return TRUE;
e7396c0f7f66 dsync: Fixed infinite loop in importer. Timo Sirainen <tss@iki.fi> parents: 15819 diff changeset	2009 }
e7396c0f7f66 dsync: Fixed infinite loop in importer. Timo Sirainen <tss@iki.fi> parents: 15819 diff changeset	2010 if (mailbox_get_last_mail_error(importer->box) == MAIL_ERROR_EXPUNGED)
e7396c0f7f66 dsync: Fixed infinite loop in importer. Timo Sirainen <tss@iki.fi> parents: 15819 diff changeset	2011 seq_range_array_remove(unwanted_uids, range[count-1].seq2);
e7396c0f7f66 dsync: Fixed infinite loop in importer. Timo Sirainen <tss@iki.fi> parents: 15819 diff changeset	2012
e7396c0f7f66 dsync: Fixed infinite loop in importer. Timo Sirainen <tss@iki.fi> parents: 15819 diff changeset	2013 /* now try to use any of them by iterating through them. (would be
e7396c0f7f66 dsync: Fixed infinite loop in importer. Timo Sirainen <tss@iki.fi> parents: 15819 diff changeset	2014 easier&faster to just iterate backwards, but probably too much
e7396c0f7f66 dsync: Fixed infinite loop in importer. Timo Sirainen <tss@iki.fi> parents: 15819 diff changeset	2015 trouble to add such API) */
e7396c0f7f66 dsync: Fixed infinite loop in importer. Timo Sirainen <tss@iki.fi> parents: 15819 diff changeset	2016 n = 0; seq_range_array_iter_init(&iter, unwanted_uids);
e7396c0f7f66 dsync: Fixed infinite loop in importer. Timo Sirainen <tss@iki.fi> parents: 15819 diff changeset	2017 while (seq_range_array_iter_nth(&iter, n++, &uid)) {
e7396c0f7f66 dsync: Fixed infinite loop in importer. Timo Sirainen <tss@iki.fi> parents: 15819 diff changeset	2018 if (dsync_msg_change_uid(importer, uid, wanted_uid)) {
e7396c0f7f66 dsync: Fixed infinite loop in importer. Timo Sirainen <tss@iki.fi> parents: 15819 diff changeset	2019 seq_range_array_remove(unwanted_uids, uid);
15772 360661d99c42 dsync: Various importer fixes. Timo Sirainen <tss@iki.fi> parents: 15770 diff changeset	2020 return TRUE;
360661d99c42 dsync: Various importer fixes. Timo Sirainen <tss@iki.fi> parents: 15770 diff changeset	2021 }
15820 e7396c0f7f66 dsync: Fixed infinite loop in importer. Timo Sirainen <tss@iki.fi> parents: 15819 diff changeset	2022 if (mailbox_get_last_mail_error(importer->box) == MAIL_ERROR_EXPUNGED)
e7396c0f7f66 dsync: Fixed infinite loop in importer. Timo Sirainen <tss@iki.fi> parents: 15819 diff changeset	2023 seq_range_array_remove(unwanted_uids, uid);
15772 360661d99c42 dsync: Various importer fixes. Timo Sirainen <tss@iki.fi> parents: 15770 diff changeset	2024 }
360661d99c42 dsync: Various importer fixes. Timo Sirainen <tss@iki.fi> parents: 15770 diff changeset	2025 return FALSE;
360661d99c42 dsync: Various importer fixes. Timo Sirainen <tss@iki.fi> parents: 15770 diff changeset	2026 }
360661d99c42 dsync: Various importer fixes. Timo Sirainen <tss@iki.fi> parents: 15770 diff changeset	2027
360661d99c42 dsync: Various importer fixes. Timo Sirainen <tss@iki.fi> parents: 15770 diff changeset	2028 static bool
360661d99c42 dsync: Various importer fixes. Timo Sirainen <tss@iki.fi> parents: 15770 diff changeset	2029 dsync_mailbox_import_try_local(struct dsync_mailbox_importer *importer,
360661d99c42 dsync: Various importer fixes. Timo Sirainen <tss@iki.fi> parents: 15770 diff changeset	2030 struct importer_new_mail *all_newmails,
360661d99c42 dsync: Various importer fixes. Timo Sirainen <tss@iki.fi> parents: 15770 diff changeset	2031 ARRAY_TYPE(seq_range) *local_uids,
360661d99c42 dsync: Various importer fixes. Timo Sirainen <tss@iki.fi> parents: 15770 diff changeset	2032 ARRAY_TYPE(seq_range) *wanted_uids)
360661d99c42 dsync: Various importer fixes. Timo Sirainen <tss@iki.fi> parents: 15770 diff changeset	2033 {
360661d99c42 dsync: Various importer fixes. Timo Sirainen <tss@iki.fi> parents: 15770 diff changeset	2034 ARRAY_TYPE(seq_range) assigned_uids, unwanted_uids;
360661d99c42 dsync: Various importer fixes. Timo Sirainen <tss@iki.fi> parents: 15770 diff changeset	2035 struct seq_range_iter local_iter, wanted_iter;
360661d99c42 dsync: Various importer fixes. Timo Sirainen <tss@iki.fi> parents: 15770 diff changeset	2036 unsigned int local_n, wanted_n;
360661d99c42 dsync: Various importer fixes. Timo Sirainen <tss@iki.fi> parents: 15770 diff changeset	2037 uint32_t local_uid, wanted_uid;
15766 4260244e57e2 dsync: Various importer fixes. Handle unexpectedly wrong mailbox state better. Timo Sirainen <tss@iki.fi> parents: 15754 diff changeset	2038 struct importer_new_mail *mail;
15772 360661d99c42 dsync: Various importer fixes. Timo Sirainen <tss@iki.fi> parents: 15770 diff changeset	2039 struct dsync_mail dmail;
360661d99c42 dsync: Various importer fixes. Timo Sirainen <tss@iki.fi> parents: 15770 diff changeset	2040
360661d99c42 dsync: Various importer fixes. Timo Sirainen <tss@iki.fi> parents: 15770 diff changeset	2041 if (array_count(local_uids) == 0)
360661d99c42 dsync: Various importer fixes. Timo Sirainen <tss@iki.fi> parents: 15770 diff changeset	2042 return FALSE;
360661d99c42 dsync: Various importer fixes. Timo Sirainen <tss@iki.fi> parents: 15770 diff changeset	2043
360661d99c42 dsync: Various importer fixes. Timo Sirainen <tss@iki.fi> parents: 15770 diff changeset	2044 local_n = wanted_n = 0;
360661d99c42 dsync: Various importer fixes. Timo Sirainen <tss@iki.fi> parents: 15770 diff changeset	2045 seq_range_array_iter_init(&local_iter, local_uids);
360661d99c42 dsync: Various importer fixes. Timo Sirainen <tss@iki.fi> parents: 15770 diff changeset	2046 seq_range_array_iter_init(&wanted_iter, wanted_uids);
15766 4260244e57e2 dsync: Various importer fixes. Handle unexpectedly wrong mailbox state better. Timo Sirainen <tss@iki.fi> parents: 15754 diff changeset	2047
15772 360661d99c42 dsync: Various importer fixes. Timo Sirainen <tss@iki.fi> parents: 15770 diff changeset	2048 /* wanted_uids contains UIDs that need to exist at the end. those that
360661d99c42 dsync: Various importer fixes. Timo Sirainen <tss@iki.fi> parents: 15770 diff changeset	2049 don't already exist in local_uids have a higher UID than any
360661d99c42 dsync: Various importer fixes. Timo Sirainen <tss@iki.fi> parents: 15770 diff changeset	2050 existing local UID */
360661d99c42 dsync: Various importer fixes. Timo Sirainen <tss@iki.fi> parents: 15770 diff changeset	2051 t_array_init(&assigned_uids, array_count(wanted_uids));
360661d99c42 dsync: Various importer fixes. Timo Sirainen <tss@iki.fi> parents: 15770 diff changeset	2052 t_array_init(&unwanted_uids, 8);
360661d99c42 dsync: Various importer fixes. Timo Sirainen <tss@iki.fi> parents: 15770 diff changeset	2053 while (seq_range_array_iter_nth(&local_iter, local_n++, &local_uid)) {
360661d99c42 dsync: Various importer fixes. Timo Sirainen <tss@iki.fi> parents: 15770 diff changeset	2054 if (seq_range_array_iter_nth(&wanted_iter, wanted_n,
360661d99c42 dsync: Various importer fixes. Timo Sirainen <tss@iki.fi> parents: 15770 diff changeset	2055 &wanted_uid)) {
360661d99c42 dsync: Various importer fixes. Timo Sirainen <tss@iki.fi> parents: 15770 diff changeset	2056 if (local_uid == wanted_uid) {
360661d99c42 dsync: Various importer fixes. Timo Sirainen <tss@iki.fi> parents: 15770 diff changeset	2057 /* we have exactly the UID we want. keep it. */
360661d99c42 dsync: Various importer fixes. Timo Sirainen <tss@iki.fi> parents: 15770 diff changeset	2058 seq_range_array_add(&assigned_uids, wanted_uid);
360661d99c42 dsync: Various importer fixes. Timo Sirainen <tss@iki.fi> parents: 15770 diff changeset	2059 wanted_n++;
360661d99c42 dsync: Various importer fixes. Timo Sirainen <tss@iki.fi> parents: 15770 diff changeset	2060 continue;
360661d99c42 dsync: Various importer fixes. Timo Sirainen <tss@iki.fi> parents: 15770 diff changeset	2061 }
360661d99c42 dsync: Various importer fixes. Timo Sirainen <tss@iki.fi> parents: 15770 diff changeset	2062 i_assert(local_uid < wanted_uid);
360661d99c42 dsync: Various importer fixes. Timo Sirainen <tss@iki.fi> parents: 15770 diff changeset	2063 }
360661d99c42 dsync: Various importer fixes. Timo Sirainen <tss@iki.fi> parents: 15770 diff changeset	2064 /* we no longer want this local UID. */
360661d99c42 dsync: Various importer fixes. Timo Sirainen <tss@iki.fi> parents: 15770 diff changeset	2065 seq_range_array_add(&unwanted_uids, local_uid);
360661d99c42 dsync: Various importer fixes. Timo Sirainen <tss@iki.fi> parents: 15770 diff changeset	2066 }
360661d99c42 dsync: Various importer fixes. Timo Sirainen <tss@iki.fi> parents: 15770 diff changeset	2067
360661d99c42 dsync: Various importer fixes. Timo Sirainen <tss@iki.fi> parents: 15770 diff changeset	2068 /* reuse as many existing messages as possible by changing their UIDs */
360661d99c42 dsync: Various importer fixes. Timo Sirainen <tss@iki.fi> parents: 15770 diff changeset	2069 while (seq_range_array_iter_nth(&wanted_iter, wanted_n, &wanted_uid)) {
360661d99c42 dsync: Various importer fixes. Timo Sirainen <tss@iki.fi> parents: 15770 diff changeset	2070 if (!dsync_mailbox_import_change_uid(importer, &unwanted_uids,
360661d99c42 dsync: Various importer fixes. Timo Sirainen <tss@iki.fi> parents: 15770 diff changeset	2071 wanted_uid))
360661d99c42 dsync: Various importer fixes. Timo Sirainen <tss@iki.fi> parents: 15770 diff changeset	2072 break;
360661d99c42 dsync: Various importer fixes. Timo Sirainen <tss@iki.fi> parents: 15770 diff changeset	2073 seq_range_array_add(&assigned_uids, wanted_uid);
360661d99c42 dsync: Various importer fixes. Timo Sirainen <tss@iki.fi> parents: 15770 diff changeset	2074 wanted_n++;
360661d99c42 dsync: Various importer fixes. Timo Sirainen <tss@iki.fi> parents: 15770 diff changeset	2075 }
360661d99c42 dsync: Various importer fixes. Timo Sirainen <tss@iki.fi> parents: 15770 diff changeset	2076
360661d99c42 dsync: Various importer fixes. Timo Sirainen <tss@iki.fi> parents: 15770 diff changeset	2077 /* expunge all unwanted messages */
360661d99c42 dsync: Various importer fixes. Timo Sirainen <tss@iki.fi> parents: 15770 diff changeset	2078 local_n = 0; seq_range_array_iter_init(&local_iter, &unwanted_uids);
360661d99c42 dsync: Various importer fixes. Timo Sirainen <tss@iki.fi> parents: 15770 diff changeset	2079 while (seq_range_array_iter_nth(&local_iter, local_n++, &local_uid)) {
360661d99c42 dsync: Various importer fixes. Timo Sirainen <tss@iki.fi> parents: 15770 diff changeset	2080 IMPORTER_DEBUG_CHANGE(importer);
360661d99c42 dsync: Various importer fixes. Timo Sirainen <tss@iki.fi> parents: 15770 diff changeset	2081 if (mail_set_uid(importer->mail, local_uid))
360661d99c42 dsync: Various importer fixes. Timo Sirainen <tss@iki.fi> parents: 15770 diff changeset	2082 mail_expunge(importer->mail);
360661d99c42 dsync: Various importer fixes. Timo Sirainen <tss@iki.fi> parents: 15770 diff changeset	2083 }
360661d99c42 dsync: Various importer fixes. Timo Sirainen <tss@iki.fi> parents: 15770 diff changeset	2084
360661d99c42 dsync: Various importer fixes. Timo Sirainen <tss@iki.fi> parents: 15770 diff changeset	2085 /* mark mails whose UIDs we got to be skipped over later */
15766 4260244e57e2 dsync: Various importer fixes. Handle unexpectedly wrong mailbox state better. Timo Sirainen <tss@iki.fi> parents: 15754 diff changeset	2086 for (mail = all_newmails; mail != NULL; mail = mail->next) {
15772 360661d99c42 dsync: Various importer fixes. Timo Sirainen <tss@iki.fi> parents: 15770 diff changeset	2087 if (!mail->skip &&
360661d99c42 dsync: Various importer fixes. Timo Sirainen <tss@iki.fi> parents: 15770 diff changeset	2088 seq_range_exists(&assigned_uids, mail->final_uid))
360661d99c42 dsync: Various importer fixes. Timo Sirainen <tss@iki.fi> parents: 15770 diff changeset	2089 mail->skip = TRUE;
360661d99c42 dsync: Various importer fixes. Timo Sirainen <tss@iki.fi> parents: 15770 diff changeset	2090 }
360661d99c42 dsync: Various importer fixes. Timo Sirainen <tss@iki.fi> parents: 15770 diff changeset	2091
360661d99c42 dsync: Various importer fixes. Timo Sirainen <tss@iki.fi> parents: 15770 diff changeset	2092 if (!seq_range_array_iter_nth(&wanted_iter, wanted_n, &wanted_uid)) {
360661d99c42 dsync: Various importer fixes. Timo Sirainen <tss@iki.fi> parents: 15770 diff changeset	2093 /* we've assigned all wanted UIDs */
360661d99c42 dsync: Various importer fixes. Timo Sirainen <tss@iki.fi> parents: 15770 diff changeset	2094 return TRUE;
360661d99c42 dsync: Various importer fixes. Timo Sirainen <tss@iki.fi> parents: 15770 diff changeset	2095 }
360661d99c42 dsync: Various importer fixes. Timo Sirainen <tss@iki.fi> parents: 15770 diff changeset	2096
360661d99c42 dsync: Various importer fixes. Timo Sirainen <tss@iki.fi> parents: 15770 diff changeset	2097 /* try to find one existing message that we can use to copy to the
360661d99c42 dsync: Various importer fixes. Timo Sirainen <tss@iki.fi> parents: 15770 diff changeset	2098 other instances */
360661d99c42 dsync: Various importer fixes. Timo Sirainen <tss@iki.fi> parents: 15770 diff changeset	2099 local_n = 0; seq_range_array_iter_init(&local_iter, local_uids);
360661d99c42 dsync: Various importer fixes. Timo Sirainen <tss@iki.fi> parents: 15770 diff changeset	2100 while (seq_range_array_iter_nth(&local_iter, local_n++, &local_uid)) {
18180 39d00448490f dsync: If same GUID already exists in storage, try to copy it instead of recreating the mail. Timo Sirainen <tss@iki.fi> parents: 18178 diff changeset	2101 if (dsync_mailbox_import_local_uid(importer, importer->mail,
39d00448490f dsync: If same GUID already exists in storage, try to copy it instead of recreating the mail. Timo Sirainen <tss@iki.fi> parents: 18178 diff changeset	2102 local_uid, all_newmails->guid,
15772 360661d99c42 dsync: Various importer fixes. Timo Sirainen <tss@iki.fi> parents: 15770 diff changeset	2103 &dmail) > 0) {
18180 39d00448490f dsync: If same GUID already exists in storage, try to copy it instead of recreating the mail. Timo Sirainen <tss@iki.fi> parents: 18178 diff changeset	2104 if (dsync_mailbox_save_newmails(importer, &dmail,
39d00448490f dsync: If same GUID already exists in storage, try to copy it instead of recreating the mail. Timo Sirainen <tss@iki.fi> parents: 18178 diff changeset	2105 all_newmails, FALSE))
39d00448490f dsync: If same GUID already exists in storage, try to copy it instead of recreating the mail. Timo Sirainen <tss@iki.fi> parents: 18178 diff changeset	2106 return TRUE;
39d00448490f dsync: If same GUID already exists in storage, try to copy it instead of recreating the mail. Timo Sirainen <tss@iki.fi> parents: 18178 diff changeset	2107 }
39d00448490f dsync: If same GUID already exists in storage, try to copy it instead of recreating the mail. Timo Sirainen <tss@iki.fi> parents: 18178 diff changeset	2108 }
39d00448490f dsync: If same GUID already exists in storage, try to copy it instead of recreating the mail. Timo Sirainen <tss@iki.fi> parents: 18178 diff changeset	2109 return FALSE;
39d00448490f dsync: If same GUID already exists in storage, try to copy it instead of recreating the mail. Timo Sirainen <tss@iki.fi> parents: 18178 diff changeset	2110 }
39d00448490f dsync: If same GUID already exists in storage, try to copy it instead of recreating the mail. Timo Sirainen <tss@iki.fi> parents: 18178 diff changeset	2111
39d00448490f dsync: If same GUID already exists in storage, try to copy it instead of recreating the mail. Timo Sirainen <tss@iki.fi> parents: 18178 diff changeset	2112 static bool
39d00448490f dsync: If same GUID already exists in storage, try to copy it instead of recreating the mail. Timo Sirainen <tss@iki.fi> parents: 18178 diff changeset	2113 dsync_mailbox_import_try_virtual_all(struct dsync_mailbox_importer *importer,
39d00448490f dsync: If same GUID already exists in storage, try to copy it instead of recreating the mail. Timo Sirainen <tss@iki.fi> parents: 18178 diff changeset	2114 struct importer_new_mail *all_newmails)
39d00448490f dsync: If same GUID already exists in storage, try to copy it instead of recreating the mail. Timo Sirainen <tss@iki.fi> parents: 18178 diff changeset	2115 {
39d00448490f dsync: If same GUID already exists in storage, try to copy it instead of recreating the mail. Timo Sirainen <tss@iki.fi> parents: 18178 diff changeset	2116 struct dsync_mail dmail;
39d00448490f dsync: If same GUID already exists in storage, try to copy it instead of recreating the mail. Timo Sirainen <tss@iki.fi> parents: 18178 diff changeset	2117
39d00448490f dsync: If same GUID already exists in storage, try to copy it instead of recreating the mail. Timo Sirainen <tss@iki.fi> parents: 18178 diff changeset	2118 if (all_newmails->virtual_all_uid == 0)
39d00448490f dsync: If same GUID already exists in storage, try to copy it instead of recreating the mail. Timo Sirainen <tss@iki.fi> parents: 18178 diff changeset	2119 return FALSE;
39d00448490f dsync: If same GUID already exists in storage, try to copy it instead of recreating the mail. Timo Sirainen <tss@iki.fi> parents: 18178 diff changeset	2120
39d00448490f dsync: If same GUID already exists in storage, try to copy it instead of recreating the mail. Timo Sirainen <tss@iki.fi> parents: 18178 diff changeset	2121 if (dsync_mailbox_import_local_uid(importer, importer->virtual_mail,
39d00448490f dsync: If same GUID already exists in storage, try to copy it instead of recreating the mail. Timo Sirainen <tss@iki.fi> parents: 18178 diff changeset	2122 all_newmails->virtual_all_uid,
39d00448490f dsync: If same GUID already exists in storage, try to copy it instead of recreating the mail. Timo Sirainen <tss@iki.fi> parents: 18178 diff changeset	2123 all_newmails->guid, &dmail) > 0) {
39d00448490f dsync: If same GUID already exists in storage, try to copy it instead of recreating the mail. Timo Sirainen <tss@iki.fi> parents: 18178 diff changeset	2124 if (dsync_mailbox_save_newmails(importer, &dmail,
39d00448490f dsync: If same GUID already exists in storage, try to copy it instead of recreating the mail. Timo Sirainen <tss@iki.fi> parents: 18178 diff changeset	2125 all_newmails, FALSE))
15766 4260244e57e2 dsync: Various importer fixes. Handle unexpectedly wrong mailbox state better. Timo Sirainen <tss@iki.fi> parents: 15754 diff changeset	2126 return TRUE;
4260244e57e2 dsync: Various importer fixes. Handle unexpectedly wrong mailbox state better. Timo Sirainen <tss@iki.fi> parents: 15754 diff changeset	2127 }
4260244e57e2 dsync: Various importer fixes. Handle unexpectedly wrong mailbox state better. Timo Sirainen <tss@iki.fi> parents: 15754 diff changeset	2128 return FALSE;
4260244e57e2 dsync: Various importer fixes. Handle unexpectedly wrong mailbox state better. Timo Sirainen <tss@iki.fi> parents: 15754 diff changeset	2129 }
4260244e57e2 dsync: Various importer fixes. Handle unexpectedly wrong mailbox state better. Timo Sirainen <tss@iki.fi> parents: 15754 diff changeset	2130
4260244e57e2 dsync: Various importer fixes. Handle unexpectedly wrong mailbox state better. Timo Sirainen <tss@iki.fi> parents: 15754 diff changeset	2131 static bool
4260244e57e2 dsync: Various importer fixes. Handle unexpectedly wrong mailbox state better. Timo Sirainen <tss@iki.fi> parents: 15754 diff changeset	2132 dsync_mailbox_import_handle_mail(struct dsync_mailbox_importer *importer,
4260244e57e2 dsync: Various importer fixes. Handle unexpectedly wrong mailbox state better. Timo Sirainen <tss@iki.fi> parents: 15754 diff changeset	2133 struct importer_new_mail *all_newmails)
4260244e57e2 dsync: Various importer fixes. Handle unexpectedly wrong mailbox state better. Timo Sirainen <tss@iki.fi> parents: 15754 diff changeset	2134 {
15772 360661d99c42 dsync: Various importer fixes. Timo Sirainen <tss@iki.fi> parents: 15770 diff changeset	2135 ARRAY_TYPE(seq_range) local_uids, wanted_uids;
15766 4260244e57e2 dsync: Various importer fixes. Handle unexpectedly wrong mailbox state better. Timo Sirainen <tss@iki.fi> parents: 15754 diff changeset	2136 struct dsync_mail_request *request;
4260244e57e2 dsync: Various importer fixes. Handle unexpectedly wrong mailbox state better. Timo Sirainen <tss@iki.fi> parents: 15754 diff changeset	2137 struct importer_new_mail *mail;
4260244e57e2 dsync: Various importer fixes. Handle unexpectedly wrong mailbox state better. Timo Sirainen <tss@iki.fi> parents: 15754 diff changeset	2138 const char *request_guid = NULL;
4260244e57e2 dsync: Various importer fixes. Handle unexpectedly wrong mailbox state better. Timo Sirainen <tss@iki.fi> parents: 15754 diff changeset	2139 uint32_t request_uid = 0;
4260244e57e2 dsync: Various importer fixes. Handle unexpectedly wrong mailbox state better. Timo Sirainen <tss@iki.fi> parents: 15754 diff changeset	2140
16206 83568daabff6 Make static analyzer happier. Timo Sirainen <tss@iki.fi> parents: 16132 diff changeset	2141 i_assert(all_newmails != NULL);
83568daabff6 Make static analyzer happier. Timo Sirainen <tss@iki.fi> parents: 16132 diff changeset	2142
15772 360661d99c42 dsync: Various importer fixes. Timo Sirainen <tss@iki.fi> parents: 15770 diff changeset	2143 /* get the list of the current local UIDs and the wanted UIDs.
360661d99c42 dsync: Various importer fixes. Timo Sirainen <tss@iki.fi> parents: 15770 diff changeset	2144 find the first remote instance that we can request in case there are
360661d99c42 dsync: Various importer fixes. Timo Sirainen <tss@iki.fi> parents: 15770 diff changeset	2145 no local instances */
360661d99c42 dsync: Various importer fixes. Timo Sirainen <tss@iki.fi> parents: 15770 diff changeset	2146 t_array_init(&local_uids, 8);
360661d99c42 dsync: Various importer fixes. Timo Sirainen <tss@iki.fi> parents: 15770 diff changeset	2147 t_array_init(&wanted_uids, 8);
15766 4260244e57e2 dsync: Various importer fixes. Handle unexpectedly wrong mailbox state better. Timo Sirainen <tss@iki.fi> parents: 15754 diff changeset	2148 for (mail = all_newmails; mail != NULL; mail = mail->next) {
15772 360661d99c42 dsync: Various importer fixes. Timo Sirainen <tss@iki.fi> parents: 15770 diff changeset	2149 if (mail->uid_in_local)
360661d99c42 dsync: Various importer fixes. Timo Sirainen <tss@iki.fi> parents: 15770 diff changeset	2150 seq_range_array_add(&local_uids, mail->local_uid);
360661d99c42 dsync: Various importer fixes. Timo Sirainen <tss@iki.fi> parents: 15770 diff changeset	2151 else if (request_guid == NULL) {
15766 4260244e57e2 dsync: Various importer fixes. Handle unexpectedly wrong mailbox state better. Timo Sirainen <tss@iki.fi> parents: 15754 diff changeset	2152 if (*mail->guid != '\0')
4260244e57e2 dsync: Various importer fixes. Handle unexpectedly wrong mailbox state better. Timo Sirainen <tss@iki.fi> parents: 15754 diff changeset	2153 request_guid = mail->guid;
4260244e57e2 dsync: Various importer fixes. Handle unexpectedly wrong mailbox state better. Timo Sirainen <tss@iki.fi> parents: 15754 diff changeset	2154 request_uid = mail->remote_uid;
4260244e57e2 dsync: Various importer fixes. Handle unexpectedly wrong mailbox state better. Timo Sirainen <tss@iki.fi> parents: 15754 diff changeset	2155 i_assert(request_uid != 0);
4260244e57e2 dsync: Various importer fixes. Handle unexpectedly wrong mailbox state better. Timo Sirainen <tss@iki.fi> parents: 15754 diff changeset	2156 }
15772 360661d99c42 dsync: Various importer fixes. Timo Sirainen <tss@iki.fi> parents: 15770 diff changeset	2157 if (!mail->skip)
360661d99c42 dsync: Various importer fixes. Timo Sirainen <tss@iki.fi> parents: 15770 diff changeset	2158 seq_range_array_add(&wanted_uids, mail->final_uid);
15766 4260244e57e2 dsync: Various importer fixes. Handle unexpectedly wrong mailbox state better. Timo Sirainen <tss@iki.fi> parents: 15754 diff changeset	2159 }
15772 360661d99c42 dsync: Various importer fixes. Timo Sirainen <tss@iki.fi> parents: 15770 diff changeset	2160 i_assert(array_count(&wanted_uids) > 0);
15766 4260244e57e2 dsync: Various importer fixes. Handle unexpectedly wrong mailbox state better. Timo Sirainen <tss@iki.fi> parents: 15754 diff changeset	2161
15772 360661d99c42 dsync: Various importer fixes. Timo Sirainen <tss@iki.fi> parents: 15770 diff changeset	2162 if (!dsync_mailbox_import_try_local(importer, all_newmails,
18180 39d00448490f dsync: If same GUID already exists in storage, try to copy it instead of recreating the mail. Timo Sirainen <tss@iki.fi> parents: 18178 diff changeset	2163 &local_uids, &wanted_uids) &&
39d00448490f dsync: If same GUID already exists in storage, try to copy it instead of recreating the mail. Timo Sirainen <tss@iki.fi> parents: 18178 diff changeset	2164 !dsync_mailbox_import_try_virtual_all(importer, all_newmails)) {
15772 360661d99c42 dsync: Various importer fixes. Timo Sirainen <tss@iki.fi> parents: 15770 diff changeset	2165 /* no local instance. request from remote */
15766 4260244e57e2 dsync: Various importer fixes. Handle unexpectedly wrong mailbox state better. Timo Sirainen <tss@iki.fi> parents: 15754 diff changeset	2166 IMPORTER_DEBUG_CHANGE(importer);
15772 360661d99c42 dsync: Various importer fixes. Timo Sirainen <tss@iki.fi> parents: 15770 diff changeset	2167 if (importer->want_mail_requests) {
360661d99c42 dsync: Various importer fixes. Timo Sirainen <tss@iki.fi> parents: 15770 diff changeset	2168 request = array_append_space(&importer->mail_requests);
360661d99c42 dsync: Various importer fixes. Timo Sirainen <tss@iki.fi> parents: 15770 diff changeset	2169 request->guid = request_guid;
360661d99c42 dsync: Various importer fixes. Timo Sirainen <tss@iki.fi> parents: 15770 diff changeset	2170 request->uid = request_uid;
360661d99c42 dsync: Various importer fixes. Timo Sirainen <tss@iki.fi> parents: 15770 diff changeset	2171 }
360661d99c42 dsync: Various importer fixes. Timo Sirainen <tss@iki.fi> parents: 15770 diff changeset	2172 return FALSE;
15766 4260244e57e2 dsync: Various importer fixes. Handle unexpectedly wrong mailbox state better. Timo Sirainen <tss@iki.fi> parents: 15754 diff changeset	2173 }
15772 360661d99c42 dsync: Various importer fixes. Timo Sirainen <tss@iki.fi> parents: 15770 diff changeset	2174 /* successfully handled all the mails locally */
16539 b5876fa03b0e dsync: If verbose_proctitle=yes, show the current state in it. Timo Sirainen <tss@iki.fi> parents: 16259 diff changeset	2175 importer->import_pos++;
15772 360661d99c42 dsync: Various importer fixes. Timo Sirainen <tss@iki.fi> parents: 15770 diff changeset	2176 return TRUE;
15766 4260244e57e2 dsync: Various importer fixes. Handle unexpectedly wrong mailbox state better. Timo Sirainen <tss@iki.fi> parents: 15754 diff changeset	2177 }
4260244e57e2 dsync: Various importer fixes. Handle unexpectedly wrong mailbox state better. Timo Sirainen <tss@iki.fi> parents: 15754 diff changeset	2178
4260244e57e2 dsync: Various importer fixes. Handle unexpectedly wrong mailbox state better. Timo Sirainen <tss@iki.fi> parents: 15754 diff changeset	2179 static void
18180 39d00448490f dsync: If same GUID already exists in storage, try to copy it instead of recreating the mail. Timo Sirainen <tss@iki.fi> parents: 18178 diff changeset	2180 dsync_mailbox_import_find_virtual_uids(struct dsync_mailbox_importer *importer)
39d00448490f dsync: If same GUID already exists in storage, try to copy it instead of recreating the mail. Timo Sirainen <tss@iki.fi> parents: 18178 diff changeset	2181 {
39d00448490f dsync: If same GUID already exists in storage, try to copy it instead of recreating the mail. Timo Sirainen <tss@iki.fi> parents: 18178 diff changeset	2182 struct mail_search_context *search_ctx;
39d00448490f dsync: If same GUID already exists in storage, try to copy it instead of recreating the mail. Timo Sirainen <tss@iki.fi> parents: 18178 diff changeset	2183 struct mail_search_args *search_args;
39d00448490f dsync: If same GUID already exists in storage, try to copy it instead of recreating the mail. Timo Sirainen <tss@iki.fi> parents: 18178 diff changeset	2184 struct importer_new_mail *newmail;
39d00448490f dsync: If same GUID already exists in storage, try to copy it instead of recreating the mail. Timo Sirainen <tss@iki.fi> parents: 18178 diff changeset	2185 struct mail *mail;
39d00448490f dsync: If same GUID already exists in storage, try to copy it instead of recreating the mail. Timo Sirainen <tss@iki.fi> parents: 18178 diff changeset	2186 const char *guid;
39d00448490f dsync: If same GUID already exists in storage, try to copy it instead of recreating the mail. Timo Sirainen <tss@iki.fi> parents: 18178 diff changeset	2187
39d00448490f dsync: If same GUID already exists in storage, try to copy it instead of recreating the mail. Timo Sirainen <tss@iki.fi> parents: 18178 diff changeset	2188 if (mailbox_sync(importer->virtual_all_box, 0) < 0) {
39d00448490f dsync: If same GUID already exists in storage, try to copy it instead of recreating the mail. Timo Sirainen <tss@iki.fi> parents: 18178 diff changeset	2189 i_error("Couldn't sync \\All mailbox '%s': %s",
39d00448490f dsync: If same GUID already exists in storage, try to copy it instead of recreating the mail. Timo Sirainen <tss@iki.fi> parents: 18178 diff changeset	2190 mailbox_get_vname(importer->virtual_all_box),
21814 759962e70148 global: Log internal storage error on failure Martti Rannanjärvi <martti.rannanjarvi@dovecot.fi> parents: 21390 diff changeset	2191 mailbox_get_last_internal_error(importer->virtual_all_box, NULL));
18180 39d00448490f dsync: If same GUID already exists in storage, try to copy it instead of recreating the mail. Timo Sirainen <tss@iki.fi> parents: 18178 diff changeset	2192 return;
39d00448490f dsync: If same GUID already exists in storage, try to copy it instead of recreating the mail. Timo Sirainen <tss@iki.fi> parents: 18178 diff changeset	2193 }
39d00448490f dsync: If same GUID already exists in storage, try to copy it instead of recreating the mail. Timo Sirainen <tss@iki.fi> parents: 18178 diff changeset	2194
39d00448490f dsync: If same GUID already exists in storage, try to copy it instead of recreating the mail. Timo Sirainen <tss@iki.fi> parents: 18178 diff changeset	2195 search_args = mail_search_build_init();
39d00448490f dsync: If same GUID already exists in storage, try to copy it instead of recreating the mail. Timo Sirainen <tss@iki.fi> parents: 18178 diff changeset	2196 mail_search_build_add_all(search_args);
39d00448490f dsync: If same GUID already exists in storage, try to copy it instead of recreating the mail. Timo Sirainen <tss@iki.fi> parents: 18178 diff changeset	2197
39d00448490f dsync: If same GUID already exists in storage, try to copy it instead of recreating the mail. Timo Sirainen <tss@iki.fi> parents: 18178 diff changeset	2198 importer->virtual_trans =
39d00448490f dsync: If same GUID already exists in storage, try to copy it instead of recreating the mail. Timo Sirainen <tss@iki.fi> parents: 18178 diff changeset	2199 mailbox_transaction_begin(importer->virtual_all_box,
19318 f9a143c630a5 dsync: Added DSYNC_BRAIN_FLAG_NO_NOTIFY to enable MAILBOX_TRANSACTION_FLAG_NO_NOTIFY Timo Sirainen <tss@iki.fi> parents: 19228 diff changeset	2200 importer->transaction_flags);
18180 39d00448490f dsync: If same GUID already exists in storage, try to copy it instead of recreating the mail. Timo Sirainen <tss@iki.fi> parents: 18178 diff changeset	2201 search_ctx = mailbox_search_init(importer->virtual_trans, search_args,
39d00448490f dsync: If same GUID already exists in storage, try to copy it instead of recreating the mail. Timo Sirainen <tss@iki.fi> parents: 18178 diff changeset	2202 NULL, MAIL_FETCH_GUID, NULL);
39d00448490f dsync: If same GUID already exists in storage, try to copy it instead of recreating the mail. Timo Sirainen <tss@iki.fi> parents: 18178 diff changeset	2203 mail_search_args_unref(&search_args);
39d00448490f dsync: If same GUID already exists in storage, try to copy it instead of recreating the mail. Timo Sirainen <tss@iki.fi> parents: 18178 diff changeset	2204
39d00448490f dsync: If same GUID already exists in storage, try to copy it instead of recreating the mail. Timo Sirainen <tss@iki.fi> parents: 18178 diff changeset	2205 while (mailbox_search_next(search_ctx, &mail)) {
39d00448490f dsync: If same GUID already exists in storage, try to copy it instead of recreating the mail. Timo Sirainen <tss@iki.fi> parents: 18178 diff changeset	2206 if (mail_get_special(mail, MAIL_FETCH_GUID, &guid) < 0) {
39d00448490f dsync: If same GUID already exists in storage, try to copy it instead of recreating the mail. Timo Sirainen <tss@iki.fi> parents: 18178 diff changeset	2207 /* ignore errors */
39d00448490f dsync: If same GUID already exists in storage, try to copy it instead of recreating the mail. Timo Sirainen <tss@iki.fi> parents: 18178 diff changeset	2208 continue;
39d00448490f dsync: If same GUID already exists in storage, try to copy it instead of recreating the mail. Timo Sirainen <tss@iki.fi> parents: 18178 diff changeset	2209 }
39d00448490f dsync: If same GUID already exists in storage, try to copy it instead of recreating the mail. Timo Sirainen <tss@iki.fi> parents: 18178 diff changeset	2210 newmail = hash_table_lookup(importer->import_guids, guid);
39d00448490f dsync: If same GUID already exists in storage, try to copy it instead of recreating the mail. Timo Sirainen <tss@iki.fi> parents: 18178 diff changeset	2211 if (newmail != NULL && newmail->virtual_all_uid == 0)
39d00448490f dsync: If same GUID already exists in storage, try to copy it instead of recreating the mail. Timo Sirainen <tss@iki.fi> parents: 18178 diff changeset	2212 newmail->virtual_all_uid = mail->uid;
39d00448490f dsync: If same GUID already exists in storage, try to copy it instead of recreating the mail. Timo Sirainen <tss@iki.fi> parents: 18178 diff changeset	2213 }
39d00448490f dsync: If same GUID already exists in storage, try to copy it instead of recreating the mail. Timo Sirainen <tss@iki.fi> parents: 18178 diff changeset	2214 if (mailbox_search_deinit(&search_ctx) < 0) {
39d00448490f dsync: If same GUID already exists in storage, try to copy it instead of recreating the mail. Timo Sirainen <tss@iki.fi> parents: 18178 diff changeset	2215 i_error("Couldn't search \\All mailbox '%s': %s",
39d00448490f dsync: If same GUID already exists in storage, try to copy it instead of recreating the mail. Timo Sirainen <tss@iki.fi> parents: 18178 diff changeset	2216 mailbox_get_vname(importer->virtual_all_box),
21814 759962e70148 global: Log internal storage error on failure Martti Rannanjärvi <martti.rannanjarvi@dovecot.fi> parents: 21390 diff changeset	2217 mailbox_get_last_internal_error(importer->virtual_all_box, NULL));
18180 39d00448490f dsync: If same GUID already exists in storage, try to copy it instead of recreating the mail. Timo Sirainen <tss@iki.fi> parents: 18178 diff changeset	2218 }
39d00448490f dsync: If same GUID already exists in storage, try to copy it instead of recreating the mail. Timo Sirainen <tss@iki.fi> parents: 18178 diff changeset	2219
39d00448490f dsync: If same GUID already exists in storage, try to copy it instead of recreating the mail. Timo Sirainen <tss@iki.fi> parents: 18178 diff changeset	2220 importer->virtual_mail = mail_alloc(importer->virtual_trans, 0, NULL);
39d00448490f dsync: If same GUID already exists in storage, try to copy it instead of recreating the mail. Timo Sirainen <tss@iki.fi> parents: 18178 diff changeset	2221 }
39d00448490f dsync: If same GUID already exists in storage, try to copy it instead of recreating the mail. Timo Sirainen <tss@iki.fi> parents: 18178 diff changeset	2222
39d00448490f dsync: If same GUID already exists in storage, try to copy it instead of recreating the mail. Timo Sirainen <tss@iki.fi> parents: 18178 diff changeset	2223 static void
15766 4260244e57e2 dsync: Various importer fixes. Handle unexpectedly wrong mailbox state better. Timo Sirainen <tss@iki.fi> parents: 15754 diff changeset	2224 dsync_mailbox_import_handle_local_mails(struct dsync_mailbox_importer *importer)
4260244e57e2 dsync: Various importer fixes. Handle unexpectedly wrong mailbox state better. Timo Sirainen <tss@iki.fi> parents: 15754 diff changeset	2225 {
4260244e57e2 dsync: Various importer fixes. Handle unexpectedly wrong mailbox state better. Timo Sirainen <tss@iki.fi> parents: 15754 diff changeset	2226 struct hash_iterate_context *iter;
4260244e57e2 dsync: Various importer fixes. Handle unexpectedly wrong mailbox state better. Timo Sirainen <tss@iki.fi> parents: 15754 diff changeset	2227 const char *key;
4260244e57e2 dsync: Various importer fixes. Handle unexpectedly wrong mailbox state better. Timo Sirainen <tss@iki.fi> parents: 15754 diff changeset	2228 void *key2;
4260244e57e2 dsync: Various importer fixes. Handle unexpectedly wrong mailbox state better. Timo Sirainen <tss@iki.fi> parents: 15754 diff changeset	2229 struct importer_new_mail *mail;
4260244e57e2 dsync: Various importer fixes. Handle unexpectedly wrong mailbox state better. Timo Sirainen <tss@iki.fi> parents: 15754 diff changeset	2230
18180 39d00448490f dsync: If same GUID already exists in storage, try to copy it instead of recreating the mail. Timo Sirainen <tss@iki.fi> parents: 18178 diff changeset	2231 if (importer->virtual_all_box != NULL &&
39d00448490f dsync: If same GUID already exists in storage, try to copy it instead of recreating the mail. Timo Sirainen <tss@iki.fi> parents: 18178 diff changeset	2232 hash_table_count(importer->import_guids) > 0) {
39d00448490f dsync: If same GUID already exists in storage, try to copy it instead of recreating the mail. Timo Sirainen <tss@iki.fi> parents: 18178 diff changeset	2233 /* find UIDs in \All mailbox for all wanted GUIDs. */
39d00448490f dsync: If same GUID already exists in storage, try to copy it instead of recreating the mail. Timo Sirainen <tss@iki.fi> parents: 18178 diff changeset	2234 dsync_mailbox_import_find_virtual_uids(importer);
39d00448490f dsync: If same GUID already exists in storage, try to copy it instead of recreating the mail. Timo Sirainen <tss@iki.fi> parents: 18178 diff changeset	2235 }
39d00448490f dsync: If same GUID already exists in storage, try to copy it instead of recreating the mail. Timo Sirainen <tss@iki.fi> parents: 18178 diff changeset	2236
15766 4260244e57e2 dsync: Various importer fixes. Handle unexpectedly wrong mailbox state better. Timo Sirainen <tss@iki.fi> parents: 15754 diff changeset	2237 iter = hash_table_iterate_init(importer->import_guids);
4260244e57e2 dsync: Various importer fixes. Handle unexpectedly wrong mailbox state better. Timo Sirainen <tss@iki.fi> parents: 15754 diff changeset	2238 while (hash_table_iterate(iter, importer->import_guids, &key, &mail)) {
15772 360661d99c42 dsync: Various importer fixes. Timo Sirainen <tss@iki.fi> parents: 15770 diff changeset	2239 T_BEGIN {
360661d99c42 dsync: Various importer fixes. Timo Sirainen <tss@iki.fi> parents: 15770 diff changeset	2240 if (dsync_mailbox_import_handle_mail(importer, mail))
360661d99c42 dsync: Various importer fixes. Timo Sirainen <tss@iki.fi> parents: 15770 diff changeset	2241 hash_table_remove(importer->import_guids, key);
360661d99c42 dsync: Various importer fixes. Timo Sirainen <tss@iki.fi> parents: 15770 diff changeset	2242 } T_END;
15766 4260244e57e2 dsync: Various importer fixes. Handle unexpectedly wrong mailbox state better. Timo Sirainen <tss@iki.fi> parents: 15754 diff changeset	2243 }
4260244e57e2 dsync: Various importer fixes. Handle unexpectedly wrong mailbox state better. Timo Sirainen <tss@iki.fi> parents: 15754 diff changeset	2244 hash_table_iterate_deinit(&iter);
4260244e57e2 dsync: Various importer fixes. Handle unexpectedly wrong mailbox state better. Timo Sirainen <tss@iki.fi> parents: 15754 diff changeset	2245
4260244e57e2 dsync: Various importer fixes. Handle unexpectedly wrong mailbox state better. Timo Sirainen <tss@iki.fi> parents: 15754 diff changeset	2246 iter = hash_table_iterate_init(importer->import_uids);
4260244e57e2 dsync: Various importer fixes. Handle unexpectedly wrong mailbox state better. Timo Sirainen <tss@iki.fi> parents: 15754 diff changeset	2247 while (hash_table_iterate(iter, importer->import_uids, &key2, &mail)) {
15772 360661d99c42 dsync: Various importer fixes. Timo Sirainen <tss@iki.fi> parents: 15770 diff changeset	2248 T_BEGIN {
360661d99c42 dsync: Various importer fixes. Timo Sirainen <tss@iki.fi> parents: 15770 diff changeset	2249 if (dsync_mailbox_import_handle_mail(importer, mail))
360661d99c42 dsync: Various importer fixes. Timo Sirainen <tss@iki.fi> parents: 15770 diff changeset	2250 hash_table_remove(importer->import_uids, key2);
360661d99c42 dsync: Various importer fixes. Timo Sirainen <tss@iki.fi> parents: 15770 diff changeset	2251 } T_END;
15766 4260244e57e2 dsync: Various importer fixes. Handle unexpectedly wrong mailbox state better. Timo Sirainen <tss@iki.fi> parents: 15754 diff changeset	2252 }
4260244e57e2 dsync: Various importer fixes. Handle unexpectedly wrong mailbox state better. Timo Sirainen <tss@iki.fi> parents: 15754 diff changeset	2253 hash_table_iterate_deinit(&iter);
4260244e57e2 dsync: Various importer fixes. Handle unexpectedly wrong mailbox state better. Timo Sirainen <tss@iki.fi> parents: 15754 diff changeset	2254 }
4260244e57e2 dsync: Various importer fixes. Handle unexpectedly wrong mailbox state better. Timo Sirainen <tss@iki.fi> parents: 15754 diff changeset	2255
19660 7a3a138a4986 doveadm backup: Detect and handle conflicts earlier. Timo Sirainen <timo.sirainen@dovecot.fi> parents: 19646 diff changeset	2256 int dsync_mailbox_import_changes_finish(struct dsync_mailbox_importer *importer)
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2257 {
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2258 i_assert(!importer->new_uids_assigned);
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2259
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2260 if (!importer->last_common_uid_found) {
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2261 /* handle pending expunges and flag updates */
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2262 dsync_mailbox_common_uid_found(importer);
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2263 }
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2264 /* skip common local mails */
14682 d0d7b810646b Make sure we check all the functions' return values. Minor API changes to simplify this. Timo Sirainen <tss@iki.fi> parents: 14676 diff changeset	2265 (void)importer_next_mail(importer, importer->last_common_uid+1);
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2266 /* if there are any local mails left, add them to newmails list */
17518 25326be366c5 dsync: Fixed infinite looping on error condition. Timo Sirainen <tss@iki.fi> parents: 17464 diff changeset	2267 while (importer->cur_mail != NULL && !importer->failed)
14682 d0d7b810646b Make sure we check all the functions' return values. Minor API changes to simplify this. Timo Sirainen <tss@iki.fi> parents: 14676 diff changeset	2268 (void)dsync_mailbox_try_save(importer, NULL);
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2269
16255 548e59794f2e dsync: Commit large transactions every 100 new messages. Timo Sirainen <tss@iki.fi> parents: 16254 diff changeset	2270 if (importer->search_ctx != NULL) {
548e59794f2e dsync: Commit large transactions every 100 new messages. Timo Sirainen <tss@iki.fi> parents: 16254 diff changeset	2271 if (mailbox_search_deinit(&importer->search_ctx) < 0) {
548e59794f2e dsync: Commit large transactions every 100 new messages. Timo Sirainen <tss@iki.fi> parents: 16254 diff changeset	2272 i_error("Mailbox %s: Search failed: %s",
548e59794f2e dsync: Commit large transactions every 100 new messages. Timo Sirainen <tss@iki.fi> parents: 16254 diff changeset	2273 mailbox_get_vname(importer->box),
21814 759962e70148 global: Log internal storage error on failure Martti Rannanjärvi <martti.rannanjarvi@dovecot.fi> parents: 21390 diff changeset	2274 mailbox_get_last_internal_error(importer->box,
759962e70148 global: Log internal storage error on failure Martti Rannanjärvi <martti.rannanjarvi@dovecot.fi> parents: 21390 diff changeset	2275 &importer->mail_error));
16255 548e59794f2e dsync: Commit large transactions every 100 new messages. Timo Sirainen <tss@iki.fi> parents: 16254 diff changeset	2276 importer->failed = TRUE;
548e59794f2e dsync: Commit large transactions every 100 new messages. Timo Sirainen <tss@iki.fi> parents: 16254 diff changeset	2277 }
548e59794f2e dsync: Commit large transactions every 100 new messages. Timo Sirainen <tss@iki.fi> parents: 16254 diff changeset	2278 }
16539 b5876fa03b0e dsync: If verbose_proctitle=yes, show the current state in it. Timo Sirainen <tss@iki.fi> parents: 16259 diff changeset	2279 importer->import_count = hash_table_count(importer->import_guids) +
b5876fa03b0e dsync: If verbose_proctitle=yes, show the current state in it. Timo Sirainen <tss@iki.fi> parents: 16259 diff changeset	2280 hash_table_count(importer->import_uids);
16256 194df093a05f dsync: Fixed assert-crash caused by previous change Timo Sirainen <tss@iki.fi> parents: 16255 diff changeset	2281
194df093a05f dsync: Fixed assert-crash caused by previous change Timo Sirainen <tss@iki.fi> parents: 16255 diff changeset	2282 dsync_mailbox_import_assign_new_uids(importer);
194df093a05f dsync: Fixed assert-crash caused by previous change Timo Sirainen <tss@iki.fi> parents: 16255 diff changeset	2283 /* save mails from local sources where possible,
194df093a05f dsync: Fixed assert-crash caused by previous change Timo Sirainen <tss@iki.fi> parents: 16255 diff changeset	2284 request the rest from remote */
19660 7a3a138a4986 doveadm backup: Detect and handle conflicts earlier. Timo Sirainen <timo.sirainen@dovecot.fi> parents: 19646 diff changeset	2285 if (!importer->failed)
7a3a138a4986 doveadm backup: Detect and handle conflicts earlier. Timo Sirainen <timo.sirainen@dovecot.fi> parents: 19646 diff changeset	2286 dsync_mailbox_import_handle_local_mails(importer);
7a3a138a4986 doveadm backup: Detect and handle conflicts earlier. Timo Sirainen <timo.sirainen@dovecot.fi> parents: 19646 diff changeset	2287 return importer->failed ? -1 : 0;
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2288 }
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2289
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2290 const struct dsync_mail_request *
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2291 dsync_mailbox_import_next_request(struct dsync_mailbox_importer *importer)
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2292 {
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2293 const struct dsync_mail_request *requests;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2294 unsigned int count;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2295
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2296 requests = array_get(&importer->mail_requests, &count);
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2297 if (importer->mail_request_idx == count)
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2298 return NULL;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2299 return &requests[importer->mail_request_idx++];
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2300 }
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2301
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2302 static const char const
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2303 dsync_mailbox_get_final_keywords(const struct dsync_mail_change *change)
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2304 {
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2305 ARRAY_TYPE(const_string) keywords;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2306 const char const changes;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2307 unsigned int i, count;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2308
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2309 if (!array_is_created(&change->keyword_changes))
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2310 return NULL;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2311
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2312 changes = array_get(&change->keyword_changes, &count);
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2313 t_array_init(&keywords, count);
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2314 for (i = 0; i < count; i++) {
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2315 if (changes[i][0] == KEYWORD_CHANGE_ADD \|\|
15700 067179cbabc2 dsync: Fixed syncing message keywords. Timo Sirainen <tss@iki.fi> parents: 15699 diff changeset	2316 changes[i][0] == KEYWORD_CHANGE_ADD_AND_FINAL) {
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2317 const char *name = changes[i]+1;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2318
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2319 array_append(&keywords, &name, 1);
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2320 }
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2321 }
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2322 if (array_count(&keywords) == 0)
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2323 return NULL;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2324
14686 9ff19c1d5f69 Added array_append_zero() to write a zero-filled record to an array. Timo Sirainen <tss@iki.fi> parents: 14682 diff changeset	2325 array_append_zero(&keywords);
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2326 return array_idx(&keywords, 0);
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2327 }
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2328
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2329 static void
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2330 dsync_mailbox_save_set_metadata(struct dsync_mailbox_importer *importer,
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2331 struct mail_save_context *save_ctx,
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2332 const struct dsync_mail_change *change)
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2333 {
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2334 const char const keyword_names;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2335 struct mail_keywords *keywords;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2336
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2337 keyword_names = dsync_mailbox_get_final_keywords(change);
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2338 keywords = keyword_names == NULL ? NULL :
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2339 mailbox_keywords_create_valid(importer->box,
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2340 keyword_names);
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2341 mailbox_save_set_flags(save_ctx, change->final_flags, keywords);
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2342 if (keywords != NULL)
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2343 mailbox_keywords_unref(&keywords);
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2344
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2345 if (change->modseq > 1) {
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2346 (void)mailbox_enable(importer->box, MAILBOX_FEATURE_CONDSTORE);
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2347 mailbox_save_set_min_modseq(save_ctx, change->modseq);
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2348 }
15496 5d90e2aa1ba8 dsync: Use private modseqs to support syncing private flags in shared mailboxes. Timo Sirainen <tss@iki.fi> parents: 15257 diff changeset	2349 /* FIXME: if there already are private flags, they get lost because
5d90e2aa1ba8 dsync: Use private modseqs to support syncing private flags in shared mailboxes. Timo Sirainen <tss@iki.fi> parents: 15257 diff changeset	2350 saving can't handle updating private index. they get added on the
5d90e2aa1ba8 dsync: Use private modseqs to support syncing private flags in shared mailboxes. Timo Sirainen <tss@iki.fi> parents: 15257 diff changeset	2351 next sync though. if this is fixed here, set min_pvt_modseq also. */
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2352 }
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2353
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2354 static int
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2355 dsync_msg_try_copy(struct dsync_mailbox_importer *importer,
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2356 struct mail_save_context **save_ctx_p,
17174 ea737947fac8 dsync: Optimization for a large number of mails with the same GUID. Timo Sirainen <tss@iki.fi> parents: 17130 diff changeset	2357 struct importer_new_mail **all_newmails_forcopy)
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2358 {
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2359 struct importer_new_mail *inst;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2360
17174 ea737947fac8 dsync: Optimization for a large number of mails with the same GUID. Timo Sirainen <tss@iki.fi> parents: 17130 diff changeset	2361 for (inst = *all_newmails_forcopy; inst != NULL; inst = inst->next) {
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2362 if (inst->uid_in_local && !inst->copy_failed &&
15766 4260244e57e2 dsync: Various importer fixes. Handle unexpectedly wrong mailbox state better. Timo Sirainen <tss@iki.fi> parents: 15754 diff changeset	2363 mail_set_uid(importer->mail, inst->local_uid)) {
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2364 if (mailbox_copy(save_ctx_p, importer->mail) < 0) {
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2365 inst->copy_failed = TRUE;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2366 return -1;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2367 }
17174 ea737947fac8 dsync: Optimization for a large number of mails with the same GUID. Timo Sirainen <tss@iki.fi> parents: 17130 diff changeset	2368 *all_newmails_forcopy = inst;
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2369 return 1;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2370 }
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2371 }
17174 ea737947fac8 dsync: Optimization for a large number of mails with the same GUID. Timo Sirainen <tss@iki.fi> parents: 17130 diff changeset	2372 *all_newmails_forcopy = NULL;
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2373 return 0;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2374 }
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2375
17845 d2e1b3f6d13b dsync: Added DSYNC_BRAIN_FLAG_NO_MAIL_PREFETCH to avoid opening mails unnecessarily. Timo Sirainen <tss@iki.fi> parents: 17518 diff changeset	2376 static void
d2e1b3f6d13b dsync: Added DSYNC_BRAIN_FLAG_NO_MAIL_PREFETCH to avoid opening mails unnecessarily. Timo Sirainen <tss@iki.fi> parents: 17518 diff changeset	2377 dsync_mailbox_save_set_nonminimal(struct mail_save_context *save_ctx,
d2e1b3f6d13b dsync: Added DSYNC_BRAIN_FLAG_NO_MAIL_PREFETCH to avoid opening mails unnecessarily. Timo Sirainen <tss@iki.fi> parents: 17518 diff changeset	2378 const struct dsync_mail *mail)
d2e1b3f6d13b dsync: Added DSYNC_BRAIN_FLAG_NO_MAIL_PREFETCH to avoid opening mails unnecessarily. Timo Sirainen <tss@iki.fi> parents: 17518 diff changeset	2379 {
d2e1b3f6d13b dsync: Added DSYNC_BRAIN_FLAG_NO_MAIL_PREFETCH to avoid opening mails unnecessarily. Timo Sirainen <tss@iki.fi> parents: 17518 diff changeset	2380 if (mail->pop3_uidl != NULL && *mail->pop3_uidl != '\0')
d2e1b3f6d13b dsync: Added DSYNC_BRAIN_FLAG_NO_MAIL_PREFETCH to avoid opening mails unnecessarily. Timo Sirainen <tss@iki.fi> parents: 17518 diff changeset	2381 mailbox_save_set_pop3_uidl(save_ctx, mail->pop3_uidl);
d2e1b3f6d13b dsync: Added DSYNC_BRAIN_FLAG_NO_MAIL_PREFETCH to avoid opening mails unnecessarily. Timo Sirainen <tss@iki.fi> parents: 17518 diff changeset	2382 if (mail->pop3_order > 0)
d2e1b3f6d13b dsync: Added DSYNC_BRAIN_FLAG_NO_MAIL_PREFETCH to avoid opening mails unnecessarily. Timo Sirainen <tss@iki.fi> parents: 17518 diff changeset	2383 mailbox_save_set_pop3_order(save_ctx, mail->pop3_order);
d2e1b3f6d13b dsync: Added DSYNC_BRAIN_FLAG_NO_MAIL_PREFETCH to avoid opening mails unnecessarily. Timo Sirainen <tss@iki.fi> parents: 17518 diff changeset	2384 mailbox_save_set_received_date(save_ctx, mail->received_date, 0);
d2e1b3f6d13b dsync: Added DSYNC_BRAIN_FLAG_NO_MAIL_PREFETCH to avoid opening mails unnecessarily. Timo Sirainen <tss@iki.fi> parents: 17518 diff changeset	2385 }
d2e1b3f6d13b dsync: Added DSYNC_BRAIN_FLAG_NO_MAIL_PREFETCH to avoid opening mails unnecessarily. Timo Sirainen <tss@iki.fi> parents: 17518 diff changeset	2386
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2387 static struct mail_save_context *
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2388 dsync_mailbox_save_init(struct dsync_mailbox_importer *importer,
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2389 const struct dsync_mail *mail,
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2390 struct importer_new_mail *newmail)
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2391 {
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2392 struct mail_save_context *save_ctx;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2393
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2394 save_ctx = mailbox_save_alloc(importer->ext_trans);
15766 4260244e57e2 dsync: Various importer fixes. Handle unexpectedly wrong mailbox state better. Timo Sirainen <tss@iki.fi> parents: 15754 diff changeset	2395 mailbox_save_set_uid(save_ctx, newmail->final_uid);
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2396 if (*mail->guid != '\0')
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2397 mailbox_save_set_guid(save_ctx, mail->guid);
17296 99a4788770cb dsync: saved-date doesn't need to be looked up until mail body is being read. Timo Sirainen <tss@iki.fi> parents: 17282 diff changeset	2398 if (mail->saved_date != 0)
99a4788770cb dsync: saved-date doesn't need to be looked up until mail body is being read. Timo Sirainen <tss@iki.fi> parents: 17282 diff changeset	2399 mailbox_save_set_save_date(save_ctx, mail->saved_date);
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2400 dsync_mailbox_save_set_metadata(importer, save_ctx, newmail->change);
17845 d2e1b3f6d13b dsync: Added DSYNC_BRAIN_FLAG_NO_MAIL_PREFETCH to avoid opening mails unnecessarily. Timo Sirainen <tss@iki.fi> parents: 17518 diff changeset	2401
d2e1b3f6d13b dsync: Added DSYNC_BRAIN_FLAG_NO_MAIL_PREFETCH to avoid opening mails unnecessarily. Timo Sirainen <tss@iki.fi> parents: 17518 diff changeset	2402 if (!mail->minimal_fields)
d2e1b3f6d13b dsync: Added DSYNC_BRAIN_FLAG_NO_MAIL_PREFETCH to avoid opening mails unnecessarily. Timo Sirainen <tss@iki.fi> parents: 17518 diff changeset	2403 dsync_mailbox_save_set_nonminimal(save_ctx, mail);
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2404 return save_ctx;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2405 }
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2406
18180 39d00448490f dsync: If same GUID already exists in storage, try to copy it instead of recreating the mail. Timo Sirainen <tss@iki.fi> parents: 18178 diff changeset	2407 static bool
17174 ea737947fac8 dsync: Optimization for a large number of mails with the same GUID. Timo Sirainen <tss@iki.fi> parents: 17130 diff changeset	2408 dsync_mailbox_save_body(struct dsync_mailbox_importer *importer,
ea737947fac8 dsync: Optimization for a large number of mails with the same GUID. Timo Sirainen <tss@iki.fi> parents: 17130 diff changeset	2409 const struct dsync_mail *mail,
ea737947fac8 dsync: Optimization for a large number of mails with the same GUID. Timo Sirainen <tss@iki.fi> parents: 17130 diff changeset	2410 struct importer_new_mail *newmail,
18180 39d00448490f dsync: If same GUID already exists in storage, try to copy it instead of recreating the mail. Timo Sirainen <tss@iki.fi> parents: 18178 diff changeset	2411 struct importer_new_mail **all_newmails_forcopy,
39d00448490f dsync: If same GUID already exists in storage, try to copy it instead of recreating the mail. Timo Sirainen <tss@iki.fi> parents: 18178 diff changeset	2412 bool remote_mail)
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2413 {
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2414 struct mail_save_context *save_ctx;
17845 d2e1b3f6d13b dsync: Added DSYNC_BRAIN_FLAG_NO_MAIL_PREFETCH to avoid opening mails unnecessarily. Timo Sirainen <tss@iki.fi> parents: 17518 diff changeset	2415 struct istream *input;
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2416 ssize_t ret;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2417 bool save_failed = FALSE;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2418
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2419 /* try to save the mail by copying an existing mail */
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2420 save_ctx = dsync_mailbox_save_init(importer, mail, newmail);
17174 ea737947fac8 dsync: Optimization for a large number of mails with the same GUID. Timo Sirainen <tss@iki.fi> parents: 17130 diff changeset	2421 if ((ret = dsync_msg_try_copy(importer, &save_ctx, all_newmails_forcopy)) < 0) {
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2422 if (save_ctx == NULL)
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2423 save_ctx = dsync_mailbox_save_init(importer, mail, newmail);
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2424 }
15676 d3cda5a567be dsync: When syncing locally, mailbox_copy() the mail to attempt a fast copy. Timo Sirainen <tss@iki.fi> parents: 15668 diff changeset	2425 if (ret <= 0 && mail->input_mail != NULL) {
d3cda5a567be dsync: When syncing locally, mailbox_copy() the mail to attempt a fast copy. Timo Sirainen <tss@iki.fi> parents: 15668 diff changeset	2426 /* copy using the source mail */
d3cda5a567be dsync: When syncing locally, mailbox_copy() the mail to attempt a fast copy. Timo Sirainen <tss@iki.fi> parents: 15668 diff changeset	2427 i_assert(mail->input_mail->uid == mail->input_mail_uid);
d3cda5a567be dsync: When syncing locally, mailbox_copy() the mail to attempt a fast copy. Timo Sirainen <tss@iki.fi> parents: 15668 diff changeset	2428 if (mailbox_copy(&save_ctx, mail->input_mail) == 0)
d3cda5a567be dsync: When syncing locally, mailbox_copy() the mail to attempt a fast copy. Timo Sirainen <tss@iki.fi> parents: 15668 diff changeset	2429 ret = 1;
d3cda5a567be dsync: When syncing locally, mailbox_copy() the mail to attempt a fast copy. Timo Sirainen <tss@iki.fi> parents: 15668 diff changeset	2430 else {
d3cda5a567be dsync: When syncing locally, mailbox_copy() the mail to attempt a fast copy. Timo Sirainen <tss@iki.fi> parents: 15668 diff changeset	2431 ret = -1;
d3cda5a567be dsync: When syncing locally, mailbox_copy() the mail to attempt a fast copy. Timo Sirainen <tss@iki.fi> parents: 15668 diff changeset	2432 save_ctx = dsync_mailbox_save_init(importer, mail, newmail);
d3cda5a567be dsync: When syncing locally, mailbox_copy() the mail to attempt a fast copy. Timo Sirainen <tss@iki.fi> parents: 15668 diff changeset	2433 }
d3cda5a567be dsync: When syncing locally, mailbox_copy() the mail to attempt a fast copy. Timo Sirainen <tss@iki.fi> parents: 15668 diff changeset	2434
d3cda5a567be dsync: When syncing locally, mailbox_copy() the mail to attempt a fast copy. Timo Sirainen <tss@iki.fi> parents: 15668 diff changeset	2435 }
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2436 if (ret > 0) {
15772 360661d99c42 dsync: Various importer fixes. Timo Sirainen <tss@iki.fi> parents: 15770 diff changeset	2437 i_assert(save_ctx == NULL);
22043 81e013b3207d dsync: Try to commit transactions every dsync_commit_msgs_interval messages Timo Sirainen <timo.sirainen@dovecot.fi> parents: 21814 diff changeset	2438 dsync_mailbox_import_saved_newmail(importer, newmail);
18180 39d00448490f dsync: If same GUID already exists in storage, try to copy it instead of recreating the mail. Timo Sirainen <tss@iki.fi> parents: 18178 diff changeset	2439 return TRUE;
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2440 }
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2441 /* fallback to saving from remote stream */
18180 39d00448490f dsync: If same GUID already exists in storage, try to copy it instead of recreating the mail. Timo Sirainen <tss@iki.fi> parents: 18178 diff changeset	2442 if (!remote_mail) {
39d00448490f dsync: If same GUID already exists in storage, try to copy it instead of recreating the mail. Timo Sirainen <tss@iki.fi> parents: 18178 diff changeset	2443 /* the mail isn't remote yet. we were just trying to copy a
39d00448490f dsync: If same GUID already exists in storage, try to copy it instead of recreating the mail. Timo Sirainen <tss@iki.fi> parents: 18178 diff changeset	2444 local mail to avoid downloading the remote mail. */
19084 5c8be83ada6c dsync: Assert-crashfix Timo Sirainen <tss@iki.fi> parents: 19080 diff changeset	2445 mailbox_save_cancel(&save_ctx);
18180 39d00448490f dsync: If same GUID already exists in storage, try to copy it instead of recreating the mail. Timo Sirainen <tss@iki.fi> parents: 18178 diff changeset	2446 return FALSE;
39d00448490f dsync: If same GUID already exists in storage, try to copy it instead of recreating the mail. Timo Sirainen <tss@iki.fi> parents: 18178 diff changeset	2447 }
17845 d2e1b3f6d13b dsync: Added DSYNC_BRAIN_FLAG_NO_MAIL_PREFETCH to avoid opening mails unnecessarily. Timo Sirainen <tss@iki.fi> parents: 17518 diff changeset	2448 if (mail->minimal_fields) {
d2e1b3f6d13b dsync: Added DSYNC_BRAIN_FLAG_NO_MAIL_PREFETCH to avoid opening mails unnecessarily. Timo Sirainen <tss@iki.fi> parents: 17518 diff changeset	2449 struct dsync_mail mail2;
d2e1b3f6d13b dsync: Added DSYNC_BRAIN_FLAG_NO_MAIL_PREFETCH to avoid opening mails unnecessarily. Timo Sirainen <tss@iki.fi> parents: 17518 diff changeset	2450 const char *error_field;
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2451
17845 d2e1b3f6d13b dsync: Added DSYNC_BRAIN_FLAG_NO_MAIL_PREFETCH to avoid opening mails unnecessarily. Timo Sirainen <tss@iki.fi> parents: 17518 diff changeset	2452 i_assert(mail->input_mail != NULL);
d2e1b3f6d13b dsync: Added DSYNC_BRAIN_FLAG_NO_MAIL_PREFETCH to avoid opening mails unnecessarily. Timo Sirainen <tss@iki.fi> parents: 17518 diff changeset	2453
d2e1b3f6d13b dsync: Added DSYNC_BRAIN_FLAG_NO_MAIL_PREFETCH to avoid opening mails unnecessarily. Timo Sirainen <tss@iki.fi> parents: 17518 diff changeset	2454 if (dsync_mail_fill_nonminimal(mail->input_mail, &mail2,
d2e1b3f6d13b dsync: Added DSYNC_BRAIN_FLAG_NO_MAIL_PREFETCH to avoid opening mails unnecessarily. Timo Sirainen <tss@iki.fi> parents: 17518 diff changeset	2455 &error_field) < 0) {
d2e1b3f6d13b dsync: Added DSYNC_BRAIN_FLAG_NO_MAIL_PREFETCH to avoid opening mails unnecessarily. Timo Sirainen <tss@iki.fi> parents: 17518 diff changeset	2456 i_error("Mailbox %s: Failed to read mail %s uid=%u: %s",
d2e1b3f6d13b dsync: Added DSYNC_BRAIN_FLAG_NO_MAIL_PREFETCH to avoid opening mails unnecessarily. Timo Sirainen <tss@iki.fi> parents: 17518 diff changeset	2457 mailbox_get_vname(importer->box),
d2e1b3f6d13b dsync: Added DSYNC_BRAIN_FLAG_NO_MAIL_PREFETCH to avoid opening mails unnecessarily. Timo Sirainen <tss@iki.fi> parents: 17518 diff changeset	2458 error_field, mail->uid,
21814 759962e70148 global: Log internal storage error on failure Martti Rannanjärvi <martti.rannanjarvi@dovecot.fi> parents: 21390 diff changeset	2459 mailbox_get_last_internal_error(importer->box,
759962e70148 global: Log internal storage error on failure Martti Rannanjärvi <martti.rannanjarvi@dovecot.fi> parents: 21390 diff changeset	2460 &importer->mail_error));
17845 d2e1b3f6d13b dsync: Added DSYNC_BRAIN_FLAG_NO_MAIL_PREFETCH to avoid opening mails unnecessarily. Timo Sirainen <tss@iki.fi> parents: 17518 diff changeset	2461 importer->failed = TRUE;
17982 d92c1c666bc3 dsync: Fixed assert-crash / memory leak on error handling path. Timo Sirainen <tss@iki.fi> parents: 17890 diff changeset	2462 mailbox_save_cancel(&save_ctx);
18180 39d00448490f dsync: If same GUID already exists in storage, try to copy it instead of recreating the mail. Timo Sirainen <tss@iki.fi> parents: 18178 diff changeset	2463 return TRUE;
17845 d2e1b3f6d13b dsync: Added DSYNC_BRAIN_FLAG_NO_MAIL_PREFETCH to avoid opening mails unnecessarily. Timo Sirainen <tss@iki.fi> parents: 17518 diff changeset	2464 }
d2e1b3f6d13b dsync: Added DSYNC_BRAIN_FLAG_NO_MAIL_PREFETCH to avoid opening mails unnecessarily. Timo Sirainen <tss@iki.fi> parents: 17518 diff changeset	2465 dsync_mailbox_save_set_nonminimal(save_ctx, &mail2);
d2e1b3f6d13b dsync: Added DSYNC_BRAIN_FLAG_NO_MAIL_PREFETCH to avoid opening mails unnecessarily. Timo Sirainen <tss@iki.fi> parents: 17518 diff changeset	2466 input = mail2.input;
d2e1b3f6d13b dsync: Added DSYNC_BRAIN_FLAG_NO_MAIL_PREFETCH to avoid opening mails unnecessarily. Timo Sirainen <tss@iki.fi> parents: 17518 diff changeset	2467 } else {
d2e1b3f6d13b dsync: Added DSYNC_BRAIN_FLAG_NO_MAIL_PREFETCH to avoid opening mails unnecessarily. Timo Sirainen <tss@iki.fi> parents: 17518 diff changeset	2468 input = mail->input;
d2e1b3f6d13b dsync: Added DSYNC_BRAIN_FLAG_NO_MAIL_PREFETCH to avoid opening mails unnecessarily. Timo Sirainen <tss@iki.fi> parents: 17518 diff changeset	2469 }
d2e1b3f6d13b dsync: Added DSYNC_BRAIN_FLAG_NO_MAIL_PREFETCH to avoid opening mails unnecessarily. Timo Sirainen <tss@iki.fi> parents: 17518 diff changeset	2470
d2e1b3f6d13b dsync: Added DSYNC_BRAIN_FLAG_NO_MAIL_PREFETCH to avoid opening mails unnecessarily. Timo Sirainen <tss@iki.fi> parents: 17518 diff changeset	2471 if (input == NULL) {
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2472 /* it was just expunged in remote, skip it */
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2473 mailbox_save_cancel(&save_ctx);
18180 39d00448490f dsync: If same GUID already exists in storage, try to copy it instead of recreating the mail. Timo Sirainen <tss@iki.fi> parents: 18178 diff changeset	2474 return TRUE;
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2475 }
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2476
17845 d2e1b3f6d13b dsync: Added DSYNC_BRAIN_FLAG_NO_MAIL_PREFETCH to avoid opening mails unnecessarily. Timo Sirainen <tss@iki.fi> parents: 17518 diff changeset	2477 i_stream_seek(input, 0);
d2e1b3f6d13b dsync: Added DSYNC_BRAIN_FLAG_NO_MAIL_PREFETCH to avoid opening mails unnecessarily. Timo Sirainen <tss@iki.fi> parents: 17518 diff changeset	2478 if (mailbox_save_begin(&save_ctx, input) < 0) {
15766 4260244e57e2 dsync: Various importer fixes. Handle unexpectedly wrong mailbox state better. Timo Sirainen <tss@iki.fi> parents: 15754 diff changeset	2479 i_error("Mailbox %s: Saving failed: %s",
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2480 mailbox_get_vname(importer->box),
21814 759962e70148 global: Log internal storage error on failure Martti Rannanjärvi <martti.rannanjarvi@dovecot.fi> parents: 21390 diff changeset	2481 mailbox_get_last_internal_error(importer->box,
759962e70148 global: Log internal storage error on failure Martti Rannanjärvi <martti.rannanjarvi@dovecot.fi> parents: 21390 diff changeset	2482 &importer->mail_error));
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2483 importer->failed = TRUE;
18180 39d00448490f dsync: If same GUID already exists in storage, try to copy it instead of recreating the mail. Timo Sirainen <tss@iki.fi> parents: 18178 diff changeset	2484 return TRUE;
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2485 }
17845 d2e1b3f6d13b dsync: Added DSYNC_BRAIN_FLAG_NO_MAIL_PREFETCH to avoid opening mails unnecessarily. Timo Sirainen <tss@iki.fi> parents: 17518 diff changeset	2486 while ((ret = i_stream_read(input)) > 0 \|\| ret == -2) {
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2487 if (mailbox_save_continue(save_ctx) < 0) {
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2488 save_failed = TRUE;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2489 ret = -1;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2490 break;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2491 }
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2492 }
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2493 i_assert(ret == -1);
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2494
17845 d2e1b3f6d13b dsync: Added DSYNC_BRAIN_FLAG_NO_MAIL_PREFETCH to avoid opening mails unnecessarily. Timo Sirainen <tss@iki.fi> parents: 17518 diff changeset	2495 if (input->stream_errno != 0) {
16940 38f404297728 dsync: Use i_stream_get_error() instead of just errno in stream error messages. Timo Sirainen <tss@iki.fi> parents: 16832 diff changeset	2496 i_error("Mailbox %s: read(msg input) failed: %s",
38f404297728 dsync: Use i_stream_get_error() instead of just errno in stream error messages. Timo Sirainen <tss@iki.fi> parents: 16832 diff changeset	2497 mailbox_get_vname(importer->box),
17845 d2e1b3f6d13b dsync: Added DSYNC_BRAIN_FLAG_NO_MAIL_PREFETCH to avoid opening mails unnecessarily. Timo Sirainen <tss@iki.fi> parents: 17518 diff changeset	2498 i_stream_get_error(input));
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2499 mailbox_save_cancel(&save_ctx);
18371 b900b50085fc dsync: Use storage's mail_error to choose the doveadm exit code. Timo Sirainen <tss@iki.fi> parents: 18183 diff changeset	2500 importer->mail_error = MAIL_ERROR_TEMP;
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2501 importer->failed = TRUE;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2502 } else if (save_failed) {
15766 4260244e57e2 dsync: Various importer fixes. Handle unexpectedly wrong mailbox state better. Timo Sirainen <tss@iki.fi> parents: 15754 diff changeset	2503 i_error("Mailbox %s: Saving failed: %s",
4260244e57e2 dsync: Various importer fixes. Handle unexpectedly wrong mailbox state better. Timo Sirainen <tss@iki.fi> parents: 15754 diff changeset	2504 mailbox_get_vname(importer->box),
21814 759962e70148 global: Log internal storage error on failure Martti Rannanjärvi <martti.rannanjarvi@dovecot.fi> parents: 21390 diff changeset	2505 mailbox_get_last_internal_error(importer->box,
759962e70148 global: Log internal storage error on failure Martti Rannanjärvi <martti.rannanjarvi@dovecot.fi> parents: 21390 diff changeset	2506 &importer->mail_error));
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2507 mailbox_save_cancel(&save_ctx);
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2508 importer->failed = TRUE;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2509 } else {
17845 d2e1b3f6d13b dsync: Added DSYNC_BRAIN_FLAG_NO_MAIL_PREFETCH to avoid opening mails unnecessarily. Timo Sirainen <tss@iki.fi> parents: 17518 diff changeset	2510 i_assert(input->eof);
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2511 if (mailbox_save_finish(&save_ctx) < 0) {
15766 4260244e57e2 dsync: Various importer fixes. Handle unexpectedly wrong mailbox state better. Timo Sirainen <tss@iki.fi> parents: 15754 diff changeset	2512 i_error("Mailbox %s: Saving failed: %s",
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2513 mailbox_get_vname(importer->box),
21814 759962e70148 global: Log internal storage error on failure Martti Rannanjärvi <martti.rannanjarvi@dovecot.fi> parents: 21390 diff changeset	2514 mailbox_get_last_internal_error(importer->box,
759962e70148 global: Log internal storage error on failure Martti Rannanjärvi <martti.rannanjarvi@dovecot.fi> parents: 21390 diff changeset	2515 &importer->mail_error));
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2516 importer->failed = TRUE;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2517 } else {
22043 81e013b3207d dsync: Try to commit transactions every dsync_commit_msgs_interval messages Timo Sirainen <timo.sirainen@dovecot.fi> parents: 21814 diff changeset	2518 dsync_mailbox_import_saved_newmail(importer, newmail);
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2519 }
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2520 }
18180 39d00448490f dsync: If same GUID already exists in storage, try to copy it instead of recreating the mail. Timo Sirainen <tss@iki.fi> parents: 18178 diff changeset	2521 return TRUE;
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2522 }
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2523
18180 39d00448490f dsync: If same GUID already exists in storage, try to copy it instead of recreating the mail. Timo Sirainen <tss@iki.fi> parents: 18178 diff changeset	2524 static bool dsync_mailbox_save_newmails(struct dsync_mailbox_importer *importer,
15766 4260244e57e2 dsync: Various importer fixes. Handle unexpectedly wrong mailbox state better. Timo Sirainen <tss@iki.fi> parents: 15754 diff changeset	2525 const struct dsync_mail *mail,
18180 39d00448490f dsync: If same GUID already exists in storage, try to copy it instead of recreating the mail. Timo Sirainen <tss@iki.fi> parents: 18178 diff changeset	2526 struct importer_new_mail *all_newmails,
39d00448490f dsync: If same GUID already exists in storage, try to copy it instead of recreating the mail. Timo Sirainen <tss@iki.fi> parents: 18178 diff changeset	2527 bool remote_mail)
15766 4260244e57e2 dsync: Various importer fixes. Handle unexpectedly wrong mailbox state better. Timo Sirainen <tss@iki.fi> parents: 15754 diff changeset	2528 {
17174 ea737947fac8 dsync: Optimization for a large number of mails with the same GUID. Timo Sirainen <tss@iki.fi> parents: 17130 diff changeset	2529 struct importer_new_mail newmail, all_newmails_forcopy;
18180 39d00448490f dsync: If same GUID already exists in storage, try to copy it instead of recreating the mail. Timo Sirainen <tss@iki.fi> parents: 18178 diff changeset	2530 bool ret = TRUE;
17174 ea737947fac8 dsync: Optimization for a large number of mails with the same GUID. Timo Sirainen <tss@iki.fi> parents: 17130 diff changeset	2531
ea737947fac8 dsync: Optimization for a large number of mails with the same GUID. Timo Sirainen <tss@iki.fi> parents: 17130 diff changeset	2532 /* if all_newmails list is large, avoid scanning through the
ea737947fac8 dsync: Optimization for a large number of mails with the same GUID. Timo Sirainen <tss@iki.fi> parents: 17130 diff changeset	2533 uninteresting ones for each newmail */
ea737947fac8 dsync: Optimization for a large number of mails with the same GUID. Timo Sirainen <tss@iki.fi> parents: 17130 diff changeset	2534 all_newmails_forcopy = all_newmails;
15766 4260244e57e2 dsync: Various importer fixes. Handle unexpectedly wrong mailbox state better. Timo Sirainen <tss@iki.fi> parents: 15754 diff changeset	2535
4260244e57e2 dsync: Various importer fixes. Handle unexpectedly wrong mailbox state better. Timo Sirainen <tss@iki.fi> parents: 15754 diff changeset	2536 /* save all instances of the message */
18180 39d00448490f dsync: If same GUID already exists in storage, try to copy it instead of recreating the mail. Timo Sirainen <tss@iki.fi> parents: 18178 diff changeset	2537 for (newmail = all_newmails; newmail != NULL && ret; newmail = newmail->next) {
15772 360661d99c42 dsync: Various importer fixes. Timo Sirainen <tss@iki.fi> parents: 15770 diff changeset	2538 if (!newmail->skip) T_BEGIN {
18180 39d00448490f dsync: If same GUID already exists in storage, try to copy it instead of recreating the mail. Timo Sirainen <tss@iki.fi> parents: 18178 diff changeset	2539 if (!dsync_mailbox_save_body(importer, mail, newmail,
39d00448490f dsync: If same GUID already exists in storage, try to copy it instead of recreating the mail. Timo Sirainen <tss@iki.fi> parents: 18178 diff changeset	2540 &all_newmails_forcopy,
39d00448490f dsync: If same GUID already exists in storage, try to copy it instead of recreating the mail. Timo Sirainen <tss@iki.fi> parents: 18178 diff changeset	2541 remote_mail))
39d00448490f dsync: If same GUID already exists in storage, try to copy it instead of recreating the mail. Timo Sirainen <tss@iki.fi> parents: 18178 diff changeset	2542 ret = FALSE;
15766 4260244e57e2 dsync: Various importer fixes. Handle unexpectedly wrong mailbox state better. Timo Sirainen <tss@iki.fi> parents: 15754 diff changeset	2543 } T_END;
4260244e57e2 dsync: Various importer fixes. Handle unexpectedly wrong mailbox state better. Timo Sirainen <tss@iki.fi> parents: 15754 diff changeset	2544 }
18180 39d00448490f dsync: If same GUID already exists in storage, try to copy it instead of recreating the mail. Timo Sirainen <tss@iki.fi> parents: 18178 diff changeset	2545 return ret;
15766 4260244e57e2 dsync: Various importer fixes. Handle unexpectedly wrong mailbox state better. Timo Sirainen <tss@iki.fi> parents: 15754 diff changeset	2546 }
4260244e57e2 dsync: Various importer fixes. Handle unexpectedly wrong mailbox state better. Timo Sirainen <tss@iki.fi> parents: 15754 diff changeset	2547
19696 3cc82c8c1b72 dsync: If mailbox importing fails, stop immediately without waiting for exporter to finish. Timo Sirainen <timo.sirainen@dovecot.fi> parents: 19670 diff changeset	2548 int dsync_mailbox_import_mail(struct dsync_mailbox_importer *importer,
3cc82c8c1b72 dsync: If mailbox importing fails, stop immediately without waiting for exporter to finish. Timo Sirainen <timo.sirainen@dovecot.fi> parents: 19670 diff changeset	2549 const struct dsync_mail *mail)
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2550 {
15766 4260244e57e2 dsync: Various importer fixes. Handle unexpectedly wrong mailbox state better. Timo Sirainen <tss@iki.fi> parents: 15754 diff changeset	2551 struct importer_new_mail *all_newmails;
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2552
15658 3a395a61f109 dsync: Fixed crashes when sending already expunged messages. Timo Sirainen <tss@iki.fi> parents: 15657 diff changeset	2553 i_assert(mail->input == NULL \|\| mail->input->seekable);
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2554 i_assert(importer->new_uids_assigned);
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2555
16254 e0acf38f6199 dsync: If saving mails fail, stop trying to save more of them and flooding logs. Timo Sirainen <tss@iki.fi> parents: 16206 diff changeset	2556 if (importer->failed)
19696 3cc82c8c1b72 dsync: If mailbox importing fails, stop immediately without waiting for exporter to finish. Timo Sirainen <timo.sirainen@dovecot.fi> parents: 19670 diff changeset	2557 return -1;
20074 7a29a0930b63 dsync: If state is invalid, exit with code 2 instead of tempfail. Timo Sirainen <timo.sirainen@dovecot.fi> parents: 20072 diff changeset	2558 if (importer->require_full_resync)
7a29a0930b63 dsync: If state is invalid, exit with code 2 instead of tempfail. Timo Sirainen <timo.sirainen@dovecot.fi> parents: 20072 diff changeset	2559 return 0;
16254 e0acf38f6199 dsync: If saving mails fail, stop trying to save more of them and flooding logs. Timo Sirainen <tss@iki.fi> parents: 16206 diff changeset	2560
16792 4a969a2e9399 dsync: Added some more debug logging. Timo Sirainen <tss@iki.fi> parents: 16762 diff changeset	2561 imp_debug(importer, "Import mail body for GUID=%s UID=%u",
4a969a2e9399 dsync: Added some more debug logging. Timo Sirainen <tss@iki.fi> parents: 16762 diff changeset	2562 mail->guid, mail->uid);
4a969a2e9399 dsync: Added some more debug logging. Timo Sirainen <tss@iki.fi> parents: 16762 diff changeset	2563
15766 4260244e57e2 dsync: Various importer fixes. Handle unexpectedly wrong mailbox state better. Timo Sirainen <tss@iki.fi> parents: 15754 diff changeset	2564 all_newmails = *mail->guid != '\0' ?
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2565 hash_table_lookup(importer->import_guids, mail->guid) :
14923 96fd2c3bf932 Reverted "support for non-pointers" part of the hash table API changes. Timo Sirainen <tss@iki.fi> parents: 14922 diff changeset	2566 hash_table_lookup(importer->import_uids, POINTER_CAST(mail->uid));
15766 4260244e57e2 dsync: Various importer fixes. Handle unexpectedly wrong mailbox state better. Timo Sirainen <tss@iki.fi> parents: 15754 diff changeset	2567 if (all_newmails == NULL) {
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2568 if (importer->want_mail_requests) {
15766 4260244e57e2 dsync: Various importer fixes. Handle unexpectedly wrong mailbox state better. Timo Sirainen <tss@iki.fi> parents: 15754 diff changeset	2569 i_error("Mailbox %s: Remote sent unwanted message body for "
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2570 "GUID=%s UID=%u",
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2571 mailbox_get_vname(importer->box),
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2572 mail->guid, mail->uid);
16792 4a969a2e9399 dsync: Added some more debug logging. Timo Sirainen <tss@iki.fi> parents: 16762 diff changeset	2573 } else {
4a969a2e9399 dsync: Added some more debug logging. Timo Sirainen <tss@iki.fi> parents: 16762 diff changeset	2574 imp_debug(importer, "Skip unwanted mail body for "
4a969a2e9399 dsync: Added some more debug logging. Timo Sirainen <tss@iki.fi> parents: 16762 diff changeset	2575 "GUID=%s UID=%u", mail->guid, mail->uid);
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2576 }
19696 3cc82c8c1b72 dsync: If mailbox importing fails, stop immediately without waiting for exporter to finish. Timo Sirainen <timo.sirainen@dovecot.fi> parents: 19670 diff changeset	2577 return 0;
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2578 }
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2579 if (*mail->guid != '\0')
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2580 hash_table_remove(importer->import_guids, mail->guid);
14923 96fd2c3bf932 Reverted "support for non-pointers" part of the hash table API changes. Timo Sirainen <tss@iki.fi> parents: 14922 diff changeset	2581 else {
96fd2c3bf932 Reverted "support for non-pointers" part of the hash table API changes. Timo Sirainen <tss@iki.fi> parents: 14922 diff changeset	2582 hash_table_remove(importer->import_uids,
96fd2c3bf932 Reverted "support for non-pointers" part of the hash table API changes. Timo Sirainen <tss@iki.fi> parents: 14922 diff changeset	2583 POINTER_CAST(mail->uid));
96fd2c3bf932 Reverted "support for non-pointers" part of the hash table API changes. Timo Sirainen <tss@iki.fi> parents: 14922 diff changeset	2584 }
16539 b5876fa03b0e dsync: If verbose_proctitle=yes, show the current state in it. Timo Sirainen <tss@iki.fi> parents: 16259 diff changeset	2585 importer->import_pos++;
18180 39d00448490f dsync: If same GUID already exists in storage, try to copy it instead of recreating the mail. Timo Sirainen <tss@iki.fi> parents: 18178 diff changeset	2586 if (!dsync_mailbox_save_newmails(importer, mail, all_newmails, TRUE))
39d00448490f dsync: If same GUID already exists in storage, try to copy it instead of recreating the mail. Timo Sirainen <tss@iki.fi> parents: 18178 diff changeset	2587 i_unreached();
19696 3cc82c8c1b72 dsync: If mailbox importing fails, stop immediately without waiting for exporter to finish. Timo Sirainen <timo.sirainen@dovecot.fi> parents: 19670 diff changeset	2588 return importer->failed ? -1 : 0;
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2589 }
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2590
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2591 static int
16762 393c389b1540 dsync: Improved debug/error logging for changes during sync. Timo Sirainen <tss@iki.fi> parents: 16680 diff changeset	2592 reassign_uids_in_seq_range(struct dsync_mailbox_importer *importer,
15824 953594b35bb3 dsync: Fixes to handling local changes during dsync. Timo Sirainen <tss@iki.fi> parents: 15820 diff changeset	2593 const ARRAY_TYPE(seq_range) *unwanted_uids)
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2594 {
16762 393c389b1540 dsync: Improved debug/error logging for changes during sync. Timo Sirainen <tss@iki.fi> parents: 16680 diff changeset	2595 struct mailbox *box = importer->box;
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2596 const enum mailbox_transaction_flags trans_flags =
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2597 MAILBOX_TRANSACTION_FLAG_EXTERNAL \|
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2598 MAILBOX_TRANSACTION_FLAG_ASSIGN_UIDS;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2599 struct mailbox_transaction_context *trans;
15824 953594b35bb3 dsync: Fixes to handling local changes during dsync. Timo Sirainen <tss@iki.fi> parents: 15820 diff changeset	2600 struct mail_search_args *search_args;
953594b35bb3 dsync: Fixes to handling local changes during dsync. Timo Sirainen <tss@iki.fi> parents: 15820 diff changeset	2601 struct mail_search_arg *arg;
953594b35bb3 dsync: Fixes to handling local changes during dsync. Timo Sirainen <tss@iki.fi> parents: 15820 diff changeset	2602 struct mail_search_context *search_ctx;
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2603 struct mail_save_context *save_ctx;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2604 struct mail *mail;
16762 393c389b1540 dsync: Improved debug/error logging for changes during sync. Timo Sirainen <tss@iki.fi> parents: 16680 diff changeset	2605 unsigned int renumber_count = 0;
15824 953594b35bb3 dsync: Fixes to handling local changes during dsync. Timo Sirainen <tss@iki.fi> parents: 15820 diff changeset	2606 int ret = 1;
953594b35bb3 dsync: Fixes to handling local changes during dsync. Timo Sirainen <tss@iki.fi> parents: 15820 diff changeset	2607
953594b35bb3 dsync: Fixes to handling local changes during dsync. Timo Sirainen <tss@iki.fi> parents: 15820 diff changeset	2608 if (array_count(unwanted_uids) == 0)
953594b35bb3 dsync: Fixes to handling local changes during dsync. Timo Sirainen <tss@iki.fi> parents: 15820 diff changeset	2609 return 1;
953594b35bb3 dsync: Fixes to handling local changes during dsync. Timo Sirainen <tss@iki.fi> parents: 15820 diff changeset	2610
16792 4a969a2e9399 dsync: Added some more debug logging. Timo Sirainen <tss@iki.fi> parents: 16762 diff changeset	2611 if (importer->debug) T_BEGIN {
4a969a2e9399 dsync: Added some more debug logging. Timo Sirainen <tss@iki.fi> parents: 16762 diff changeset	2612 string_t *str = t_str_new(256);
4a969a2e9399 dsync: Added some more debug logging. Timo Sirainen <tss@iki.fi> parents: 16762 diff changeset	2613 imap_write_seq_range(str, unwanted_uids);
4a969a2e9399 dsync: Added some more debug logging. Timo Sirainen <tss@iki.fi> parents: 16762 diff changeset	2614 imp_debug(importer, "Reassign UIDs: %s", str_c(str));
4a969a2e9399 dsync: Added some more debug logging. Timo Sirainen <tss@iki.fi> parents: 16762 diff changeset	2615 } T_END;
4a969a2e9399 dsync: Added some more debug logging. Timo Sirainen <tss@iki.fi> parents: 16762 diff changeset	2616
15824 953594b35bb3 dsync: Fixes to handling local changes during dsync. Timo Sirainen <tss@iki.fi> parents: 15820 diff changeset	2617 search_args = mail_search_build_init();
953594b35bb3 dsync: Fixes to handling local changes during dsync. Timo Sirainen <tss@iki.fi> parents: 15820 diff changeset	2618 arg = mail_search_build_add(search_args, SEARCH_UIDSET);
953594b35bb3 dsync: Fixes to handling local changes during dsync. Timo Sirainen <tss@iki.fi> parents: 15820 diff changeset	2619 p_array_init(&arg->value.seqset, search_args->pool,
953594b35bb3 dsync: Fixes to handling local changes during dsync. Timo Sirainen <tss@iki.fi> parents: 15820 diff changeset	2620 array_count(unwanted_uids));
953594b35bb3 dsync: Fixes to handling local changes during dsync. Timo Sirainen <tss@iki.fi> parents: 15820 diff changeset	2621 array_append_array(&arg->value.seqset, unwanted_uids);
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2622
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2623 trans = mailbox_transaction_begin(box, trans_flags);
15824 953594b35bb3 dsync: Fixes to handling local changes during dsync. Timo Sirainen <tss@iki.fi> parents: 15820 diff changeset	2624 search_ctx = mailbox_search_init(trans, search_args, NULL, 0, NULL);
953594b35bb3 dsync: Fixes to handling local changes during dsync. Timo Sirainen <tss@iki.fi> parents: 15820 diff changeset	2625 mail_search_args_unref(&search_args);
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2626
15824 953594b35bb3 dsync: Fixes to handling local changes during dsync. Timo Sirainen <tss@iki.fi> parents: 15820 diff changeset	2627 while (mailbox_search_next(search_ctx, &mail)) {
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2628 save_ctx = mailbox_save_alloc(trans);
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2629 mailbox_save_copy_flags(save_ctx, mail);
15233 f417cab3a21f dsync: Use mailbox_move() to reassign UIDs instead of copy+expunge. Timo Sirainen <tss@iki.fi> parents: 15232 diff changeset	2630 if (mailbox_move(&save_ctx, mail) < 0) {
15766 4260244e57e2 dsync: Various importer fixes. Handle unexpectedly wrong mailbox state better. Timo Sirainen <tss@iki.fi> parents: 15754 diff changeset	2631 i_error("Mailbox %s: Couldn't move mail within mailbox: %s",
15233 f417cab3a21f dsync: Use mailbox_move() to reassign UIDs instead of copy+expunge. Timo Sirainen <tss@iki.fi> parents: 15232 diff changeset	2632 mailbox_get_vname(box),
21814 759962e70148 global: Log internal storage error on failure Martti Rannanjärvi <martti.rannanjarvi@dovecot.fi> parents: 21390 diff changeset	2633 mailbox_get_last_internal_error(box, &importer->mail_error));
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2634 ret = -1;
15824 953594b35bb3 dsync: Fixes to handling local changes during dsync. Timo Sirainen <tss@iki.fi> parents: 15820 diff changeset	2635 } else if (ret > 0) {
953594b35bb3 dsync: Fixes to handling local changes during dsync. Timo Sirainen <tss@iki.fi> parents: 15820 diff changeset	2636 ret = 0;
15233 f417cab3a21f dsync: Use mailbox_move() to reassign UIDs instead of copy+expunge. Timo Sirainen <tss@iki.fi> parents: 15232 diff changeset	2637 }
16762 393c389b1540 dsync: Improved debug/error logging for changes during sync. Timo Sirainen <tss@iki.fi> parents: 16680 diff changeset	2638 renumber_count++;
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2639 }
15824 953594b35bb3 dsync: Fixes to handling local changes during dsync. Timo Sirainen <tss@iki.fi> parents: 15820 diff changeset	2640 if (mailbox_search_deinit(&search_ctx) < 0) {
953594b35bb3 dsync: Fixes to handling local changes during dsync. Timo Sirainen <tss@iki.fi> parents: 15820 diff changeset	2641 i_error("Mailbox %s: mail search failed: %s",
953594b35bb3 dsync: Fixes to handling local changes during dsync. Timo Sirainen <tss@iki.fi> parents: 15820 diff changeset	2642 mailbox_get_vname(box),
21814 759962e70148 global: Log internal storage error on failure Martti Rannanjärvi <martti.rannanjarvi@dovecot.fi> parents: 21390 diff changeset	2643 mailbox_get_last_internal_error(box, &importer->mail_error));
15824 953594b35bb3 dsync: Fixes to handling local changes during dsync. Timo Sirainen <tss@iki.fi> parents: 15820 diff changeset	2644 ret = -1;
953594b35bb3 dsync: Fixes to handling local changes during dsync. Timo Sirainen <tss@iki.fi> parents: 15820 diff changeset	2645 }
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2646
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2647 if (mailbox_transaction_commit(&trans) < 0) {
15766 4260244e57e2 dsync: Various importer fixes. Handle unexpectedly wrong mailbox state better. Timo Sirainen <tss@iki.fi> parents: 15754 diff changeset	2648 i_error("Mailbox %s: UID reassign commit failed: %s",
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2649 mailbox_get_vname(box),
21814 759962e70148 global: Log internal storage error on failure Martti Rannanjärvi <martti.rannanjarvi@dovecot.fi> parents: 21390 diff changeset	2650 mailbox_get_last_internal_error(box, &importer->mail_error));
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2651 ret = -1;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2652 }
16793 2ce07adab319 dsync: Added more consistency when debug logging about changes during sync. Timo Sirainen <tss@iki.fi> parents: 16792 diff changeset	2653 if (ret == 0) {
2ce07adab319 dsync: Added more consistency when debug logging about changes during sync. Timo Sirainen <tss@iki.fi> parents: 16792 diff changeset	2654 imp_debug(importer, "Mailbox %s: Change during sync: "
2ce07adab319 dsync: Added more consistency when debug logging about changes during sync. Timo Sirainen <tss@iki.fi> parents: 16792 diff changeset	2655 "Renumbered %u of %u unwanted UIDs",
2ce07adab319 dsync: Added more consistency when debug logging about changes during sync. Timo Sirainen <tss@iki.fi> parents: 16792 diff changeset	2656 mailbox_get_vname(box),
2ce07adab319 dsync: Added more consistency when debug logging about changes during sync. Timo Sirainen <tss@iki.fi> parents: 16792 diff changeset	2657 renumber_count, array_count(unwanted_uids));
16762 393c389b1540 dsync: Improved debug/error logging for changes during sync. Timo Sirainen <tss@iki.fi> parents: 16680 diff changeset	2658 }
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2659 return ret;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2660 }
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2661
15617 2c4394d0360b dsync: Error handling fix. Timo Sirainen <tss@iki.fi> parents: 15612 diff changeset	2662 static int
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2663 reassign_unwanted_uids(struct dsync_mailbox_importer *importer,
21256 4ac3461df334 dsync: When logging "Mailbox changed caused a desync", log also the reason. Timo Sirainen <timo.sirainen@dovecot.fi> parents: 20831 diff changeset	2664 const char **changes_during_sync_r)
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2665 {
15824 953594b35bb3 dsync: Fixes to handling local changes during dsync. Timo Sirainen <tss@iki.fi> parents: 15820 diff changeset	2666 ARRAY_TYPE(seq_range) unwanted_uids;
16255 548e59794f2e dsync: Commit large transactions every 100 new messages. Timo Sirainen <tss@iki.fi> parents: 16254 diff changeset	2667 const uint32_t wanted_uids, saved_uids;
548e59794f2e dsync: Commit large transactions every 100 new messages. Timo Sirainen <tss@iki.fi> parents: 16254 diff changeset	2668 uint32_t highest_seen_uid;
548e59794f2e dsync: Commit large transactions every 100 new messages. Timo Sirainen <tss@iki.fi> parents: 16254 diff changeset	2669 unsigned int i, wanted_count, saved_count;
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2670 int ret = 0;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2671
15824 953594b35bb3 dsync: Fixes to handling local changes during dsync. Timo Sirainen <tss@iki.fi> parents: 15820 diff changeset	2672 wanted_uids = array_get(&importer->wanted_uids, &wanted_count);
16255 548e59794f2e dsync: Commit large transactions every 100 new messages. Timo Sirainen <tss@iki.fi> parents: 16254 diff changeset	2673 saved_uids = array_get(&importer->saved_uids, &saved_count);
548e59794f2e dsync: Commit large transactions every 100 new messages. Timo Sirainen <tss@iki.fi> parents: 16254 diff changeset	2674 i_assert(wanted_count == saved_count);
548e59794f2e dsync: Commit large transactions every 100 new messages. Timo Sirainen <tss@iki.fi> parents: 16254 diff changeset	2675 if (wanted_count == 0)
15824 953594b35bb3 dsync: Fixes to handling local changes during dsync. Timo Sirainen <tss@iki.fi> parents: 15820 diff changeset	2676 return 0;
15612 a0493fa018fa dsync: Fixed handling UID renumbering when new mail was saved during dsync. Timo Sirainen <tss@iki.fi> parents: 15611 diff changeset	2677 /* wanted_uids contains the UIDs we tried to save mails with.
a0493fa018fa dsync: Fixed handling UID renumbering when new mail was saved during dsync. Timo Sirainen <tss@iki.fi> parents: 15611 diff changeset	2678 if nothing changed during dsync, we should have the expected UIDs
16255 548e59794f2e dsync: Commit large transactions every 100 new messages. Timo Sirainen <tss@iki.fi> parents: 16254 diff changeset	2679 (saved_uids) and all is well.
15612 a0493fa018fa dsync: Fixed handling UID renumbering when new mail was saved during dsync. Timo Sirainen <tss@iki.fi> parents: 15611 diff changeset	2680
a0493fa018fa dsync: Fixed handling UID renumbering when new mail was saved during dsync. Timo Sirainen <tss@iki.fi> parents: 15611 diff changeset	2681 if any new messages got inserted during dsync, we'll need to fix up
a0493fa018fa dsync: Fixed handling UID renumbering when new mail was saved during dsync. Timo Sirainen <tss@iki.fi> parents: 15611 diff changeset	2682 the UIDs and let the next dsync fix up the other side. for example:
a0493fa018fa dsync: Fixed handling UID renumbering when new mail was saved during dsync. Timo Sirainen <tss@iki.fi> parents: 15611 diff changeset	2683
a0493fa018fa dsync: Fixed handling UID renumbering when new mail was saved during dsync. Timo Sirainen <tss@iki.fi> parents: 15611 diff changeset	2684 remote uids = 5,7,9 = wanted_uids
a0493fa018fa dsync: Fixed handling UID renumbering when new mail was saved during dsync. Timo Sirainen <tss@iki.fi> parents: 15611 diff changeset	2685 remote uidnext = 12
a0493fa018fa dsync: Fixed handling UID renumbering when new mail was saved during dsync. Timo Sirainen <tss@iki.fi> parents: 15611 diff changeset	2686 locally added new uid=5 ->
a0493fa018fa dsync: Fixed handling UID renumbering when new mail was saved during dsync. Timo Sirainen <tss@iki.fi> parents: 15611 diff changeset	2687 saved_uids = 10,7,9
a0493fa018fa dsync: Fixed handling UID renumbering when new mail was saved during dsync. Timo Sirainen <tss@iki.fi> parents: 15611 diff changeset	2688
15824 953594b35bb3 dsync: Fixes to handling local changes during dsync. Timo Sirainen <tss@iki.fi> parents: 15820 diff changeset	2689 we'll now need to reassign UIDs 5 and 10. to be fully future-proof
953594b35bb3 dsync: Fixes to handling local changes during dsync. Timo Sirainen <tss@iki.fi> parents: 15820 diff changeset	2690 we'll reassign all UIDs between [original local uidnext .. highest
953594b35bb3 dsync: Fixes to handling local changes during dsync. Timo Sirainen <tss@iki.fi> parents: 15820 diff changeset	2691 UID we think we know] that aren't in saved_uids. */
15612 a0493fa018fa dsync: Fixed handling UID renumbering when new mail was saved during dsync. Timo Sirainen <tss@iki.fi> parents: 15611 diff changeset	2692
15824 953594b35bb3 dsync: Fixes to handling local changes during dsync. Timo Sirainen <tss@iki.fi> parents: 15820 diff changeset	2693 /* create uidset for the list of UIDs we don't want to exist */
953594b35bb3 dsync: Fixes to handling local changes during dsync. Timo Sirainen <tss@iki.fi> parents: 15820 diff changeset	2694 t_array_init(&unwanted_uids, 8);
953594b35bb3 dsync: Fixes to handling local changes during dsync. Timo Sirainen <tss@iki.fi> parents: 15820 diff changeset	2695 highest_seen_uid = I_MAX(importer->remote_uid_next-1,
953594b35bb3 dsync: Fixes to handling local changes during dsync. Timo Sirainen <tss@iki.fi> parents: 15820 diff changeset	2696 importer->highest_wanted_uid);
953594b35bb3 dsync: Fixes to handling local changes during dsync. Timo Sirainen <tss@iki.fi> parents: 15820 diff changeset	2697 i_assert(importer->local_uid_next <= highest_seen_uid);
953594b35bb3 dsync: Fixes to handling local changes during dsync. Timo Sirainen <tss@iki.fi> parents: 15820 diff changeset	2698 seq_range_array_add_range(&unwanted_uids,
953594b35bb3 dsync: Fixes to handling local changes during dsync. Timo Sirainen <tss@iki.fi> parents: 15820 diff changeset	2699 importer->local_uid_next, highest_seen_uid);
16255 548e59794f2e dsync: Commit large transactions every 100 new messages. Timo Sirainen <tss@iki.fi> parents: 16254 diff changeset	2700 for (i = 0; i < wanted_count; i++) {
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2701 i_assert(i < wanted_count);
16255 548e59794f2e dsync: Commit large transactions every 100 new messages. Timo Sirainen <tss@iki.fi> parents: 16254 diff changeset	2702 if (saved_uids[i] == wanted_uids[i])
548e59794f2e dsync: Commit large transactions every 100 new messages. Timo Sirainen <tss@iki.fi> parents: 16254 diff changeset	2703 seq_range_array_remove(&unwanted_uids, saved_uids[i]);
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2704 }
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2705
16762 393c389b1540 dsync: Improved debug/error logging for changes during sync. Timo Sirainen <tss@iki.fi> parents: 16680 diff changeset	2706 ret = reassign_uids_in_seq_range(importer, &unwanted_uids);
15824 953594b35bb3 dsync: Fixes to handling local changes during dsync. Timo Sirainen <tss@iki.fi> parents: 15820 diff changeset	2707 if (ret == 0) {
21256 4ac3461df334 dsync: When logging "Mailbox changed caused a desync", log also the reason. Timo Sirainen <timo.sirainen@dovecot.fi> parents: 20831 diff changeset	2708 *changes_during_sync_r = t_strdup_printf(
4ac3461df334 dsync: When logging "Mailbox changed caused a desync", log also the reason. Timo Sirainen <timo.sirainen@dovecot.fi> parents: 20831 diff changeset	2709 "%u UIDs changed due to UID conflicts",
4ac3461df334 dsync: When logging "Mailbox changed caused a desync", log also the reason. Timo Sirainen <timo.sirainen@dovecot.fi> parents: 20831 diff changeset	2710 seq_range_count(&unwanted_uids));
15781 5a3586ffc644 dsync: If unexpected changes happen during sync, return a safe last-common-uid for state. Timo Sirainen <tss@iki.fi> parents: 15779 diff changeset	2711 /* conflicting changes during sync, revert our last-common-uid
5a3586ffc644 dsync: If unexpected changes happen during sync, return a safe last-common-uid for state. Timo Sirainen <tss@iki.fi> parents: 15779 diff changeset	2712 back to a safe value. */
5a3586ffc644 dsync: If unexpected changes happen during sync, return a safe last-common-uid for state. Timo Sirainen <tss@iki.fi> parents: 15779 diff changeset	2713 importer->last_common_uid = importer->local_uid_next - 1;
5a3586ffc644 dsync: If unexpected changes happen during sync, return a safe last-common-uid for state. Timo Sirainen <tss@iki.fi> parents: 15779 diff changeset	2714 }
15824 953594b35bb3 dsync: Fixes to handling local changes during dsync. Timo Sirainen <tss@iki.fi> parents: 15820 diff changeset	2715 return ret < 0 ? -1 : 0;
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2716 }
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2717
16255 548e59794f2e dsync: Commit large transactions every 100 new messages. Timo Sirainen <tss@iki.fi> parents: 16254 diff changeset	2718 static int
548e59794f2e dsync: Commit large transactions every 100 new messages. Timo Sirainen <tss@iki.fi> parents: 16254 diff changeset	2719 dsync_mailbox_import_commit(struct dsync_mailbox_importer *importer, bool final)
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2720 {
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2721 struct mail_transaction_commit_changes changes;
16255 548e59794f2e dsync: Commit large transactions every 100 new messages. Timo Sirainen <tss@iki.fi> parents: 16254 diff changeset	2722 struct seq_range_iter iter;
548e59794f2e dsync: Commit large transactions every 100 new messages. Timo Sirainen <tss@iki.fi> parents: 16254 diff changeset	2723 uint32_t uid;
548e59794f2e dsync: Commit large transactions every 100 new messages. Timo Sirainen <tss@iki.fi> parents: 16254 diff changeset	2724 unsigned int n;
548e59794f2e dsync: Commit large transactions every 100 new messages. Timo Sirainen <tss@iki.fi> parents: 16254 diff changeset	2725 int ret = importer->failed ? -1 : 0;
548e59794f2e dsync: Commit large transactions every 100 new messages. Timo Sirainen <tss@iki.fi> parents: 16254 diff changeset	2726
548e59794f2e dsync: Commit large transactions every 100 new messages. Timo Sirainen <tss@iki.fi> parents: 16254 diff changeset	2727 mail_free(&importer->mail);
548e59794f2e dsync: Commit large transactions every 100 new messages. Timo Sirainen <tss@iki.fi> parents: 16254 diff changeset	2728 mail_free(&importer->ext_mail);
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2729
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2730 /* commit saves */
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2731 if (mailbox_transaction_commit_get_changes(&importer->ext_trans,
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2732 &changes) < 0) {
15766 4260244e57e2 dsync: Various importer fixes. Handle unexpectedly wrong mailbox state better. Timo Sirainen <tss@iki.fi> parents: 15754 diff changeset	2733 i_error("Mailbox %s: Save commit failed: %s",
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2734 mailbox_get_vname(importer->box),
21814 759962e70148 global: Log internal storage error on failure Martti Rannanjärvi <martti.rannanjarvi@dovecot.fi> parents: 21390 diff changeset	2735 mailbox_get_last_internal_error(importer->box, &importer->mail_error));
16255 548e59794f2e dsync: Commit large transactions every 100 new messages. Timo Sirainen <tss@iki.fi> parents: 16254 diff changeset	2736 /* removed wanted_uids that weren't actually saved */
548e59794f2e dsync: Commit large transactions every 100 new messages. Timo Sirainen <tss@iki.fi> parents: 16254 diff changeset	2737 array_delete(&importer->wanted_uids,
548e59794f2e dsync: Commit large transactions every 100 new messages. Timo Sirainen <tss@iki.fi> parents: 16254 diff changeset	2738 array_count(&importer->saved_uids),
548e59794f2e dsync: Commit large transactions every 100 new messages. Timo Sirainen <tss@iki.fi> parents: 16254 diff changeset	2739 array_count(&importer->wanted_uids) -
548e59794f2e dsync: Commit large transactions every 100 new messages. Timo Sirainen <tss@iki.fi> parents: 16254 diff changeset	2740 array_count(&importer->saved_uids));
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2741 mailbox_transaction_rollback(&importer->trans);
16255 548e59794f2e dsync: Commit large transactions every 100 new messages. Timo Sirainen <tss@iki.fi> parents: 16254 diff changeset	2742 ret = -1;
548e59794f2e dsync: Commit large transactions every 100 new messages. Timo Sirainen <tss@iki.fi> parents: 16254 diff changeset	2743 } else {
548e59794f2e dsync: Commit large transactions every 100 new messages. Timo Sirainen <tss@iki.fi> parents: 16254 diff changeset	2744 /* remember the UIDs that were successfully saved */
16792 4a969a2e9399 dsync: Added some more debug logging. Timo Sirainen <tss@iki.fi> parents: 16762 diff changeset	2745 if (importer->debug) T_BEGIN {
4a969a2e9399 dsync: Added some more debug logging. Timo Sirainen <tss@iki.fi> parents: 16762 diff changeset	2746 string_t *str = t_str_new(256);
4a969a2e9399 dsync: Added some more debug logging. Timo Sirainen <tss@iki.fi> parents: 16762 diff changeset	2747 imap_write_seq_range(str, &changes.saved_uids);
4a969a2e9399 dsync: Added some more debug logging. Timo Sirainen <tss@iki.fi> parents: 16762 diff changeset	2748 imp_debug(importer, "Saved UIDs: %s", str_c(str));
4a969a2e9399 dsync: Added some more debug logging. Timo Sirainen <tss@iki.fi> parents: 16762 diff changeset	2749 } T_END;
16255 548e59794f2e dsync: Commit large transactions every 100 new messages. Timo Sirainen <tss@iki.fi> parents: 16254 diff changeset	2750 seq_range_array_iter_init(&iter, &changes.saved_uids); n = 0;
548e59794f2e dsync: Commit large transactions every 100 new messages. Timo Sirainen <tss@iki.fi> parents: 16254 diff changeset	2751 while (seq_range_array_iter_nth(&iter, n++, &uid))
548e59794f2e dsync: Commit large transactions every 100 new messages. Timo Sirainen <tss@iki.fi> parents: 16254 diff changeset	2752 array_append(&importer->saved_uids, &uid, 1);
548e59794f2e dsync: Commit large transactions every 100 new messages. Timo Sirainen <tss@iki.fi> parents: 16254 diff changeset	2753 pool_unref(&changes.pool);
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2754
16255 548e59794f2e dsync: Commit large transactions every 100 new messages. Timo Sirainen <tss@iki.fi> parents: 16254 diff changeset	2755 /* commit flag changes and expunges */
548e59794f2e dsync: Commit large transactions every 100 new messages. Timo Sirainen <tss@iki.fi> parents: 16254 diff changeset	2756 if (mailbox_transaction_commit(&importer->trans) < 0) {
548e59794f2e dsync: Commit large transactions every 100 new messages. Timo Sirainen <tss@iki.fi> parents: 16254 diff changeset	2757 i_error("Mailbox %s: Commit failed: %s",
548e59794f2e dsync: Commit large transactions every 100 new messages. Timo Sirainen <tss@iki.fi> parents: 16254 diff changeset	2758 mailbox_get_vname(importer->box),
21814 759962e70148 global: Log internal storage error on failure Martti Rannanjärvi <martti.rannanjarvi@dovecot.fi> parents: 21390 diff changeset	2759 mailbox_get_last_internal_error(importer->box,
759962e70148 global: Log internal storage error on failure Martti Rannanjärvi <martti.rannanjarvi@dovecot.fi> parents: 21390 diff changeset	2760 &importer->mail_error));
16255 548e59794f2e dsync: Commit large transactions every 100 new messages. Timo Sirainen <tss@iki.fi> parents: 16254 diff changeset	2761 ret = -1;
548e59794f2e dsync: Commit large transactions every 100 new messages. Timo Sirainen <tss@iki.fi> parents: 16254 diff changeset	2762 }
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2763 }
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2764
16255 548e59794f2e dsync: Commit large transactions every 100 new messages. Timo Sirainen <tss@iki.fi> parents: 16254 diff changeset	2765 if (!final)
548e59794f2e dsync: Commit large transactions every 100 new messages. Timo Sirainen <tss@iki.fi> parents: 16254 diff changeset	2766 dsync_mailbox_import_transaction_begin(importer);
548e59794f2e dsync: Commit large transactions every 100 new messages. Timo Sirainen <tss@iki.fi> parents: 16254 diff changeset	2767 return ret;
548e59794f2e dsync: Commit large transactions every 100 new messages. Timo Sirainen <tss@iki.fi> parents: 16254 diff changeset	2768 }
548e59794f2e dsync: Commit large transactions every 100 new messages. Timo Sirainen <tss@iki.fi> parents: 16254 diff changeset	2769
548e59794f2e dsync: Commit large transactions every 100 new messages. Timo Sirainen <tss@iki.fi> parents: 16254 diff changeset	2770 static int dsync_mailbox_import_finish(struct dsync_mailbox_importer *importer,
21256 4ac3461df334 dsync: When logging "Mailbox changed caused a desync", log also the reason. Timo Sirainen <timo.sirainen@dovecot.fi> parents: 20831 diff changeset	2771 const char **changes_during_sync_r)
16255 548e59794f2e dsync: Commit large transactions every 100 new messages. Timo Sirainen <tss@iki.fi> parents: 16254 diff changeset	2772 {
548e59794f2e dsync: Commit large transactions every 100 new messages. Timo Sirainen <tss@iki.fi> parents: 16254 diff changeset	2773 struct mailbox_update update;
548e59794f2e dsync: Commit large transactions every 100 new messages. Timo Sirainen <tss@iki.fi> parents: 16254 diff changeset	2774 int ret;
548e59794f2e dsync: Commit large transactions every 100 new messages. Timo Sirainen <tss@iki.fi> parents: 16254 diff changeset	2775
548e59794f2e dsync: Commit large transactions every 100 new messages. Timo Sirainen <tss@iki.fi> parents: 16254 diff changeset	2776 ret = dsync_mailbox_import_commit(importer, TRUE);
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2777
16255 548e59794f2e dsync: Commit large transactions every 100 new messages. Timo Sirainen <tss@iki.fi> parents: 16254 diff changeset	2778 if (ret == 0) {
548e59794f2e dsync: Commit large transactions every 100 new messages. Timo Sirainen <tss@iki.fi> parents: 16254 diff changeset	2779 /* update mailbox metadata if we successfully saved
548e59794f2e dsync: Commit large transactions every 100 new messages. Timo Sirainen <tss@iki.fi> parents: 16254 diff changeset	2780 everything. */
21389 59437f8764c6 global: Replaced all instances of memset(p, 0, sizeof(p)) with the new i_zero() macro. Stephan Bosch <stephan.bosch@dovecot.fi>* parents: 21267 diff changeset	2781 i_zero(&update);
16255 548e59794f2e dsync: Commit large transactions every 100 new messages. Timo Sirainen <tss@iki.fi> parents: 16254 diff changeset	2782 update.min_next_uid = importer->remote_uid_next;
548e59794f2e dsync: Commit large transactions every 100 new messages. Timo Sirainen <tss@iki.fi> parents: 16254 diff changeset	2783 update.min_first_recent_uid =
548e59794f2e dsync: Commit large transactions every 100 new messages. Timo Sirainen <tss@iki.fi> parents: 16254 diff changeset	2784 I_MIN(importer->last_common_uid+1,
548e59794f2e dsync: Commit large transactions every 100 new messages. Timo Sirainen <tss@iki.fi> parents: 16254 diff changeset	2785 importer->remote_first_recent_uid);
548e59794f2e dsync: Commit large transactions every 100 new messages. Timo Sirainen <tss@iki.fi> parents: 16254 diff changeset	2786 update.min_highest_modseq = importer->remote_highest_modseq;
548e59794f2e dsync: Commit large transactions every 100 new messages. Timo Sirainen <tss@iki.fi> parents: 16254 diff changeset	2787 update.min_highest_pvt_modseq = importer->remote_highest_pvt_modseq;
548e59794f2e dsync: Commit large transactions every 100 new messages. Timo Sirainen <tss@iki.fi> parents: 16254 diff changeset	2788
16792 4a969a2e9399 dsync: Added some more debug logging. Timo Sirainen <tss@iki.fi> parents: 16762 diff changeset	2789 imp_debug(importer, "Finish update: min_next_uid=%u "
4a969a2e9399 dsync: Added some more debug logging. Timo Sirainen <tss@iki.fi> parents: 16762 diff changeset	2790 "min_first_recent_uid=%u min_highest_modseq=%llu "
4a969a2e9399 dsync: Added some more debug logging. Timo Sirainen <tss@iki.fi> parents: 16762 diff changeset	2791 "min_highest_pvt_modseq=%llu",
4a969a2e9399 dsync: Added some more debug logging. Timo Sirainen <tss@iki.fi> parents: 16762 diff changeset	2792 update.min_next_uid, update.min_first_recent_uid,
17272 767acd575d06 Compiler warning fixes Timo Sirainen <tss@iki.fi> parents: 17261 diff changeset	2793 (unsigned long long)update.min_highest_modseq,
767acd575d06 Compiler warning fixes Timo Sirainen <tss@iki.fi> parents: 17261 diff changeset	2794 (unsigned long long)update.min_highest_pvt_modseq);
16792 4a969a2e9399 dsync: Added some more debug logging. Timo Sirainen <tss@iki.fi> parents: 16762 diff changeset	2795
16255 548e59794f2e dsync: Commit large transactions every 100 new messages. Timo Sirainen <tss@iki.fi> parents: 16254 diff changeset	2796 if (mailbox_update(importer->box, &update) < 0) {
548e59794f2e dsync: Commit large transactions every 100 new messages. Timo Sirainen <tss@iki.fi> parents: 16254 diff changeset	2797 i_error("Mailbox %s: Update failed: %s",
548e59794f2e dsync: Commit large transactions every 100 new messages. Timo Sirainen <tss@iki.fi> parents: 16254 diff changeset	2798 mailbox_get_vname(importer->box),
21814 759962e70148 global: Log internal storage error on failure Martti Rannanjärvi <martti.rannanjarvi@dovecot.fi> parents: 21390 diff changeset	2799 mailbox_get_last_internal_error(importer->box,
759962e70148 global: Log internal storage error on failure Martti Rannanjärvi <martti.rannanjarvi@dovecot.fi> parents: 21390 diff changeset	2800 &importer->mail_error));
16255 548e59794f2e dsync: Commit large transactions every 100 new messages. Timo Sirainen <tss@iki.fi> parents: 16254 diff changeset	2801 ret = -1;
548e59794f2e dsync: Commit large transactions every 100 new messages. Timo Sirainen <tss@iki.fi> parents: 16254 diff changeset	2802 }
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2803 }
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2804
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2805 /* sync mailbox to finish flag changes and expunges. */
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2806 if (mailbox_sync(importer->box, 0) < 0) {
15766 4260244e57e2 dsync: Various importer fixes. Handle unexpectedly wrong mailbox state better. Timo Sirainen <tss@iki.fi> parents: 15754 diff changeset	2807 i_error("Mailbox %s: Sync failed: %s",
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2808 mailbox_get_vname(importer->box),
21814 759962e70148 global: Log internal storage error on failure Martti Rannanjärvi <martti.rannanjarvi@dovecot.fi> parents: 21390 diff changeset	2809 mailbox_get_last_internal_error(importer->box,
759962e70148 global: Log internal storage error on failure Martti Rannanjärvi <martti.rannanjarvi@dovecot.fi> parents: 21390 diff changeset	2810 &importer->mail_error));
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2811 ret = -1;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2812 }
16255 548e59794f2e dsync: Commit large transactions every 100 new messages. Timo Sirainen <tss@iki.fi> parents: 16254 diff changeset	2813 if (ret == 0) {
548e59794f2e dsync: Commit large transactions every 100 new messages. Timo Sirainen <tss@iki.fi> parents: 16254 diff changeset	2814 /* give new UIDs to messages that got saved with unwanted UIDs.
548e59794f2e dsync: Commit large transactions every 100 new messages. Timo Sirainen <tss@iki.fi> parents: 16254 diff changeset	2815 do it only if the whole transaction succeeded. */
548e59794f2e dsync: Commit large transactions every 100 new messages. Timo Sirainen <tss@iki.fi> parents: 16254 diff changeset	2816 if (reassign_unwanted_uids(importer, changes_during_sync_r) < 0)
548e59794f2e dsync: Commit large transactions every 100 new messages. Timo Sirainen <tss@iki.fi> parents: 16254 diff changeset	2817 ret = -1;
548e59794f2e dsync: Commit large transactions every 100 new messages. Timo Sirainen <tss@iki.fi> parents: 16254 diff changeset	2818 }
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2819 return ret;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2820 }
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2821
15662 fc5c3281d6d3 dsync: If remote doesn't send some message bodies, log an error about each one separately. Timo Sirainen <tss@iki.fi> parents: 15661 diff changeset	2822 static void
fc5c3281d6d3 dsync: If remote doesn't send some message bodies, log an error about each one separately. Timo Sirainen <tss@iki.fi> parents: 15661 diff changeset	2823 dsync_mailbox_import_check_missing_guid_imports(struct dsync_mailbox_importer *importer)
14918 8eae4e205c82 Hash table API is now (mostly) type safe. Timo Sirainen <tss@iki.fi> parents: 14917 diff changeset	2824 {
8eae4e205c82 Hash table API is now (mostly) type safe. Timo Sirainen <tss@iki.fi> parents: 14917 diff changeset	2825 struct hash_iterate_context *iter;
8eae4e205c82 Hash table API is now (mostly) type safe. Timo Sirainen <tss@iki.fi> parents: 14917 diff changeset	2826 const char *key;
8eae4e205c82 Hash table API is now (mostly) type safe. Timo Sirainen <tss@iki.fi> parents: 14917 diff changeset	2827 struct importer_new_mail *mail;
8eae4e205c82 Hash table API is now (mostly) type safe. Timo Sirainen <tss@iki.fi> parents: 14917 diff changeset	2828
15611 cc77431b09b4 dsync: Don't wrongly log errors about missing message bodies. Timo Sirainen <tss@iki.fi> parents: 15591 diff changeset	2829 iter = hash_table_iterate_init(importer->import_guids);
cc77431b09b4 dsync: Don't wrongly log errors about missing message bodies. Timo Sirainen <tss@iki.fi> parents: 15591 diff changeset	2830 while (hash_table_iterate(iter, importer->import_guids, &key, &mail)) {
14918 8eae4e205c82 Hash table API is now (mostly) type safe. Timo Sirainen <tss@iki.fi> parents: 14917 diff changeset	2831 for (; mail != NULL; mail = mail->next) {
15772 360661d99c42 dsync: Various importer fixes. Timo Sirainen <tss@iki.fi> parents: 15770 diff changeset	2832 if (mail->skip)
15611 cc77431b09b4 dsync: Don't wrongly log errors about missing message bodies. Timo Sirainen <tss@iki.fi> parents: 15591 diff changeset	2833 continue;
cc77431b09b4 dsync: Don't wrongly log errors about missing message bodies. Timo Sirainen <tss@iki.fi> parents: 15591 diff changeset	2834
15662 fc5c3281d6d3 dsync: If remote doesn't send some message bodies, log an error about each one separately. Timo Sirainen <tss@iki.fi> parents: 15661 diff changeset	2835 i_error("Mailbox %s: Remote didn't send mail GUID=%s (UID=%u)",
fc5c3281d6d3 dsync: If remote doesn't send some message bodies, log an error about each one separately. Timo Sirainen <tss@iki.fi> parents: 15661 diff changeset	2836 mailbox_get_vname(importer->box),
15766 4260244e57e2 dsync: Various importer fixes. Handle unexpectedly wrong mailbox state better. Timo Sirainen <tss@iki.fi> parents: 15754 diff changeset	2837 mail->guid, mail->remote_uid);
18371 b900b50085fc dsync: Use storage's mail_error to choose the doveadm exit code. Timo Sirainen <tss@iki.fi> parents: 18183 diff changeset	2838 importer->mail_error = MAIL_ERROR_TEMP;
16132 0eef38d78069 dsync: If remote doesn't send some mails, don't exit with code 0. Timo Sirainen <tss@iki.fi> parents: 16117 diff changeset	2839 importer->failed = TRUE;
14918 8eae4e205c82 Hash table API is now (mostly) type safe. Timo Sirainen <tss@iki.fi> parents: 14917 diff changeset	2840 }
8eae4e205c82 Hash table API is now (mostly) type safe. Timo Sirainen <tss@iki.fi> parents: 14917 diff changeset	2841 }
8eae4e205c82 Hash table API is now (mostly) type safe. Timo Sirainen <tss@iki.fi> parents: 14917 diff changeset	2842 hash_table_iterate_deinit(&iter);
8eae4e205c82 Hash table API is now (mostly) type safe. Timo Sirainen <tss@iki.fi> parents: 14917 diff changeset	2843 }
8eae4e205c82 Hash table API is now (mostly) type safe. Timo Sirainen <tss@iki.fi> parents: 14917 diff changeset	2844
15662 fc5c3281d6d3 dsync: If remote doesn't send some message bodies, log an error about each one separately. Timo Sirainen <tss@iki.fi> parents: 15661 diff changeset	2845 static void
fc5c3281d6d3 dsync: If remote doesn't send some message bodies, log an error about each one separately. Timo Sirainen <tss@iki.fi> parents: 15661 diff changeset	2846 dsync_mailbox_import_check_missing_uid_imports(struct dsync_mailbox_importer *importer)
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2847 {
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2848 struct hash_iterate_context *iter;
14923 96fd2c3bf932 Reverted "support for non-pointers" part of the hash table API changes. Timo Sirainen <tss@iki.fi> parents: 14922 diff changeset	2849 void *key;
96fd2c3bf932 Reverted "support for non-pointers" part of the hash table API changes. Timo Sirainen <tss@iki.fi> parents: 14922 diff changeset	2850 struct importer_new_mail *mail;
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2851
15611 cc77431b09b4 dsync: Don't wrongly log errors about missing message bodies. Timo Sirainen <tss@iki.fi> parents: 15591 diff changeset	2852 iter = hash_table_iterate_init(importer->import_uids);
cc77431b09b4 dsync: Don't wrongly log errors about missing message bodies. Timo Sirainen <tss@iki.fi> parents: 15591 diff changeset	2853 while (hash_table_iterate(iter, importer->import_uids, &key, &mail)) {
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2854 for (; mail != NULL; mail = mail->next) {
15772 360661d99c42 dsync: Various importer fixes. Timo Sirainen <tss@iki.fi> parents: 15770 diff changeset	2855 if (mail->skip)
15611 cc77431b09b4 dsync: Don't wrongly log errors about missing message bodies. Timo Sirainen <tss@iki.fi> parents: 15591 diff changeset	2856 continue;
cc77431b09b4 dsync: Don't wrongly log errors about missing message bodies. Timo Sirainen <tss@iki.fi> parents: 15591 diff changeset	2857
15662 fc5c3281d6d3 dsync: If remote doesn't send some message bodies, log an error about each one separately. Timo Sirainen <tss@iki.fi> parents: 15661 diff changeset	2858 i_error("Mailbox %s: Remote didn't send mail UID=%u",
fc5c3281d6d3 dsync: If remote doesn't send some message bodies, log an error about each one separately. Timo Sirainen <tss@iki.fi> parents: 15661 diff changeset	2859 mailbox_get_vname(importer->box),
15766 4260244e57e2 dsync: Various importer fixes. Handle unexpectedly wrong mailbox state better. Timo Sirainen <tss@iki.fi> parents: 15754 diff changeset	2860 mail->remote_uid);
18371 b900b50085fc dsync: Use storage's mail_error to choose the doveadm exit code. Timo Sirainen <tss@iki.fi> parents: 18183 diff changeset	2861 importer->mail_error = MAIL_ERROR_TEMP;
16132 0eef38d78069 dsync: If remote doesn't send some mails, don't exit with code 0. Timo Sirainen <tss@iki.fi> parents: 16117 diff changeset	2862 importer->failed = TRUE;
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2863 }
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2864 }
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2865 hash_table_iterate_deinit(&iter);
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2866 }
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2867
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2868 int dsync_mailbox_import_deinit(struct dsync_mailbox_importer **_importer,
15766 4260244e57e2 dsync: Various importer fixes. Handle unexpectedly wrong mailbox state better. Timo Sirainen <tss@iki.fi> parents: 15754 diff changeset	2869 bool success,
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2870 uint32_t *last_common_uid_r,
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2871 uint64_t *last_common_modseq_r,
15496 5d90e2aa1ba8 dsync: Use private modseqs to support syncing private flags in shared mailboxes. Timo Sirainen <tss@iki.fi> parents: 15257 diff changeset	2872 uint64_t *last_common_pvt_modseq_r,
17282 9669c9a8984f dsync: Include messages_count in the mailbox states. Timo Sirainen <tss@iki.fi> parents: 17272 diff changeset	2873 uint32_t *last_messages_count_r,
21256 4ac3461df334 dsync: When logging "Mailbox changed caused a desync", log also the reason. Timo Sirainen <timo.sirainen@dovecot.fi> parents: 20831 diff changeset	2874 const char **changes_during_sync_r,
20074 7a29a0930b63 dsync: If state is invalid, exit with code 2 instead of tempfail. Timo Sirainen <timo.sirainen@dovecot.fi> parents: 20072 diff changeset	2875 bool *require_full_resync_r,
18371 b900b50085fc dsync: Use storage's mail_error to choose the doveadm exit code. Timo Sirainen <tss@iki.fi> parents: 18183 diff changeset	2876 enum mail_error *error_r)
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2877 {
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2878 struct dsync_mailbox_importer importer = _importer;
17282 9669c9a8984f dsync: Include messages_count in the mailbox states. Timo Sirainen <tss@iki.fi> parents: 17272 diff changeset	2879 struct mailbox_status status;
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2880 int ret;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2881
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2882 *_importer = NULL;
21256 4ac3461df334 dsync: When logging "Mailbox changed caused a desync", log also the reason. Timo Sirainen <timo.sirainen@dovecot.fi> parents: 20831 diff changeset	2883 *changes_during_sync_r = NULL;
20074 7a29a0930b63 dsync: If state is invalid, exit with code 2 instead of tempfail. Timo Sirainen <timo.sirainen@dovecot.fi> parents: 20072 diff changeset	2884 *require_full_resync_r = importer->require_full_resync;
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2885
20074 7a29a0930b63 dsync: If state is invalid, exit with code 2 instead of tempfail. Timo Sirainen <timo.sirainen@dovecot.fi> parents: 20072 diff changeset	2886 if ((!success \|\| importer->require_full_resync) && !importer->failed) {
18371 b900b50085fc dsync: Use storage's mail_error to choose the doveadm exit code. Timo Sirainen <tss@iki.fi> parents: 18183 diff changeset	2887 importer->mail_error = MAIL_ERROR_TEMP;
15766 4260244e57e2 dsync: Various importer fixes. Handle unexpectedly wrong mailbox state better. Timo Sirainen <tss@iki.fi> parents: 15754 diff changeset	2888 importer->failed = TRUE;
18371 b900b50085fc dsync: Use storage's mail_error to choose the doveadm exit code. Timo Sirainen <tss@iki.fi> parents: 18183 diff changeset	2889 }
15766 4260244e57e2 dsync: Various importer fixes. Handle unexpectedly wrong mailbox state better. Timo Sirainen <tss@iki.fi> parents: 15754 diff changeset	2890
4260244e57e2 dsync: Various importer fixes. Handle unexpectedly wrong mailbox state better. Timo Sirainen <tss@iki.fi> parents: 15754 diff changeset	2891 if (!importer->new_uids_assigned && !importer->failed)
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2892 dsync_mailbox_import_assign_new_uids(importer);
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2893
15766 4260244e57e2 dsync: Various importer fixes. Handle unexpectedly wrong mailbox state better. Timo Sirainen <tss@iki.fi> parents: 15754 diff changeset	2894 if (!importer->failed) {
4260244e57e2 dsync: Various importer fixes. Handle unexpectedly wrong mailbox state better. Timo Sirainen <tss@iki.fi> parents: 15754 diff changeset	2895 dsync_mailbox_import_check_missing_guid_imports(importer);
4260244e57e2 dsync: Various importer fixes. Handle unexpectedly wrong mailbox state better. Timo Sirainen <tss@iki.fi> parents: 15754 diff changeset	2896 dsync_mailbox_import_check_missing_uid_imports(importer);
4260244e57e2 dsync: Various importer fixes. Handle unexpectedly wrong mailbox state better. Timo Sirainen <tss@iki.fi> parents: 15754 diff changeset	2897 }
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2898
14682 d0d7b810646b Make sure we check all the functions' return values. Minor API changes to simplify this. Timo Sirainen <tss@iki.fi> parents: 14676 diff changeset	2899 if (importer->search_ctx != NULL) {
15766 4260244e57e2 dsync: Various importer fixes. Handle unexpectedly wrong mailbox state better. Timo Sirainen <tss@iki.fi> parents: 15754 diff changeset	2900 if (mailbox_search_deinit(&importer->search_ctx) < 0) {
4260244e57e2 dsync: Various importer fixes. Handle unexpectedly wrong mailbox state better. Timo Sirainen <tss@iki.fi> parents: 15754 diff changeset	2901 i_error("Mailbox %s: Search failed: %s",
4260244e57e2 dsync: Various importer fixes. Handle unexpectedly wrong mailbox state better. Timo Sirainen <tss@iki.fi> parents: 15754 diff changeset	2902 mailbox_get_vname(importer->box),
21814 759962e70148 global: Log internal storage error on failure Martti Rannanjärvi <martti.rannanjarvi@dovecot.fi> parents: 21390 diff changeset	2903 mailbox_get_last_internal_error(importer->box,
759962e70148 global: Log internal storage error on failure Martti Rannanjärvi <martti.rannanjarvi@dovecot.fi> parents: 21390 diff changeset	2904 &importer->mail_error));
14682 d0d7b810646b Make sure we check all the functions' return values. Minor API changes to simplify this. Timo Sirainen <tss@iki.fi> parents: 14676 diff changeset	2905 importer->failed = TRUE;
15766 4260244e57e2 dsync: Various importer fixes. Handle unexpectedly wrong mailbox state better. Timo Sirainen <tss@iki.fi> parents: 15754 diff changeset	2906 }
14682 d0d7b810646b Make sure we check all the functions' return values. Minor API changes to simplify this. Timo Sirainen <tss@iki.fi> parents: 14676 diff changeset	2907 }
16255 548e59794f2e dsync: Commit large transactions every 100 new messages. Timo Sirainen <tss@iki.fi> parents: 16254 diff changeset	2908 if (dsync_mailbox_import_finish(importer, changes_during_sync_r) < 0)
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2909 importer->failed = TRUE;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2910
18180 39d00448490f dsync: If same GUID already exists in storage, try to copy it instead of recreating the mail. Timo Sirainen <tss@iki.fi> parents: 18178 diff changeset	2911 if (importer->virtual_mail != NULL)
39d00448490f dsync: If same GUID already exists in storage, try to copy it instead of recreating the mail. Timo Sirainen <tss@iki.fi> parents: 18178 diff changeset	2912 mail_free(&importer->virtual_mail);
39d00448490f dsync: If same GUID already exists in storage, try to copy it instead of recreating the mail. Timo Sirainen <tss@iki.fi> parents: 18178 diff changeset	2913 if (importer->virtual_trans != NULL)
39d00448490f dsync: If same GUID already exists in storage, try to copy it instead of recreating the mail. Timo Sirainen <tss@iki.fi> parents: 18178 diff changeset	2914 (void)mailbox_transaction_commit(&importer->virtual_trans);
39d00448490f dsync: If same GUID already exists in storage, try to copy it instead of recreating the mail. Timo Sirainen <tss@iki.fi> parents: 18178 diff changeset	2915
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2916 hash_table_destroy(&importer->import_guids);
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2917 hash_table_destroy(&importer->import_uids);
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2918 array_free(&importer->maybe_expunge_uids);
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2919 array_free(&importer->maybe_saves);
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2920 array_free(&importer->wanted_uids);
16255 548e59794f2e dsync: Commit large transactions every 100 new messages. Timo Sirainen <tss@iki.fi> parents: 16254 diff changeset	2921 array_free(&importer->saved_uids);
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2922 array_free(&importer->newmails);
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2923 if (array_is_created(&importer->mail_requests))
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2924 array_free(&importer->mail_requests);
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2925
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2926 *last_common_uid_r = importer->last_common_uid;
21267 b60c7b697838 dsync: Fixed boolean expression in dsync_mailbox_import_deinit(). Stephan Bosch <stephan.bosch@dovecot.fi> parents: 21256 diff changeset	2927 if (*changes_during_sync_r == NULL) {
15232 6c850258002f doveadm sync/backup: Added -s <state> parameter to do a fast stateful sync. Timo Sirainen <tss@iki.fi> parents: 14926 diff changeset	2928 *last_common_modseq_r = importer->remote_highest_modseq;
15496 5d90e2aa1ba8 dsync: Use private modseqs to support syncing private flags in shared mailboxes. Timo Sirainen <tss@iki.fi> parents: 15257 diff changeset	2929 *last_common_pvt_modseq_r = importer->remote_highest_pvt_modseq;
5d90e2aa1ba8 dsync: Use private modseqs to support syncing private flags in shared mailboxes. Timo Sirainen <tss@iki.fi> parents: 15257 diff changeset	2930 } else {
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2931 /* local changes occurred during dsync. we exported changes up
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2932 to local_initial_highestmodseq, so all of the changes have
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2933 happened after it. we want the next run to see those changes,
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2934 so return it as the last common modseq */
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2935 *last_common_modseq_r = importer->local_initial_highestmodseq;
15496 5d90e2aa1ba8 dsync: Use private modseqs to support syncing private flags in shared mailboxes. Timo Sirainen <tss@iki.fi> parents: 15257 diff changeset	2936 *last_common_pvt_modseq_r = importer->local_initial_highestpvtmodseq;
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2937 }
17859 c02918434721 doveadm backup: Locally expunged mails weren't always added back. Timo Sirainen <tss@iki.fi> parents: 17845 diff changeset	2938 if (importer->delete_mailbox) {
c02918434721 doveadm backup: Locally expunged mails weren't always added back. Timo Sirainen <tss@iki.fi> parents: 17845 diff changeset	2939 if (mailbox_delete(importer->box) < 0) {
c02918434721 doveadm backup: Locally expunged mails weren't always added back. Timo Sirainen <tss@iki.fi> parents: 17845 diff changeset	2940 i_error("Couldn't delete mailbox %s: %s",
c02918434721 doveadm backup: Locally expunged mails weren't always added back. Timo Sirainen <tss@iki.fi> parents: 17845 diff changeset	2941 mailbox_get_vname(importer->box),
21814 759962e70148 global: Log internal storage error on failure Martti Rannanjärvi <martti.rannanjarvi@dovecot.fi> parents: 21390 diff changeset	2942 mailbox_get_last_internal_error(importer->box,
759962e70148 global: Log internal storage error on failure Martti Rannanjärvi <martti.rannanjarvi@dovecot.fi> parents: 21390 diff changeset	2943 &importer->mail_error));
17859 c02918434721 doveadm backup: Locally expunged mails weren't always added back. Timo Sirainen <tss@iki.fi> parents: 17845 diff changeset	2944 importer->failed = TRUE;
c02918434721 doveadm backup: Locally expunged mails weren't always added back. Timo Sirainen <tss@iki.fi> parents: 17845 diff changeset	2945 }
c02918434721 doveadm backup: Locally expunged mails weren't always added back. Timo Sirainen <tss@iki.fi> parents: 17845 diff changeset	2946 *last_messages_count_r = 0;
c02918434721 doveadm backup: Locally expunged mails weren't always added back. Timo Sirainen <tss@iki.fi> parents: 17845 diff changeset	2947 } else {
c02918434721 doveadm backup: Locally expunged mails weren't always added back. Timo Sirainen <tss@iki.fi> parents: 17845 diff changeset	2948 mailbox_get_open_status(importer->box, STATUS_MESSAGES, &status);
c02918434721 doveadm backup: Locally expunged mails weren't always added back. Timo Sirainen <tss@iki.fi> parents: 17845 diff changeset	2949 *last_messages_count_r = status.messages;
c02918434721 doveadm backup: Locally expunged mails weren't always added back. Timo Sirainen <tss@iki.fi> parents: 17845 diff changeset	2950 }
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2951
18371 b900b50085fc dsync: Use storage's mail_error to choose the doveadm exit code. Timo Sirainen <tss@iki.fi> parents: 18183 diff changeset	2952 i_assert(importer->failed == (importer->mail_error != 0));
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2953 ret = importer->failed ? -1 : 0;
18371 b900b50085fc dsync: Use storage's mail_error to choose the doveadm exit code. Timo Sirainen <tss@iki.fi> parents: 18183 diff changeset	2954 *error_r = importer->mail_error;
14584 b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2955 pool_unref(&importer->pool);
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2956 return ret;
b2076acc3715 Initial version of dsync rewrite. Timo Sirainen <tss@iki.fi> parents: diff changeset	2957 }
16539 b5876fa03b0e dsync: If verbose_proctitle=yes, show the current state in it. Timo Sirainen <tss@iki.fi> parents: 16259 diff changeset	2958
b5876fa03b0e dsync: If verbose_proctitle=yes, show the current state in it. Timo Sirainen <tss@iki.fi> parents: 16259 diff changeset	2959 const char dsync_mailbox_import_get_proctitle(struct dsync_mailbox_importer importer)
b5876fa03b0e dsync: If verbose_proctitle=yes, show the current state in it. Timo Sirainen <tss@iki.fi> parents: 16259 diff changeset	2960 {
b5876fa03b0e dsync: If verbose_proctitle=yes, show the current state in it. Timo Sirainen <tss@iki.fi> parents: 16259 diff changeset	2961 if (importer->search_ctx != NULL)
b5876fa03b0e dsync: If verbose_proctitle=yes, show the current state in it. Timo Sirainen <tss@iki.fi> parents: 16259 diff changeset	2962 return "";
b5876fa03b0e dsync: If verbose_proctitle=yes, show the current state in it. Timo Sirainen <tss@iki.fi> parents: 16259 diff changeset	2963 return t_strdup_printf("%u/%u", importer->import_pos,
b5876fa03b0e dsync: If verbose_proctitle=yes, show the current state in it. Timo Sirainen <tss@iki.fi> parents: 16259 diff changeset	2964 importer->import_count);
b5876fa03b0e dsync: If verbose_proctitle=yes, show the current state in it. Timo Sirainen <tss@iki.fi> parents: 16259 diff changeset	2965 }

Mercurial > dovecot > core-2.2

annotate src/doveadm/dsync/dsync-mailbox-import.c @ 22713:cb108f786fb4