view src/doveadm/dsync/dsync-brain-private.h @ 22715:20415dd0b85a

dsync: Add per-mailbox sync lock that is always used. Both importing and exporting gets the lock before they even sync the mailbox. The lock is kept until the import/export finishes. This guarantees that no matter how dsync is run, two dsyncs can't be working on the same mailbox at the same time. This lock is in addition to the optional per-user lock enabled by the -l parameter. If the -l parameter is used, the same lock timeout is used for the per-mailbox lock. Otherwise 30s timeout is used. This should help to avoid email duplication when replication is enabled for public namespaces, and maybe in some other rare situations as well.
author Timo Sirainen <>
date Thu, 28 Dec 2017 14:10:23 +0200
parents 400ff84f109d
line wrap: on
line source


#include "hash.h"
#include "dsync-brain.h"
#include "dsync-mailbox.h"
#include "dsync-mailbox-state.h"

#define DSYNC_LOCK_FILENAME ".dovecot-sync.lock"
#define DSYNC_MAILBOX_LOCK_FILENAME ".dovecot-box-sync.lock"

struct dsync_mailbox_tree_sync_change;

enum dsync_state {
	/* if sync_type=STATE, the master brain knows the saved "last common
	   mailbox state". this state is sent to the slave. */

	/* both sides send their mailbox trees */

	/* master decides in which order mailboxes are synced (it knows the
	   slave's mailboxes by looking at the received mailbox tree) */
	/* once mailbox is selected, the mails inside it are synced.
	   after the mails are synced, another mailbox is synced. */


enum dsync_box_state {

struct dsync_brain {
	pool_t pool;
	struct mail_user *user;
	struct dsync_ibc *ibc;
	const char *process_title_prefix;
	ARRAY(struct mail_namespace *) sync_namespaces;
	const char *sync_box;
	struct mailbox *virtual_all_box;
	guid_128_t sync_box_guid;
	const char *const *exclude_mailboxes;
	enum dsync_brain_sync_type sync_type;
	time_t sync_since_timestamp;
	time_t sync_until_timestamp;
	uoff_t sync_max_size;
	const char *sync_flag;
	char alt_char;
	unsigned int import_commit_msgs_interval;
	unsigned int hdr_hash_version;

	unsigned int lock_timeout;
	int lock_fd;
	const char *lock_path;
	struct file_lock *lock;

	char hierarchy_sep;
	struct dsync_mailbox_tree *local_mailbox_tree;
	struct dsync_mailbox_tree *remote_mailbox_tree;
	struct dsync_mailbox_tree_iter *local_tree_iter;

	enum dsync_state state, pre_box_state;
	enum dsync_box_state box_recv_state;
	enum dsync_box_state box_send_state;
	unsigned int proctitle_update_counter;

	struct dsync_transaction_log_scan *log_scan;
	struct dsync_mailbox_importer *box_importer;
	struct dsync_mailbox_exporter *box_exporter;

	struct mailbox *box;
	struct file_lock *box_lock;
	unsigned int mailbox_lock_timeout_secs;
	struct dsync_mailbox local_dsync_box, remote_dsync_box;
	pool_t dsync_box_pool;
	/* list of mailbox states
	   for master brain: given to brain at init and
	   for slave brain: received from DSYNC_STATE_SLAVE_RECV_LAST_COMMON */
	HASH_TABLE_TYPE(dsync_mailbox_state) mailbox_states;
	/* DSYNC_STATE_MASTER_SEND_LAST_COMMON: current send position */
	struct hash_iterate_context *mailbox_states_iter;
	/* state of the mailbox we're currently syncing, changed at
	   init and deinit */
	struct dsync_mailbox_state mailbox_state;
	/* new states for synced mailboxes */
	ARRAY_TYPE(dsync_mailbox_state) remote_mailbox_states;

	const char *changes_during_sync;
	enum mail_error mail_error;
	const char *const *hashed_headers;

	unsigned int master_brain:1;
	unsigned int mail_requests:1;
	unsigned int backup_send:1;
	unsigned int backup_recv:1;
	unsigned int purge:1;
	unsigned int debug:1;
	unsigned int sync_visible_namespaces:1;
	unsigned int no_mail_sync:1;
	unsigned int no_backup_overwrite:1;
	unsigned int no_mail_prefetch:1;
	unsigned int no_mailbox_renames:1;
	unsigned int changes_during_remote_sync:1;
	unsigned int require_full_resync:1;
	unsigned int verbose_proctitle:1;
	unsigned int no_notify:1;
	unsigned int failed:1;
	unsigned int empty_hdr_workaround:1;

extern const char *dsync_box_state_names[DSYNC_BOX_STATE_DONE+1];

void dsync_brain_mailbox_trees_init(struct dsync_brain *brain);
void dsync_brain_send_mailbox_tree(struct dsync_brain *brain);
void dsync_brain_send_mailbox_tree_deletes(struct dsync_brain *brain);
bool dsync_brain_recv_mailbox_tree(struct dsync_brain *brain);
bool dsync_brain_recv_mailbox_tree_deletes(struct dsync_brain *brain);
int dsync_brain_mailbox_tree_sync_change(struct dsync_brain *brain,
			const struct dsync_mailbox_tree_sync_change *change,
			enum mail_error *error_r);

void dsync_brain_sync_mailbox_deinit(struct dsync_brain *brain);
int dsync_brain_mailbox_alloc(struct dsync_brain *brain, const guid_128_t guid,
			      struct mailbox **box_r, const char **errstr_r,
			      enum mail_error *error_r);
bool dsync_brain_mailbox_update_pre(struct dsync_brain *brain,
				    struct mailbox *box,
				    const struct dsync_mailbox *local_box,
				    const struct dsync_mailbox *remote_box,
				    const char **reason_r);
bool dsync_boxes_need_sync(struct dsync_brain *brain,
			   const struct dsync_mailbox *box1,
			   const struct dsync_mailbox *box2);
void dsync_brain_sync_init_box_states(struct dsync_brain *brain);
void dsync_brain_set_changes_during_sync(struct dsync_brain *brain,
					 const char *reason);

void dsync_brain_master_send_mailbox(struct dsync_brain *brain);
bool dsync_brain_slave_recv_mailbox(struct dsync_brain *brain);
int dsync_brain_sync_mailbox_open(struct dsync_brain *brain,
				  const struct dsync_mailbox *remote_dsync_box);
bool dsync_brain_sync_mails(struct dsync_brain *brain);