Mercurial > dovecot > core-2.2

annotate src/doveadm/dsync/dsync-brain.c @ 22549:400ff84f109d

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
dsync: Add hashed_headers setting This makes it possible to configure them
author Aki Tuomi <aki.tuomi@dovecot.fi>
date Fri, 22 Sep 2017 13:30:43 +0300
parents e95435889161
children cb108f786fb4
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
21390
2e2563132d5f Updated copyright notices to include the year 2017.
Stephan Bosch <stephan.bosch@dovecot.fi>
parents: 21389
diff changeset
1 /* Copyright (c) 2013-2017 Dovecot authors, see the included COPYING file */
9559
34eef8a2716b dsync: Initial implementation.
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
2
34eef8a2716b dsync: Initial implementation.
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
3 #include "lib.h"
34eef8a2716b dsync: Initial implementation.
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
4 #include "array.h"
34eef8a2716b dsync: Initial implementation.
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
5 #include "hash.h"
15782
6f024e0289da dsync: -l parameter locking is now done on the server with "lower" hostname.
Timo Sirainen <tss@iki.fi>
parents: 15750
diff changeset
6 #include "hostpid.h"
16539
b5876fa03b0e dsync: If verbose_proctitle=yes, show the current state in it.
Timo Sirainen <tss@iki.fi>
parents: 16538
diff changeset
7 #include "str.h"
21289
08eec0b1aae6 dsync: Fix .dovecot-sync.lock timeout checking
Timo Sirainen <timo.sirainen@dovecot.fi>
parents: 21288
diff changeset
8 #include "file-create-locked.h"
16539
b5876fa03b0e dsync: If verbose_proctitle=yes, show the current state in it.
Timo Sirainen <tss@iki.fi>
parents: 16538
diff changeset
9 #include "process-title.h"
16540
84014270ee37 dsync: Don't try to find mailboxes from unwanted namespaces.
Timo Sirainen <tss@iki.fi>
parents: 16539
diff changeset
10 #include "settings-parser.h"
16539
b5876fa03b0e dsync: If verbose_proctitle=yes, show the current state in it.
Timo Sirainen <tss@iki.fi>
parents: 16538
diff changeset
11 #include "master-service.h"
b5876fa03b0e dsync: If verbose_proctitle=yes, show the current state in it.
Timo Sirainen <tss@iki.fi>
parents: 16538
diff changeset
12 #include "master-service-settings.h"
14584
b2076acc3715 Initial version of dsync rewrite.
Timo Sirainen <tss@iki.fi>
parents: 14253
diff changeset
13 #include "mail-namespace.h"
b2076acc3715 Initial version of dsync rewrite.
Timo Sirainen <tss@iki.fi>
parents: 14253
diff changeset
14 #include "dsync-mailbox-tree.h"
15037
920756cd29b8 dsync: Renamed "slave" to "ibc" (= inter-brain communicator)
Timo Sirainen <tss@iki.fi>
parents: 14939
diff changeset
15 #include "dsync-ibc.h"
9559
34eef8a2716b dsync: Initial implementation.
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
16 #include "dsync-brain-private.h"
16539
b5876fa03b0e dsync: If verbose_proctitle=yes, show the current state in it.
Timo Sirainen <tss@iki.fi>
parents: 16538
diff changeset
17 #include "dsync-mailbox-import.h"
b5876fa03b0e dsync: If verbose_proctitle=yes, show the current state in it.
Timo Sirainen <tss@iki.fi>
parents: 16538
diff changeset
18 #include "dsync-mailbox-export.h"
9559
34eef8a2716b dsync: Initial implementation.
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
19
15782
6f024e0289da dsync: -l parameter locking is now done on the server with "lower" hostname.
Timo Sirainen <tss@iki.fi>
parents: 15750
diff changeset
20 #include <sys/stat.h>
6f024e0289da dsync: -l parameter locking is now done on the server with "lower" hostname.
Timo Sirainen <tss@iki.fi>
parents: 15750
diff changeset
21
21288
9316919c5932 dsync: Improve process title during initialization
Timo Sirainen <timo.sirainen@dovecot.fi>
parents: 21256
diff changeset
22 enum dsync_brain_title {
9316919c5932 dsync: Improve process title during initialization
Timo Sirainen <timo.sirainen@dovecot.fi>
parents: 21256
diff changeset
23 DSYNC_BRAIN_TITLE_NONE = 0,
9316919c5932 dsync: Improve process title during initialization
Timo Sirainen <timo.sirainen@dovecot.fi>
parents: 21256
diff changeset
24 DSYNC_BRAIN_TITLE_LOCKING,
9316919c5932 dsync: Improve process title during initialization
Timo Sirainen <timo.sirainen@dovecot.fi>
parents: 21256
diff changeset
25 };
9316919c5932 dsync: Improve process title during initialization
Timo Sirainen <timo.sirainen@dovecot.fi>
parents: 21256
diff changeset
26
16538
83bea7e1f136 dsync: State names were wrong in debug/error messages.
Timo Sirainen <tss@iki.fi>
parents: 16503
diff changeset
27 static const char *dsync_state_names[] = {
83bea7e1f136 dsync: State names were wrong in debug/error messages.
Timo Sirainen <tss@iki.fi>
parents: 16503
diff changeset
28 "master_recv_handshake",
83bea7e1f136 dsync: State names were wrong in debug/error messages.
Timo Sirainen <tss@iki.fi>
parents: 16503
diff changeset
29 "slave_recv_handshake",
83bea7e1f136 dsync: State names were wrong in debug/error messages.
Timo Sirainen <tss@iki.fi>
parents: 16503
diff changeset
30 "master_send_last_common",
83bea7e1f136 dsync: State names were wrong in debug/error messages.
Timo Sirainen <tss@iki.fi>
parents: 16503
diff changeset
31 "slave_recv_last_common",
15464
af3b86a16ac3 dsync: Added debugging and rawlogging support.
Timo Sirainen <tss@iki.fi>
parents: 15355
diff changeset
32 "send_mailbox_tree",
af3b86a16ac3 dsync: Added debugging and rawlogging support.
Timo Sirainen <tss@iki.fi>
parents: 15355
diff changeset
33 "send_mailbox_tree_deletes",
af3b86a16ac3 dsync: Added debugging and rawlogging support.
Timo Sirainen <tss@iki.fi>
parents: 15355
diff changeset
34 "recv_mailbox_tree",
af3b86a16ac3 dsync: Added debugging and rawlogging support.
Timo Sirainen <tss@iki.fi>
parents: 15355
diff changeset
35 "recv_mailbox_tree_deletes",
af3b86a16ac3 dsync: Added debugging and rawlogging support.
Timo Sirainen <tss@iki.fi>
parents: 15355
diff changeset
36 "master_send_mailbox",
af3b86a16ac3 dsync: Added debugging and rawlogging support.
Timo Sirainen <tss@iki.fi>
parents: 15355
diff changeset
37 "slave_recv_mailbox",
af3b86a16ac3 dsync: Added debugging and rawlogging support.
Timo Sirainen <tss@iki.fi>
parents: 15355
diff changeset
38 "sync_mails",
18351
b4dbe64c0032 dsync: Added an extra "finish" state to allow slave-dsync to report error to master-dsync.
Timo Sirainen <tss@iki.fi>
parents: 18287
diff changeset
39 "finish",
15464
af3b86a16ac3 dsync: Added debugging and rawlogging support.
Timo Sirainen <tss@iki.fi>
parents: 15355
diff changeset
40 "done"
af3b86a16ac3 dsync: Added debugging and rawlogging support.
Timo Sirainen <tss@iki.fi>
parents: 15355
diff changeset
41 };
af3b86a16ac3 dsync: Added debugging and rawlogging support.
Timo Sirainen <tss@iki.fi>
parents: 15355
diff changeset
42
17271
bb5deb884303 dsync: Added more debug output
Timo Sirainen <tss@iki.fi>
parents: 17221
diff changeset
43 static void dsync_brain_mailbox_states_dump(struct dsync_brain *brain);
bb5deb884303 dsync: Added more debug output
Timo Sirainen <tss@iki.fi>
parents: 17221
diff changeset
44
21288
9316919c5932 dsync: Improve process title during initialization
Timo Sirainen <timo.sirainen@dovecot.fi>
parents: 21256
diff changeset
45 static const char *
9316919c5932 dsync: Improve process title during initialization
Timo Sirainen <timo.sirainen@dovecot.fi>
parents: 21256
diff changeset
46 dsync_brain_get_proctitle_full(struct dsync_brain *brain,
9316919c5932 dsync: Improve process title during initialization
Timo Sirainen <timo.sirainen@dovecot.fi>
parents: 21256
diff changeset
47 enum dsync_brain_title title)
16539
b5876fa03b0e dsync: If verbose_proctitle=yes, show the current state in it.
Timo Sirainen <tss@iki.fi>
parents: 16538
diff changeset
48 {
b5876fa03b0e dsync: If verbose_proctitle=yes, show the current state in it.
Timo Sirainen <tss@iki.fi>
parents: 16538
diff changeset
49 string_t *str = t_str_new(128);
b5876fa03b0e dsync: If verbose_proctitle=yes, show the current state in it.
Timo Sirainen <tss@iki.fi>
parents: 16538
diff changeset
50 const char *import_title, *export_title;
b5876fa03b0e dsync: If verbose_proctitle=yes, show the current state in it.
Timo Sirainen <tss@iki.fi>
parents: 16538
diff changeset
51
b5876fa03b0e dsync: If verbose_proctitle=yes, show the current state in it.
Timo Sirainen <tss@iki.fi>
parents: 16538
diff changeset
52 str_append_c(str, '[');
17279
fac4e7320676 dsync: If dsync was started via doveadm-server, show the remote client's IP address in ps output.
Timo Sirainen <tss@iki.fi>
parents: 17271
diff changeset
53 if (brain->process_title_prefix != NULL)
fac4e7320676 dsync: If dsync was started via doveadm-server, show the remote client's IP address in ps output.
Timo Sirainen <tss@iki.fi>
parents: 17271
diff changeset
54 str_append(str, brain->process_title_prefix);
16539
b5876fa03b0e dsync: If verbose_proctitle=yes, show the current state in it.
Timo Sirainen <tss@iki.fi>
parents: 16538
diff changeset
55 str_append(str, brain->user->username);
b5876fa03b0e dsync: If verbose_proctitle=yes, show the current state in it.
Timo Sirainen <tss@iki.fi>
parents: 16538
diff changeset
56 if (brain->box == NULL) {
b5876fa03b0e dsync: If verbose_proctitle=yes, show the current state in it.
Timo Sirainen <tss@iki.fi>
parents: 16538
diff changeset
57 str_append_c(str, ' ');
b5876fa03b0e dsync: If verbose_proctitle=yes, show the current state in it.
Timo Sirainen <tss@iki.fi>
parents: 16538
diff changeset
58 str_append(str, dsync_state_names[brain->state]);
b5876fa03b0e dsync: If verbose_proctitle=yes, show the current state in it.
Timo Sirainen <tss@iki.fi>
parents: 16538
diff changeset
59 } else {
b5876fa03b0e dsync: If verbose_proctitle=yes, show the current state in it.
Timo Sirainen <tss@iki.fi>
parents: 16538
diff changeset
60 str_append_c(str, ' ');
b5876fa03b0e dsync: If verbose_proctitle=yes, show the current state in it.
Timo Sirainen <tss@iki.fi>
parents: 16538
diff changeset
61 str_append(str, mailbox_get_vname(brain->box));
b5876fa03b0e dsync: If verbose_proctitle=yes, show the current state in it.
Timo Sirainen <tss@iki.fi>
parents: 16538
diff changeset
62 import_title = brain->box_importer == NULL ? "" :
b5876fa03b0e dsync: If verbose_proctitle=yes, show the current state in it.
Timo Sirainen <tss@iki.fi>
parents: 16538
diff changeset
63 dsync_mailbox_import_get_proctitle(brain->box_importer);
b5876fa03b0e dsync: If verbose_proctitle=yes, show the current state in it.
Timo Sirainen <tss@iki.fi>
parents: 16538
diff changeset
64 export_title = brain->box_exporter == NULL ? "" :
b5876fa03b0e dsync: If verbose_proctitle=yes, show the current state in it.
Timo Sirainen <tss@iki.fi>
parents: 16538
diff changeset
65 dsync_mailbox_export_get_proctitle(brain->box_exporter);
b5876fa03b0e dsync: If verbose_proctitle=yes, show the current state in it.
Timo Sirainen <tss@iki.fi>
parents: 16538
diff changeset
66 if (import_title[0] == '\0' && export_title[0] == '\0') {
b5876fa03b0e dsync: If verbose_proctitle=yes, show the current state in it.
Timo Sirainen <tss@iki.fi>
parents: 16538
diff changeset
67 str_printfa(str, " send:%s recv:%s",
b5876fa03b0e dsync: If verbose_proctitle=yes, show the current state in it.
Timo Sirainen <tss@iki.fi>
parents: 16538
diff changeset
68 dsync_box_state_names[brain->box_send_state],
b5876fa03b0e dsync: If verbose_proctitle=yes, show the current state in it.
Timo Sirainen <tss@iki.fi>
parents: 16538
diff changeset
69 dsync_box_state_names[brain->box_recv_state]);
b5876fa03b0e dsync: If verbose_proctitle=yes, show the current state in it.
Timo Sirainen <tss@iki.fi>
parents: 16538
diff changeset
70 } else {
b5876fa03b0e dsync: If verbose_proctitle=yes, show the current state in it.
Timo Sirainen <tss@iki.fi>
parents: 16538
diff changeset
71 if (import_title[0] != '\0') {
b5876fa03b0e dsync: If verbose_proctitle=yes, show the current state in it.
Timo Sirainen <tss@iki.fi>
parents: 16538
diff changeset
72 str_append(str, " import:");
b5876fa03b0e dsync: If verbose_proctitle=yes, show the current state in it.
Timo Sirainen <tss@iki.fi>
parents: 16538
diff changeset
73 str_append(str, import_title);
b5876fa03b0e dsync: If verbose_proctitle=yes, show the current state in it.
Timo Sirainen <tss@iki.fi>
parents: 16538
diff changeset
74 }
b5876fa03b0e dsync: If verbose_proctitle=yes, show the current state in it.
Timo Sirainen <tss@iki.fi>
parents: 16538
diff changeset
75 if (export_title[0] != '\0') {
b5876fa03b0e dsync: If verbose_proctitle=yes, show the current state in it.
Timo Sirainen <tss@iki.fi>
parents: 16538
diff changeset
76 str_append(str, " export:");
b5876fa03b0e dsync: If verbose_proctitle=yes, show the current state in it.
Timo Sirainen <tss@iki.fi>
parents: 16538
diff changeset
77 str_append(str, export_title);
b5876fa03b0e dsync: If verbose_proctitle=yes, show the current state in it.
Timo Sirainen <tss@iki.fi>
parents: 16538
diff changeset
78 }
b5876fa03b0e dsync: If verbose_proctitle=yes, show the current state in it.
Timo Sirainen <tss@iki.fi>
parents: 16538
diff changeset
79 }
b5876fa03b0e dsync: If verbose_proctitle=yes, show the current state in it.
Timo Sirainen <tss@iki.fi>
parents: 16538
diff changeset
80 }
21288
9316919c5932 dsync: Improve process title during initialization
Timo Sirainen <timo.sirainen@dovecot.fi>
parents: 21256
diff changeset
81 switch (title) {
9316919c5932 dsync: Improve process title during initialization
Timo Sirainen <timo.sirainen@dovecot.fi>
parents: 21256
diff changeset
82 case DSYNC_BRAIN_TITLE_NONE:
9316919c5932 dsync: Improve process title during initialization
Timo Sirainen <timo.sirainen@dovecot.fi>
parents: 21256
diff changeset
83 break;
9316919c5932 dsync: Improve process title during initialization
Timo Sirainen <timo.sirainen@dovecot.fi>
parents: 21256
diff changeset
84 case DSYNC_BRAIN_TITLE_LOCKING:
9316919c5932 dsync: Improve process title during initialization
Timo Sirainen <timo.sirainen@dovecot.fi>
parents: 21256
diff changeset
85 str_append(str, " locking "DSYNC_LOCK_FILENAME);
9316919c5932 dsync: Improve process title during initialization
Timo Sirainen <timo.sirainen@dovecot.fi>
parents: 21256
diff changeset
86 break;
9316919c5932 dsync: Improve process title during initialization
Timo Sirainen <timo.sirainen@dovecot.fi>
parents: 21256
diff changeset
87 }
16539
b5876fa03b0e dsync: If verbose_proctitle=yes, show the current state in it.
Timo Sirainen <tss@iki.fi>
parents: 16538
diff changeset
88 str_append_c(str, ']');
b5876fa03b0e dsync: If verbose_proctitle=yes, show the current state in it.
Timo Sirainen <tss@iki.fi>
parents: 16538
diff changeset
89 return str_c(str);
b5876fa03b0e dsync: If verbose_proctitle=yes, show the current state in it.
Timo Sirainen <tss@iki.fi>
parents: 16538
diff changeset
90 }
b5876fa03b0e dsync: If verbose_proctitle=yes, show the current state in it.
Timo Sirainen <tss@iki.fi>
parents: 16538
diff changeset
91
21288
9316919c5932 dsync: Improve process title during initialization
Timo Sirainen <timo.sirainen@dovecot.fi>
parents: 21256
diff changeset
92 static const char *dsync_brain_get_proctitle(struct dsync_brain *brain)
9316919c5932 dsync: Improve process title during initialization
Timo Sirainen <timo.sirainen@dovecot.fi>
parents: 21256
diff changeset
93 {
9316919c5932 dsync: Improve process title during initialization
Timo Sirainen <timo.sirainen@dovecot.fi>
parents: 21256
diff changeset
94 return dsync_brain_get_proctitle_full(brain, DSYNC_BRAIN_TITLE_NONE);
9316919c5932 dsync: Improve process title during initialization
Timo Sirainen <timo.sirainen@dovecot.fi>
parents: 21256
diff changeset
95 }
9316919c5932 dsync: Improve process title during initialization
Timo Sirainen <timo.sirainen@dovecot.fi>
parents: 21256
diff changeset
96
14584
b2076acc3715 Initial version of dsync rewrite.
Timo Sirainen <tss@iki.fi>
parents: 14253
diff changeset
97 static void dsync_brain_run_io(void *context)
b2076acc3715 Initial version of dsync rewrite.
Timo Sirainen <tss@iki.fi>
parents: 14253
diff changeset
98 {
b2076acc3715 Initial version of dsync rewrite.
Timo Sirainen <tss@iki.fi>
parents: 14253
diff changeset
99 struct dsync_brain *brain = context;
b2076acc3715 Initial version of dsync rewrite.
Timo Sirainen <tss@iki.fi>
parents: 14253
diff changeset
100 bool changed, try_pending;
b2076acc3715 Initial version of dsync rewrite.
Timo Sirainen <tss@iki.fi>
parents: 14253
diff changeset
101
15037
920756cd29b8 dsync: Renamed "slave" to "ibc" (= inter-brain communicator)
Timo Sirainen <tss@iki.fi>
parents: 14939
diff changeset
102 if (dsync_ibc_has_failed(brain->ibc)) {
14584
b2076acc3715 Initial version of dsync rewrite.
Timo Sirainen <tss@iki.fi>
parents: 14253
diff changeset
103 io_loop_stop(current_ioloop);
b2076acc3715 Initial version of dsync rewrite.
Timo Sirainen <tss@iki.fi>
parents: 14253
diff changeset
104 brain->failed = TRUE;
b2076acc3715 Initial version of dsync rewrite.
Timo Sirainen <tss@iki.fi>
parents: 14253
diff changeset
105 return;
b2076acc3715 Initial version of dsync rewrite.
Timo Sirainen <tss@iki.fi>
parents: 14253
diff changeset
106 }
11687
b37c85676f8e dsync: When -v parameter is given, show progress counter of saving new messages.
Timo Sirainen <tss@iki.fi>
parents: 11684
diff changeset
107
14584
b2076acc3715 Initial version of dsync rewrite.
Timo Sirainen <tss@iki.fi>
parents: 14253
diff changeset
108 try_pending = TRUE;
b2076acc3715 Initial version of dsync rewrite.
Timo Sirainen <tss@iki.fi>
parents: 14253
diff changeset
109 do {
b2076acc3715 Initial version of dsync rewrite.
Timo Sirainen <tss@iki.fi>
parents: 14253
diff changeset
110 if (!dsync_brain_run(brain, &changed)) {
b2076acc3715 Initial version of dsync rewrite.
Timo Sirainen <tss@iki.fi>
parents: 14253
diff changeset
111 io_loop_stop(current_ioloop);
b2076acc3715 Initial version of dsync rewrite.
Timo Sirainen <tss@iki.fi>
parents: 14253
diff changeset
112 break;
b2076acc3715 Initial version of dsync rewrite.
Timo Sirainen <tss@iki.fi>
parents: 14253
diff changeset
113 }
b2076acc3715 Initial version of dsync rewrite.
Timo Sirainen <tss@iki.fi>
parents: 14253
diff changeset
114 if (changed)
b2076acc3715 Initial version of dsync rewrite.
Timo Sirainen <tss@iki.fi>
parents: 14253
diff changeset
115 try_pending = TRUE;
b2076acc3715 Initial version of dsync rewrite.
Timo Sirainen <tss@iki.fi>
parents: 14253
diff changeset
116 else if (try_pending) {
15037
920756cd29b8 dsync: Renamed "slave" to "ibc" (= inter-brain communicator)
Timo Sirainen <tss@iki.fi>
parents: 14939
diff changeset
117 if (dsync_ibc_has_pending_data(brain->ibc))
14584
b2076acc3715 Initial version of dsync rewrite.
Timo Sirainen <tss@iki.fi>
parents: 14253
diff changeset
118 changed = TRUE;
b2076acc3715 Initial version of dsync rewrite.
Timo Sirainen <tss@iki.fi>
parents: 14253
diff changeset
119 try_pending = FALSE;
b2076acc3715 Initial version of dsync rewrite.
Timo Sirainen <tss@iki.fi>
parents: 14253
diff changeset
120 }
b2076acc3715 Initial version of dsync rewrite.
Timo Sirainen <tss@iki.fi>
parents: 14253
diff changeset
121 } while (changed);
b2076acc3715 Initial version of dsync rewrite.
Timo Sirainen <tss@iki.fi>
parents: 14253
diff changeset
122 }
11727
42dfcf9c896b dsync backup: Fail if it looks like backup is running in wrong direction.
Timo Sirainen <tss@iki.fi>
parents: 11688
diff changeset
123
14584
b2076acc3715 Initial version of dsync rewrite.
Timo Sirainen <tss@iki.fi>
parents: 14253
diff changeset
124 static struct dsync_brain *
15037
920756cd29b8 dsync: Renamed "slave" to "ibc" (= inter-brain communicator)
Timo Sirainen <tss@iki.fi>
parents: 14939
diff changeset
125 dsync_brain_common_init(struct mail_user *user, struct dsync_ibc *ibc)
14584
b2076acc3715 Initial version of dsync rewrite.
Timo Sirainen <tss@iki.fi>
parents: 14253
diff changeset
126 {
b2076acc3715 Initial version of dsync rewrite.
Timo Sirainen <tss@iki.fi>
parents: 14253
diff changeset
127 struct dsync_brain *brain;
16539
b5876fa03b0e dsync: If verbose_proctitle=yes, show the current state in it.
Timo Sirainen <tss@iki.fi>
parents: 16538
diff changeset
128 const struct master_service_settings *service_set;
14584
b2076acc3715 Initial version of dsync rewrite.
Timo Sirainen <tss@iki.fi>
parents: 14253
diff changeset
129 pool_t pool;
b2076acc3715 Initial version of dsync rewrite.
Timo Sirainen <tss@iki.fi>
parents: 14253
diff changeset
130
16539
b5876fa03b0e dsync: If verbose_proctitle=yes, show the current state in it.
Timo Sirainen <tss@iki.fi>
parents: 16538
diff changeset
131 service_set = master_service_settings_get(master_service);
16801
56be613e8ece dsync: Added -P parameter to do a purge for the remote storage after syncing.
Timo Sirainen <tss@iki.fi>
parents: 16561
diff changeset
132 mail_user_ref(user);
16539
b5876fa03b0e dsync: If verbose_proctitle=yes, show the current state in it.
Timo Sirainen <tss@iki.fi>
parents: 16538
diff changeset
133
14584
b2076acc3715 Initial version of dsync rewrite.
Timo Sirainen <tss@iki.fi>
parents: 14253
diff changeset
134 pool = pool_alloconly_create("dsync brain", 10240);
b2076acc3715 Initial version of dsync rewrite.
Timo Sirainen <tss@iki.fi>
parents: 14253
diff changeset
135 brain = p_new(pool, struct dsync_brain, 1);
b2076acc3715 Initial version of dsync rewrite.
Timo Sirainen <tss@iki.fi>
parents: 14253
diff changeset
136 brain->pool = pool;
b2076acc3715 Initial version of dsync rewrite.
Timo Sirainen <tss@iki.fi>
parents: 14253
diff changeset
137 brain->user = user;
15037
920756cd29b8 dsync: Renamed "slave" to "ibc" (= inter-brain communicator)
Timo Sirainen <tss@iki.fi>
parents: 14939
diff changeset
138 brain->ibc = ibc;
14584
b2076acc3715 Initial version of dsync rewrite.
Timo Sirainen <tss@iki.fi>
parents: 14253
diff changeset
139 brain->sync_type = DSYNC_BRAIN_SYNC_TYPE_UNKNOWN;
15782
6f024e0289da dsync: -l parameter locking is now done on the server with "lower" hostname.
Timo Sirainen <tss@iki.fi>
parents: 15750
diff changeset
140 brain->lock_fd = -1;
16539
b5876fa03b0e dsync: If verbose_proctitle=yes, show the current state in it.
Timo Sirainen <tss@iki.fi>
parents: 16538
diff changeset
141 brain->verbose_proctitle = service_set->verbose_proctitle;
15232
6c850258002f doveadm sync/backup: Added -s <state> parameter to do a fast stateful sync.
Timo Sirainen <tss@iki.fi>
parents: 15231
diff changeset
142 hash_table_create(&brain->mailbox_states, pool, 0,
6c850258002f doveadm sync/backup: Added -s <state> parameter to do a fast stateful sync.
Timo Sirainen <tss@iki.fi>
parents: 15231
diff changeset
143 guid_128_hash, guid_128_cmp);
6c850258002f doveadm sync/backup: Added -s <state> parameter to do a fast stateful sync.
Timo Sirainen <tss@iki.fi>
parents: 15231
diff changeset
144 p_array_init(&brain->remote_mailbox_states, pool, 64);
14584
b2076acc3715 Initial version of dsync rewrite.
Timo Sirainen <tss@iki.fi>
parents: 14253
diff changeset
145 return brain;
b2076acc3715 Initial version of dsync rewrite.
Timo Sirainen <tss@iki.fi>
parents: 14253
diff changeset
146 }
9559
34eef8a2716b dsync: Initial implementation.
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
147
15231
454d0563927d doveadm: "backup" command is working again.
Timo Sirainen <tss@iki.fi>
parents: 15037
diff changeset
148 static void
454d0563927d doveadm: "backup" command is working again.
Timo Sirainen <tss@iki.fi>
parents: 15037
diff changeset
149 dsync_brain_set_flags(struct dsync_brain *brain, enum dsync_brain_flags flags)
454d0563927d doveadm: "backup" command is working again.
Timo Sirainen <tss@iki.fi>
parents: 15037
diff changeset
150 {
15744
b4e2b3b54f0a dsync: Renamed "guid_requests" to "mail_requests"
Timo Sirainen <tss@iki.fi>
parents: 15715
diff changeset
151 brain->mail_requests =
b4e2b3b54f0a dsync: Renamed "guid_requests" to "mail_requests"
Timo Sirainen <tss@iki.fi>
parents: 15715
diff changeset
152 (flags & DSYNC_BRAIN_FLAG_SEND_MAIL_REQUESTS) != 0;
15231
454d0563927d doveadm: "backup" command is working again.
Timo Sirainen <tss@iki.fi>
parents: 15037
diff changeset
153 brain->backup_send = (flags & DSYNC_BRAIN_FLAG_BACKUP_SEND) != 0;
454d0563927d doveadm: "backup" command is working again.
Timo Sirainen <tss@iki.fi>
parents: 15037
diff changeset
154 brain->backup_recv = (flags & DSYNC_BRAIN_FLAG_BACKUP_RECV) != 0;
15464
af3b86a16ac3 dsync: Added debugging and rawlogging support.
Timo Sirainen <tss@iki.fi>
parents: 15355
diff changeset
155 brain->debug = (flags & DSYNC_BRAIN_FLAG_DEBUG) != 0;
15836
607758d425e7 dsync: Renamed -a parameter to -N. It also now skips invisible namespaces.
Timo Sirainen <tss@iki.fi>
parents: 15825
diff changeset
156 brain->sync_visible_namespaces =
607758d425e7 dsync: Renamed -a parameter to -N. It also now skips invisible namespaces.
Timo Sirainen <tss@iki.fi>
parents: 15825
diff changeset
157 (flags & DSYNC_BRAIN_FLAG_SYNC_VISIBLE_NAMESPACES) != 0;
16084
38774adaddaf dsync: -m '' parameter now syncs mailbox list, but no actual mails.
Timo Sirainen <tss@iki.fi>
parents: 15986
diff changeset
158 brain->no_mail_sync = (flags & DSYNC_BRAIN_FLAG_NO_MAIL_SYNC) != 0;
16561
0144704e1b99 dsync: Added -1 parameter to do a "one way sync" without reverting changes.
Timo Sirainen <tss@iki.fi>
parents: 16541
diff changeset
159 brain->no_backup_overwrite =
0144704e1b99 dsync: Added -1 parameter to do a "one way sync" without reverting changes.
Timo Sirainen <tss@iki.fi>
parents: 16541
diff changeset
160 (flags & DSYNC_BRAIN_FLAG_NO_BACKUP_OVERWRITE) != 0;
17845
d2e1b3f6d13b dsync: Added DSYNC_BRAIN_FLAG_NO_MAIL_PREFETCH to avoid opening mails unnecessarily.
Timo Sirainen <tss@iki.fi>
parents: 17800
diff changeset
161 brain->no_mail_prefetch =
d2e1b3f6d13b dsync: Added DSYNC_BRAIN_FLAG_NO_MAIL_PREFETCH to avoid opening mails unnecessarily.
Timo Sirainen <tss@iki.fi>
parents: 17800
diff changeset
162 (flags & DSYNC_BRAIN_FLAG_NO_MAIL_PREFETCH) != 0;
19021
3fc9658c9712 dsync: Added -D parameter to disable mailbox renaming.
Timo Sirainen <tss@iki.fi>
parents: 18371
diff changeset
163 brain->no_mailbox_renames =
3fc9658c9712 dsync: Added -D parameter to disable mailbox renaming.
Timo Sirainen <tss@iki.fi>
parents: 18371
diff changeset
164 (flags & DSYNC_BRAIN_FLAG_NO_MAILBOX_RENAMES) != 0;
19318
f9a143c630a5 dsync: Added DSYNC_BRAIN_FLAG_NO_NOTIFY to enable MAILBOX_TRANSACTION_FLAG_NO_NOTIFY
Timo Sirainen <tss@iki.fi>
parents: 19136
diff changeset
165 brain->no_notify = (flags & DSYNC_BRAIN_FLAG_NO_NOTIFY) != 0;
20595
40ce04c672a4 dsync: Add support for features
Aki Tuomi <aki.tuomi@dovecot.fi>
parents: 20458
diff changeset
166 brain->empty_hdr_workaround = (flags & DSYNC_BRAIN_FLAG_EMPTY_HDR_WORKAROUND) != 0;
15231
454d0563927d doveadm: "backup" command is working again.
Timo Sirainen <tss@iki.fi>
parents: 15037
diff changeset
167 }
454d0563927d doveadm: "backup" command is working again.
Timo Sirainen <tss@iki.fi>
parents: 15037
diff changeset
168
18180
39d00448490f dsync: If same GUID already exists in storage, try to copy it instead of recreating the mail.
Timo Sirainen <tss@iki.fi>
parents: 18177
diff changeset
169 static void
39d00448490f dsync: If same GUID already exists in storage, try to copy it instead of recreating the mail.
Timo Sirainen <tss@iki.fi>
parents: 18177
diff changeset
170 dsync_brain_open_virtual_all_box(struct dsync_brain *brain,
39d00448490f dsync: If same GUID already exists in storage, try to copy it instead of recreating the mail.
Timo Sirainen <tss@iki.fi>
parents: 18177
diff changeset
171 const char *vname)
39d00448490f dsync: If same GUID already exists in storage, try to copy it instead of recreating the mail.
Timo Sirainen <tss@iki.fi>
parents: 18177
diff changeset
172 {
39d00448490f dsync: If same GUID already exists in storage, try to copy it instead of recreating the mail.
Timo Sirainen <tss@iki.fi>
parents: 18177
diff changeset
173 struct mail_namespace *ns;
39d00448490f dsync: If same GUID already exists in storage, try to copy it instead of recreating the mail.
Timo Sirainen <tss@iki.fi>
parents: 18177
diff changeset
174
39d00448490f dsync: If same GUID already exists in storage, try to copy it instead of recreating the mail.
Timo Sirainen <tss@iki.fi>
parents: 18177
diff changeset
175 ns = mail_namespace_find(brain->user->namespaces, vname);
18287
06c4c42549a7 dsync: Open mailboxes with readonly-flag whenever possible.
Timo Sirainen <tss@iki.fi>
parents: 18181
diff changeset
176 brain->virtual_all_box =
06c4c42549a7 dsync: Open mailboxes with readonly-flag whenever possible.
Timo Sirainen <tss@iki.fi>
parents: 18181
diff changeset
177 mailbox_alloc(ns->list, vname, MAILBOX_FLAG_READONLY);
18180
39d00448490f dsync: If same GUID already exists in storage, try to copy it instead of recreating the mail.
Timo Sirainen <tss@iki.fi>
parents: 18177
diff changeset
178 }
39d00448490f dsync: If same GUID already exists in storage, try to copy it instead of recreating the mail.
Timo Sirainen <tss@iki.fi>
parents: 18177
diff changeset
179
9736
d9a96da46d4a dsync: Lots of updates and fixes.
Timo Sirainen <tss@iki.fi>
parents: 9686
diff changeset
180 struct dsync_brain *
15037
920756cd29b8 dsync: Renamed "slave" to "ibc" (= inter-brain communicator)
Timo Sirainen <tss@iki.fi>
parents: 14939
diff changeset
181 dsync_brain_master_init(struct mail_user *user, struct dsync_ibc *ibc,
14584
b2076acc3715 Initial version of dsync rewrite.
Timo Sirainen <tss@iki.fi>
parents: 14253
diff changeset
182 enum dsync_brain_sync_type sync_type,
16396
2d6a3035a6f7 dsync: Small code cleanup.
Timo Sirainen <tss@iki.fi>
parents: 16253
diff changeset
183 enum dsync_brain_flags flags,
2d6a3035a6f7 dsync: Small code cleanup.
Timo Sirainen <tss@iki.fi>
parents: 16253
diff changeset
184 const struct dsync_brain_settings *set)
14584
b2076acc3715 Initial version of dsync rewrite.
Timo Sirainen <tss@iki.fi>
parents: 14253
diff changeset
185 {
15037
920756cd29b8 dsync: Renamed "slave" to "ibc" (= inter-brain communicator)
Timo Sirainen <tss@iki.fi>
parents: 14939
diff changeset
186 struct dsync_ibc_settings ibc_set;
14584
b2076acc3715 Initial version of dsync rewrite.
Timo Sirainen <tss@iki.fi>
parents: 14253
diff changeset
187 struct dsync_brain *brain;
17170
08f1c7af0ac0 dsync: Support multiple -n parameters.
Timo Sirainen <tss@iki.fi>
parents: 17130
diff changeset
188 struct mail_namespace *const *nsp;
08f1c7af0ac0 dsync: Support multiple -n parameters.
Timo Sirainen <tss@iki.fi>
parents: 17130
diff changeset
189 string_t *sync_ns_str = NULL;
14584
b2076acc3715 Initial version of dsync rewrite.
Timo Sirainen <tss@iki.fi>
parents: 14253
diff changeset
190 const char *error;
b2076acc3715 Initial version of dsync rewrite.
Timo Sirainen <tss@iki.fi>
parents: 14253
diff changeset
191
b2076acc3715 Initial version of dsync rewrite.
Timo Sirainen <tss@iki.fi>
parents: 14253
diff changeset
192 i_assert(sync_type != DSYNC_BRAIN_SYNC_TYPE_UNKNOWN);
16396
2d6a3035a6f7 dsync: Small code cleanup.
Timo Sirainen <tss@iki.fi>
parents: 16253
diff changeset
193 i_assert(sync_type != DSYNC_BRAIN_SYNC_TYPE_STATE ||
2d6a3035a6f7 dsync: Small code cleanup.
Timo Sirainen <tss@iki.fi>
parents: 16253
diff changeset
194 (set->state != NULL && *set->state != '\0'));
16538
83bea7e1f136 dsync: State names were wrong in debug/error messages.
Timo Sirainen <tss@iki.fi>
parents: 16503
diff changeset
195 i_assert(N_ELEMENTS(dsync_state_names) == DSYNC_STATE_DONE+1);
14584
b2076acc3715 Initial version of dsync rewrite.
Timo Sirainen <tss@iki.fi>
parents: 14253
diff changeset
196
15037
920756cd29b8 dsync: Renamed "slave" to "ibc" (= inter-brain communicator)
Timo Sirainen <tss@iki.fi>
parents: 14939
diff changeset
197 brain = dsync_brain_common_init(user, ibc);
17279
fac4e7320676 dsync: If dsync was started via doveadm-server, show the remote client's IP address in ps output.
Timo Sirainen <tss@iki.fi>
parents: 17271
diff changeset
198 brain->process_title_prefix =
fac4e7320676 dsync: If dsync was started via doveadm-server, show the remote client's IP address in ps output.
Timo Sirainen <tss@iki.fi>
parents: 17271
diff changeset
199 p_strdup(brain->pool, set->process_title_prefix);
14584
b2076acc3715 Initial version of dsync rewrite.
Timo Sirainen <tss@iki.fi>
parents: 14253
diff changeset
200 brain->sync_type = sync_type;
17170
08f1c7af0ac0 dsync: Support multiple -n parameters.
Timo Sirainen <tss@iki.fi>
parents: 17130
diff changeset
201 if (array_count(&set->sync_namespaces) > 0) {
08f1c7af0ac0 dsync: Support multiple -n parameters.
Timo Sirainen <tss@iki.fi>
parents: 17130
diff changeset
202 sync_ns_str = t_str_new(128);
08f1c7af0ac0 dsync: Support multiple -n parameters.
Timo Sirainen <tss@iki.fi>
parents: 17130
diff changeset
203 p_array_init(&brain->sync_namespaces, brain->pool,
08f1c7af0ac0 dsync: Support multiple -n parameters.
Timo Sirainen <tss@iki.fi>
parents: 17130
diff changeset
204 array_count(&set->sync_namespaces));
08f1c7af0ac0 dsync: Support multiple -n parameters.
Timo Sirainen <tss@iki.fi>
parents: 17130
diff changeset
205 array_foreach(&set->sync_namespaces, nsp) {
08f1c7af0ac0 dsync: Support multiple -n parameters.
Timo Sirainen <tss@iki.fi>
parents: 17130
diff changeset
206 str_append(sync_ns_str, (*nsp)->prefix);
17219
f0b8cf119b7f dsync: Fixed giving -n "" followed by other -n parameters.
Timo Sirainen <tss@iki.fi>
parents: 17200
diff changeset
207 str_append_c(sync_ns_str, '\n');
17170
08f1c7af0ac0 dsync: Support multiple -n parameters.
Timo Sirainen <tss@iki.fi>
parents: 17130
diff changeset
208 array_append(&brain->sync_namespaces, nsp, 1);
08f1c7af0ac0 dsync: Support multiple -n parameters.
Timo Sirainen <tss@iki.fi>
parents: 17130
diff changeset
209 }
17221
e75851b65900 dsync: Fix to previous commit to actually compile..
Timo Sirainen <tss@iki.fi>
parents: 17219
diff changeset
210 str_delete(sync_ns_str, str_len(sync_ns_str)-1, 1);
17170
08f1c7af0ac0 dsync: Support multiple -n parameters.
Timo Sirainen <tss@iki.fi>
parents: 17130
diff changeset
211 }
17800
eff79a80e0c9 dsync: Moved all doveadm-specific code to doveadm-dsync.c
Timo Sirainen <tss@iki.fi>
parents: 17377
diff changeset
212 brain->alt_char = set->mailbox_alt_char == '\0' ? '_' :
eff79a80e0c9 dsync: Moved all doveadm-specific code to doveadm-dsync.c
Timo Sirainen <tss@iki.fi>
parents: 17377
diff changeset
213 set->mailbox_alt_char;
18177
f393f63764e0 dsync: Added -t <timestamp> parameter to save only mails newer than <timestamp>
Timo Sirainen <tss@iki.fi>
parents: 18137
diff changeset
214 brain->sync_since_timestamp = set->sync_since_timestamp;
20831
0c4e5c2725a3 doveadm-sync: Add end-date support
Aki Tuomi <aki.tuomi@dovecot.fi>
parents: 20633
diff changeset
215 brain->sync_until_timestamp = set->sync_until_timestamp;
20633
d54651ba988a doveadm sync/backup: Added -S <max size> parameter to skip too large mails.
Timo Sirainen <timo.sirainen@dovecot.fi>
parents: 20595
diff changeset
216 brain->sync_max_size = set->sync_max_size;
18181
35e4a6ae8d85 dsync: Added -F parameter to sync only mails with[out] specific flag.
Timo Sirainen <tss@iki.fi>
parents: 18180
diff changeset
217 brain->sync_flag = p_strdup(brain->pool, set->sync_flag);
16396
2d6a3035a6f7 dsync: Small code cleanup.
Timo Sirainen <tss@iki.fi>
parents: 16253
diff changeset
218 brain->sync_box = p_strdup(brain->pool, set->sync_box);
16398
4883a8e1db13 dsync: Added -x parameter to exclude mailboxes from sync.
Timo Sirainen <tss@iki.fi>
parents: 16396
diff changeset
219 brain->exclude_mailboxes = set->exclude_mailboxes == NULL ? NULL :
4883a8e1db13 dsync: Added -x parameter to exclude mailboxes from sync.
Timo Sirainen <tss@iki.fi>
parents: 16396
diff changeset
220 p_strarray_dup(brain->pool, set->exclude_mailboxes);
16396
2d6a3035a6f7 dsync: Small code cleanup.
Timo Sirainen <tss@iki.fi>
parents: 16253
diff changeset
221 memcpy(brain->sync_box_guid, set->sync_box_guid,
2d6a3035a6f7 dsync: Small code cleanup.
Timo Sirainen <tss@iki.fi>
parents: 16253
diff changeset
222 sizeof(brain->sync_box_guid));
2d6a3035a6f7 dsync: Small code cleanup.
Timo Sirainen <tss@iki.fi>
parents: 16253
diff changeset
223 brain->lock_timeout = set->lock_timeout_secs;
22043
81e013b3207d dsync: Try to commit transactions every dsync_commit_msgs_interval messages
Timo Sirainen <timo.sirainen@dovecot.fi>
parents: 21814
diff changeset
224 brain->import_commit_msgs_interval = set->import_commit_msgs_interval;
14584
b2076acc3715 Initial version of dsync rewrite.
Timo Sirainen <tss@iki.fi>
parents: 14253
diff changeset
225 brain->master_brain = TRUE;
22549
400ff84f109d dsync: Add hashed_headers setting
Aki Tuomi <aki.tuomi@dovecot.fi>
parents: 22264
diff changeset
226 brain->hashed_headers =
400ff84f109d dsync: Add hashed_headers setting
Aki Tuomi <aki.tuomi@dovecot.fi>
parents: 22264
diff changeset
227 (const char*const*)p_strarray_dup(brain->pool, set->hashed_headers);
15231
454d0563927d doveadm: "backup" command is working again.
Timo Sirainen <tss@iki.fi>
parents: 15037
diff changeset
228 dsync_brain_set_flags(brain, flags);
14584
b2076acc3715 Initial version of dsync rewrite.
Timo Sirainen <tss@iki.fi>
parents: 14253
diff changeset
229
18180
39d00448490f dsync: If same GUID already exists in storage, try to copy it instead of recreating the mail.
Timo Sirainen <tss@iki.fi>
parents: 18177
diff changeset
230 if (set->virtual_all_box != NULL)
39d00448490f dsync: If same GUID already exists in storage, try to copy it instead of recreating the mail.
Timo Sirainen <tss@iki.fi>
parents: 18177
diff changeset
231 dsync_brain_open_virtual_all_box(brain, set->virtual_all_box);
39d00448490f dsync: If same GUID already exists in storage, try to copy it instead of recreating the mail.
Timo Sirainen <tss@iki.fi>
parents: 18177
diff changeset
232
17271
bb5deb884303 dsync: Added more debug output
Timo Sirainen <tss@iki.fi>
parents: 17221
diff changeset
233 if (sync_type != DSYNC_BRAIN_SYNC_TYPE_STATE)
bb5deb884303 dsync: Added more debug output
Timo Sirainen <tss@iki.fi>
parents: 17221
diff changeset
234 ;
bb5deb884303 dsync: Added more debug output
Timo Sirainen <tss@iki.fi>
parents: 17221
diff changeset
235 else if (dsync_mailbox_states_import(brain->mailbox_states, brain->pool,
bb5deb884303 dsync: Added more debug output
Timo Sirainen <tss@iki.fi>
parents: 17221
diff changeset
236 set->state, &error) < 0) {
15782
6f024e0289da dsync: -l parameter locking is now done on the server with "lower" hostname.
Timo Sirainen <tss@iki.fi>
parents: 15750
diff changeset
237 hash_table_clear(brain->mailbox_states, FALSE);
6f024e0289da dsync: -l parameter locking is now done on the server with "lower" hostname.
Timo Sirainen <tss@iki.fi>
parents: 15750
diff changeset
238 i_error("Saved sync state is invalid, "
6f024e0289da dsync: -l parameter locking is now done on the server with "lower" hostname.
Timo Sirainen <tss@iki.fi>
parents: 15750
diff changeset
239 "falling back to full sync: %s", error);
6f024e0289da dsync: -l parameter locking is now done on the server with "lower" hostname.
Timo Sirainen <tss@iki.fi>
parents: 15750
diff changeset
240 brain->sync_type = sync_type = DSYNC_BRAIN_SYNC_TYPE_FULL;
17271
bb5deb884303 dsync: Added more debug output
Timo Sirainen <tss@iki.fi>
parents: 17221
diff changeset
241 } else {
bb5deb884303 dsync: Added more debug output
Timo Sirainen <tss@iki.fi>
parents: 17221
diff changeset
242 if (brain->debug) {
bb5deb884303 dsync: Added more debug output
Timo Sirainen <tss@iki.fi>
parents: 17221
diff changeset
243 i_debug("brain %c: Imported mailbox states:",
bb5deb884303 dsync: Added more debug output
Timo Sirainen <tss@iki.fi>
parents: 17221
diff changeset
244 brain->master_brain ? 'M' : 'S');
bb5deb884303 dsync: Added more debug output
Timo Sirainen <tss@iki.fi>
parents: 17221
diff changeset
245 dsync_brain_mailbox_states_dump(brain);
bb5deb884303 dsync: Added more debug output
Timo Sirainen <tss@iki.fi>
parents: 17221
diff changeset
246 }
14584
b2076acc3715 Initial version of dsync rewrite.
Timo Sirainen <tss@iki.fi>
parents: 14253
diff changeset
247 }
b2076acc3715 Initial version of dsync rewrite.
Timo Sirainen <tss@iki.fi>
parents: 14253
diff changeset
248 dsync_brain_mailbox_trees_init(brain);
b2076acc3715 Initial version of dsync rewrite.
Timo Sirainen <tss@iki.fi>
parents: 14253
diff changeset
249
21389
59437f8764c6 global: Replaced all instances of memset(p, 0, sizeof(*p)) with the new i_zero() macro.
Stephan Bosch <stephan.bosch@dovecot.fi>
parents: 21289
diff changeset
250 i_zero(&ibc_set);
15815
e7aabd79c9d5 dsync: Use full hostname+domain when comparing if hosts are different in locking.
Timo Sirainen <tss@iki.fi>
parents: 15782
diff changeset
251 ibc_set.hostname = my_hostdomain();
17170
08f1c7af0ac0 dsync: Support multiple -n parameters.
Timo Sirainen <tss@iki.fi>
parents: 17130
diff changeset
252 ibc_set.sync_ns_prefixes = sync_ns_str == NULL ?
08f1c7af0ac0 dsync: Support multiple -n parameters.
Timo Sirainen <tss@iki.fi>
parents: 17130
diff changeset
253 NULL : str_c(sync_ns_str);
16396
2d6a3035a6f7 dsync: Small code cleanup.
Timo Sirainen <tss@iki.fi>
parents: 16253
diff changeset
254 ibc_set.sync_box = set->sync_box;
18180
39d00448490f dsync: If same GUID already exists in storage, try to copy it instead of recreating the mail.
Timo Sirainen <tss@iki.fi>
parents: 18177
diff changeset
255 ibc_set.virtual_all_box = set->virtual_all_box;
16398
4883a8e1db13 dsync: Added -x parameter to exclude mailboxes from sync.
Timo Sirainen <tss@iki.fi>
parents: 16396
diff changeset
256 ibc_set.exclude_mailboxes = set->exclude_mailboxes;
18177
f393f63764e0 dsync: Added -t <timestamp> parameter to save only mails newer than <timestamp>
Timo Sirainen <tss@iki.fi>
parents: 18137
diff changeset
257 ibc_set.sync_since_timestamp = set->sync_since_timestamp;
20831
0c4e5c2725a3 doveadm-sync: Add end-date support
Aki Tuomi <aki.tuomi@dovecot.fi>
parents: 20633
diff changeset
258 ibc_set.sync_until_timestamp = set->sync_until_timestamp;
20633
d54651ba988a doveadm sync/backup: Added -S <max size> parameter to skip too large mails.
Timo Sirainen <timo.sirainen@dovecot.fi>
parents: 20595
diff changeset
259 ibc_set.sync_max_size = set->sync_max_size;
18181
35e4a6ae8d85 dsync: Added -F parameter to sync only mails with[out] specific flag.
Timo Sirainen <tss@iki.fi>
parents: 18180
diff changeset
260 ibc_set.sync_flags = set->sync_flag;
16396
2d6a3035a6f7 dsync: Small code cleanup.
Timo Sirainen <tss@iki.fi>
parents: 16253
diff changeset
261 memcpy(ibc_set.sync_box_guid, set->sync_box_guid,
16110
c51873a8e0d9 doveadm sync/backup: Added -g <guid> to sync only the specified mailbox (by GUID)
Timo Sirainen <tss@iki.fi>
parents: 16089
diff changeset
262 sizeof(ibc_set.sync_box_guid));
15037
920756cd29b8 dsync: Renamed "slave" to "ibc" (= inter-brain communicator)
Timo Sirainen <tss@iki.fi>
parents: 14939
diff changeset
263 ibc_set.sync_type = sync_type;
19646
25f06710e671 dsync: When comparing headers' hashes to match messages, try to normalize the input.
Timo Sirainen <timo.sirainen@dovecot.fi>
parents: 19552
diff changeset
264 ibc_set.hdr_hash_v2 = TRUE;
16396
2d6a3035a6f7 dsync: Small code cleanup.
Timo Sirainen <tss@iki.fi>
parents: 16253
diff changeset
265 ibc_set.lock_timeout = set->lock_timeout_secs;
22043
81e013b3207d dsync: Try to commit transactions every dsync_commit_msgs_interval messages
Timo Sirainen <timo.sirainen@dovecot.fi>
parents: 21814
diff changeset
266 ibc_set.import_commit_msgs_interval = set->import_commit_msgs_interval;
22549
400ff84f109d dsync: Add hashed_headers setting
Aki Tuomi <aki.tuomi@dovecot.fi>
parents: 22264
diff changeset
267 ibc_set.hashed_headers = set->hashed_headers;
15231
454d0563927d doveadm: "backup" command is working again.
Timo Sirainen <tss@iki.fi>
parents: 15037
diff changeset
268 /* reverse the backup direction for the slave */
454d0563927d doveadm: "backup" command is working again.
Timo Sirainen <tss@iki.fi>
parents: 15037
diff changeset
269 ibc_set.brain_flags = flags & ~(DSYNC_BRAIN_FLAG_BACKUP_SEND |
454d0563927d doveadm: "backup" command is working again.
Timo Sirainen <tss@iki.fi>
parents: 15037
diff changeset
270 DSYNC_BRAIN_FLAG_BACKUP_RECV);
454d0563927d doveadm: "backup" command is working again.
Timo Sirainen <tss@iki.fi>
parents: 15037
diff changeset
271 if ((flags & DSYNC_BRAIN_FLAG_BACKUP_SEND) != 0)
454d0563927d doveadm: "backup" command is working again.
Timo Sirainen <tss@iki.fi>
parents: 15037
diff changeset
272 ibc_set.brain_flags |= DSYNC_BRAIN_FLAG_BACKUP_RECV;
454d0563927d doveadm: "backup" command is working again.
Timo Sirainen <tss@iki.fi>
parents: 15037
diff changeset
273 else if ((flags & DSYNC_BRAIN_FLAG_BACKUP_RECV) != 0)
454d0563927d doveadm: "backup" command is working again.
Timo Sirainen <tss@iki.fi>
parents: 15037
diff changeset
274 ibc_set.brain_flags |= DSYNC_BRAIN_FLAG_BACKUP_SEND;
15037
920756cd29b8 dsync: Renamed "slave" to "ibc" (= inter-brain communicator)
Timo Sirainen <tss@iki.fi>
parents: 14939
diff changeset
275 dsync_ibc_send_handshake(ibc, &ibc_set);
14584
b2076acc3715 Initial version of dsync rewrite.
Timo Sirainen <tss@iki.fi>
parents: 14253
diff changeset
276
15037
920756cd29b8 dsync: Renamed "slave" to "ibc" (= inter-brain communicator)
Timo Sirainen <tss@iki.fi>
parents: 14939
diff changeset
277 dsync_ibc_set_io_callback(ibc, dsync_brain_run_io, brain);
15782
6f024e0289da dsync: -l parameter locking is now done on the server with "lower" hostname.
Timo Sirainen <tss@iki.fi>
parents: 15750
diff changeset
278 brain->state = DSYNC_STATE_MASTER_RECV_HANDSHAKE;
21288
9316919c5932 dsync: Improve process title during initialization
Timo Sirainen <timo.sirainen@dovecot.fi>
parents: 21256
diff changeset
279
9316919c5932 dsync: Improve process title during initialization
Timo Sirainen <timo.sirainen@dovecot.fi>
parents: 21256
diff changeset
280 if (brain->verbose_proctitle)
9316919c5932 dsync: Improve process title during initialization
Timo Sirainen <timo.sirainen@dovecot.fi>
parents: 21256
diff changeset
281 process_title_set(dsync_brain_get_proctitle(brain));
14584
b2076acc3715 Initial version of dsync rewrite.
Timo Sirainen <tss@iki.fi>
parents: 14253
diff changeset
282 return brain;
b2076acc3715 Initial version of dsync rewrite.
Timo Sirainen <tss@iki.fi>
parents: 14253
diff changeset
283 }
b2076acc3715 Initial version of dsync rewrite.
Timo Sirainen <tss@iki.fi>
parents: 14253
diff changeset
284
b2076acc3715 Initial version of dsync rewrite.
Timo Sirainen <tss@iki.fi>
parents: 14253
diff changeset
285 struct dsync_brain *
16539
b5876fa03b0e dsync: If verbose_proctitle=yes, show the current state in it.
Timo Sirainen <tss@iki.fi>
parents: 16538
diff changeset
286 dsync_brain_slave_init(struct mail_user *user, struct dsync_ibc *ibc,
17279
fac4e7320676 dsync: If dsync was started via doveadm-server, show the remote client's IP address in ps output.
Timo Sirainen <tss@iki.fi>
parents: 17271
diff changeset
287 bool local, const char *process_title_prefix)
9559
34eef8a2716b dsync: Initial implementation.
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
288 {
15782
6f024e0289da dsync: -l parameter locking is now done on the server with "lower" hostname.
Timo Sirainen <tss@iki.fi>
parents: 15750
diff changeset
289 struct dsync_ibc_settings ibc_set;
9559
34eef8a2716b dsync: Initial implementation.
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
290 struct dsync_brain *brain;
34eef8a2716b dsync: Initial implementation.
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
291
15037
920756cd29b8 dsync: Renamed "slave" to "ibc" (= inter-brain communicator)
Timo Sirainen <tss@iki.fi>
parents: 14939
diff changeset
292 brain = dsync_brain_common_init(user, ibc);
17279
fac4e7320676 dsync: If dsync was started via doveadm-server, show the remote client's IP address in ps output.
Timo Sirainen <tss@iki.fi>
parents: 17271
diff changeset
293 brain->process_title_prefix =
fac4e7320676 dsync: If dsync was started via doveadm-server, show the remote client's IP address in ps output.
Timo Sirainen <tss@iki.fi>
parents: 17271
diff changeset
294 p_strdup(brain->pool, process_title_prefix);
14584
b2076acc3715 Initial version of dsync rewrite.
Timo Sirainen <tss@iki.fi>
parents: 14253
diff changeset
295 brain->state = DSYNC_STATE_SLAVE_RECV_HANDSHAKE;
11734
ba9f33f9c6c3 dsync: If verbosity is enabled, log why desyncing was caused.
Timo Sirainen <tss@iki.fi>
parents: 11727
diff changeset
296
16539
b5876fa03b0e dsync: If verbose_proctitle=yes, show the current state in it.
Timo Sirainen <tss@iki.fi>
parents: 16538
diff changeset
297 if (local) {
b5876fa03b0e dsync: If verbose_proctitle=yes, show the current state in it.
Timo Sirainen <tss@iki.fi>
parents: 16538
diff changeset
298 /* both master and slave are running within the same process,
b5876fa03b0e dsync: If verbose_proctitle=yes, show the current state in it.
Timo Sirainen <tss@iki.fi>
parents: 16538
diff changeset
299 update the proctitle only for master. */
b5876fa03b0e dsync: If verbose_proctitle=yes, show the current state in it.
Timo Sirainen <tss@iki.fi>
parents: 16538
diff changeset
300 brain->verbose_proctitle = FALSE;
b5876fa03b0e dsync: If verbose_proctitle=yes, show the current state in it.
Timo Sirainen <tss@iki.fi>
parents: 16538
diff changeset
301 }
b5876fa03b0e dsync: If verbose_proctitle=yes, show the current state in it.
Timo Sirainen <tss@iki.fi>
parents: 16538
diff changeset
302
21389
59437f8764c6 global: Replaced all instances of memset(p, 0, sizeof(*p)) with the new i_zero() macro.
Stephan Bosch <stephan.bosch@dovecot.fi>
parents: 21289
diff changeset
303 i_zero(&ibc_set);
19646
25f06710e671 dsync: When comparing headers' hashes to match messages, try to normalize the input.
Timo Sirainen <timo.sirainen@dovecot.fi>
parents: 19552
diff changeset
304 ibc_set.hdr_hash_v2 = TRUE;
15815
e7aabd79c9d5 dsync: Use full hostname+domain when comparing if hosts are different in locking.
Timo Sirainen <tss@iki.fi>
parents: 15782
diff changeset
305 ibc_set.hostname = my_hostdomain();
15782
6f024e0289da dsync: -l parameter locking is now done on the server with "lower" hostname.
Timo Sirainen <tss@iki.fi>
parents: 15750
diff changeset
306 dsync_ibc_send_handshake(ibc, &ibc_set);
6f024e0289da dsync: -l parameter locking is now done on the server with "lower" hostname.
Timo Sirainen <tss@iki.fi>
parents: 15750
diff changeset
307
21288
9316919c5932 dsync: Improve process title during initialization
Timo Sirainen <timo.sirainen@dovecot.fi>
parents: 21256
diff changeset
308 if (brain->verbose_proctitle)
9316919c5932 dsync: Improve process title during initialization
Timo Sirainen <timo.sirainen@dovecot.fi>
parents: 21256
diff changeset
309 process_title_set(dsync_brain_get_proctitle(brain));
15037
920756cd29b8 dsync: Renamed "slave" to "ibc" (= inter-brain communicator)
Timo Sirainen <tss@iki.fi>
parents: 14939
diff changeset
310 dsync_ibc_set_io_callback(ibc, dsync_brain_run_io, brain);
9559
34eef8a2716b dsync: Initial implementation.
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
311 return brain;
34eef8a2716b dsync: Initial implementation.
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
312 }
34eef8a2716b dsync: Initial implementation.
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
313
16801
56be613e8ece dsync: Added -P parameter to do a purge for the remote storage after syncing.
Timo Sirainen <tss@iki.fi>
parents: 16561
diff changeset
314 static void dsync_brain_purge(struct dsync_brain *brain)
56be613e8ece dsync: Added -P parameter to do a purge for the remote storage after syncing.
Timo Sirainen <tss@iki.fi>
parents: 16561
diff changeset
315 {
56be613e8ece dsync: Added -P parameter to do a purge for the remote storage after syncing.
Timo Sirainen <tss@iki.fi>
parents: 16561
diff changeset
316 struct mail_namespace *ns;
56be613e8ece dsync: Added -P parameter to do a purge for the remote storage after syncing.
Timo Sirainen <tss@iki.fi>
parents: 16561
diff changeset
317 struct mail_storage *storage;
56be613e8ece dsync: Added -P parameter to do a purge for the remote storage after syncing.
Timo Sirainen <tss@iki.fi>
parents: 16561
diff changeset
318
56be613e8ece dsync: Added -P parameter to do a purge for the remote storage after syncing.
Timo Sirainen <tss@iki.fi>
parents: 16561
diff changeset
319 for (ns = brain->user->namespaces; ns != NULL; ns = ns->next) {
56be613e8ece dsync: Added -P parameter to do a purge for the remote storage after syncing.
Timo Sirainen <tss@iki.fi>
parents: 16561
diff changeset
320 if (!dsync_brain_want_namespace(brain, ns))
56be613e8ece dsync: Added -P parameter to do a purge for the remote storage after syncing.
Timo Sirainen <tss@iki.fi>
parents: 16561
diff changeset
321 continue;
56be613e8ece dsync: Added -P parameter to do a purge for the remote storage after syncing.
Timo Sirainen <tss@iki.fi>
parents: 16561
diff changeset
322
56be613e8ece dsync: Added -P parameter to do a purge for the remote storage after syncing.
Timo Sirainen <tss@iki.fi>
parents: 16561
diff changeset
323 storage = mail_namespace_get_default_storage(ns);
56be613e8ece dsync: Added -P parameter to do a purge for the remote storage after syncing.
Timo Sirainen <tss@iki.fi>
parents: 16561
diff changeset
324 if (mail_storage_purge(storage) < 0) {
56be613e8ece dsync: Added -P parameter to do a purge for the remote storage after syncing.
Timo Sirainen <tss@iki.fi>
parents: 16561
diff changeset
325 i_error("Purging namespace '%s' failed: %s", ns->prefix,
21814
759962e70148 global: Log internal storage error on failure
Martti Rannanjärvi <martti.rannanjarvi@dovecot.fi>
parents: 21390
diff changeset
326 mail_storage_get_last_internal_error(storage, NULL));
16801
56be613e8ece dsync: Added -P parameter to do a purge for the remote storage after syncing.
Timo Sirainen <tss@iki.fi>
parents: 16561
diff changeset
327 }
56be613e8ece dsync: Added -P parameter to do a purge for the remote storage after syncing.
Timo Sirainen <tss@iki.fi>
parents: 16561
diff changeset
328 }
56be613e8ece dsync: Added -P parameter to do a purge for the remote storage after syncing.
Timo Sirainen <tss@iki.fi>
parents: 16561
diff changeset
329 }
56be613e8ece dsync: Added -P parameter to do a purge for the remote storage after syncing.
Timo Sirainen <tss@iki.fi>
parents: 16561
diff changeset
330
18371
b900b50085fc dsync: Use storage's mail_error to choose the doveadm exit code.
Timo Sirainen <tss@iki.fi>
parents: 18351
diff changeset
331 int dsync_brain_deinit(struct dsync_brain **_brain, enum mail_error *error_r)
9559
34eef8a2716b dsync: Initial implementation.
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
332 {
34eef8a2716b dsync: Initial implementation.
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
333 struct dsync_brain *brain = *_brain;
14584
b2076acc3715 Initial version of dsync rewrite.
Timo Sirainen <tss@iki.fi>
parents: 14253
diff changeset
334 int ret;
11680
49b226835cd2 dsync: Another try at not increasing mailbox uidnext/highestmodseq on failure
Timo Sirainen <tss@iki.fi>
parents: 11677
diff changeset
335
9559
34eef8a2716b dsync: Initial implementation.
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
336 *_brain = NULL;
14584
b2076acc3715 Initial version of dsync rewrite.
Timo Sirainen <tss@iki.fi>
parents: 14253
diff changeset
337
15986
f6377e089dee dsync: If I/O gets stalled, log the state in which it happened.
Timo Sirainen <tss@iki.fi>
parents: 15836
diff changeset
338 if (dsync_ibc_has_timed_out(brain->ibc)) {
f6377e089dee dsync: If I/O gets stalled, log the state in which it happened.
Timo Sirainen <tss@iki.fi>
parents: 15836
diff changeset
339 i_error("Timeout during state=%s%s",
f6377e089dee dsync: If I/O gets stalled, log the state in which it happened.
Timo Sirainen <tss@iki.fi>
parents: 15836
diff changeset
340 dsync_state_names[brain->state],
f6377e089dee dsync: If I/O gets stalled, log the state in which it happened.
Timo Sirainen <tss@iki.fi>
parents: 15836
diff changeset
341 brain->state != DSYNC_STATE_SYNC_MAILS ? "" :
f6377e089dee dsync: If I/O gets stalled, log the state in which it happened.
Timo Sirainen <tss@iki.fi>
parents: 15836
diff changeset
342 t_strdup_printf(" (send=%s recv=%s)",
f6377e089dee dsync: If I/O gets stalled, log the state in which it happened.
Timo Sirainen <tss@iki.fi>
parents: 15836
diff changeset
343 dsync_box_state_names[brain->box_send_state],
f6377e089dee dsync: If I/O gets stalled, log the state in which it happened.
Timo Sirainen <tss@iki.fi>
parents: 15836
diff changeset
344 dsync_box_state_names[brain->box_recv_state]));
f6377e089dee dsync: If I/O gets stalled, log the state in which it happened.
Timo Sirainen <tss@iki.fi>
parents: 15836
diff changeset
345 }
15037
920756cd29b8 dsync: Renamed "slave" to "ibc" (= inter-brain communicator)
Timo Sirainen <tss@iki.fi>
parents: 14939
diff changeset
346 if (dsync_ibc_has_failed(brain->ibc) ||
14584
b2076acc3715 Initial version of dsync rewrite.
Timo Sirainen <tss@iki.fi>
parents: 14253
diff changeset
347 brain->state != DSYNC_STATE_DONE)
b2076acc3715 Initial version of dsync rewrite.
Timo Sirainen <tss@iki.fi>
parents: 14253
diff changeset
348 brain->failed = TRUE;
15632
40dd9a080b6e dsync: Close mail streams earlier on failures to avoid assert-crashing
Timo Sirainen <tss@iki.fi>
parents: 15594
diff changeset
349 dsync_ibc_close_mail_streams(brain->ibc);
14584
b2076acc3715 Initial version of dsync rewrite.
Timo Sirainen <tss@iki.fi>
parents: 14253
diff changeset
350
16801
56be613e8ece dsync: Added -P parameter to do a purge for the remote storage after syncing.
Timo Sirainen <tss@iki.fi>
parents: 16561
diff changeset
351 if (brain->purge && !brain->failed)
56be613e8ece dsync: Added -P parameter to do a purge for the remote storage after syncing.
Timo Sirainen <tss@iki.fi>
parents: 16561
diff changeset
352 dsync_brain_purge(brain);
56be613e8ece dsync: Added -P parameter to do a purge for the remote storage after syncing.
Timo Sirainen <tss@iki.fi>
parents: 16561
diff changeset
353
14584
b2076acc3715 Initial version of dsync rewrite.
Timo Sirainen <tss@iki.fi>
parents: 14253
diff changeset
354 if (brain->box != NULL)
b2076acc3715 Initial version of dsync rewrite.
Timo Sirainen <tss@iki.fi>
parents: 14253
diff changeset
355 dsync_brain_sync_mailbox_deinit(brain);
18180
39d00448490f dsync: If same GUID already exists in storage, try to copy it instead of recreating the mail.
Timo Sirainen <tss@iki.fi>
parents: 18177
diff changeset
356 if (brain->virtual_all_box != NULL)
39d00448490f dsync: If same GUID already exists in storage, try to copy it instead of recreating the mail.
Timo Sirainen <tss@iki.fi>
parents: 18177
diff changeset
357 mailbox_free(&brain->virtual_all_box);
14584
b2076acc3715 Initial version of dsync rewrite.
Timo Sirainen <tss@iki.fi>
parents: 14253
diff changeset
358 if (brain->local_tree_iter != NULL)
b2076acc3715 Initial version of dsync rewrite.
Timo Sirainen <tss@iki.fi>
parents: 14253
diff changeset
359 dsync_mailbox_tree_iter_deinit(&brain->local_tree_iter);
16541
07e314199f56 dsync: Don't crash at deinit when dsync fails early.
Timo Sirainen <tss@iki.fi>
parents: 16540
diff changeset
360 if (brain->local_mailbox_tree != NULL)
07e314199f56 dsync: Don't crash at deinit when dsync fails early.
Timo Sirainen <tss@iki.fi>
parents: 16540
diff changeset
361 dsync_mailbox_tree_deinit(&brain->local_mailbox_tree);
07e314199f56 dsync: Don't crash at deinit when dsync fails early.
Timo Sirainen <tss@iki.fi>
parents: 16540
diff changeset
362 if (brain->remote_mailbox_tree != NULL)
07e314199f56 dsync: Don't crash at deinit when dsync fails early.
Timo Sirainen <tss@iki.fi>
parents: 16540
diff changeset
363 dsync_mailbox_tree_deinit(&brain->remote_mailbox_tree);
15232
6c850258002f doveadm sync/backup: Added -s <state> parameter to do a fast stateful sync.
Timo Sirainen <tss@iki.fi>
parents: 15231
diff changeset
364 if (brain->mailbox_states_iter != NULL)
6c850258002f doveadm sync/backup: Added -s <state> parameter to do a fast stateful sync.
Timo Sirainen <tss@iki.fi>
parents: 15231
diff changeset
365 hash_table_iterate_deinit(&brain->mailbox_states_iter);
6c850258002f doveadm sync/backup: Added -s <state> parameter to do a fast stateful sync.
Timo Sirainen <tss@iki.fi>
parents: 15231
diff changeset
366 hash_table_destroy(&brain->mailbox_states);
14584
b2076acc3715 Initial version of dsync rewrite.
Timo Sirainen <tss@iki.fi>
parents: 14253
diff changeset
367
17377
a21ac1261487 dsync: mailbox_metadata.cache_fields must be copied to permanent memory.
Timo Sirainen <tss@iki.fi>
parents: 17282
diff changeset
368 if (brain->dsync_box_pool != NULL)
a21ac1261487 dsync: mailbox_metadata.cache_fields must be copied to permanent memory.
Timo Sirainen <tss@iki.fi>
parents: 17282
diff changeset
369 pool_unref(&brain->dsync_box_pool);
a21ac1261487 dsync: mailbox_metadata.cache_fields must be copied to permanent memory.
Timo Sirainen <tss@iki.fi>
parents: 17282
diff changeset
370
15782
6f024e0289da dsync: -l parameter locking is now done on the server with "lower" hostname.
Timo Sirainen <tss@iki.fi>
parents: 15750
diff changeset
371 if (brain->lock_fd != -1) {
6f024e0289da dsync: -l parameter locking is now done on the server with "lower" hostname.
Timo Sirainen <tss@iki.fi>
parents: 15750
diff changeset
372 /* unlink the lock file before it gets unlocked */
19136
fefaa6d09a81 Replaced unlink() calls with i_unlink*() wherever possible.
Timo Sirainen <tss@iki.fi>
parents: 19021
diff changeset
373 i_unlink(brain->lock_path);
15782
6f024e0289da dsync: -l parameter locking is now done on the server with "lower" hostname.
Timo Sirainen <tss@iki.fi>
parents: 15750
diff changeset
374 file_lock_free(&brain->lock);
6f024e0289da dsync: -l parameter locking is now done on the server with "lower" hostname.
Timo Sirainen <tss@iki.fi>
parents: 15750
diff changeset
375 i_close_fd(&brain->lock_fd);
6f024e0289da dsync: -l parameter locking is now done on the server with "lower" hostname.
Timo Sirainen <tss@iki.fi>
parents: 15750
diff changeset
376 }
6f024e0289da dsync: -l parameter locking is now done on the server with "lower" hostname.
Timo Sirainen <tss@iki.fi>
parents: 15750
diff changeset
377
14584
b2076acc3715 Initial version of dsync rewrite.
Timo Sirainen <tss@iki.fi>
parents: 14253
diff changeset
378 ret = brain->failed ? -1 : 0;
16801
56be613e8ece dsync: Added -P parameter to do a purge for the remote storage after syncing.
Timo Sirainen <tss@iki.fi>
parents: 16561
diff changeset
379 mail_user_unref(&brain->user);
18371
b900b50085fc dsync: Use storage's mail_error to choose the doveadm exit code.
Timo Sirainen <tss@iki.fi>
parents: 18351
diff changeset
380
b900b50085fc dsync: Use storage's mail_error to choose the doveadm exit code.
Timo Sirainen <tss@iki.fi>
parents: 18351
diff changeset
381 *error_r = !brain->failed ? 0 :
b900b50085fc dsync: Use storage's mail_error to choose the doveadm exit code.
Timo Sirainen <tss@iki.fi>
parents: 18351
diff changeset
382 (brain->mail_error == 0 ? MAIL_ERROR_TEMP : brain->mail_error);
14584
b2076acc3715 Initial version of dsync rewrite.
Timo Sirainen <tss@iki.fi>
parents: 14253
diff changeset
383 pool_unref(&brain->pool);
9559
34eef8a2716b dsync: Initial implementation.
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
384 return ret;
34eef8a2716b dsync: Initial implementation.
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
385 }
34eef8a2716b dsync: Initial implementation.
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
386
15782
6f024e0289da dsync: -l parameter locking is now done on the server with "lower" hostname.
Timo Sirainen <tss@iki.fi>
parents: 15750
diff changeset
387 static int
6f024e0289da dsync: -l parameter locking is now done on the server with "lower" hostname.
Timo Sirainen <tss@iki.fi>
parents: 15750
diff changeset
388 dsync_brain_lock(struct dsync_brain *brain, const char *remote_hostname)
6f024e0289da dsync: -l parameter locking is now done on the server with "lower" hostname.
Timo Sirainen <tss@iki.fi>
parents: 15750
diff changeset
389 {
21289
08eec0b1aae6 dsync: Fix .dovecot-sync.lock timeout checking
Timo Sirainen <timo.sirainen@dovecot.fi>
parents: 21288
diff changeset
390 const struct file_create_settings lock_set = {
08eec0b1aae6 dsync: Fix .dovecot-sync.lock timeout checking
Timo Sirainen <timo.sirainen@dovecot.fi>
parents: 21288
diff changeset
391 .lock_timeout_secs = brain->lock_timeout,
08eec0b1aae6 dsync: Fix .dovecot-sync.lock timeout checking
Timo Sirainen <timo.sirainen@dovecot.fi>
parents: 21288
diff changeset
392 .lock_method = FILE_LOCK_METHOD_FCNTL,
08eec0b1aae6 dsync: Fix .dovecot-sync.lock timeout checking
Timo Sirainen <timo.sirainen@dovecot.fi>
parents: 21288
diff changeset
393 };
08eec0b1aae6 dsync: Fix .dovecot-sync.lock timeout checking
Timo Sirainen <timo.sirainen@dovecot.fi>
parents: 21288
diff changeset
394 const char *home, *error;
08eec0b1aae6 dsync: Fix .dovecot-sync.lock timeout checking
Timo Sirainen <timo.sirainen@dovecot.fi>
parents: 21288
diff changeset
395 bool created;
15782
6f024e0289da dsync: -l parameter locking is now done on the server with "lower" hostname.
Timo Sirainen <tss@iki.fi>
parents: 15750
diff changeset
396 int ret;
6f024e0289da dsync: -l parameter locking is now done on the server with "lower" hostname.
Timo Sirainen <tss@iki.fi>
parents: 15750
diff changeset
397
15825
df29194c1ab6 dsync: Fix to checking which side should do the locking.
Timo Sirainen <tss@iki.fi>
parents: 15816
diff changeset
398 if ((ret = strcmp(remote_hostname, my_hostdomain())) < 0) {
15782
6f024e0289da dsync: -l parameter locking is now done on the server with "lower" hostname.
Timo Sirainen <tss@iki.fi>
parents: 15750
diff changeset
399 /* locking done by remote */
6f024e0289da dsync: -l parameter locking is now done on the server with "lower" hostname.
Timo Sirainen <tss@iki.fi>
parents: 15750
diff changeset
400 return 0;
6f024e0289da dsync: -l parameter locking is now done on the server with "lower" hostname.
Timo Sirainen <tss@iki.fi>
parents: 15750
diff changeset
401 }
6f024e0289da dsync: -l parameter locking is now done on the server with "lower" hostname.
Timo Sirainen <tss@iki.fi>
parents: 15750
diff changeset
402 if (ret == 0 && !brain->master_brain) {
6f024e0289da dsync: -l parameter locking is now done on the server with "lower" hostname.
Timo Sirainen <tss@iki.fi>
parents: 15750
diff changeset
403 /* running dsync within the same server.
6f024e0289da dsync: -l parameter locking is now done on the server with "lower" hostname.
Timo Sirainen <tss@iki.fi>
parents: 15750
diff changeset
404 locking done by master brain. */
6f024e0289da dsync: -l parameter locking is now done on the server with "lower" hostname.
Timo Sirainen <tss@iki.fi>
parents: 15750
diff changeset
405 return 0;
6f024e0289da dsync: -l parameter locking is now done on the server with "lower" hostname.
Timo Sirainen <tss@iki.fi>
parents: 15750
diff changeset
406 }
6f024e0289da dsync: -l parameter locking is now done on the server with "lower" hostname.
Timo Sirainen <tss@iki.fi>
parents: 15750
diff changeset
407
6f024e0289da dsync: -l parameter locking is now done on the server with "lower" hostname.
Timo Sirainen <tss@iki.fi>
parents: 15750
diff changeset
408 if ((ret = mail_user_get_home(brain->user, &home)) < 0) {
6f024e0289da dsync: -l parameter locking is now done on the server with "lower" hostname.
Timo Sirainen <tss@iki.fi>
parents: 15750
diff changeset
409 i_error("Couldn't look up user's home dir");
6f024e0289da dsync: -l parameter locking is now done on the server with "lower" hostname.
Timo Sirainen <tss@iki.fi>
parents: 15750
diff changeset
410 return -1;
6f024e0289da dsync: -l parameter locking is now done on the server with "lower" hostname.
Timo Sirainen <tss@iki.fi>
parents: 15750
diff changeset
411 }
6f024e0289da dsync: -l parameter locking is now done on the server with "lower" hostname.
Timo Sirainen <tss@iki.fi>
parents: 15750
diff changeset
412 if (ret == 0) {
6f024e0289da dsync: -l parameter locking is now done on the server with "lower" hostname.
Timo Sirainen <tss@iki.fi>
parents: 15750
diff changeset
413 i_error("User has no home directory");
6f024e0289da dsync: -l parameter locking is now done on the server with "lower" hostname.
Timo Sirainen <tss@iki.fi>
parents: 15750
diff changeset
414 return -1;
6f024e0289da dsync: -l parameter locking is now done on the server with "lower" hostname.
Timo Sirainen <tss@iki.fi>
parents: 15750
diff changeset
415 }
6f024e0289da dsync: -l parameter locking is now done on the server with "lower" hostname.
Timo Sirainen <tss@iki.fi>
parents: 15750
diff changeset
416
21288
9316919c5932 dsync: Improve process title during initialization
Timo Sirainen <timo.sirainen@dovecot.fi>
parents: 21256
diff changeset
417 if (brain->verbose_proctitle)
9316919c5932 dsync: Improve process title during initialization
Timo Sirainen <timo.sirainen@dovecot.fi>
parents: 21256
diff changeset
418 process_title_set(dsync_brain_get_proctitle_full(brain, DSYNC_BRAIN_TITLE_LOCKING));
15782
6f024e0289da dsync: -l parameter locking is now done on the server with "lower" hostname.
Timo Sirainen <tss@iki.fi>
parents: 15750
diff changeset
419 brain->lock_path = p_strconcat(brain->pool, home,
6f024e0289da dsync: -l parameter locking is now done on the server with "lower" hostname.
Timo Sirainen <tss@iki.fi>
parents: 15750
diff changeset
420 "/"DSYNC_LOCK_FILENAME, NULL);
21289
08eec0b1aae6 dsync: Fix .dovecot-sync.lock timeout checking
Timo Sirainen <timo.sirainen@dovecot.fi>
parents: 21288
diff changeset
421 brain->lock_fd = file_create_locked(brain->lock_path, &lock_set,
08eec0b1aae6 dsync: Fix .dovecot-sync.lock timeout checking
Timo Sirainen <timo.sirainen@dovecot.fi>
parents: 21288
diff changeset
422 &brain->lock, &created, &error);
08eec0b1aae6 dsync: Fix .dovecot-sync.lock timeout checking
Timo Sirainen <timo.sirainen@dovecot.fi>
parents: 21288
diff changeset
423 if (brain->lock_fd == -1)
08eec0b1aae6 dsync: Fix .dovecot-sync.lock timeout checking
Timo Sirainen <timo.sirainen@dovecot.fi>
parents: 21288
diff changeset
424 i_error("Couldn't lock %s: %s", brain->lock_path, error);
21288
9316919c5932 dsync: Improve process title during initialization
Timo Sirainen <timo.sirainen@dovecot.fi>
parents: 21256
diff changeset
425 if (brain->verbose_proctitle)
9316919c5932 dsync: Improve process title during initialization
Timo Sirainen <timo.sirainen@dovecot.fi>
parents: 21256
diff changeset
426 process_title_set(dsync_brain_get_proctitle(brain));
21289
08eec0b1aae6 dsync: Fix .dovecot-sync.lock timeout checking
Timo Sirainen <timo.sirainen@dovecot.fi>
parents: 21288
diff changeset
427 return brain->lock_fd == -1 ? -1 : 0;
15782
6f024e0289da dsync: -l parameter locking is now done on the server with "lower" hostname.
Timo Sirainen <tss@iki.fi>
parents: 15750
diff changeset
428 }
6f024e0289da dsync: -l parameter locking is now done on the server with "lower" hostname.
Timo Sirainen <tss@iki.fi>
parents: 15750
diff changeset
429
22264
e95435889161 dsync: Use header hashing version 3
Timo Sirainen <timo.sirainen@dovecot.fi>
parents: 22043
diff changeset
430 static void
e95435889161 dsync: Use header hashing version 3
Timo Sirainen <timo.sirainen@dovecot.fi>
parents: 22043
diff changeset
431 dsync_brain_set_hdr_hash_version(struct dsync_brain *brain,
e95435889161 dsync: Use header hashing version 3
Timo Sirainen <timo.sirainen@dovecot.fi>
parents: 22043
diff changeset
432 const struct dsync_ibc_settings *ibc_set)
e95435889161 dsync: Use header hashing version 3
Timo Sirainen <timo.sirainen@dovecot.fi>
parents: 22043
diff changeset
433 {
e95435889161 dsync: Use header hashing version 3
Timo Sirainen <timo.sirainen@dovecot.fi>
parents: 22043
diff changeset
434 if (ibc_set->hdr_hash_v3)
e95435889161 dsync: Use header hashing version 3
Timo Sirainen <timo.sirainen@dovecot.fi>
parents: 22043
diff changeset
435 brain->hdr_hash_version = 3;
e95435889161 dsync: Use header hashing version 3
Timo Sirainen <timo.sirainen@dovecot.fi>
parents: 22043
diff changeset
436 else if (ibc_set->hdr_hash_v2)
e95435889161 dsync: Use header hashing version 3
Timo Sirainen <timo.sirainen@dovecot.fi>
parents: 22043
diff changeset
437 brain->hdr_hash_version = 3;
e95435889161 dsync: Use header hashing version 3
Timo Sirainen <timo.sirainen@dovecot.fi>
parents: 22043
diff changeset
438 else
e95435889161 dsync: Use header hashing version 3
Timo Sirainen <timo.sirainen@dovecot.fi>
parents: 22043
diff changeset
439 brain->hdr_hash_version = 1;
e95435889161 dsync: Use header hashing version 3
Timo Sirainen <timo.sirainen@dovecot.fi>
parents: 22043
diff changeset
440 }
e95435889161 dsync: Use header hashing version 3
Timo Sirainen <timo.sirainen@dovecot.fi>
parents: 22043
diff changeset
441
15782
6f024e0289da dsync: -l parameter locking is now done on the server with "lower" hostname.
Timo Sirainen <tss@iki.fi>
parents: 15750
diff changeset
442 static bool dsync_brain_master_recv_handshake(struct dsync_brain *brain)
6f024e0289da dsync: -l parameter locking is now done on the server with "lower" hostname.
Timo Sirainen <tss@iki.fi>
parents: 15750
diff changeset
443 {
6f024e0289da dsync: -l parameter locking is now done on the server with "lower" hostname.
Timo Sirainen <tss@iki.fi>
parents: 15750
diff changeset
444 const struct dsync_ibc_settings *ibc_set;
6f024e0289da dsync: -l parameter locking is now done on the server with "lower" hostname.
Timo Sirainen <tss@iki.fi>
parents: 15750
diff changeset
445
6f024e0289da dsync: -l parameter locking is now done on the server with "lower" hostname.
Timo Sirainen <tss@iki.fi>
parents: 15750
diff changeset
446 i_assert(brain->master_brain);
6f024e0289da dsync: -l parameter locking is now done on the server with "lower" hostname.
Timo Sirainen <tss@iki.fi>
parents: 15750
diff changeset
447
6f024e0289da dsync: -l parameter locking is now done on the server with "lower" hostname.
Timo Sirainen <tss@iki.fi>
parents: 15750
diff changeset
448 if (dsync_ibc_recv_handshake(brain->ibc, &ibc_set) == 0)
6f024e0289da dsync: -l parameter locking is now done on the server with "lower" hostname.
Timo Sirainen <tss@iki.fi>
parents: 15750
diff changeset
449 return FALSE;
6f024e0289da dsync: -l parameter locking is now done on the server with "lower" hostname.
Timo Sirainen <tss@iki.fi>
parents: 15750
diff changeset
450
6f024e0289da dsync: -l parameter locking is now done on the server with "lower" hostname.
Timo Sirainen <tss@iki.fi>
parents: 15750
diff changeset
451 if (brain->lock_timeout > 0) {
15816
adb9857a3879 dsync: If locking fails, fail instead of continuing.
Timo Sirainen <tss@iki.fi>
parents: 15815
diff changeset
452 if (dsync_brain_lock(brain, ibc_set->hostname) < 0) {
adb9857a3879 dsync: If locking fails, fail instead of continuing.
Timo Sirainen <tss@iki.fi>
parents: 15815
diff changeset
453 brain->failed = TRUE;
15782
6f024e0289da dsync: -l parameter locking is now done on the server with "lower" hostname.
Timo Sirainen <tss@iki.fi>
parents: 15750
diff changeset
454 return FALSE;
15816
adb9857a3879 dsync: If locking fails, fail instead of continuing.
Timo Sirainen <tss@iki.fi>
parents: 15815
diff changeset
455 }
15782
6f024e0289da dsync: -l parameter locking is now done on the server with "lower" hostname.
Timo Sirainen <tss@iki.fi>
parents: 15750
diff changeset
456 }
22264
e95435889161 dsync: Use header hashing version 3
Timo Sirainen <timo.sirainen@dovecot.fi>
parents: 22043
diff changeset
457 dsync_brain_set_hdr_hash_version(brain, ibc_set);
15782
6f024e0289da dsync: -l parameter locking is now done on the server with "lower" hostname.
Timo Sirainen <tss@iki.fi>
parents: 15750
diff changeset
458
6f024e0289da dsync: -l parameter locking is now done on the server with "lower" hostname.
Timo Sirainen <tss@iki.fi>
parents: 15750
diff changeset
459 brain->state = brain->sync_type == DSYNC_BRAIN_SYNC_TYPE_STATE ?
6f024e0289da dsync: -l parameter locking is now done on the server with "lower" hostname.
Timo Sirainen <tss@iki.fi>
parents: 15750
diff changeset
460 DSYNC_STATE_MASTER_SEND_LAST_COMMON :
6f024e0289da dsync: -l parameter locking is now done on the server with "lower" hostname.
Timo Sirainen <tss@iki.fi>
parents: 15750
diff changeset
461 DSYNC_STATE_SEND_MAILBOX_TREE;
6f024e0289da dsync: -l parameter locking is now done on the server with "lower" hostname.
Timo Sirainen <tss@iki.fi>
parents: 15750
diff changeset
462 return TRUE;
6f024e0289da dsync: -l parameter locking is now done on the server with "lower" hostname.
Timo Sirainen <tss@iki.fi>
parents: 15750
diff changeset
463 }
6f024e0289da dsync: -l parameter locking is now done on the server with "lower" hostname.
Timo Sirainen <tss@iki.fi>
parents: 15750
diff changeset
464
14584
b2076acc3715 Initial version of dsync rewrite.
Timo Sirainen <tss@iki.fi>
parents: 14253
diff changeset
465 static bool dsync_brain_slave_recv_handshake(struct dsync_brain *brain)
9559
34eef8a2716b dsync: Initial implementation.
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
466 {
15037
920756cd29b8 dsync: Renamed "slave" to "ibc" (= inter-brain communicator)
Timo Sirainen <tss@iki.fi>
parents: 14939
diff changeset
467 const struct dsync_ibc_settings *ibc_set;
17170
08f1c7af0ac0 dsync: Support multiple -n parameters.
Timo Sirainen <tss@iki.fi>
parents: 17130
diff changeset
468 struct mail_namespace *ns;
08f1c7af0ac0 dsync: Support multiple -n parameters.
Timo Sirainen <tss@iki.fi>
parents: 17130
diff changeset
469 const char *const *prefixes;
9559
34eef8a2716b dsync: Initial implementation.
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
470
14584
b2076acc3715 Initial version of dsync rewrite.
Timo Sirainen <tss@iki.fi>
parents: 14253
diff changeset
471 i_assert(!brain->master_brain);
9559
34eef8a2716b dsync: Initial implementation.
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
472
15037
920756cd29b8 dsync: Renamed "slave" to "ibc" (= inter-brain communicator)
Timo Sirainen <tss@iki.fi>
parents: 14939
diff changeset
473 if (dsync_ibc_recv_handshake(brain->ibc, &ibc_set) == 0)
14584
b2076acc3715 Initial version of dsync rewrite.
Timo Sirainen <tss@iki.fi>
parents: 14253
diff changeset
474 return FALSE;
22264
e95435889161 dsync: Use header hashing version 3
Timo Sirainen <timo.sirainen@dovecot.fi>
parents: 22043
diff changeset
475 dsync_brain_set_hdr_hash_version(brain, ibc_set);
10331
b5b253d35612 dsync: Fixed subscription syncing to work with namespace prefixes.
Timo Sirainen <tss@iki.fi>
parents: 10328
diff changeset
476
15782
6f024e0289da dsync: -l parameter locking is now done on the server with "lower" hostname.
Timo Sirainen <tss@iki.fi>
parents: 15750
diff changeset
477 if (ibc_set->lock_timeout > 0) {
6f024e0289da dsync: -l parameter locking is now done on the server with "lower" hostname.
Timo Sirainen <tss@iki.fi>
parents: 15750
diff changeset
478 brain->lock_timeout = ibc_set->lock_timeout;
15816
adb9857a3879 dsync: If locking fails, fail instead of continuing.
Timo Sirainen <tss@iki.fi>
parents: 15815
diff changeset
479 if (dsync_brain_lock(brain, ibc_set->hostname) < 0) {
adb9857a3879 dsync: If locking fails, fail instead of continuing.
Timo Sirainen <tss@iki.fi>
parents: 15815
diff changeset
480 brain->failed = TRUE;
15782
6f024e0289da dsync: -l parameter locking is now done on the server with "lower" hostname.
Timo Sirainen <tss@iki.fi>
parents: 15750
diff changeset
481 return FALSE;
15816
adb9857a3879 dsync: If locking fails, fail instead of continuing.
Timo Sirainen <tss@iki.fi>
parents: 15815
diff changeset
482 }
15782
6f024e0289da dsync: -l parameter locking is now done on the server with "lower" hostname.
Timo Sirainen <tss@iki.fi>
parents: 15750
diff changeset
483 }
6f024e0289da dsync: -l parameter locking is now done on the server with "lower" hostname.
Timo Sirainen <tss@iki.fi>
parents: 15750
diff changeset
484
17170
08f1c7af0ac0 dsync: Support multiple -n parameters.
Timo Sirainen <tss@iki.fi>
parents: 17130
diff changeset
485 if (ibc_set->sync_ns_prefixes != NULL) {
08f1c7af0ac0 dsync: Support multiple -n parameters.
Timo Sirainen <tss@iki.fi>
parents: 17130
diff changeset
486 p_array_init(&brain->sync_namespaces, brain->pool, 4);
08f1c7af0ac0 dsync: Support multiple -n parameters.
Timo Sirainen <tss@iki.fi>
parents: 17130
diff changeset
487 prefixes = t_strsplit(ibc_set->sync_ns_prefixes, "\n");
17200
3e1a69e0cda9 dsync: Fixed using -n "" parameter
Timo Sirainen <tss@iki.fi>
parents: 17170
diff changeset
488 if (prefixes[0] == NULL) {
3e1a69e0cda9 dsync: Fixed using -n "" parameter
Timo Sirainen <tss@iki.fi>
parents: 17170
diff changeset
489 /* ugly workaround for strsplit API: there was one
3e1a69e0cda9 dsync: Fixed using -n "" parameter
Timo Sirainen <tss@iki.fi>
parents: 17170
diff changeset
490 prefix="" entry */
3e1a69e0cda9 dsync: Fixed using -n "" parameter
Timo Sirainen <tss@iki.fi>
parents: 17170
diff changeset
491 static const char *empty_prefix[] = { "", NULL };
3e1a69e0cda9 dsync: Fixed using -n "" parameter
Timo Sirainen <tss@iki.fi>
parents: 17170
diff changeset
492 prefixes = empty_prefix;
3e1a69e0cda9 dsync: Fixed using -n "" parameter
Timo Sirainen <tss@iki.fi>
parents: 17170
diff changeset
493 }
17170
08f1c7af0ac0 dsync: Support multiple -n parameters.
Timo Sirainen <tss@iki.fi>
parents: 17130
diff changeset
494 for (; *prefixes != NULL; prefixes++) {
08f1c7af0ac0 dsync: Support multiple -n parameters.
Timo Sirainen <tss@iki.fi>
parents: 17130
diff changeset
495 ns = mail_namespace_find(brain->user->namespaces,
08f1c7af0ac0 dsync: Support multiple -n parameters.
Timo Sirainen <tss@iki.fi>
parents: 17130
diff changeset
496 *prefixes);
08f1c7af0ac0 dsync: Support multiple -n parameters.
Timo Sirainen <tss@iki.fi>
parents: 17130
diff changeset
497 if (ns == NULL) {
08f1c7af0ac0 dsync: Support multiple -n parameters.
Timo Sirainen <tss@iki.fi>
parents: 17130
diff changeset
498 i_error("Namespace not found: '%s'", *prefixes);
08f1c7af0ac0 dsync: Support multiple -n parameters.
Timo Sirainen <tss@iki.fi>
parents: 17130
diff changeset
499 brain->failed = TRUE;
08f1c7af0ac0 dsync: Support multiple -n parameters.
Timo Sirainen <tss@iki.fi>
parents: 17130
diff changeset
500 return FALSE;
08f1c7af0ac0 dsync: Support multiple -n parameters.
Timo Sirainen <tss@iki.fi>
parents: 17130
diff changeset
501 }
08f1c7af0ac0 dsync: Support multiple -n parameters.
Timo Sirainen <tss@iki.fi>
parents: 17130
diff changeset
502 array_append(&brain->sync_namespaces, &ns, 1);
08f1c7af0ac0 dsync: Support multiple -n parameters.
Timo Sirainen <tss@iki.fi>
parents: 17130
diff changeset
503 }
11676
cf7f6912af02 dsync: Added backup command, which syncs source to destination, discarding any changes in dest.
Timo Sirainen <tss@iki.fi>
parents: 11524
diff changeset
504 }
15471
88ac919f8afe dsync: Added back support for syncing only one mailbox (-m parameter)
Timo Sirainen <tss@iki.fi>
parents: 15469
diff changeset
505 brain->sync_box = p_strdup(brain->pool, ibc_set->sync_box);
16398
4883a8e1db13 dsync: Added -x parameter to exclude mailboxes from sync.
Timo Sirainen <tss@iki.fi>
parents: 16396
diff changeset
506 brain->exclude_mailboxes = ibc_set->exclude_mailboxes == NULL ? NULL :
4883a8e1db13 dsync: Added -x parameter to exclude mailboxes from sync.
Timo Sirainen <tss@iki.fi>
parents: 16396
diff changeset
507 p_strarray_dup(brain->pool, ibc_set->exclude_mailboxes);
18177
f393f63764e0 dsync: Added -t <timestamp> parameter to save only mails newer than <timestamp>
Timo Sirainen <tss@iki.fi>
parents: 18137
diff changeset
508 brain->sync_since_timestamp = ibc_set->sync_since_timestamp;
20831
0c4e5c2725a3 doveadm-sync: Add end-date support
Aki Tuomi <aki.tuomi@dovecot.fi>
parents: 20633
diff changeset
509 brain->sync_until_timestamp = ibc_set->sync_until_timestamp;
20633
d54651ba988a doveadm sync/backup: Added -S <max size> parameter to skip too large mails.
Timo Sirainen <timo.sirainen@dovecot.fi>
parents: 20595
diff changeset
510 brain->sync_max_size = ibc_set->sync_max_size;
18181
35e4a6ae8d85 dsync: Added -F parameter to sync only mails with[out] specific flag.
Timo Sirainen <tss@iki.fi>
parents: 18180
diff changeset
511 brain->sync_flag = p_strdup(brain->pool, ibc_set->sync_flags);
16110
c51873a8e0d9 doveadm sync/backup: Added -g <guid> to sync only the specified mailbox (by GUID)
Timo Sirainen <tss@iki.fi>
parents: 16089
diff changeset
512 memcpy(brain->sync_box_guid, ibc_set->sync_box_guid,
c51873a8e0d9 doveadm sync/backup: Added -g <guid> to sync only the specified mailbox (by GUID)
Timo Sirainen <tss@iki.fi>
parents: 16089
diff changeset
513 sizeof(brain->sync_box_guid));
14584
b2076acc3715 Initial version of dsync rewrite.
Timo Sirainen <tss@iki.fi>
parents: 14253
diff changeset
514 i_assert(brain->sync_type == DSYNC_BRAIN_SYNC_TYPE_UNKNOWN);
15037
920756cd29b8 dsync: Renamed "slave" to "ibc" (= inter-brain communicator)
Timo Sirainen <tss@iki.fi>
parents: 14939
diff changeset
515 brain->sync_type = ibc_set->sync_type;
16801
56be613e8ece dsync: Added -P parameter to do a purge for the remote storage after syncing.
Timo Sirainen <tss@iki.fi>
parents: 16561
diff changeset
516
15231
454d0563927d doveadm: "backup" command is working again.
Timo Sirainen <tss@iki.fi>
parents: 15037
diff changeset
517 dsync_brain_set_flags(brain, ibc_set->brain_flags);
22549
400ff84f109d dsync: Add hashed_headers setting
Aki Tuomi <aki.tuomi@dovecot.fi>
parents: 22264
diff changeset
518 if (ibc_set->hashed_headers != NULL)
400ff84f109d dsync: Add hashed_headers setting
Aki Tuomi <aki.tuomi@dovecot.fi>
parents: 22264
diff changeset
519 brain->hashed_headers =
400ff84f109d dsync: Add hashed_headers setting
Aki Tuomi <aki.tuomi@dovecot.fi>
parents: 22264
diff changeset
520 p_strarray_dup(brain->pool, (const char*const*)ibc_set->hashed_headers);
16801
56be613e8ece dsync: Added -P parameter to do a purge for the remote storage after syncing.
Timo Sirainen <tss@iki.fi>
parents: 16561
diff changeset
521 /* this flag is only set on the remote slave brain */
56be613e8ece dsync: Added -P parameter to do a purge for the remote storage after syncing.
Timo Sirainen <tss@iki.fi>
parents: 16561
diff changeset
522 brain->purge = (ibc_set->brain_flags &
56be613e8ece dsync: Added -P parameter to do a purge for the remote storage after syncing.
Timo Sirainen <tss@iki.fi>
parents: 16561
diff changeset
523 DSYNC_BRAIN_FLAG_PURGE_REMOTE) != 0;
11676
cf7f6912af02 dsync: Added backup command, which syncs source to destination, discarding any changes in dest.
Timo Sirainen <tss@iki.fi>
parents: 11524
diff changeset
524
18180
39d00448490f dsync: If same GUID already exists in storage, try to copy it instead of recreating the mail.
Timo Sirainen <tss@iki.fi>
parents: 18177
diff changeset
525 if (ibc_set->virtual_all_box != NULL)
39d00448490f dsync: If same GUID already exists in storage, try to copy it instead of recreating the mail.
Timo Sirainen <tss@iki.fi>
parents: 18177
diff changeset
526 dsync_brain_open_virtual_all_box(brain, ibc_set->virtual_all_box);
14584
b2076acc3715 Initial version of dsync rewrite.
Timo Sirainen <tss@iki.fi>
parents: 14253
diff changeset
527 dsync_brain_mailbox_trees_init(brain);
11676
cf7f6912af02 dsync: Added backup command, which syncs source to destination, discarding any changes in dest.
Timo Sirainen <tss@iki.fi>
parents: 11524
diff changeset
528
14584
b2076acc3715 Initial version of dsync rewrite.
Timo Sirainen <tss@iki.fi>
parents: 14253
diff changeset
529 if (brain->sync_type == DSYNC_BRAIN_SYNC_TYPE_STATE)
b2076acc3715 Initial version of dsync rewrite.
Timo Sirainen <tss@iki.fi>
parents: 14253
diff changeset
530 brain->state = DSYNC_STATE_SLAVE_RECV_LAST_COMMON;
b2076acc3715 Initial version of dsync rewrite.
Timo Sirainen <tss@iki.fi>
parents: 14253
diff changeset
531 else
b2076acc3715 Initial version of dsync rewrite.
Timo Sirainen <tss@iki.fi>
parents: 14253
diff changeset
532 brain->state = DSYNC_STATE_SEND_MAILBOX_TREE;
b2076acc3715 Initial version of dsync rewrite.
Timo Sirainen <tss@iki.fi>
parents: 14253
diff changeset
533 return TRUE;
11727
42dfcf9c896b dsync backup: Fail if it looks like backup is running in wrong direction.
Timo Sirainen <tss@iki.fi>
parents: 11688
diff changeset
534 }
42dfcf9c896b dsync backup: Fail if it looks like backup is running in wrong direction.
Timo Sirainen <tss@iki.fi>
parents: 11688
diff changeset
535
14584
b2076acc3715 Initial version of dsync rewrite.
Timo Sirainen <tss@iki.fi>
parents: 14253
diff changeset
536 static void dsync_brain_master_send_last_common(struct dsync_brain *brain)
9559
34eef8a2716b dsync: Initial implementation.
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
537 {
15232
6c850258002f doveadm sync/backup: Added -s <state> parameter to do a fast stateful sync.
Timo Sirainen <tss@iki.fi>
parents: 15231
diff changeset
538 struct dsync_mailbox_state *state;
6c850258002f doveadm sync/backup: Added -s <state> parameter to do a fast stateful sync.
Timo Sirainen <tss@iki.fi>
parents: 15231
diff changeset
539 uint8_t *guid;
15037
920756cd29b8 dsync: Renamed "slave" to "ibc" (= inter-brain communicator)
Timo Sirainen <tss@iki.fi>
parents: 14939
diff changeset
540 enum dsync_ibc_send_ret ret = DSYNC_IBC_SEND_RET_OK;
11727
42dfcf9c896b dsync backup: Fail if it looks like backup is running in wrong direction.
Timo Sirainen <tss@iki.fi>
parents: 11688
diff changeset
541
14584
b2076acc3715 Initial version of dsync rewrite.
Timo Sirainen <tss@iki.fi>
parents: 14253
diff changeset
542 i_assert(brain->master_brain);
9736
d9a96da46d4a dsync: Lots of updates and fixes.
Timo Sirainen <tss@iki.fi>
parents: 9686
diff changeset
543
15232
6c850258002f doveadm sync/backup: Added -s <state> parameter to do a fast stateful sync.
Timo Sirainen <tss@iki.fi>
parents: 15231
diff changeset
544 if (brain->mailbox_states_iter == NULL) {
6c850258002f doveadm sync/backup: Added -s <state> parameter to do a fast stateful sync.
Timo Sirainen <tss@iki.fi>
parents: 15231
diff changeset
545 brain->mailbox_states_iter =
6c850258002f doveadm sync/backup: Added -s <state> parameter to do a fast stateful sync.
Timo Sirainen <tss@iki.fi>
parents: 15231
diff changeset
546 hash_table_iterate_init(brain->mailbox_states);
6c850258002f doveadm sync/backup: Added -s <state> parameter to do a fast stateful sync.
Timo Sirainen <tss@iki.fi>
parents: 15231
diff changeset
547 }
6c850258002f doveadm sync/backup: Added -s <state> parameter to do a fast stateful sync.
Timo Sirainen <tss@iki.fi>
parents: 15231
diff changeset
548
6c850258002f doveadm sync/backup: Added -s <state> parameter to do a fast stateful sync.
Timo Sirainen <tss@iki.fi>
parents: 15231
diff changeset
549 for (;;) {
15037
920756cd29b8 dsync: Renamed "slave" to "ibc" (= inter-brain communicator)
Timo Sirainen <tss@iki.fi>
parents: 14939
diff changeset
550 if (ret == DSYNC_IBC_SEND_RET_FULL)
14584
b2076acc3715 Initial version of dsync rewrite.
Timo Sirainen <tss@iki.fi>
parents: 14253
diff changeset
551 return;
15232
6c850258002f doveadm sync/backup: Added -s <state> parameter to do a fast stateful sync.
Timo Sirainen <tss@iki.fi>
parents: 15231
diff changeset
552 if (!hash_table_iterate(brain->mailbox_states_iter,
6c850258002f doveadm sync/backup: Added -s <state> parameter to do a fast stateful sync.
Timo Sirainen <tss@iki.fi>
parents: 15231
diff changeset
553 brain->mailbox_states, &guid, &state))
6c850258002f doveadm sync/backup: Added -s <state> parameter to do a fast stateful sync.
Timo Sirainen <tss@iki.fi>
parents: 15231
diff changeset
554 break;
6c850258002f doveadm sync/backup: Added -s <state> parameter to do a fast stateful sync.
Timo Sirainen <tss@iki.fi>
parents: 15231
diff changeset
555 ret = dsync_ibc_send_mailbox_state(brain->ibc, state);
9638
b11a3eda2477 dsync: Use expunged messages' GUIDs to determine what to do with missing messages at end of mailbox.
Timo Sirainen <tss@iki.fi>
parents: 9592
diff changeset
556 }
15232
6c850258002f doveadm sync/backup: Added -s <state> parameter to do a fast stateful sync.
Timo Sirainen <tss@iki.fi>
parents: 15231
diff changeset
557 hash_table_iterate_deinit(&brain->mailbox_states_iter);
6c850258002f doveadm sync/backup: Added -s <state> parameter to do a fast stateful sync.
Timo Sirainen <tss@iki.fi>
parents: 15231
diff changeset
558
16222
d79cf48f1072 dsync: Fixed talking to earlier dsync without mailbox attribute support.
Timo Sirainen <tss@iki.fi>
parents: 16110
diff changeset
559 dsync_ibc_send_end_of_list(brain->ibc, DSYNC_IBC_EOL_MAILBOX_STATE);
14584
b2076acc3715 Initial version of dsync rewrite.
Timo Sirainen <tss@iki.fi>
parents: 14253
diff changeset
560 brain->state = DSYNC_STATE_SEND_MAILBOX_TREE;
15232
6c850258002f doveadm sync/backup: Added -s <state> parameter to do a fast stateful sync.
Timo Sirainen <tss@iki.fi>
parents: 15231
diff changeset
561 }
6c850258002f doveadm sync/backup: Added -s <state> parameter to do a fast stateful sync.
Timo Sirainen <tss@iki.fi>
parents: 15231
diff changeset
562
6c850258002f doveadm sync/backup: Added -s <state> parameter to do a fast stateful sync.
Timo Sirainen <tss@iki.fi>
parents: 15231
diff changeset
563 static void dsync_mailbox_state_add(struct dsync_brain *brain,
6c850258002f doveadm sync/backup: Added -s <state> parameter to do a fast stateful sync.
Timo Sirainen <tss@iki.fi>
parents: 15231
diff changeset
564 const struct dsync_mailbox_state *state)
6c850258002f doveadm sync/backup: Added -s <state> parameter to do a fast stateful sync.
Timo Sirainen <tss@iki.fi>
parents: 15231
diff changeset
565 {
6c850258002f doveadm sync/backup: Added -s <state> parameter to do a fast stateful sync.
Timo Sirainen <tss@iki.fi>
parents: 15231
diff changeset
566 struct dsync_mailbox_state *dupstate;
6c850258002f doveadm sync/backup: Added -s <state> parameter to do a fast stateful sync.
Timo Sirainen <tss@iki.fi>
parents: 15231
diff changeset
567 uint8_t *guid_p;
6c850258002f doveadm sync/backup: Added -s <state> parameter to do a fast stateful sync.
Timo Sirainen <tss@iki.fi>
parents: 15231
diff changeset
568
6c850258002f doveadm sync/backup: Added -s <state> parameter to do a fast stateful sync.
Timo Sirainen <tss@iki.fi>
parents: 15231
diff changeset
569 dupstate = p_new(brain->pool, struct dsync_mailbox_state, 1);
6c850258002f doveadm sync/backup: Added -s <state> parameter to do a fast stateful sync.
Timo Sirainen <tss@iki.fi>
parents: 15231
diff changeset
570 *dupstate = *state;
6c850258002f doveadm sync/backup: Added -s <state> parameter to do a fast stateful sync.
Timo Sirainen <tss@iki.fi>
parents: 15231
diff changeset
571 guid_p = dupstate->mailbox_guid;
6c850258002f doveadm sync/backup: Added -s <state> parameter to do a fast stateful sync.
Timo Sirainen <tss@iki.fi>
parents: 15231
diff changeset
572 hash_table_insert(brain->mailbox_states, guid_p, dupstate);
11524
c955d4789553 dsync: Fixed syncing \noselect mailboxes.
Timo Sirainen <tss@iki.fi>
parents: 10976
diff changeset
573 }
c955d4789553 dsync: Fixed syncing \noselect mailboxes.
Timo Sirainen <tss@iki.fi>
parents: 10976
diff changeset
574
14584
b2076acc3715 Initial version of dsync rewrite.
Timo Sirainen <tss@iki.fi>
parents: 14253
diff changeset
575 static bool dsync_brain_slave_recv_last_common(struct dsync_brain *brain)
10328
b63fd6156663 dsync: Added support for subscription syncing.
Timo Sirainen <tss@iki.fi>
parents: 10323
diff changeset
576 {
14584
b2076acc3715 Initial version of dsync rewrite.
Timo Sirainen <tss@iki.fi>
parents: 14253
diff changeset
577 struct dsync_mailbox_state state;
15037
920756cd29b8 dsync: Renamed "slave" to "ibc" (= inter-brain communicator)
Timo Sirainen <tss@iki.fi>
parents: 14939
diff changeset
578 enum dsync_ibc_recv_ret ret;
14584
b2076acc3715 Initial version of dsync rewrite.
Timo Sirainen <tss@iki.fi>
parents: 14253
diff changeset
579 bool changed = FALSE;
10328
b63fd6156663 dsync: Added support for subscription syncing.
Timo Sirainen <tss@iki.fi>
parents: 10323
diff changeset
580
14584
b2076acc3715 Initial version of dsync rewrite.
Timo Sirainen <tss@iki.fi>
parents: 14253
diff changeset
581 i_assert(!brain->master_brain);
10328
b63fd6156663 dsync: Added support for subscription syncing.
Timo Sirainen <tss@iki.fi>
parents: 10323
diff changeset
582
15037
920756cd29b8 dsync: Renamed "slave" to "ibc" (= inter-brain communicator)
Timo Sirainen <tss@iki.fi>
parents: 14939
diff changeset
583 while ((ret = dsync_ibc_recv_mailbox_state(brain->ibc, &state)) > 0) {
15232
6c850258002f doveadm sync/backup: Added -s <state> parameter to do a fast stateful sync.
Timo Sirainen <tss@iki.fi>
parents: 15231
diff changeset
584 dsync_mailbox_state_add(brain, &state);
14584
b2076acc3715 Initial version of dsync rewrite.
Timo Sirainen <tss@iki.fi>
parents: 14253
diff changeset
585 changed = TRUE;
10328
b63fd6156663 dsync: Added support for subscription syncing.
Timo Sirainen <tss@iki.fi>
parents: 10323
diff changeset
586 }
15037
920756cd29b8 dsync: Renamed "slave" to "ibc" (= inter-brain communicator)
Timo Sirainen <tss@iki.fi>
parents: 14939
diff changeset
587 if (ret == DSYNC_IBC_RECV_RET_FINISHED) {
14584
b2076acc3715 Initial version of dsync rewrite.
Timo Sirainen <tss@iki.fi>
parents: 14253
diff changeset
588 brain->state = DSYNC_STATE_SEND_MAILBOX_TREE;
b2076acc3715 Initial version of dsync rewrite.
Timo Sirainen <tss@iki.fi>
parents: 14253
diff changeset
589 changed = TRUE;
9736
d9a96da46d4a dsync: Lots of updates and fixes.
Timo Sirainen <tss@iki.fi>
parents: 9686
diff changeset
590 }
14584
b2076acc3715 Initial version of dsync rewrite.
Timo Sirainen <tss@iki.fi>
parents: 14253
diff changeset
591 return changed;
10371
b99a19d5a93c dsync: Sync mailbox renames.
Timo Sirainen <tss@iki.fi>
parents: 10368
diff changeset
592 }
b99a19d5a93c dsync: Sync mailbox renames.
Timo Sirainen <tss@iki.fi>
parents: 10368
diff changeset
593
18351
b4dbe64c0032 dsync: Added an extra "finish" state to allow slave-dsync to report error to master-dsync.
Timo Sirainen <tss@iki.fi>
parents: 18287
diff changeset
594 static bool dsync_brain_finish(struct dsync_brain *brain)
b4dbe64c0032 dsync: Added an extra "finish" state to allow slave-dsync to report error to master-dsync.
Timo Sirainen <tss@iki.fi>
parents: 18287
diff changeset
595 {
b4dbe64c0032 dsync: Added an extra "finish" state to allow slave-dsync to report error to master-dsync.
Timo Sirainen <tss@iki.fi>
parents: 18287
diff changeset
596 const char *error;
18371
b900b50085fc dsync: Use storage's mail_error to choose the doveadm exit code.
Timo Sirainen <tss@iki.fi>
parents: 18351
diff changeset
597 enum mail_error mail_error;
20458
16f9c0a46cbb dsync: When full resync is wanted in a stateful sync, output empty state.
Timo Sirainen <timo.sirainen@dovecot.fi>
parents: 20073
diff changeset
598 bool require_full_resync;
18371
b900b50085fc dsync: Use storage's mail_error to choose the doveadm exit code.
Timo Sirainen <tss@iki.fi>
parents: 18351
diff changeset
599 enum dsync_ibc_recv_ret ret;
18351
b4dbe64c0032 dsync: Added an extra "finish" state to allow slave-dsync to report error to master-dsync.
Timo Sirainen <tss@iki.fi>
parents: 18287
diff changeset
600
b4dbe64c0032 dsync: Added an extra "finish" state to allow slave-dsync to report error to master-dsync.
Timo Sirainen <tss@iki.fi>
parents: 18287
diff changeset
601 if (!brain->master_brain) {
b4dbe64c0032 dsync: Added an extra "finish" state to allow slave-dsync to report error to master-dsync.
Timo Sirainen <tss@iki.fi>
parents: 18287
diff changeset
602 dsync_ibc_send_finish(brain->ibc,
18371
b900b50085fc dsync: Use storage's mail_error to choose the doveadm exit code.
Timo Sirainen <tss@iki.fi>
parents: 18351
diff changeset
603 brain->failed ? "dsync failed" : NULL,
20458
16f9c0a46cbb dsync: When full resync is wanted in a stateful sync, output empty state.
Timo Sirainen <timo.sirainen@dovecot.fi>
parents: 20073
diff changeset
604 brain->mail_error,
16f9c0a46cbb dsync: When full resync is wanted in a stateful sync, output empty state.
Timo Sirainen <timo.sirainen@dovecot.fi>
parents: 20073
diff changeset
605 brain->require_full_resync);
18351
b4dbe64c0032 dsync: Added an extra "finish" state to allow slave-dsync to report error to master-dsync.
Timo Sirainen <tss@iki.fi>
parents: 18287
diff changeset
606 brain->state = DSYNC_STATE_DONE;
b4dbe64c0032 dsync: Added an extra "finish" state to allow slave-dsync to report error to master-dsync.
Timo Sirainen <tss@iki.fi>
parents: 18287
diff changeset
607 return TRUE;
b4dbe64c0032 dsync: Added an extra "finish" state to allow slave-dsync to report error to master-dsync.
Timo Sirainen <tss@iki.fi>
parents: 18287
diff changeset
608 }
20458
16f9c0a46cbb dsync: When full resync is wanted in a stateful sync, output empty state.
Timo Sirainen <timo.sirainen@dovecot.fi>
parents: 20073
diff changeset
609 ret = dsync_ibc_recv_finish(brain->ibc, &error, &mail_error,
16f9c0a46cbb dsync: When full resync is wanted in a stateful sync, output empty state.
Timo Sirainen <timo.sirainen@dovecot.fi>
parents: 20073
diff changeset
610 &require_full_resync);
18371
b900b50085fc dsync: Use storage's mail_error to choose the doveadm exit code.
Timo Sirainen <tss@iki.fi>
parents: 18351
diff changeset
611 if (ret == DSYNC_IBC_RECV_RET_TRYAGAIN)
18351
b4dbe64c0032 dsync: Added an extra "finish" state to allow slave-dsync to report error to master-dsync.
Timo Sirainen <tss@iki.fi>
parents: 18287
diff changeset
612 return FALSE;
b4dbe64c0032 dsync: Added an extra "finish" state to allow slave-dsync to report error to master-dsync.
Timo Sirainen <tss@iki.fi>
parents: 18287
diff changeset
613 if (error != NULL) {
b4dbe64c0032 dsync: Added an extra "finish" state to allow slave-dsync to report error to master-dsync.
Timo Sirainen <tss@iki.fi>
parents: 18287
diff changeset
614 i_error("Remote dsync failed: %s", error);
b4dbe64c0032 dsync: Added an extra "finish" state to allow slave-dsync to report error to master-dsync.
Timo Sirainen <tss@iki.fi>
parents: 18287
diff changeset
615 brain->failed = TRUE;
18371
b900b50085fc dsync: Use storage's mail_error to choose the doveadm exit code.
Timo Sirainen <tss@iki.fi>
parents: 18351
diff changeset
616 if (mail_error != 0 &&
b900b50085fc dsync: Use storage's mail_error to choose the doveadm exit code.
Timo Sirainen <tss@iki.fi>
parents: 18351
diff changeset
617 (brain->mail_error == 0 || brain->mail_error == MAIL_ERROR_TEMP))
b900b50085fc dsync: Use storage's mail_error to choose the doveadm exit code.
Timo Sirainen <tss@iki.fi>
parents: 18351
diff changeset
618 brain->mail_error = mail_error;
18351
b4dbe64c0032 dsync: Added an extra "finish" state to allow slave-dsync to report error to master-dsync.
Timo Sirainen <tss@iki.fi>
parents: 18287
diff changeset
619 }
20458
16f9c0a46cbb dsync: When full resync is wanted in a stateful sync, output empty state.
Timo Sirainen <timo.sirainen@dovecot.fi>
parents: 20073
diff changeset
620 if (require_full_resync)
16f9c0a46cbb dsync: When full resync is wanted in a stateful sync, output empty state.
Timo Sirainen <timo.sirainen@dovecot.fi>
parents: 20073
diff changeset
621 brain->require_full_resync = TRUE;
18351
b4dbe64c0032 dsync: Added an extra "finish" state to allow slave-dsync to report error to master-dsync.
Timo Sirainen <tss@iki.fi>
parents: 18287
diff changeset
622 brain->state = DSYNC_STATE_DONE;
b4dbe64c0032 dsync: Added an extra "finish" state to allow slave-dsync to report error to master-dsync.
Timo Sirainen <tss@iki.fi>
parents: 18287
diff changeset
623 return TRUE;
b4dbe64c0032 dsync: Added an extra "finish" state to allow slave-dsync to report error to master-dsync.
Timo Sirainen <tss@iki.fi>
parents: 18287
diff changeset
624 }
b4dbe64c0032 dsync: Added an extra "finish" state to allow slave-dsync to report error to master-dsync.
Timo Sirainen <tss@iki.fi>
parents: 18287
diff changeset
625
14584
b2076acc3715 Initial version of dsync rewrite.
Timo Sirainen <tss@iki.fi>
parents: 14253
diff changeset
626 static bool dsync_brain_run_real(struct dsync_brain *brain, bool *changed_r)
9639
ffda7bd92ebc dsync: Skip syncing mailboxes whose uidvalidity/uidnext/highest-modseq hasn't changed.
Timo Sirainen <tss@iki.fi>
parents: 9638
diff changeset
627 {
16539
b5876fa03b0e dsync: If verbose_proctitle=yes, show the current state in it.
Timo Sirainen <tss@iki.fi>
parents: 16538
diff changeset
628 enum dsync_state orig_state = brain->state;
b5876fa03b0e dsync: If verbose_proctitle=yes, show the current state in it.
Timo Sirainen <tss@iki.fi>
parents: 16538
diff changeset
629 enum dsync_box_state orig_box_recv_state = brain->box_recv_state;
b5876fa03b0e dsync: If verbose_proctitle=yes, show the current state in it.
Timo Sirainen <tss@iki.fi>
parents: 16538
diff changeset
630 enum dsync_box_state orig_box_send_state = brain->box_send_state;
14584
b2076acc3715 Initial version of dsync rewrite.
Timo Sirainen <tss@iki.fi>
parents: 14253
diff changeset
631 bool changed = FALSE, ret = TRUE;
9639
ffda7bd92ebc dsync: Skip syncing mailboxes whose uidvalidity/uidnext/highest-modseq hasn't changed.
Timo Sirainen <tss@iki.fi>
parents: 9638
diff changeset
632
14584
b2076acc3715 Initial version of dsync rewrite.
Timo Sirainen <tss@iki.fi>
parents: 14253
diff changeset
633 if (brain->failed)
b2076acc3715 Initial version of dsync rewrite.
Timo Sirainen <tss@iki.fi>
parents: 14253
diff changeset
634 return FALSE;
9686
4d5cc6ce68aa dsync: Lots of improvements and fixes. Appears to be somewhat working now.
Timo Sirainen <tss@iki.fi>
parents: 9644
diff changeset
635
14584
b2076acc3715 Initial version of dsync rewrite.
Timo Sirainen <tss@iki.fi>
parents: 14253
diff changeset
636 switch (brain->state) {
15782
6f024e0289da dsync: -l parameter locking is now done on the server with "lower" hostname.
Timo Sirainen <tss@iki.fi>
parents: 15750
diff changeset
637 case DSYNC_STATE_MASTER_RECV_HANDSHAKE:
6f024e0289da dsync: -l parameter locking is now done on the server with "lower" hostname.
Timo Sirainen <tss@iki.fi>
parents: 15750
diff changeset
638 changed = dsync_brain_master_recv_handshake(brain);
6f024e0289da dsync: -l parameter locking is now done on the server with "lower" hostname.
Timo Sirainen <tss@iki.fi>
parents: 15750
diff changeset
639 break;
14584
b2076acc3715 Initial version of dsync rewrite.
Timo Sirainen <tss@iki.fi>
parents: 14253
diff changeset
640 case DSYNC_STATE_SLAVE_RECV_HANDSHAKE:
b2076acc3715 Initial version of dsync rewrite.
Timo Sirainen <tss@iki.fi>
parents: 14253
diff changeset
641 changed = dsync_brain_slave_recv_handshake(brain);
b2076acc3715 Initial version of dsync rewrite.
Timo Sirainen <tss@iki.fi>
parents: 14253
diff changeset
642 break;
b2076acc3715 Initial version of dsync rewrite.
Timo Sirainen <tss@iki.fi>
parents: 14253
diff changeset
643 case DSYNC_STATE_MASTER_SEND_LAST_COMMON:
b2076acc3715 Initial version of dsync rewrite.
Timo Sirainen <tss@iki.fi>
parents: 14253
diff changeset
644 dsync_brain_master_send_last_common(brain);
b2076acc3715 Initial version of dsync rewrite.
Timo Sirainen <tss@iki.fi>
parents: 14253
diff changeset
645 changed = TRUE;
b2076acc3715 Initial version of dsync rewrite.
Timo Sirainen <tss@iki.fi>
parents: 14253
diff changeset
646 break;
b2076acc3715 Initial version of dsync rewrite.
Timo Sirainen <tss@iki.fi>
parents: 14253
diff changeset
647 case DSYNC_STATE_SLAVE_RECV_LAST_COMMON:
b2076acc3715 Initial version of dsync rewrite.
Timo Sirainen <tss@iki.fi>
parents: 14253
diff changeset
648 changed = dsync_brain_slave_recv_last_common(brain);
b2076acc3715 Initial version of dsync rewrite.
Timo Sirainen <tss@iki.fi>
parents: 14253
diff changeset
649 break;
b2076acc3715 Initial version of dsync rewrite.
Timo Sirainen <tss@iki.fi>
parents: 14253
diff changeset
650 case DSYNC_STATE_SEND_MAILBOX_TREE:
b2076acc3715 Initial version of dsync rewrite.
Timo Sirainen <tss@iki.fi>
parents: 14253
diff changeset
651 dsync_brain_send_mailbox_tree(brain);
b2076acc3715 Initial version of dsync rewrite.
Timo Sirainen <tss@iki.fi>
parents: 14253
diff changeset
652 changed = TRUE;
b2076acc3715 Initial version of dsync rewrite.
Timo Sirainen <tss@iki.fi>
parents: 14253
diff changeset
653 break;
b2076acc3715 Initial version of dsync rewrite.
Timo Sirainen <tss@iki.fi>
parents: 14253
diff changeset
654 case DSYNC_STATE_RECV_MAILBOX_TREE:
b2076acc3715 Initial version of dsync rewrite.
Timo Sirainen <tss@iki.fi>
parents: 14253
diff changeset
655 changed = dsync_brain_recv_mailbox_tree(brain);
b2076acc3715 Initial version of dsync rewrite.
Timo Sirainen <tss@iki.fi>
parents: 14253
diff changeset
656 break;
b2076acc3715 Initial version of dsync rewrite.
Timo Sirainen <tss@iki.fi>
parents: 14253
diff changeset
657 case DSYNC_STATE_SEND_MAILBOX_TREE_DELETES:
b2076acc3715 Initial version of dsync rewrite.
Timo Sirainen <tss@iki.fi>
parents: 14253
diff changeset
658 dsync_brain_send_mailbox_tree_deletes(brain);
b2076acc3715 Initial version of dsync rewrite.
Timo Sirainen <tss@iki.fi>
parents: 14253
diff changeset
659 changed = TRUE;
b2076acc3715 Initial version of dsync rewrite.
Timo Sirainen <tss@iki.fi>
parents: 14253
diff changeset
660 break;
b2076acc3715 Initial version of dsync rewrite.
Timo Sirainen <tss@iki.fi>
parents: 14253
diff changeset
661 case DSYNC_STATE_RECV_MAILBOX_TREE_DELETES:
b2076acc3715 Initial version of dsync rewrite.
Timo Sirainen <tss@iki.fi>
parents: 14253
diff changeset
662 changed = dsync_brain_recv_mailbox_tree_deletes(brain);
b2076acc3715 Initial version of dsync rewrite.
Timo Sirainen <tss@iki.fi>
parents: 14253
diff changeset
663 break;
b2076acc3715 Initial version of dsync rewrite.
Timo Sirainen <tss@iki.fi>
parents: 14253
diff changeset
664 case DSYNC_STATE_MASTER_SEND_MAILBOX:
b2076acc3715 Initial version of dsync rewrite.
Timo Sirainen <tss@iki.fi>
parents: 14253
diff changeset
665 dsync_brain_master_send_mailbox(brain);
b2076acc3715 Initial version of dsync rewrite.
Timo Sirainen <tss@iki.fi>
parents: 14253
diff changeset
666 changed = TRUE;
b2076acc3715 Initial version of dsync rewrite.
Timo Sirainen <tss@iki.fi>
parents: 14253
diff changeset
667 break;
b2076acc3715 Initial version of dsync rewrite.
Timo Sirainen <tss@iki.fi>
parents: 14253
diff changeset
668 case DSYNC_STATE_SLAVE_RECV_MAILBOX:
b2076acc3715 Initial version of dsync rewrite.
Timo Sirainen <tss@iki.fi>
parents: 14253
diff changeset
669 changed = dsync_brain_slave_recv_mailbox(brain);
b2076acc3715 Initial version of dsync rewrite.
Timo Sirainen <tss@iki.fi>
parents: 14253
diff changeset
670 break;
b2076acc3715 Initial version of dsync rewrite.
Timo Sirainen <tss@iki.fi>
parents: 14253
diff changeset
671 case DSYNC_STATE_SYNC_MAILS:
b2076acc3715 Initial version of dsync rewrite.
Timo Sirainen <tss@iki.fi>
parents: 14253
diff changeset
672 changed = dsync_brain_sync_mails(brain);
b2076acc3715 Initial version of dsync rewrite.
Timo Sirainen <tss@iki.fi>
parents: 14253
diff changeset
673 break;
18351
b4dbe64c0032 dsync: Added an extra "finish" state to allow slave-dsync to report error to master-dsync.
Timo Sirainen <tss@iki.fi>
parents: 18287
diff changeset
674 case DSYNC_STATE_FINISH:
b4dbe64c0032 dsync: Added an extra "finish" state to allow slave-dsync to report error to master-dsync.
Timo Sirainen <tss@iki.fi>
parents: 18287
diff changeset
675 changed = dsync_brain_finish(brain);
b4dbe64c0032 dsync: Added an extra "finish" state to allow slave-dsync to report error to master-dsync.
Timo Sirainen <tss@iki.fi>
parents: 18287
diff changeset
676 break;
14584
b2076acc3715 Initial version of dsync rewrite.
Timo Sirainen <tss@iki.fi>
parents: 14253
diff changeset
677 case DSYNC_STATE_DONE:
b2076acc3715 Initial version of dsync rewrite.
Timo Sirainen <tss@iki.fi>
parents: 14253
diff changeset
678 changed = TRUE;
b2076acc3715 Initial version of dsync rewrite.
Timo Sirainen <tss@iki.fi>
parents: 14253
diff changeset
679 ret = FALSE;
b2076acc3715 Initial version of dsync rewrite.
Timo Sirainen <tss@iki.fi>
parents: 14253
diff changeset
680 break;
9639
ffda7bd92ebc dsync: Skip syncing mailboxes whose uidvalidity/uidnext/highest-modseq hasn't changed.
Timo Sirainen <tss@iki.fi>
parents: 9638
diff changeset
681 }
16539
b5876fa03b0e dsync: If verbose_proctitle=yes, show the current state in it.
Timo Sirainen <tss@iki.fi>
parents: 16538
diff changeset
682 if (brain->verbose_proctitle) {
b5876fa03b0e dsync: If verbose_proctitle=yes, show the current state in it.
Timo Sirainen <tss@iki.fi>
parents: 16538
diff changeset
683 if (orig_state != brain->state ||
b5876fa03b0e dsync: If verbose_proctitle=yes, show the current state in it.
Timo Sirainen <tss@iki.fi>
parents: 16538
diff changeset
684 orig_box_recv_state != brain->box_recv_state ||
b5876fa03b0e dsync: If verbose_proctitle=yes, show the current state in it.
Timo Sirainen <tss@iki.fi>
parents: 16538
diff changeset
685 orig_box_send_state != brain->box_send_state ||
b5876fa03b0e dsync: If verbose_proctitle=yes, show the current state in it.
Timo Sirainen <tss@iki.fi>
parents: 16538
diff changeset
686 ++brain->proctitle_update_counter % 100 == 0)
b5876fa03b0e dsync: If verbose_proctitle=yes, show the current state in it.
Timo Sirainen <tss@iki.fi>
parents: 16538
diff changeset
687 process_title_set(dsync_brain_get_proctitle(brain));
b5876fa03b0e dsync: If verbose_proctitle=yes, show the current state in it.
Timo Sirainen <tss@iki.fi>
parents: 16538
diff changeset
688 }
14584
b2076acc3715 Initial version of dsync rewrite.
Timo Sirainen <tss@iki.fi>
parents: 14253
diff changeset
689 *changed_r = changed;
b2076acc3715 Initial version of dsync rewrite.
Timo Sirainen <tss@iki.fi>
parents: 14253
diff changeset
690 return brain->failed ? FALSE : ret;
10371
b99a19d5a93c dsync: Sync mailbox renames.
Timo Sirainen <tss@iki.fi>
parents: 10368
diff changeset
691 }
b99a19d5a93c dsync: Sync mailbox renames.
Timo Sirainen <tss@iki.fi>
parents: 10368
diff changeset
692
14584
b2076acc3715 Initial version of dsync rewrite.
Timo Sirainen <tss@iki.fi>
parents: 14253
diff changeset
693 bool dsync_brain_run(struct dsync_brain *brain, bool *changed_r)
9559
34eef8a2716b dsync: Initial implementation.
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
694 {
14584
b2076acc3715 Initial version of dsync rewrite.
Timo Sirainen <tss@iki.fi>
parents: 14253
diff changeset
695 bool ret;
10373
036db604f86c dsync: Fixed problems with syncing mailbox names that are invalid on other side.
Timo Sirainen <tss@iki.fi>
parents: 10371
diff changeset
696
14584
b2076acc3715 Initial version of dsync rewrite.
Timo Sirainen <tss@iki.fi>
parents: 14253
diff changeset
697 *changed_r = FALSE;
9559
34eef8a2716b dsync: Initial implementation.
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
698
15037
920756cd29b8 dsync: Renamed "slave" to "ibc" (= inter-brain communicator)
Timo Sirainen <tss@iki.fi>
parents: 14939
diff changeset
699 if (dsync_ibc_has_failed(brain->ibc)) {
14584
b2076acc3715 Initial version of dsync rewrite.
Timo Sirainen <tss@iki.fi>
parents: 14253
diff changeset
700 brain->failed = TRUE;
b2076acc3715 Initial version of dsync rewrite.
Timo Sirainen <tss@iki.fi>
parents: 14253
diff changeset
701 return FALSE;
12028
6ef21adeb61d dsync: If worker fails, abort earlier.
Timo Sirainen <tss@iki.fi>
parents: 11971
diff changeset
702 }
6ef21adeb61d dsync: If worker fails, abort earlier.
Timo Sirainen <tss@iki.fi>
parents: 11971
diff changeset
703
14584
b2076acc3715 Initial version of dsync rewrite.
Timo Sirainen <tss@iki.fi>
parents: 14253
diff changeset
704 T_BEGIN {
b2076acc3715 Initial version of dsync rewrite.
Timo Sirainen <tss@iki.fi>
parents: 14253
diff changeset
705 ret = dsync_brain_run_real(brain, changed_r);
b2076acc3715 Initial version of dsync rewrite.
Timo Sirainen <tss@iki.fi>
parents: 14253
diff changeset
706 } T_END;
b2076acc3715 Initial version of dsync rewrite.
Timo Sirainen <tss@iki.fi>
parents: 14253
diff changeset
707 return ret;
10368
9f2e8d230bd5 dsync: If some uid/modseq changes couldn't be done, exit with different value.
Timo Sirainen <tss@iki.fi>
parents: 10356
diff changeset
708 }
12124
eb5adb193064 dsync: Don't crash/hang if syncing fails before ioloop is started.
Timo Sirainen <tss@iki.fi>
parents: 12028
diff changeset
709
17271
bb5deb884303 dsync: Added more debug output
Timo Sirainen <tss@iki.fi>
parents: 17221
diff changeset
710 static void dsync_brain_mailbox_states_dump(struct dsync_brain *brain)
bb5deb884303 dsync: Added more debug output
Timo Sirainen <tss@iki.fi>
parents: 17221
diff changeset
711 {
bb5deb884303 dsync: Added more debug output
Timo Sirainen <tss@iki.fi>
parents: 17221
diff changeset
712 struct hash_iterate_context *iter;
bb5deb884303 dsync: Added more debug output
Timo Sirainen <tss@iki.fi>
parents: 17221
diff changeset
713 struct dsync_mailbox_state *state;
bb5deb884303 dsync: Added more debug output
Timo Sirainen <tss@iki.fi>
parents: 17221
diff changeset
714 uint8_t *guid;
bb5deb884303 dsync: Added more debug output
Timo Sirainen <tss@iki.fi>
parents: 17221
diff changeset
715
bb5deb884303 dsync: Added more debug output
Timo Sirainen <tss@iki.fi>
parents: 17221
diff changeset
716 iter = hash_table_iterate_init(brain->mailbox_states);
bb5deb884303 dsync: Added more debug output
Timo Sirainen <tss@iki.fi>
parents: 17221
diff changeset
717 while (hash_table_iterate(iter, brain->mailbox_states, &guid, &state)) {
17282
9669c9a8984f dsync: Include messages_count in the mailbox states.
Timo Sirainen <tss@iki.fi>
parents: 17279
diff changeset
718 i_debug("brain %c: Mailbox %s state: uidvalidity=%u uid=%u modseq=%llu pvt_modseq=%llu messages=%u changes_during_sync=%d",
17271
bb5deb884303 dsync: Added more debug output
Timo Sirainen <tss@iki.fi>
parents: 17221
diff changeset
719 brain->master_brain ? 'M' : 'S',
bb5deb884303 dsync: Added more debug output
Timo Sirainen <tss@iki.fi>
parents: 17221
diff changeset
720 guid_128_to_string(guid),
bb5deb884303 dsync: Added more debug output
Timo Sirainen <tss@iki.fi>
parents: 17221
diff changeset
721 state->last_uidvalidity,
bb5deb884303 dsync: Added more debug output
Timo Sirainen <tss@iki.fi>
parents: 17221
diff changeset
722 state->last_common_uid,
bb5deb884303 dsync: Added more debug output
Timo Sirainen <tss@iki.fi>
parents: 17221
diff changeset
723 (unsigned long long)state->last_common_modseq,
bb5deb884303 dsync: Added more debug output
Timo Sirainen <tss@iki.fi>
parents: 17221
diff changeset
724 (unsigned long long)state->last_common_pvt_modseq,
17282
9669c9a8984f dsync: Include messages_count in the mailbox states.
Timo Sirainen <tss@iki.fi>
parents: 17279
diff changeset
725 state->last_messages_count,
17271
bb5deb884303 dsync: Added more debug output
Timo Sirainen <tss@iki.fi>
parents: 17221
diff changeset
726 state->changes_during_sync);
bb5deb884303 dsync: Added more debug output
Timo Sirainen <tss@iki.fi>
parents: 17221
diff changeset
727 }
bb5deb884303 dsync: Added more debug output
Timo Sirainen <tss@iki.fi>
parents: 17221
diff changeset
728 hash_table_iterate_deinit(&iter);
bb5deb884303 dsync: Added more debug output
Timo Sirainen <tss@iki.fi>
parents: 17221
diff changeset
729 }
bb5deb884303 dsync: Added more debug output
Timo Sirainen <tss@iki.fi>
parents: 17221
diff changeset
730
15232
6c850258002f doveadm sync/backup: Added -s <state> parameter to do a fast stateful sync.
Timo Sirainen <tss@iki.fi>
parents: 15231
diff changeset
731 void dsync_brain_get_state(struct dsync_brain *brain, string_t *output)
6c850258002f doveadm sync/backup: Added -s <state> parameter to do a fast stateful sync.
Timo Sirainen <tss@iki.fi>
parents: 15231
diff changeset
732 {
6c850258002f doveadm sync/backup: Added -s <state> parameter to do a fast stateful sync.
Timo Sirainen <tss@iki.fi>
parents: 15231
diff changeset
733 struct hash_iterate_context *iter;
6c850258002f doveadm sync/backup: Added -s <state> parameter to do a fast stateful sync.
Timo Sirainen <tss@iki.fi>
parents: 15231
diff changeset
734 struct dsync_mailbox_node *node;
6c850258002f doveadm sync/backup: Added -s <state> parameter to do a fast stateful sync.
Timo Sirainen <tss@iki.fi>
parents: 15231
diff changeset
735 const struct dsync_mailbox_state *new_state;
6c850258002f doveadm sync/backup: Added -s <state> parameter to do a fast stateful sync.
Timo Sirainen <tss@iki.fi>
parents: 15231
diff changeset
736 struct dsync_mailbox_state *state;
6c850258002f doveadm sync/backup: Added -s <state> parameter to do a fast stateful sync.
Timo Sirainen <tss@iki.fi>
parents: 15231
diff changeset
737 const uint8_t *guid_p;
6c850258002f doveadm sync/backup: Added -s <state> parameter to do a fast stateful sync.
Timo Sirainen <tss@iki.fi>
parents: 15231
diff changeset
738 uint8_t *guid;
6c850258002f doveadm sync/backup: Added -s <state> parameter to do a fast stateful sync.
Timo Sirainen <tss@iki.fi>
parents: 15231
diff changeset
739
20073
1b4e84d0a728 dsync: If full resync is requested, return empty state string.
Timo Sirainen <timo.sirainen@dovecot.fi>
parents: 19646
diff changeset
740 if (brain->require_full_resync)
1b4e84d0a728 dsync: If full resync is requested, return empty state string.
Timo Sirainen <timo.sirainen@dovecot.fi>
parents: 19646
diff changeset
741 return;
1b4e84d0a728 dsync: If full resync is requested, return empty state string.
Timo Sirainen <timo.sirainen@dovecot.fi>
parents: 19646
diff changeset
742
15232
6c850258002f doveadm sync/backup: Added -s <state> parameter to do a fast stateful sync.
Timo Sirainen <tss@iki.fi>
parents: 15231
diff changeset
743 /* update mailbox states */
6c850258002f doveadm sync/backup: Added -s <state> parameter to do a fast stateful sync.
Timo Sirainen <tss@iki.fi>
parents: 15231
diff changeset
744 array_foreach(&brain->remote_mailbox_states, new_state) {
6c850258002f doveadm sync/backup: Added -s <state> parameter to do a fast stateful sync.
Timo Sirainen <tss@iki.fi>
parents: 15231
diff changeset
745 guid_p = new_state->mailbox_guid;
6c850258002f doveadm sync/backup: Added -s <state> parameter to do a fast stateful sync.
Timo Sirainen <tss@iki.fi>
parents: 15231
diff changeset
746 state = hash_table_lookup(brain->mailbox_states, guid_p);
6c850258002f doveadm sync/backup: Added -s <state> parameter to do a fast stateful sync.
Timo Sirainen <tss@iki.fi>
parents: 15231
diff changeset
747 if (state != NULL)
6c850258002f doveadm sync/backup: Added -s <state> parameter to do a fast stateful sync.
Timo Sirainen <tss@iki.fi>
parents: 15231
diff changeset
748 *state = *new_state;
6c850258002f doveadm sync/backup: Added -s <state> parameter to do a fast stateful sync.
Timo Sirainen <tss@iki.fi>
parents: 15231
diff changeset
749 else
6c850258002f doveadm sync/backup: Added -s <state> parameter to do a fast stateful sync.
Timo Sirainen <tss@iki.fi>
parents: 15231
diff changeset
750 dsync_mailbox_state_add(brain, new_state);
6c850258002f doveadm sync/backup: Added -s <state> parameter to do a fast stateful sync.
Timo Sirainen <tss@iki.fi>
parents: 15231
diff changeset
751 }
6c850258002f doveadm sync/backup: Added -s <state> parameter to do a fast stateful sync.
Timo Sirainen <tss@iki.fi>
parents: 15231
diff changeset
752
6c850258002f doveadm sync/backup: Added -s <state> parameter to do a fast stateful sync.
Timo Sirainen <tss@iki.fi>
parents: 15231
diff changeset
753 /* remove nonexistent mailboxes */
6c850258002f doveadm sync/backup: Added -s <state> parameter to do a fast stateful sync.
Timo Sirainen <tss@iki.fi>
parents: 15231
diff changeset
754 iter = hash_table_iterate_init(brain->mailbox_states);
6c850258002f doveadm sync/backup: Added -s <state> parameter to do a fast stateful sync.
Timo Sirainen <tss@iki.fi>
parents: 15231
diff changeset
755 while (hash_table_iterate(iter, brain->mailbox_states, &guid, &state)) {
6c850258002f doveadm sync/backup: Added -s <state> parameter to do a fast stateful sync.
Timo Sirainen <tss@iki.fi>
parents: 15231
diff changeset
756 node = dsync_mailbox_tree_lookup_guid(brain->local_mailbox_tree,
6c850258002f doveadm sync/backup: Added -s <state> parameter to do a fast stateful sync.
Timo Sirainen <tss@iki.fi>
parents: 15231
diff changeset
757 guid);
6c850258002f doveadm sync/backup: Added -s <state> parameter to do a fast stateful sync.
Timo Sirainen <tss@iki.fi>
parents: 15231
diff changeset
758 if (node == NULL ||
17271
bb5deb884303 dsync: Added more debug output
Timo Sirainen <tss@iki.fi>
parents: 17221
diff changeset
759 node->existence != DSYNC_MAILBOX_NODE_EXISTS) {
bb5deb884303 dsync: Added more debug output
Timo Sirainen <tss@iki.fi>
parents: 17221
diff changeset
760 if (brain->debug) {
bb5deb884303 dsync: Added more debug output
Timo Sirainen <tss@iki.fi>
parents: 17221
diff changeset
761 i_debug("brain %c: Removed state for deleted mailbox %s",
bb5deb884303 dsync: Added more debug output
Timo Sirainen <tss@iki.fi>
parents: 17221
diff changeset
762 brain->master_brain ? 'M' : 'S',
bb5deb884303 dsync: Added more debug output
Timo Sirainen <tss@iki.fi>
parents: 17221
diff changeset
763 guid_128_to_string(guid));
bb5deb884303 dsync: Added more debug output
Timo Sirainen <tss@iki.fi>
parents: 17221
diff changeset
764 }
15232
6c850258002f doveadm sync/backup: Added -s <state> parameter to do a fast stateful sync.
Timo Sirainen <tss@iki.fi>
parents: 15231
diff changeset
765 hash_table_remove(brain->mailbox_states, guid);
17271
bb5deb884303 dsync: Added more debug output
Timo Sirainen <tss@iki.fi>
parents: 17221
diff changeset
766 }
15232
6c850258002f doveadm sync/backup: Added -s <state> parameter to do a fast stateful sync.
Timo Sirainen <tss@iki.fi>
parents: 15231
diff changeset
767 }
6c850258002f doveadm sync/backup: Added -s <state> parameter to do a fast stateful sync.
Timo Sirainen <tss@iki.fi>
parents: 15231
diff changeset
768 hash_table_iterate_deinit(&iter);
6c850258002f doveadm sync/backup: Added -s <state> parameter to do a fast stateful sync.
Timo Sirainen <tss@iki.fi>
parents: 15231
diff changeset
769
17271
bb5deb884303 dsync: Added more debug output
Timo Sirainen <tss@iki.fi>
parents: 17221
diff changeset
770 if (brain->debug) {
bb5deb884303 dsync: Added more debug output
Timo Sirainen <tss@iki.fi>
parents: 17221
diff changeset
771 i_debug("brain %c: Exported mailbox states:",
bb5deb884303 dsync: Added more debug output
Timo Sirainen <tss@iki.fi>
parents: 17221
diff changeset
772 brain->master_brain ? 'M' : 'S');
bb5deb884303 dsync: Added more debug output
Timo Sirainen <tss@iki.fi>
parents: 17221
diff changeset
773 dsync_brain_mailbox_states_dump(brain);
bb5deb884303 dsync: Added more debug output
Timo Sirainen <tss@iki.fi>
parents: 17221
diff changeset
774 }
15232
6c850258002f doveadm sync/backup: Added -s <state> parameter to do a fast stateful sync.
Timo Sirainen <tss@iki.fi>
parents: 15231
diff changeset
775 dsync_mailbox_states_export(brain->mailbox_states, output);
6c850258002f doveadm sync/backup: Added -s <state> parameter to do a fast stateful sync.
Timo Sirainen <tss@iki.fi>
parents: 15231
diff changeset
776 }
6c850258002f doveadm sync/backup: Added -s <state> parameter to do a fast stateful sync.
Timo Sirainen <tss@iki.fi>
parents: 15231
diff changeset
777
16253
1a58d4bcb49c dsync: -U parameter never updated replicator's full_sync state.
Timo Sirainen <tss@iki.fi>
parents: 16251
diff changeset
778 enum dsync_brain_sync_type dsync_brain_get_sync_type(struct dsync_brain *brain)
1a58d4bcb49c dsync: -U parameter never updated replicator's full_sync state.
Timo Sirainen <tss@iki.fi>
parents: 16251
diff changeset
779 {
1a58d4bcb49c dsync: -U parameter never updated replicator's full_sync state.
Timo Sirainen <tss@iki.fi>
parents: 16251
diff changeset
780 return brain->sync_type;
1a58d4bcb49c dsync: -U parameter never updated replicator's full_sync state.
Timo Sirainen <tss@iki.fi>
parents: 16251
diff changeset
781 }
1a58d4bcb49c dsync: -U parameter never updated replicator's full_sync state.
Timo Sirainen <tss@iki.fi>
parents: 16251
diff changeset
782
12124
eb5adb193064 dsync: Don't crash/hang if syncing fails before ioloop is started.
Timo Sirainen <tss@iki.fi>
parents: 12028
diff changeset
783 bool dsync_brain_has_failed(struct dsync_brain *brain)
eb5adb193064 dsync: Don't crash/hang if syncing fails before ioloop is started.
Timo Sirainen <tss@iki.fi>
parents: 12028
diff changeset
784 {
14584
b2076acc3715 Initial version of dsync rewrite.
Timo Sirainen <tss@iki.fi>
parents: 14253
diff changeset
785 return brain->failed;
12124
eb5adb193064 dsync: Don't crash/hang if syncing fails before ioloop is started.
Timo Sirainen <tss@iki.fi>
parents: 12028
diff changeset
786 }
16454
5593d6129712 dsync: If unexpected changes happened during sync, log a warning and exit with code 2.
Timo Sirainen <tss@iki.fi>
parents: 16398
diff changeset
787
21256
4ac3461df334 dsync: When logging "Mailbox changed caused a desync", log also the reason.
Timo Sirainen <timo.sirainen@dovecot.fi>
parents: 20831
diff changeset
788 const char *dsync_brain_get_unexpected_changes_reason(struct dsync_brain *brain,
4ac3461df334 dsync: When logging "Mailbox changed caused a desync", log also the reason.
Timo Sirainen <timo.sirainen@dovecot.fi>
parents: 20831
diff changeset
789 bool *remote_only_r)
16454
5593d6129712 dsync: If unexpected changes happened during sync, log a warning and exit with code 2.
Timo Sirainen <tss@iki.fi>
parents: 16398
diff changeset
790 {
21256
4ac3461df334 dsync: When logging "Mailbox changed caused a desync", log also the reason.
Timo Sirainen <timo.sirainen@dovecot.fi>
parents: 20831
diff changeset
791 if (brain->changes_during_sync == NULL &&
4ac3461df334 dsync: When logging "Mailbox changed caused a desync", log also the reason.
Timo Sirainen <timo.sirainen@dovecot.fi>
parents: 20831
diff changeset
792 brain->changes_during_remote_sync) {
4ac3461df334 dsync: When logging "Mailbox changed caused a desync", log also the reason.
Timo Sirainen <timo.sirainen@dovecot.fi>
parents: 20831
diff changeset
793 *remote_only_r = TRUE;
4ac3461df334 dsync: When logging "Mailbox changed caused a desync", log also the reason.
Timo Sirainen <timo.sirainen@dovecot.fi>
parents: 20831
diff changeset
794 return "Remote notified that changes happened during sync";
4ac3461df334 dsync: When logging "Mailbox changed caused a desync", log also the reason.
Timo Sirainen <timo.sirainen@dovecot.fi>
parents: 20831
diff changeset
795 }
4ac3461df334 dsync: When logging "Mailbox changed caused a desync", log also the reason.
Timo Sirainen <timo.sirainen@dovecot.fi>
parents: 20831
diff changeset
796 *remote_only_r = FALSE;
16454
5593d6129712 dsync: If unexpected changes happened during sync, log a warning and exit with code 2.
Timo Sirainen <tss@iki.fi>
parents: 16398
diff changeset
797 return brain->changes_during_sync;
5593d6129712 dsync: If unexpected changes happened during sync, log a warning and exit with code 2.
Timo Sirainen <tss@iki.fi>
parents: 16398
diff changeset
798 }
16540
84014270ee37 dsync: Don't try to find mailboxes from unwanted namespaces.
Timo Sirainen <tss@iki.fi>
parents: 16539
diff changeset
799
84014270ee37 dsync: Don't try to find mailboxes from unwanted namespaces.
Timo Sirainen <tss@iki.fi>
parents: 16539
diff changeset
800 bool dsync_brain_want_namespace(struct dsync_brain *brain,
84014270ee37 dsync: Don't try to find mailboxes from unwanted namespaces.
Timo Sirainen <tss@iki.fi>
parents: 16539
diff changeset
801 struct mail_namespace *ns)
84014270ee37 dsync: Don't try to find mailboxes from unwanted namespaces.
Timo Sirainen <tss@iki.fi>
parents: 16539
diff changeset
802 {
17170
08f1c7af0ac0 dsync: Support multiple -n parameters.
Timo Sirainen <tss@iki.fi>
parents: 17130
diff changeset
803 struct mail_namespace *const *nsp;
08f1c7af0ac0 dsync: Support multiple -n parameters.
Timo Sirainen <tss@iki.fi>
parents: 17130
diff changeset
804
08f1c7af0ac0 dsync: Support multiple -n parameters.
Timo Sirainen <tss@iki.fi>
parents: 17130
diff changeset
805 if (array_is_created(&brain->sync_namespaces)) {
08f1c7af0ac0 dsync: Support multiple -n parameters.
Timo Sirainen <tss@iki.fi>
parents: 17130
diff changeset
806 array_foreach(&brain->sync_namespaces, nsp) {
08f1c7af0ac0 dsync: Support multiple -n parameters.
Timo Sirainen <tss@iki.fi>
parents: 17130
diff changeset
807 if (ns == *nsp)
08f1c7af0ac0 dsync: Support multiple -n parameters.
Timo Sirainen <tss@iki.fi>
parents: 17130
diff changeset
808 return TRUE;
08f1c7af0ac0 dsync: Support multiple -n parameters.
Timo Sirainen <tss@iki.fi>
parents: 17130
diff changeset
809 }
08f1c7af0ac0 dsync: Support multiple -n parameters.
Timo Sirainen <tss@iki.fi>
parents: 17130
diff changeset
810 return FALSE;
08f1c7af0ac0 dsync: Support multiple -n parameters.
Timo Sirainen <tss@iki.fi>
parents: 17130
diff changeset
811 }
16540
84014270ee37 dsync: Don't try to find mailboxes from unwanted namespaces.
Timo Sirainen <tss@iki.fi>
parents: 16539
diff changeset
812 if (ns->alias_for != NULL) {
84014270ee37 dsync: Don't try to find mailboxes from unwanted namespaces.
Timo Sirainen <tss@iki.fi>
parents: 16539
diff changeset
813 /* always skip aliases */
84014270ee37 dsync: Don't try to find mailboxes from unwanted namespaces.
Timo Sirainen <tss@iki.fi>
parents: 16539
diff changeset
814 return FALSE;
84014270ee37 dsync: Don't try to find mailboxes from unwanted namespaces.
Timo Sirainen <tss@iki.fi>
parents: 16539
diff changeset
815 }
84014270ee37 dsync: Don't try to find mailboxes from unwanted namespaces.
Timo Sirainen <tss@iki.fi>
parents: 16539
diff changeset
816 if (brain->sync_visible_namespaces) {
84014270ee37 dsync: Don't try to find mailboxes from unwanted namespaces.
Timo Sirainen <tss@iki.fi>
parents: 16539
diff changeset
817 if ((ns->flags & NAMESPACE_FLAG_HIDDEN) == 0)
84014270ee37 dsync: Don't try to find mailboxes from unwanted namespaces.
Timo Sirainen <tss@iki.fi>
parents: 16539
diff changeset
818 return TRUE;
84014270ee37 dsync: Don't try to find mailboxes from unwanted namespaces.
Timo Sirainen <tss@iki.fi>
parents: 16539
diff changeset
819 if ((ns->flags & (NAMESPACE_FLAG_LIST_PREFIX |
84014270ee37 dsync: Don't try to find mailboxes from unwanted namespaces.
Timo Sirainen <tss@iki.fi>
parents: 16539
diff changeset
820 NAMESPACE_FLAG_LIST_CHILDREN)) != 0)
84014270ee37 dsync: Don't try to find mailboxes from unwanted namespaces.
Timo Sirainen <tss@iki.fi>
parents: 16539
diff changeset
821 return TRUE;
84014270ee37 dsync: Don't try to find mailboxes from unwanted namespaces.
Timo Sirainen <tss@iki.fi>
parents: 16539
diff changeset
822 return FALSE;
84014270ee37 dsync: Don't try to find mailboxes from unwanted namespaces.
Timo Sirainen <tss@iki.fi>
parents: 16539
diff changeset
823 } else {
84014270ee37 dsync: Don't try to find mailboxes from unwanted namespaces.
Timo Sirainen <tss@iki.fi>
parents: 16539
diff changeset
824 return strcmp(ns->unexpanded_set->location,
84014270ee37 dsync: Don't try to find mailboxes from unwanted namespaces.
Timo Sirainen <tss@iki.fi>
parents: 16539
diff changeset
825 SETTING_STRVAR_UNEXPANDED) == 0;
84014270ee37 dsync: Don't try to find mailboxes from unwanted namespaces.
Timo Sirainen <tss@iki.fi>
parents: 16539
diff changeset
826 }
84014270ee37 dsync: Don't try to find mailboxes from unwanted namespaces.
Timo Sirainen <tss@iki.fi>
parents: 16539
diff changeset
827 }
21256
4ac3461df334 dsync: When logging "Mailbox changed caused a desync", log also the reason.
Timo Sirainen <timo.sirainen@dovecot.fi>
parents: 20831
diff changeset
828
4ac3461df334 dsync: When logging "Mailbox changed caused a desync", log also the reason.
Timo Sirainen <timo.sirainen@dovecot.fi>
parents: 20831
diff changeset
829 void dsync_brain_set_changes_during_sync(struct dsync_brain *brain,
4ac3461df334 dsync: When logging "Mailbox changed caused a desync", log also the reason.
Timo Sirainen <timo.sirainen@dovecot.fi>
parents: 20831
diff changeset
830 const char *reason)
4ac3461df334 dsync: When logging "Mailbox changed caused a desync", log also the reason.
Timo Sirainen <timo.sirainen@dovecot.fi>
parents: 20831
diff changeset
831 {
4ac3461df334 dsync: When logging "Mailbox changed caused a desync", log also the reason.
Timo Sirainen <timo.sirainen@dovecot.fi>
parents: 20831
diff changeset
832 if (brain->debug) {
4ac3461df334 dsync: When logging "Mailbox changed caused a desync", log also the reason.
Timo Sirainen <timo.sirainen@dovecot.fi>
parents: 20831
diff changeset
833 i_debug("brain %c: Change during sync: %s",
4ac3461df334 dsync: When logging "Mailbox changed caused a desync", log also the reason.
Timo Sirainen <timo.sirainen@dovecot.fi>
parents: 20831
diff changeset
834 brain->master_brain ? 'M' : 'S', reason);
4ac3461df334 dsync: When logging "Mailbox changed caused a desync", log also the reason.
Timo Sirainen <timo.sirainen@dovecot.fi>
parents: 20831
diff changeset
835 }
4ac3461df334 dsync: When logging "Mailbox changed caused a desync", log also the reason.
Timo Sirainen <timo.sirainen@dovecot.fi>
parents: 20831
diff changeset
836 if (brain->changes_during_sync == NULL)
4ac3461df334 dsync: When logging "Mailbox changed caused a desync", log also the reason.
Timo Sirainen <timo.sirainen@dovecot.fi>
parents: 20831
diff changeset
837 brain->changes_during_sync = p_strdup(brain->pool, reason);
4ac3461df334 dsync: When logging "Mailbox changed caused a desync", log also the reason.
Timo Sirainen <timo.sirainen@dovecot.fi>
parents: 20831
diff changeset
838 }