Mercurial > dovecot > core-2.2
changeset 14584:b2076acc3715
Initial version of dsync rewrite.
* doveadm backup not implemented at all yet
* syncing mailbox renames is somewhat broken (at least renaming \noselect
mailboxes)
* saving/restoring "state" is implemented by dsync brain, but not by
doveadm. this should be easy to do, just need to figure out how the
replication code wants it.
line wrap: on
line diff
--- a/src/doveadm/doveadm-settings.c Fri May 04 05:35:36 2012 +0300 +++ b/src/doveadm/doveadm-settings.c Tue May 22 23:17:31 2012 +0300 @@ -111,15 +111,18 @@ } /* <settings checks> */ -static bool doveadm_settings_check(void *_set ATTR_UNUSED, - pool_t pool ATTR_UNUSED, - const char **error_r ATTR_UNUSED) +static bool doveadm_settings_check(void *_set, pool_t pool ATTR_UNUSED, + const char **error_r) { -#ifndef CONFIG_BINARY struct doveadm_settings *set = _set; +#ifndef CONFIG_BINARY fix_base_path(set, pool, &set->doveadm_socket_path); #endif + if (*set->dsync_alt_char == '\0') { + *error_r = "dsync_alt_char must not be empty"; + return FALSE; + } return TRUE; } /* </settings checks> */
--- a/src/doveadm/dsync/Makefile.am Fri May 04 05:35:36 2012 +0300 +++ b/src/doveadm/dsync/Makefile.am Tue May 22 23:17:31 2012 +0300 @@ -14,65 +14,34 @@ libdsync_a_SOURCES = \ doveadm-dsync.c \ dsync-brain.c \ - dsync-brain-msgs.c \ - dsync-brain-msgs-new.c \ - dsync-data.c \ - dsync-proxy.c \ - dsync-proxy-client.c \ - dsync-proxy-server.c \ - dsync-proxy-server-cmd.c \ - dsync-worker.c \ - dsync-worker-local.c + dsync-brain-mailbox.c \ + dsync-brain-mailbox-tree.c \ + dsync-brain-mailbox-tree-sync.c \ + dsync-brain-mails.c \ + dsync-deserializer.c \ + dsync-mail.c \ + dsync-mailbox-import.c \ + dsync-mailbox-export.c \ + dsync-mailbox-state.c \ + dsync-mailbox-tree.c \ + dsync-mailbox-tree-fill.c \ + dsync-mailbox-tree-sync.c \ + dsync-serializer.c \ + dsync-slave.c \ + dsync-slave-io.c \ + dsync-slave-pipe.c \ + dsync-transaction-log-scan.c noinst_HEADERS = \ doveadm-dsync.h \ dsync-brain.h \ dsync-brain-private.h \ - dsync-data.h \ - dsync-proxy.h \ - dsync-proxy-server.h \ - dsync-worker.h \ - dsync-worker-private.h \ - test-dsync-common.h \ - test-dsync-worker.h - -test_programs = \ - test-dsync-brain \ - test-dsync-brain-msgs \ - test-dsync-proxy \ - test-dsync-proxy-server-cmd - -noinst_PROGRAMS = $(test_programs) - -test_libs = \ - ../../lib-test/libtest.la \ - ../../lib-mail/libmail.la \ - ../../lib-imap/libimap.la \ - ../../lib-charset/libcharset.la \ - ../../lib/liblib.la - -test_ldadd = \ - $(test_libs) \ - $(LIBICONV) - -test_dsync_brain_SOURCES = test-dsync-brain.c test-dsync-worker.c test-dsync-common.c -test_dsync_brain_LDADD = dsync-data.o dsync-brain.o dsync-worker.o $(test_ldadd) -test_dsync_brain_DEPENDENCIES = dsync-data.o dsync-brain.o dsync-worker.o $(test_libs) - -test_dsync_brain_msgs_SOURCES = test-dsync-brain-msgs.c test-dsync-worker.c test-dsync-common.c -test_dsync_brain_msgs_LDADD = dsync-data.o dsync-brain-msgs.o dsync-worker.o $(test_ldadd) -test_dsync_brain_msgs_DEPENDENCIES = dsync-data.o dsync-brain-msgs.o dsync-worker.o $(test_libs) - -test_dsync_proxy_SOURCES = test-dsync-proxy.c test-dsync-common.c -test_dsync_proxy_LDADD = dsync-proxy.o dsync-data.o $(test_ldadd) -test_dsync_proxy_DEPENDENCIES = dsync-proxy.o dsync-data.o $(test_libs) - -test_dsync_proxy_server_cmd_SOURCES = test-dsync-proxy-server-cmd.c test-dsync-worker.c test-dsync-common.c -test_dsync_proxy_server_cmd_LDADD = dsync-worker.o dsync-proxy.o dsync-proxy-server-cmd.o dsync-data.o $(test_ldadd) -test_dsync_proxy_server_cmd_DEPENDENCIES = dsync-worker.o dsync-proxy.o dsync-proxy-server-cmd.o dsync-data.o $(test_libs) - -check: check-am check-test -check-test: all-am - for bin in $(test_programs); do \ - if ! $(RUN_TEST) ./$$bin; then exit 1; fi; \ - done + dsync-mailbox-import.h \ + dsync-mailbox-export.h \ + dsync-mailbox-state.h \ + dsync-mailbox-tree.h \ + dsync-serializer.h \ + dsync-deserializer.h \ + dsync-slave.h \ + dsync-slave-private.h \ + dsync-transaction-log-scan.h
--- a/src/doveadm/dsync/doveadm-dsync.c Fri May 04 05:35:36 2012 +0300 +++ b/src/doveadm/dsync/doveadm-dsync.c Tue May 22 23:17:31 2012 +0300 @@ -11,11 +11,11 @@ #include "mail-storage-service.h" #include "mail-user.h" #include "mail-namespace.h" +#include "mailbox-list.h" #include "doveadm-settings.h" #include "doveadm-mail.h" #include "dsync-brain.h" -#include "dsync-worker.h" -#include "dsync-proxy-server.h" +#include "dsync-slave.h" #include "doveadm-dsync.h" #include <stdio.h> @@ -23,13 +23,12 @@ #include <unistd.h> #include <ctype.h> -#define DSYNC_LOCK_FILENAME ".dovecot-sync.lock" - struct dsync_cmd_context { struct doveadm_mail_cmd_context ctx; - enum dsync_brain_flags brain_flags; + enum dsync_brain_sync_type sync_type; const char *mailbox, *namespace_prefix; + const char *remote_name; const char *local_location; int fd_in, fd_out, fd_err; @@ -39,7 +38,7 @@ unsigned int lock:1; unsigned int default_replica_location:1; - unsigned int reverse_workers:1; + unsigned int reverse_backup:1; //FIXME unsigned int remote:1; }; @@ -227,17 +226,18 @@ return TRUE; } -static struct dsync_worker * -cmd_dsync_run_local(struct dsync_cmd_context *ctx, struct mail_user *user) +static struct dsync_slave * +cmd_dsync_run_local(struct dsync_cmd_context *ctx, struct mail_user *user, + struct dsync_brain *brain, struct dsync_slave *slave2) { + struct dsync_brain *brain2; struct mail_user *user2; - struct dsync_worker *worker2; struct setting_parser_context *set_parser; const char *set_line, *path1, *path2; + bool brain1_running, brain2_running, changed1, changed2; i_assert(ctx->local_location != NULL); - ctx->brain_flags |= DSYNC_BRAIN_FLAG_LOCAL; i_set_failure_prefix(t_strdup_printf("dsync(%s): ", user->username)); /* update mail_location and create another user for the @@ -267,18 +267,30 @@ "points to same directory: %s", path1); } - worker2 = dsync_worker_init_local(user2, ctx->namespace_prefix, - *ctx->ctx.set->dsync_alt_char); + brain2 = dsync_brain_slave_init(user2, slave2); + + brain1_running = brain2_running = TRUE; + changed1 = changed2 = TRUE; + while (brain1_running || brain2_running) { + if (dsync_brain_has_failed(brain) || + dsync_brain_has_failed(brain2)) + break; + + i_assert(changed1 || changed2); + brain1_running = dsync_brain_run(brain, &changed1); + brain2_running = dsync_brain_run(brain2, &changed2); + } mail_user_unref(&user2); - return worker2; + dsync_brain_deinit(&brain2); + return slave2; } -static struct dsync_worker * -cmd_dsync_run_remote(struct dsync_cmd_context *ctx, struct mail_user *user) +static void +cmd_dsync_run_remote(struct mail_user *user) { i_set_failure_prefix(t_strdup_printf("dsync-local(%s): ", user->username)); - return dsync_worker_init_proxy_client(ctx->fd_in, ctx->fd_out); + io_loop_run(current_ioloop); } static const char *const * @@ -297,108 +309,56 @@ return get_ssh_cmd_args(ctx, host, login, username); } -static int dsync_lock(struct mail_user *user, unsigned int lock_timeout, - const char **path_r, struct file_lock **lock_r) -{ - const char *home, *path; - int ret, fd; - - if ((ret = mail_user_get_home(user, &home)) < 0) { - i_error("Couldn't look up user's home dir"); - return -1; - } - if (ret == 0) { - i_error("User has no home directory"); - return -1; - } - - path = t_strconcat(home, "/"DSYNC_LOCK_FILENAME, NULL); - fd = creat(path, 0600); - if (fd == -1) { - i_error("Couldn't create lock %s: %m", path); - return -1; - } - - if (file_wait_lock(fd, path, F_WRLCK, FILE_LOCK_METHOD_FCNTL, - lock_timeout, lock_r) <= 0) { - i_error("Couldn't lock %s: %m", path); - (void)close(fd); - return -1; - } - *path_r = path; - return fd; -} - -static int -cmd_dsync_start(struct dsync_cmd_context *ctx, struct dsync_worker *worker1, - struct dsync_worker *worker2) -{ - struct dsync_brain *brain; - - /* create and run the brain */ - brain = dsync_brain_init(worker1, worker2, ctx->mailbox, - ctx->brain_flags); - if (!ctx->remote) - dsync_brain_sync_all(brain); - else { - dsync_brain_sync(brain); - if (!dsync_brain_has_failed(brain)) - io_loop_run(current_ioloop); - } - /* deinit */ - if (dsync_brain_has_unexpected_changes(brain)) { - i_warning("Mailbox changes caused a desync. " - "You may want to run dsync again."); - ctx->ctx.exit_code = 2; - } - if (dsync_brain_deinit(&brain) < 0) { - ctx->ctx.exit_code = EX_TEMPFAIL; - return -1; - } - return 0; -} - static int cmd_dsync_run(struct doveadm_mail_cmd_context *_ctx, struct mail_user *user) { struct dsync_cmd_context *ctx = (struct dsync_cmd_context *)_ctx; - struct dsync_worker *worker1, *worker2, *workertmp; - const char *lock_path; - struct file_lock *lock; - int lock_fd, ret = 0; + struct dsync_slave *slave, *slave2 = NULL; + struct dsync_brain *brain; + struct mail_namespace *sync_ns = NULL; + int ret = 0; user->admin = TRUE; user->dsyncing = TRUE; - /* create workers */ - worker1 = dsync_worker_init_local(user, ctx->namespace_prefix, - *_ctx->set->dsync_alt_char); + if (ctx->namespace_prefix != NULL) { + sync_ns = mail_namespace_find(user->namespaces, + ctx->namespace_prefix); + if (sync_ns == NULL) { + i_fatal("Namespace prefix=%s doesn't exist", + ctx->namespace_prefix); + } + } + if (!ctx->remote) - worker2 = cmd_dsync_run_local(ctx, user); - else - worker2 = cmd_dsync_run_remote(ctx, user); - if (ctx->reverse_workers) { - workertmp = worker1; - worker1 = worker2; - worker2 = workertmp; + dsync_slave_init_pipe(&slave, &slave2); + else { + string_t *temp_prefix = t_str_new(64); + mail_user_set_get_temp_prefix(temp_prefix, user->set); + slave = dsync_slave_init_io(ctx->fd_in, ctx->fd_out, + ctx->remote_name, + str_c(temp_prefix)); } - if (!ctx->lock) - ret = cmd_dsync_start(ctx, worker1, worker2); - else { - lock_fd = dsync_lock(user, ctx->lock_timeout, &lock_path, &lock); - if (lock_fd == -1) { - _ctx->exit_code = EX_TEMPFAIL; - ret = -1; - } else { - ret = cmd_dsync_start(ctx, worker1, worker2); - file_lock_free(&lock); - if (close(lock_fd) < 0) - i_error("close(%s) failed: %m", lock_path); - } + if (doveadm_debug || doveadm_verbose) { + // FIXME } - dsync_worker_deinit(&worker1); - dsync_worker_deinit(&worker2); + brain = dsync_brain_master_init(user, slave, sync_ns, + ctx->sync_type, + DSYNC_BRAIN_FLAG_MAILS_HAVE_GUIDS | + DSYNC_BRAIN_FLAG_SEND_REQUESTS, + NULL); + + if (!ctx->remote) + cmd_dsync_run_local(ctx, user, brain, slave2); + else + cmd_dsync_run_remote(user); + + if (dsync_brain_deinit(&brain) < 0) + _ctx->exit_code = EX_TEMPFAIL; + dsync_slave_deinit(&slave); + if (slave2 != NULL) + dsync_slave_deinit(&slave2); if (ctx->io_err != NULL) io_remove(&ctx->io_err); if (ctx->fd_err != -1) { @@ -423,6 +383,7 @@ ctx->fd_out = STDOUT_FILENO; ctx->fd_err = -1; ctx->remote = FALSE; + ctx->remote_name = "remote"; if (ctx->default_replica_location) { ctx->local_location = @@ -450,7 +411,8 @@ if (remote_cmd_args == NULL && ctx->local_location != NULL && strncmp(ctx->local_location, "remote:", 7) == 0) { /* this is a remote (ssh) command */ - remote_cmd_args = parse_ssh_location(ctx, ctx->local_location+7, + ctx->remote_name = ctx->local_location+7; + remote_cmd_args = parse_ssh_location(ctx, ctx->remote_name, _ctx->cur_username); } @@ -477,9 +439,6 @@ } lib_signals_ignore(SIGHUP, TRUE); - - if (doveadm_debug || doveadm_verbose) - ctx->brain_flags |= DSYNC_BRAIN_FLAG_VERBOSE; } static void cmd_dsync_preinit(struct doveadm_mail_cmd_context *ctx) @@ -502,12 +461,7 @@ legacy_dsync = TRUE; break; case 'f': - ctx->brain_flags |= DSYNC_BRAIN_FLAG_FULL_SYNC; - break; - case 'l': - ctx->lock = TRUE; - if (str_to_uint(optarg, &ctx->lock_timeout) < 0) - i_error("Invalid -l parameter: %s", optarg); + ctx->sync_type = DSYNC_BRAIN_SYNC_TYPE_FULL; break; case 'm': ctx->mailbox = optarg; @@ -516,7 +470,7 @@ ctx->namespace_prefix = optarg; break; case 'R': - ctx->reverse_workers = TRUE; + ctx->reverse_backup = TRUE; break; default: return FALSE; @@ -535,6 +489,7 @@ ctx->ctx.v.init = cmd_dsync_init; ctx->ctx.v.prerun = cmd_dsync_prerun; ctx->ctx.v.run = cmd_dsync_run; + ctx->sync_type = DSYNC_BRAIN_SYNC_TYPE_CHANGED; return &ctx->ctx; } @@ -545,48 +500,35 @@ _ctx = cmd_dsync_alloc(); ctx = (struct dsync_cmd_context *)_ctx; - ctx->brain_flags |= DSYNC_BRAIN_FLAG_BACKUP; + //FIXME return _ctx; } static int -cmd_dsync_server_run(struct doveadm_mail_cmd_context *_ctx, +cmd_dsync_server_run(struct doveadm_mail_cmd_context *_ctx ATTR_UNUSED, struct mail_user *user) { - struct dsync_cmd_context *ctx = (struct dsync_cmd_context *)_ctx; - struct dsync_proxy_server *server; - struct dsync_worker *worker; - struct file_lock *lock; - const char *lock_path; - int lock_fd, ret = 0; + struct dsync_slave *slave; + struct dsync_brain *brain; + string_t *temp_prefix; user->admin = TRUE; user->dsyncing = TRUE; i_set_failure_prefix(t_strdup_printf("dsync-remote(%s): ", user->username)); - worker = dsync_worker_init_local(user, ctx->namespace_prefix, - *_ctx->set->dsync_alt_char); - server = dsync_proxy_server_init(STDIN_FILENO, STDOUT_FILENO, worker); + + temp_prefix = t_str_new(64); + mail_user_set_get_temp_prefix(temp_prefix, user->set); - if (!ctx->lock) - io_loop_run(current_ioloop); - else { - lock_fd = dsync_lock(user, ctx->lock_timeout, &lock_path, &lock); - if (lock_fd == -1) { - _ctx->exit_code = EX_TEMPFAIL; - ret = -1; - } else { - io_loop_run(current_ioloop); - file_lock_free(&lock); - if (close(lock_fd) < 0) - i_error("close(%s) failed: %m", lock_path); - } - } + slave = dsync_slave_init_io(STDIN_FILENO, STDOUT_FILENO, + "local", str_c(temp_prefix)); + brain = dsync_brain_slave_init(user, slave); - dsync_proxy_server_deinit(&server); - dsync_worker_deinit(&worker); - return ret; + io_loop_run(current_ioloop); + + dsync_slave_deinit(&slave); + return dsync_brain_deinit(&brain); } static bool @@ -621,6 +563,7 @@ ctx->ctx.getopt_args = "El:n:"; ctx->ctx.v.parse_arg = cmd_mailbox_dsync_server_parse_arg; ctx->ctx.v.run = cmd_dsync_server_run; + ctx->sync_type = DSYNC_BRAIN_SYNC_TYPE_CHANGED; return &ctx->ctx; }
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/doveadm/dsync/dsync-brain-mailbox-tree-sync.c Tue May 22 23:17:31 2012 +0300 @@ -0,0 +1,150 @@ +/* Copyright (c) 2012 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "mail-namespace.h" +#include "mail-storage.h" +#include "dsync-mailbox-tree.h" +#include "dsync-brain-private.h" + +static int +sync_create_box(struct mailbox *box, const guid_128_t mailbox_guid, + uint32_t uid_validity) +{ + struct mailbox_metadata metadata; + struct mailbox_update update; + enum mail_error error; + const char *errstr; + + memset(&update, 0, sizeof(update)); + memcpy(update.mailbox_guid, mailbox_guid, sizeof(update.mailbox_guid)); + update.uid_validity = uid_validity; + + if (mailbox_create(box, &update, FALSE) < 0) { + errstr = mailbox_get_last_error(box, &error); + if (error != MAIL_ERROR_EXISTS) { + i_error("Can't create mailbox %s: %s", + mailbox_get_vname(box), errstr); + return -1; + } + } + /* sync the mailbox so we can look up its latest status */ + if (mailbox_sync(box, MAILBOX_SYNC_FLAG_FULL_READ) < 0) { + i_error("Can't sync mailbox %s: %s", + mailbox_get_vname(box), + mailbox_get_last_error(box, NULL)); + return -1; + } + + /* verify that the GUID is what we wanted. if it's not, it probably + means that the mailbox had already been created. then we'll use the + GUID that is higher. + + mismatching UIDVALIDITY is handled later, because we choose it by + checking which mailbox has more messages */ + if (mailbox_get_metadata(box, MAILBOX_METADATA_GUID, &metadata) < 0) { + i_error("Can't get mailbox GUID %s: %s", + mailbox_get_vname(box), + mailbox_get_last_error(box, NULL)); + return -1; + } + + if (memcmp(mailbox_guid, metadata.guid, sizeof(metadata.guid)) > 0) { + memset(&update, 0, sizeof(update)); + memcpy(update.mailbox_guid, mailbox_guid, + sizeof(update.mailbox_guid)); + if (mailbox_update(box, &update) < 0) { + i_error("Can't update mailbox GUID %s: %s", + mailbox_get_vname(box), + mailbox_get_last_error(box, NULL)); + return -1; + } + } + return 0; +} + +int dsync_brain_mailbox_tree_sync_change(struct dsync_brain *brain, + const struct dsync_mailbox_tree_sync_change *change) +{ + struct mailbox *box = NULL, *destbox; + const char *errstr, *func_name = NULL, *storage_name; + enum mail_error error; + int ret = -1; + + switch (change->type) { + case DSYNC_MAILBOX_TREE_SYNC_TYPE_DELETE_BOX: + /* make sure we're deleting the correct mailbox */ + ret = dsync_brain_mailbox_alloc(brain, change->mailbox_guid, + &box); + if (ret <= 0) { + brain->changes_during_sync = TRUE; + return ret; + } + break; + case DSYNC_MAILBOX_TREE_SYNC_TYPE_DELETE_DIR: + storage_name = mailbox_list_get_storage_name(change->ns->list, + change->full_name); + if (mailbox_list_delete_dir(change->ns->list, storage_name) == 0) + return 0; + + errstr = mailbox_list_get_last_error(change->ns->list, &error); + if (error == MAIL_ERROR_NOTFOUND || + error == MAIL_ERROR_EXISTS) { + brain->changes_during_sync = TRUE; + return 0; + } else { + i_error("Mailbox sync: mailbox_list_delete_dir failed: %s", + errstr); + return -1; + } + default: + box = mailbox_alloc(change->ns->list, change->full_name, 0); + break; + } + switch (change->type) { + case DSYNC_MAILBOX_TREE_SYNC_TYPE_CREATE_BOX: + ret = sync_create_box(box, change->mailbox_guid, + change->uid_validity); + mailbox_free(&box); + return ret; + case DSYNC_MAILBOX_TREE_SYNC_TYPE_CREATE_DIR: + ret = mailbox_create(box, NULL, TRUE); + func_name = "mailbox_create"; + break; + case DSYNC_MAILBOX_TREE_SYNC_TYPE_DELETE_BOX: + ret = mailbox_delete(box); + func_name = "mailbox_delete"; + break; + case DSYNC_MAILBOX_TREE_SYNC_TYPE_DELETE_DIR: + i_unreached(); + case DSYNC_MAILBOX_TREE_SYNC_TYPE_RENAME: + destbox = mailbox_alloc(change->ns->list, + change->rename_dest_name, 0); + ret = mailbox_rename(box, destbox, TRUE); + func_name = "mailbox_rename"; + mailbox_free(&destbox); + break; + case DSYNC_MAILBOX_TREE_SYNC_TYPE_SUBSCRIBE: + ret = mailbox_set_subscribed(box, TRUE); + func_name = "mailbox_set_subscribed"; + break; + case DSYNC_MAILBOX_TREE_SYNC_TYPE_UNSUBSCRIBE: + ret = mailbox_set_subscribed(box, FALSE); + func_name = "mailbox_set_subscribed"; + break; + } + if (ret < 0) { + errstr = mailbox_get_last_error(box, &error); + if (error == MAIL_ERROR_EXISTS || + error == MAIL_ERROR_NOTFOUND) { + /* mailbox was already created or was already deleted. + let the next sync figure out what to do */ + brain->changes_during_sync = TRUE; + ret = 0; + } else { + i_error("Mailbox %s sync: %s failed: %s", + mailbox_get_vname(box), func_name, errstr); + } + } + mailbox_free(&box); + return ret; +}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/doveadm/dsync/dsync-brain-mailbox-tree.c Tue May 22 23:17:31 2012 +0300 @@ -0,0 +1,394 @@ +/* Copyright (c) 2012 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "str.h" +#include "settings-parser.h" +#include "mail-namespace.h" +#include "doveadm-settings.h" +#include "dsync-slave.h" +#include "dsync-mailbox-tree.h" +#include "dsync-brain-private.h" + +#include <ctype.h> + +static bool dsync_brain_want_namespace(struct dsync_brain *brain, + struct mail_namespace *ns) +{ + if (brain->sync_ns == ns) + return TRUE; + + return brain->sync_ns == NULL && + strcmp(ns->unexpanded_set->location, + SETTING_STRVAR_UNEXPANDED) == 0; +} + +static void dsync_brain_check_namespaces(struct dsync_brain *brain) +{ + struct mail_namespace *ns, *first_ns = NULL; + char sep; + + i_assert(brain->hierarchy_sep == '\0'); + + if (brain->sync_ns != NULL) { + brain->hierarchy_sep = mail_namespace_get_sep(brain->sync_ns); + return; + } + + for (ns = brain->user->namespaces; ns != NULL; ns = ns->next) { + if (!dsync_brain_want_namespace(brain, ns)) + continue; + + sep = mail_namespace_get_sep(ns); + if (brain->hierarchy_sep == '\0') { + brain->hierarchy_sep = sep; + first_ns = ns; + } else if (brain->hierarchy_sep != sep) { + i_fatal("Synced namespaces have conflicting separators " + "('%c' for prefix=\"%s\", '%c' for prefix=\"%s\")", + brain->hierarchy_sep, first_ns->prefix, + sep, ns->prefix); + } + } + if (brain->hierarchy_sep != '\0') + return; + + i_fatal("All your namespaces have a location setting. " + "Only namespaces with empty location settings are converted. " + "(One namespace should default to mail_location setting)"); +} + +void dsync_brain_mailbox_trees_init(struct dsync_brain *brain) +{ + struct mail_namespace *ns; + + dsync_brain_check_namespaces(brain); + + brain->local_mailbox_tree = + dsync_mailbox_tree_init(brain->hierarchy_sep); + /* we'll convert remote mailbox names to use our own separator */ + brain->remote_mailbox_tree = + dsync_mailbox_tree_init(brain->hierarchy_sep); + + /* fill the local mailbox tree */ + if (brain->sync_ns != NULL) { + if (dsync_mailbox_tree_fill(brain->local_mailbox_tree, + brain->sync_ns) < 0) + brain->failed = TRUE; + } else { + for (ns = brain->user->namespaces; ns != NULL; ns = ns->next) { + if (!dsync_brain_want_namespace(brain, ns)) + continue; + if (dsync_mailbox_tree_fill(brain->local_mailbox_tree, + ns) < 0) + brain->failed = TRUE; + } + } + + brain->local_tree_iter = + dsync_mailbox_tree_iter_init(brain->local_mailbox_tree); +} + + +void dsync_brain_send_mailbox_tree(struct dsync_brain *brain) +{ + struct dsync_mailbox_node *node; + enum dsync_slave_send_ret ret; + const char *full_name; + char sep[2]; + + sep[0] = brain->hierarchy_sep; sep[1] = '\0'; + while (dsync_mailbox_tree_iter_next(brain->local_tree_iter, + &full_name, &node)) { + T_BEGIN { + const char *const *parts; + + parts = t_strsplit(full_name, sep); + ret = dsync_slave_send_mailbox_tree_node(brain->slave, + parts, node); + } T_END; + if (ret == DSYNC_SLAVE_SEND_RET_FULL) + return; + } + dsync_mailbox_tree_iter_deinit(&brain->local_tree_iter); + dsync_slave_send_end_of_list(brain->slave); + + brain->state = DSYNC_STATE_SEND_MAILBOX_TREE_DELETES; +} + +void dsync_brain_send_mailbox_tree_deletes(struct dsync_brain *brain) +{ + const struct dsync_mailbox_delete *deletes; + unsigned int count; + + deletes = dsync_mailbox_tree_get_deletes(brain->local_mailbox_tree, + &count); + dsync_slave_send_mailbox_deletes(brain->slave, deletes, count, + brain->hierarchy_sep); + + brain->state = DSYNC_STATE_RECV_MAILBOX_TREE; +} + +static bool +dsync_namespace_match_parts(struct mail_namespace *ns, + const char *const *name_parts) +{ + const char *part, *prefix = ns->prefix; + unsigned int part_len; + char ns_sep = mail_namespace_get_sep(ns); + + for (; *name_parts != NULL && *prefix != '\0'; name_parts++) { + part = *name_parts; + part_len = strlen(part); + + if (strncmp(prefix, part, part_len) != 0) + return FALSE; + if (prefix[part_len] != ns_sep) + return FALSE; + prefix += part_len + 1; + } + return *name_parts != NULL; +} + +static struct mail_namespace * +dsync_find_namespace(struct dsync_brain *brain, const char *const *name_parts) +{ + struct mail_namespace *ns, *best_ns = NULL; + + for (ns = brain->user->namespaces; ns != NULL; ns = ns->next) { + if (!dsync_brain_want_namespace(brain, ns)) + continue; + + if (ns->prefix_len == 0) { + /* prefix="" is the fallback namespace */ + if (best_ns == NULL) + best_ns = ns; + } else if (dsync_namespace_match_parts(ns, name_parts)) { + if (best_ns == NULL || + best_ns->prefix_len < ns->prefix_len) + best_ns = ns; + } + } + return best_ns; +} + +static const char * +mailbox_name_cleanup(const char *input, char real_sep, char alt_char) +{ + char *output, *p; + + output = t_strdup_noconst(input); + for (p = output; *p != '\0'; p++) { + if (*p == real_sep || (uint8_t)*input < 32 || + (uint8_t)*input >= 0x80) + *p = alt_char; + } + return output; +} + +static const char *mailbox_name_force_cleanup(const char *input, char alt_char) +{ + char *output, *p; + + output = t_strdup_noconst(input); + for (p = output; *p != '\0'; p++) { + if (!i_isalnum(*p)) + *p = alt_char; + } + return output; +} + +static const char * +dsync_fix_mailbox_name(struct mail_namespace *ns, const char *vname, + char alt_char) +{ + const char *name; + char list_sep; + + name = mailbox_list_get_storage_name(ns->list, vname); + + list_sep = mailbox_list_get_hierarchy_sep(ns->list); + if (!mailbox_list_is_valid_create_name(ns->list, name)) { + /* change any real separators to alt separators, + drop any potentially invalid characters */ + name = mailbox_name_cleanup(name, list_sep, alt_char); + } + if (!mailbox_list_is_valid_create_name(ns->list, name)) { + /* still not working, apparently it's not valid mUTF-7. + just drop all non-alphanumeric characters. */ + name = mailbox_name_force_cleanup(name, alt_char); + } + if (!mailbox_list_is_valid_create_name(ns->list, name)) { + /* probably some reserved name (e.g. dbox-Mails) */ + name = t_strconcat("_", name, NULL); + } + if (!mailbox_list_is_valid_create_name(ns->list, name)) { + /* name is too long? just give up and generate a + unique name */ + guid_128_t guid; + + guid_128_generate(guid); + name = guid_128_to_string(guid); + } + i_assert(mailbox_list_is_valid_create_name(ns->list, name)); + return mailbox_list_get_vname(ns->list, name); +} + +static int +dsync_get_mailbox_name(struct dsync_brain *brain, const char *const *name_parts, + const char **name_r, struct mail_namespace **ns_r) +{ + struct mail_namespace *ns; + const char *p; + string_t *vname; + char ns_sep, alt_char = doveadm_settings->dsync_alt_char[0]; + + i_assert(*name_parts != NULL); + + ns = dsync_find_namespace(brain, name_parts); + if (ns == NULL) + return -1; + ns_sep = mail_namespace_get_sep(ns); + + /* build the mailbox name */ + vname = t_str_new(128); + for (; *name_parts != NULL; name_parts++) { + for (p = *name_parts; *p != '\0'; p++) { + if (*p != ns_sep) + str_append_c(vname, *p); + else + str_append_c(vname, alt_char); + } + str_append_c(vname, ns_sep); + } + str_truncate(vname, str_len(vname)-1); + + *name_r = dsync_fix_mailbox_name(ns, str_c(vname), alt_char); + *ns_r = ns; + return 0; +} + +static void dsync_brain_mailbox_trees_sync(struct dsync_brain *brain) +{ + struct dsync_mailbox_tree_sync_ctx *ctx; + const struct dsync_mailbox_tree_sync_change *change; + + ctx = dsync_mailbox_trees_sync_init(brain->local_mailbox_tree, + brain->remote_mailbox_tree); + while ((change = dsync_mailbox_trees_sync_next(ctx)) != NULL) { + if (dsync_brain_mailbox_tree_sync_change(brain, change) < 0) + brain->failed = TRUE; + } + dsync_mailbox_trees_sync_deinit(&ctx); +} + +bool dsync_brain_recv_mailbox_tree(struct dsync_brain *brain) +{ + const struct dsync_mailbox_node *remote_node; + struct dsync_mailbox_node *node; + const char *const *parts, *name; + struct mail_namespace *ns; + enum dsync_slave_recv_ret ret; + char sep[2]; + bool changed = FALSE; + + while ((ret = dsync_slave_recv_mailbox_tree_node(brain->slave, &parts, + &remote_node)) > 0) { + if (dsync_get_mailbox_name(brain, parts, &name, &ns) < 0) { + sep[0] = brain->hierarchy_sep; sep[1] = '\0'; + i_error("Couldn't find namespace for mailbox %s", + t_strarray_join(parts, sep)); + brain->failed = TRUE; + return TRUE; + } + node = dsync_mailbox_tree_get(brain->remote_mailbox_tree, name); + node->ns = ns; + dsync_mailbox_node_copy_data(node, remote_node); + } + if (ret == DSYNC_SLAVE_RECV_RET_FINISHED) { + if (dsync_mailbox_tree_build_guid_hash(brain->remote_mailbox_tree) < 0) + brain->failed = TRUE; + + brain->state = DSYNC_STATE_RECV_MAILBOX_TREE_DELETES; + changed = TRUE; + } + return changed; +} + +static void +dsync_brain_mailbox_tree_add_delete(struct dsync_mailbox_tree *tree, + struct dsync_mailbox_tree *other_tree, + const struct dsync_mailbox_delete *other_del) +{ + const struct dsync_mailbox_node *node; + struct dsync_mailbox_node *other_node; + const char *name; + + /* see if we can find the deletion based on mailbox tree that should + still have the mailbox */ + node = dsync_mailbox_tree_find_delete(tree, other_del); + if (node == NULL) + return; + + /* make a node for it in the other mailbox tree */ + name = dsync_mailbox_node_get_full_name(tree, node); + other_node = dsync_mailbox_tree_get(other_tree, name); + + if (!guid_128_is_empty(other_node->mailbox_guid) || + (other_node->existence == DSYNC_MAILBOX_NODE_EXISTS && + !other_del->delete_mailbox)) { + /* other side has already created a new mailbox or + directory with this name, we can't delete it */ + return; + } + + /* ok, mark the other node deleted */ + if (other_del->delete_mailbox) { + memcpy(other_node->mailbox_guid, node->mailbox_guid, + sizeof(other_node->mailbox_guid)); + } + i_assert(other_node->ns == NULL || other_node->ns == node->ns); + other_node->ns = node->ns; + other_node->existence = DSYNC_MAILBOX_NODE_DELETED; + + if (dsync_mailbox_tree_guid_hash_add(other_tree, other_node) < 0) + i_unreached(); +} + +bool dsync_brain_recv_mailbox_tree_deletes(struct dsync_brain *brain) +{ + const struct dsync_mailbox_delete *deletes; + unsigned int i, count; + char sep; + + if (dsync_slave_recv_mailbox_deletes(brain->slave, &deletes, &count, + &sep) == 0) + return FALSE; + + /* apply remote's mailbox deletions based on our local tree */ + dsync_mailbox_tree_set_remote_sep(brain->local_mailbox_tree, sep); + for (i = 0; i < count; i++) { + dsync_brain_mailbox_tree_add_delete(brain->local_mailbox_tree, + brain->remote_mailbox_tree, + &deletes[i]); + } + + /* apply local mailbox deletions based on remote tree */ + deletes = dsync_mailbox_tree_get_deletes(brain->local_mailbox_tree, + &count); + dsync_mailbox_tree_set_remote_sep(brain->remote_mailbox_tree, + brain->hierarchy_sep); + for (i = 0; i < count; i++) { + dsync_brain_mailbox_tree_add_delete(brain->remote_mailbox_tree, + brain->local_mailbox_tree, + &deletes[i]); + } + + dsync_brain_mailbox_trees_sync(brain); + brain->state = brain->master_brain ? + DSYNC_STATE_MASTER_SEND_MAILBOX : + DSYNC_STATE_SLAVE_RECV_MAILBOX; + i_assert(brain->local_tree_iter == NULL); + brain->local_tree_iter = + dsync_mailbox_tree_iter_init(brain->local_mailbox_tree); + return TRUE; +}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/doveadm/dsync/dsync-brain-mailbox.c Tue May 22 23:17:31 2012 +0300 @@ -0,0 +1,587 @@ +/* Copyright (c) 2012 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "hash.h" +#include "mail-cache-private.h" +#include "mail-namespace.h" +#include "mail-storage-private.h" +#include "dsync-slave.h" +#include "dsync-mailbox-tree.h" +#include "dsync-mailbox-import.h" +#include "dsync-mailbox-export.h" +#include "dsync-transaction-log-scan.h" +#include "dsync-brain-private.h" + +static int +ns_mailbox_try_alloc(struct mail_namespace *ns, const guid_128_t guid, + struct mailbox **box_r) +{ + struct mailbox *box; + enum mailbox_existence existence; + int ret; + + box = mailbox_alloc_guid(ns->list, guid, 0); + ret = mailbox_exists(box, FALSE, &existence); + if (ret < 0) { + mailbox_free(&box); + return -1; + } + if (existence != MAILBOX_EXISTENCE_SELECT) { + mailbox_free(&box); + return 0; + } + *box_r = box; + return 1; +} + +int dsync_brain_mailbox_alloc(struct dsync_brain *brain, const guid_128_t guid, + struct mailbox **box_r) +{ + struct mail_namespace *ns; + int ret; + + *box_r = NULL; + if (brain->sync_ns != NULL) { + ret = ns_mailbox_try_alloc(brain->sync_ns, guid, box_r); + if (ret < 0) + brain->failed = TRUE; + return ret; + } + + for (ns = brain->user->namespaces; ns != NULL; ns = ns->next) { + if ((ret = ns_mailbox_try_alloc(ns, guid, box_r)) != 0) { + if (ret < 0) + brain->failed = TRUE; + return ret; + } + } + return 0; +} + +static const struct dsync_mailbox_state * +dsync_mailbox_state_find(struct dsync_brain *brain, + const guid_128_t mailbox_guid) +{ + const struct dsync_mailbox_state *state; + + array_foreach(&brain->mailbox_states, state) { + if (memcmp(state->mailbox_guid, mailbox_guid, + sizeof(state->mailbox_guid)) == 0) + return state; + } + return NULL; +} + +static void +dsync_mailbox_state_remove(struct dsync_brain *brain, + const guid_128_t mailbox_guid) +{ + const struct dsync_mailbox_state *states; + unsigned int i, count; + + states = array_get(&brain->mailbox_states, &count); + for (i = 0; i < count; i++) { + if (memcmp(states[i].mailbox_guid, mailbox_guid, + sizeof(states[i].mailbox_guid)) == 0) { + array_delete(&brain->mailbox_states, i, 1); + break; + } + } +} + +static int +dsync_brain_sync_mailbox_init(struct dsync_brain *brain, + struct mailbox *box, + const struct dsync_mailbox *local_dsync_box, + enum dsync_box_state box_state) +{ + enum dsync_mailbox_exporter_flags exporter_flags = 0; + const struct dsync_mailbox_state *state; + uint32_t last_common_uid, highest_wanted_uid; + uint64_t last_common_modseq; + + i_assert(brain->box_importer == NULL); + i_assert(brain->box_exporter == NULL); + i_assert(box->synced); + + brain->box = box; + brain->pre_box_state = brain->state; + brain->box_recv_state = box_state; + brain->box_send_state = box_state; + brain->local_dsync_box = *local_dsync_box; + memset(&brain->remote_dsync_box, 0, sizeof(brain->remote_dsync_box)); + + state = dsync_mailbox_state_find(brain, local_dsync_box->mailbox_guid); + if (state != NULL) { + brain->mailbox_state = *state; + last_common_uid = state->last_common_uid; + last_common_modseq = state->last_common_modseq; + } else { + memset(&brain->mailbox_state, 0, sizeof(brain->mailbox_state)); + memcpy(brain->mailbox_state.mailbox_guid, + local_dsync_box->mailbox_guid, + sizeof(brain->mailbox_state.mailbox_guid)); + brain->mailbox_state.last_uidvalidity = + local_dsync_box->uid_validity; + last_common_uid = 0; + last_common_modseq = 0; + } + + highest_wanted_uid = last_common_uid == 0 ? + (uint32_t)-1 : last_common_uid; + if (dsync_transaction_log_scan_init(box->view, highest_wanted_uid, + last_common_modseq, + &brain->log_scan) < 0) { + i_error("Failed to read transaction log for mailbox %s", + mailbox_get_vname(box)); + brain->failed = TRUE; + return -1; + } + + if (!brain->guid_requests) + exporter_flags |= DSYNC_MAILBOX_EXPORTER_FLAG_AUTO_EXPORT_MAILS; + if (brain->mails_have_guids) + exporter_flags |= DSYNC_MAILBOX_EXPORTER_FLAG_MAILS_HAVE_GUIDS; + brain->box_exporter = + dsync_mailbox_export_init(box, brain->log_scan, last_common_uid, + last_common_modseq, exporter_flags); + return 0; +} + +void dsync_brain_sync_mailbox_init_remote(struct dsync_brain *brain, + const struct dsync_mailbox *remote_dsync_box) +{ + enum dsync_mailbox_import_flags import_flags = 0; + const struct dsync_mailbox_state *state; + uint32_t last_common_uid; + uint64_t last_common_modseq; + + i_assert(brain->box_importer == NULL); + + i_assert(memcmp(brain->local_dsync_box.mailbox_guid, + remote_dsync_box->mailbox_guid, + sizeof(remote_dsync_box->mailbox_guid)) == 0); + + brain->remote_dsync_box = *remote_dsync_box; + + state = dsync_mailbox_state_find(brain, remote_dsync_box->mailbox_guid); + if (state != NULL) { + last_common_uid = state->last_common_uid; + last_common_modseq = state->last_common_modseq; + } else { + last_common_uid = 0; + last_common_modseq = 0; + } + + if (brain->guid_requests) + import_flags |= DSYNC_MAILBOX_IMPORT_FLAG_WANT_MAIL_REQUESTS; + if (brain->master_brain) + import_flags |= DSYNC_MAILBOX_IMPORT_FLAG_MASTER_BRAIN; + if (brain->mails_have_guids) + import_flags |= DSYNC_MAILBOX_IMPORT_FLAG_MAILS_HAVE_GUIDS; + + brain->box_importer = + dsync_mailbox_import_init(brain->box, brain->log_scan, + last_common_uid, last_common_modseq, + remote_dsync_box->uid_next, + remote_dsync_box->first_recent_uid, + remote_dsync_box->highest_modseq, + import_flags); +} + +void dsync_brain_sync_mailbox_deinit(struct dsync_brain *brain) +{ + struct dsync_mailbox_state *state; + + i_assert(brain->box != NULL); + + state = p_new(brain->pool, struct dsync_mailbox_state, 1); + *state = brain->mailbox_state; + hash_table_insert(brain->remote_mailbox_states, + state->mailbox_guid, state); + + if (brain->box_exporter != NULL) { + const char *error; + + i_assert(brain->failed || + brain->sync_type == DSYNC_BRAIN_SYNC_TYPE_CHANGED); + (void)dsync_mailbox_export_deinit(&brain->box_exporter, &error); + } + if (brain->box_importer != NULL) { + uint32_t last_common_uid; + uint64_t last_common_modseq; + bool changes_during_sync; + + i_assert(brain->failed); + (void)dsync_mailbox_import_deinit(&brain->box_importer, + &last_common_uid, + &last_common_modseq, + &changes_during_sync); + } + if (brain->log_scan != NULL) + dsync_transaction_log_scan_deinit(&brain->log_scan); + mailbox_free(&brain->box); + + brain->state = brain->pre_box_state; +} + +static int dsync_box_get(struct mailbox *box, struct dsync_mailbox *dsync_box_r) +{ + const enum mailbox_status_items status_items = + STATUS_UIDVALIDITY | STATUS_UIDNEXT | STATUS_MESSAGES | + STATUS_FIRST_RECENT_UID | STATUS_HIGHESTMODSEQ; + const enum mailbox_metadata_items metadata_items = + MAILBOX_METADATA_CACHE_FIELDS | MAILBOX_METADATA_GUID; + struct mailbox_status status; + struct mailbox_metadata metadata; + const char *errstr; + enum mail_error error; + + /* get metadata first, since it may autocreate the mailbox */ + if (mailbox_get_metadata(box, metadata_items, &metadata) < 0 || + mailbox_get_status(box, status_items, &status) < 0) { + errstr = mailbox_get_last_error(box, &error); + if (error == MAIL_ERROR_NOTFOUND || + error == MAIL_ERROR_NOTPOSSIBLE) { + /* Mailbox isn't selectable, try the next one. We + should have already caught \Noselect mailboxes, but + check them anyway here. The NOTPOSSIBLE check is + mainly for invalid mbox files. */ + return 0; + } + i_error("Failed to access mailbox %s: %s", + mailbox_get_vname(box), errstr); + return -1; + } + + i_assert(status.uidvalidity != 0 || status.messages == 0); + + memset(dsync_box_r, 0, sizeof(*dsync_box_r)); + memcpy(dsync_box_r->mailbox_guid, metadata.guid, + sizeof(dsync_box_r->mailbox_guid)); + dsync_box_r->uid_validity = status.uidvalidity; + dsync_box_r->uid_next = status.uidnext; + dsync_box_r->messages_count = status.messages; + dsync_box_r->first_recent_uid = status.first_recent_uid; + dsync_box_r->highest_modseq = status.highest_modseq; + dsync_box_r->cache_fields = *metadata.cache_fields; + return 1; +} + +static bool +dsync_brain_has_mailbox_state_changed(struct dsync_brain *brain, + const struct dsync_mailbox *dsync_box) +{ + const struct dsync_mailbox_state *state; + + if (brain->sync_type != DSYNC_BRAIN_SYNC_TYPE_STATE) + return TRUE; + + state = dsync_mailbox_state_find(brain, dsync_box->mailbox_guid); + return state == NULL || + state->last_uidvalidity != dsync_box->uid_validity || + state->last_common_uid+1 != dsync_box->uid_next || + state->last_common_modseq != dsync_box->highest_modseq; +} + +static int +dsync_brain_try_next_mailbox(struct dsync_brain *brain, struct mailbox **box_r, + struct dsync_mailbox *dsync_box_r) +{ + struct dsync_mailbox dsync_box; + struct mailbox *box; + struct dsync_mailbox_node *node; + const char *vname = NULL; + bool synced = FALSE; + int ret; + + while (dsync_mailbox_tree_iter_next(brain->local_tree_iter, &vname, &node)) { + if (node->existence == DSYNC_MAILBOX_NODE_EXISTS && + !guid_128_is_empty(node->mailbox_guid)) + break; + vname = NULL; + } + if (vname == NULL) { + /* no more mailboxes */ + dsync_mailbox_tree_iter_deinit(&brain->local_tree_iter); + return -1; + } + + box = mailbox_alloc(node->ns->list, vname, 0); + for (;;) { + if ((ret = dsync_box_get(box, &dsync_box)) <= 0) { + if (ret < 0) + brain->failed = TRUE; + mailbox_free(&box); + return ret; + } + + /* if mailbox's last_common_* state equals the current state, + we can skip the mailbox */ + if (!dsync_brain_has_mailbox_state_changed(brain, &dsync_box)) { + mailbox_free(&box); + return 0; + } + if (synced) { + /* ok, the mailbox really changed */ + break; + } + + /* mailbox appears to have changed. do a full sync here and get the + state again */ + if (mailbox_sync(box, MAILBOX_SYNC_FLAG_FULL_READ) < 0) { + i_error("Can't sync mailbox %s: %s", + mailbox_get_vname(box), + mailbox_get_last_error(box, NULL)); + brain->failed = TRUE; + mailbox_free(&box); + return -1; + } + synced = TRUE; + } + + *box_r = box; + *dsync_box_r = dsync_box; + return 1; +} + +static bool +dsync_brain_next_mailbox(struct dsync_brain *brain, struct mailbox **box_r, + struct dsync_mailbox *dsync_box_r) +{ + int ret; + + while ((ret = dsync_brain_try_next_mailbox(brain, box_r, dsync_box_r)) == 0) + ; + return ret > 0; +} + +void dsync_brain_master_send_mailbox(struct dsync_brain *brain) +{ + struct dsync_mailbox dsync_box; + struct mailbox *box; + + i_assert(brain->master_brain); + i_assert(brain->box == NULL); + + if (!dsync_brain_next_mailbox(brain, &box, &dsync_box)) { + brain->state = DSYNC_STATE_DONE; + dsync_slave_send_end_of_list(brain->slave); + return; + } + + /* start exporting this mailbox (wait for remote to start importing) */ + dsync_slave_send_mailbox(brain->slave, &dsync_box); + dsync_brain_sync_mailbox_init(brain, box, &dsync_box, + DSYNC_BOX_STATE_MAILBOX); + brain->state = DSYNC_STATE_SYNC_MAILS; +} + +bool dsync_boxes_need_sync(const struct dsync_mailbox *box1, + const struct dsync_mailbox *box2) +{ + return box1->highest_modseq != box2->highest_modseq || + box1->messages_count != box2->messages_count || + box1->uid_next != box2->uid_next || + box1->uid_validity != box2->uid_validity || + box1->first_recent_uid != box2->first_recent_uid; +} + +static int +mailbox_cache_field_name_cmp(const struct mailbox_cache_field *f1, + const struct mailbox_cache_field *f2) +{ + return strcmp(f1->name, f2->name); +} + +static void +dsync_cache_fields_update(const struct dsync_mailbox *local_box, + const struct dsync_mailbox *remote_box, + struct mailbox_update *update) +{ + ARRAY_TYPE(mailbox_cache_field) local_sorted, remote_sorted, changes; + const struct mailbox_cache_field *local_fields, *remote_fields; + unsigned int li, ri, local_count, remote_count; + time_t drop_older_timestamp; + int ret; + + if (array_count(&remote_box->cache_fields) == 0) { + /* remote has no cached fields. there's nothing to update. */ + return; + } + + t_array_init(&local_sorted, array_count(&local_box->cache_fields)); + t_array_init(&remote_sorted, array_count(&remote_box->cache_fields)); + array_append_array(&local_sorted, &local_box->cache_fields); + array_append_array(&remote_sorted, &remote_box->cache_fields); + array_sort(&local_sorted, mailbox_cache_field_name_cmp); + array_sort(&remote_sorted, mailbox_cache_field_name_cmp); + + if (array_count(&local_sorted) == 0) { + /* local has no cached fields. set them to same as remote. */ + (void)array_append_space(&remote_sorted); + update->cache_updates = array_idx(&remote_sorted, 0); + return; + } + + /* figure out what to change */ + local_fields = array_get(&local_sorted, &local_count); + remote_fields = array_get(&remote_sorted, &remote_count); + t_array_init(&changes, local_count + remote_count); + drop_older_timestamp = ioloop_time - MAIL_CACHE_FIELD_DROP_SECS; + + for (li = ri = 0; li < local_count || ri < remote_count; ) { + ret = li == local_count ? 1 : + ri == remote_count ? -1 : + strcmp(local_fields[li].name, remote_fields[ri].name); + if (ret == 0) { + /* field exists in both local and remote */ + const struct mailbox_cache_field *lf = &local_fields[li]; + const struct mailbox_cache_field *rf = &remote_fields[ri]; + + if (lf->last_used > rf->last_used || + (lf->last_used == rf->last_used && + lf->decision > rf->decision)) { + /* use local decision and timestamp */ + } else { + array_append(&changes, rf, 1); + } + li++; ri++; + } else if (ret < 0) { + /* remote field doesn't exist */ + li++; + } else { + /* local field doesn't exist */ + if (remote_fields[ri].last_used < drop_older_timestamp) { + /* field hasn't be used for a long time, remote + will probably drop this soon as well */ + } else { + array_append(&changes, &remote_fields[ri], 1); + } + ri++; + } + } + i_assert(li == local_count && ri == remote_count); + if (array_count(&changes) > 0) { + (void)array_append_space(&changes); + update->cache_updates = array_idx(&changes, 0); + } +} + +void dsync_brain_mailbox_update_pre(struct dsync_brain *brain, + struct mailbox *box, + const struct dsync_mailbox *local_box, + const struct dsync_mailbox *remote_box) +{ + struct mailbox_update update; + const struct dsync_mailbox_state *state; + + memset(&update, 0, sizeof(update)); + + if (local_box->uid_validity != remote_box->uid_validity) { + /* Keep the UIDVALIDITY for the mailbox that has more + messages. If they equal, use the higher UIDVALIDITY. */ + if (remote_box->messages_count > local_box->messages_count || + (remote_box->messages_count == local_box->messages_count && + remote_box->uid_validity > local_box->uid_validity)) + update.uid_validity = remote_box->uid_validity; + + state = dsync_mailbox_state_find(brain, local_box->mailbox_guid); + if (state != NULL && state->last_common_uid > 0) { + /* we can't continue syncing this mailbox in this + session, because the other side already started + sending mailbox changes, but not for all mails. */ + dsync_mailbox_state_remove(brain, local_box->mailbox_guid); + // FIXME: handle this properly + } + } + + dsync_cache_fields_update(local_box, remote_box, &update); + + if (update.uid_validity == 0 && + update.cache_updates == NULL) { + /* no changes */ + return; + } + + if (mailbox_update(box, &update) < 0) { + i_error("Couldn't update mailbox %s metadata: %s", + mailbox_get_vname(brain->box), + mailbox_get_last_error(brain->box, NULL)); + brain->failed = TRUE; + } +} + +bool dsync_brain_slave_recv_mailbox(struct dsync_brain *brain) +{ + const struct dsync_mailbox *dsync_box; + struct dsync_mailbox local_dsync_box, delete_box; + struct mailbox *box; + int ret; + + i_assert(!brain->master_brain); + i_assert(brain->box == NULL); + + if ((ret = dsync_slave_recv_mailbox(brain->slave, &dsync_box)) == 0) + return FALSE; + if (ret < 0) { + brain->state = DSYNC_STATE_DONE; + return TRUE; + } + + if (dsync_brain_mailbox_alloc(brain, dsync_box->mailbox_guid, &box) < 0) { + i_assert(brain->failed); + return TRUE; + } + if (box == NULL) { + /* mailbox was probably deleted/renamed during sync */ + //FIXME: in case it wasn't, do error handling + brain->changes_during_sync = TRUE; + return TRUE; + } + if (mailbox_sync(box, MAILBOX_SYNC_FLAG_FULL_READ) < 0) { + i_error("Can't sync mailbox %s: %s", + mailbox_get_vname(box), + mailbox_get_last_error(box, NULL)); + mailbox_free(&box); + brain->failed = TRUE; + return TRUE; + } + + if ((ret = dsync_box_get(box, &local_dsync_box)) <= 0) { + mailbox_free(&box); + if (ret < 0) { + brain->failed = TRUE; + return TRUE; + } + /* another process just deleted this mailbox? */ + memset(&delete_box, 0, sizeof(delete_box)); + memcpy(delete_box.mailbox_guid, dsync_box->mailbox_guid, + sizeof(delete_box.mailbox_guid)); + delete_box.mailbox_lost = TRUE; + dsync_slave_send_mailbox(brain->slave, &delete_box); + return TRUE; + } + i_assert(local_dsync_box.uid_validity != 0); + i_assert(memcmp(dsync_box->mailbox_guid, local_dsync_box.mailbox_guid, + sizeof(dsync_box->mailbox_guid)) == 0); + dsync_slave_send_mailbox(brain->slave, &local_dsync_box); + + dsync_brain_mailbox_update_pre(brain, box, &local_dsync_box, dsync_box); + + if (brain->sync_type == DSYNC_BRAIN_SYNC_TYPE_CHANGED && + !dsync_boxes_need_sync(&local_dsync_box, dsync_box)) { + /* no fields appear to have changed, skip this mailbox */ + mailbox_free(&box); + return TRUE; + } + + /* start export/import */ + if (dsync_brain_sync_mailbox_init(brain, box, &local_dsync_box, + DSYNC_BOX_STATE_CHANGES) == 0) + dsync_brain_sync_mailbox_init_remote(brain, dsync_box); + + brain->state = DSYNC_STATE_SYNC_MAILS; + return TRUE; +}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/doveadm/dsync/dsync-brain-mails.c Tue May 22 23:17:31 2012 +0300 @@ -0,0 +1,274 @@ +/* Copyright (c) 2012 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "istream.h" +#include "dsync-slave.h" +#include "dsync-mail.h" +#include "dsync-mailbox-import.h" +#include "dsync-mailbox-export.h" +#include "dsync-brain-private.h" + +static bool dsync_brain_master_sync_recv_mailbox(struct dsync_brain *brain) +{ + const struct dsync_mailbox *dsync_box; + enum dsync_slave_recv_ret ret; + + i_assert(brain->master_brain); + + if ((ret = dsync_slave_recv_mailbox(brain->slave, &dsync_box)) == 0) + return FALSE; + if (ret == DSYNC_SLAVE_RECV_RET_FINISHED) { + i_error("Remote sent end-of-list instead of a mailbox"); + brain->failed = TRUE; + return TRUE; + } + if (memcmp(dsync_box->mailbox_guid, brain->local_dsync_box.mailbox_guid, + sizeof(dsync_box->mailbox_guid)) != 0) { + i_error("Remote sent mailbox with a wrong GUID"); + brain->failed = TRUE; + return TRUE; + } + + if (dsync_box->mailbox_lost) { + /* remote lost the mailbox. it's probably already deleted, but + verify it on next sync just to be sure */ + dsync_brain_sync_mailbox_deinit(brain); + return TRUE; + } + dsync_brain_mailbox_update_pre(brain, brain->box, + &brain->local_dsync_box, dsync_box); + + if (brain->sync_type == DSYNC_BRAIN_SYNC_TYPE_CHANGED && + !dsync_boxes_need_sync(&brain->local_dsync_box, dsync_box)) { + /* no fields appear to have changed, skip this mailbox */ + dsync_brain_sync_mailbox_deinit(brain); + return TRUE; + } + dsync_brain_sync_mailbox_init_remote(brain, dsync_box); + brain->box_recv_state = DSYNC_BOX_STATE_CHANGES; + brain->box_send_state = DSYNC_BOX_STATE_CHANGES; + + i_assert(brain->box_importer != NULL); + return TRUE; +} + +static bool dsync_brain_recv_mail_change(struct dsync_brain *brain) +{ + const struct dsync_mail_change *change; + enum dsync_slave_recv_ret ret; + + if ((ret = dsync_slave_recv_change(brain->slave, &change)) == 0) + return FALSE; + if (ret == DSYNC_SLAVE_RECV_RET_FINISHED) { + dsync_mailbox_import_changes_finish(brain->box_importer); + brain->box_recv_state = brain->guid_requests ? + DSYNC_BOX_STATE_MAIL_REQUESTS : DSYNC_BOX_STATE_MAILS; + return TRUE; + } + dsync_mailbox_import_change(brain->box_importer, change); + return TRUE; +} + +static void dsync_brain_send_mail_change(struct dsync_brain *brain) +{ + const struct dsync_mail_change *change; + + while ((change = dsync_mailbox_export_next(brain->box_exporter)) != NULL) { + if (dsync_slave_send_change(brain->slave, change) == 0) + return; + } + dsync_slave_send_end_of_list(brain->slave); + brain->box_send_state = brain->guid_requests ? + DSYNC_BOX_STATE_MAIL_REQUESTS : DSYNC_BOX_STATE_MAILS; +} + +static bool dsync_brain_recv_mail_request(struct dsync_brain *brain) +{ + const struct dsync_mail_request *request; + enum dsync_slave_recv_ret ret; + + i_assert(brain->guid_requests); + + if ((ret = dsync_slave_recv_mail_request(brain->slave, &request)) == 0) + return FALSE; + if (ret == DSYNC_SLAVE_RECV_RET_FINISHED) { + brain->box_recv_state = DSYNC_BOX_STATE_MAILS; + return TRUE; + } + dsync_mailbox_export_want_mail(brain->box_exporter, request); + return TRUE; +} + +static void dsync_brain_send_mail_request(struct dsync_brain *brain) +{ + const struct dsync_mail_request *request; + + i_assert(brain->guid_requests); + + while ((request = dsync_mailbox_import_next_request(brain->box_importer)) != NULL) { + if (dsync_slave_send_mail_request(brain->slave, request) == 0) + return; + } + if (brain->box_recv_state > DSYNC_BOX_STATE_CHANGES) { + dsync_slave_send_end_of_list(brain->slave); + brain->box_send_state = DSYNC_BOX_STATE_MAILS; + } +} + +static void dsync_brain_sync_half_finished(struct dsync_brain *brain) +{ + struct dsync_mailbox_state state; + bool changes_during_sync; + const char *error; + + if (brain->box_recv_state < DSYNC_BOX_STATE_RECV_LAST_COMMON || + brain->box_send_state < DSYNC_BOX_STATE_RECV_LAST_COMMON) + return; + + /* finished with this mailbox */ + if (dsync_mailbox_export_deinit(&brain->box_exporter, &error) < 0) { + i_error("Exporting mailbox %s failed: %s", + mailbox_get_vname(brain->box), error); + brain->failed = TRUE; + return; + } + + memset(&state, 0, sizeof(state)); + memcpy(state.mailbox_guid, brain->local_dsync_box.mailbox_guid, + sizeof(state.mailbox_guid)); + state.last_uidvalidity = brain->local_dsync_box.uid_validity; + if (brain->box_importer == NULL) { + /* this mailbox didn't exist on remote */ + state.last_common_uid = brain->local_dsync_box.uid_next-1; + state.last_common_modseq = + brain->local_dsync_box.highest_modseq; + } else { + if (dsync_mailbox_import_deinit(&brain->box_importer, + &state.last_common_uid, + &state.last_common_modseq, + &changes_during_sync) < 0) { + i_error("Importing mailbox %s failed", + mailbox_get_vname(brain->box)); + brain->failed = TRUE; + return; + } + if (changes_during_sync) + brain->changes_during_sync = TRUE; + } + dsync_slave_send_mailbox_state(brain->slave, &state); +} + +static bool dsync_brain_recv_mail(struct dsync_brain *brain) +{ + struct dsync_mail *mail; + enum dsync_slave_recv_ret ret; + + if ((ret = dsync_slave_recv_mail(brain->slave, &mail)) == 0) + return FALSE; + if (ret == DSYNC_SLAVE_RECV_RET_FINISHED) { + brain->box_recv_state = DSYNC_BOX_STATE_RECV_LAST_COMMON; + dsync_brain_sync_half_finished(brain); + return TRUE; + } + dsync_mailbox_import_mail(brain->box_importer, mail); + if (mail->input != NULL) + i_stream_unref(&mail->input); + return TRUE; +} + +static bool dsync_brain_send_mail(struct dsync_brain *brain) +{ + const struct dsync_mail *mail; + bool changed = FALSE; + + while ((mail = dsync_mailbox_export_next_mail(brain->box_exporter)) != NULL) { + changed = TRUE; + if (dsync_slave_send_mail(brain->slave, mail) == 0) + return TRUE; + } + if (brain->guid_requests && + brain->box_recv_state < DSYNC_BOX_STATE_MAILS) { + /* wait for mail requests to finish */ + return changed; + } + + brain->box_send_state = DSYNC_BOX_STATE_DONE; + dsync_slave_send_end_of_list(brain->slave); + + dsync_brain_sync_half_finished(brain); + return TRUE; +} + +static bool dsync_brain_recv_last_common(struct dsync_brain *brain) +{ + enum dsync_slave_recv_ret ret; + struct dsync_mailbox_state state; + + if ((ret = dsync_slave_recv_mailbox_state(brain->slave, &state)) == 0) + return FALSE; + if (ret == DSYNC_SLAVE_RECV_RET_FINISHED) { + i_error("Remote sent end-of-list instead of a mailbox state"); + brain->failed = TRUE; + return TRUE; + } + i_assert(brain->box_send_state == DSYNC_BOX_STATE_DONE); + brain->mailbox_state = state; + + dsync_brain_sync_mailbox_deinit(brain); + return TRUE; +} + +bool dsync_brain_sync_mails(struct dsync_brain *brain) +{ + bool changed = FALSE; + + i_assert(brain->box != NULL); + + switch (brain->box_recv_state) { + case DSYNC_BOX_STATE_MAILBOX: + changed = dsync_brain_master_sync_recv_mailbox(brain); + break; + case DSYNC_BOX_STATE_CHANGES: + changed = dsync_brain_recv_mail_change(brain); + break; + case DSYNC_BOX_STATE_MAIL_REQUESTS: + changed = dsync_brain_recv_mail_request(brain); + break; + case DSYNC_BOX_STATE_MAILS: + changed = dsync_brain_recv_mail(brain); + break; + case DSYNC_BOX_STATE_RECV_LAST_COMMON: + changed = dsync_brain_recv_last_common(brain); + break; + case DSYNC_BOX_STATE_DONE: + break; + } + + if (brain->failed) + return TRUE; + + switch (brain->box_send_state) { + case DSYNC_BOX_STATE_MAILBOX: + /* wait for mailbox to be received first */ + break; + case DSYNC_BOX_STATE_CHANGES: + dsync_brain_send_mail_change(brain); + changed = TRUE; + break; + case DSYNC_BOX_STATE_MAIL_REQUESTS: + dsync_brain_send_mail_request(brain); + changed = TRUE; + break; + case DSYNC_BOX_STATE_MAILS: + if (!dsync_slave_is_send_queue_full(brain->slave)) { + if (dsync_brain_send_mail(brain)) + changed = TRUE; + } + break; + case DSYNC_BOX_STATE_RECV_LAST_COMMON: + i_unreached(); + case DSYNC_BOX_STATE_DONE: + break; + } + return changed; +}
--- a/src/doveadm/dsync/dsync-brain-msgs-new.c Fri May 04 05:35:36 2012 +0300 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,393 +0,0 @@ -/* Copyright (c) 2009-2012 Dovecot authors, see the included COPYING file */ - -/* - This code contains the step 6 explained in dsync-brain-msgs.c: - It saves/copies new messages and gives new UIDs for conflicting messages. - - The input is both workers' msg iterators' new_msgs and uid_conflicts - variables. They're first sorted by mailbox and secondarily by wanted - destination UID. Destination UIDs of conflicts should always be higher - than new messages'. - - Mailboxes are handled one at a time: - - 1. Go through all saved messages. If we've already seen an instance of this - message, try to copy it. Otherwise save a new instance of it. - 2. Some of the copies may fail because they're already expunged by that - time. A list of these failed copies are saved to copy_retry_indexes. - 3. UID conflicts are resolved by assigning a new UID to the message. - To avoid delays with remote dsync, this is done via worker API. - Internally the local worker copies the message to its new UID and - once the copy succeeds, the old UID is expunged. If the copy fails, it's - either due to message already being expunged or something more fatal. - 4. Once all messages are saved/copied, see if there are any failed copies. - If so, goto 1, but going through only the failed messages. - 5. If there are more mailboxes left, go to next one and goto 1. - - Step 4 may require waiting for remote worker to send all replies. -*/ - -#include "lib.h" -#include "array.h" -#include "istream.h" -#include "hash.h" -#include "dsync-worker.h" -#include "dsync-brain-private.h" - -struct dsync_brain_msg_copy_context { - struct dsync_brain_msg_iter *iter; - unsigned int msg_idx; -}; - -struct dsync_brain_msg_save_context { - struct dsync_brain_msg_iter *iter; - const struct dsync_message *msg; - unsigned int mailbox_idx; -}; - -static void -dsync_brain_msg_sync_add_new_msgs(struct dsync_brain_msg_iter *iter); - -static void msg_save_callback(void *context) -{ - struct dsync_brain_msg_save_context *ctx = context; - - dsync_brain_guid_add(ctx->iter, ctx->mailbox_idx, ctx->msg); - if (--ctx->iter->save_results_left == 0 && !ctx->iter->adding_msgs) - dsync_brain_msg_sync_add_new_msgs(ctx->iter); - i_free(ctx); -} - -static void msg_get_callback(enum dsync_msg_get_result result, - const struct dsync_msg_static_data *data, - void *context) -{ - struct dsync_brain_msg_save_context *ctx = context; - const struct dsync_brain_mailbox *mailbox; - struct istream *input; - - i_assert(ctx->iter->save_results_left > 0); - - mailbox = array_idx(&ctx->iter->sync->mailboxes, ctx->mailbox_idx); - switch (result) { - case DSYNC_MSG_GET_RESULT_SUCCESS: - /* the mailbox may have changed, make sure we've the - right one */ - dsync_worker_select_mailbox(ctx->iter->worker, &mailbox->box); - - input = data->input; - dsync_worker_msg_save(ctx->iter->worker, ctx->msg, data, - msg_save_callback, ctx); - i_stream_unref(&input); - break; - case DSYNC_MSG_GET_RESULT_EXPUNGED: - /* mail got expunged during sync. just skip this. */ - msg_save_callback(ctx); - break; - case DSYNC_MSG_GET_RESULT_FAILED: - i_error("msg-get failed: box=%s uid=%u guid=%s", - mailbox->box.name, ctx->msg->uid, ctx->msg->guid); - dsync_brain_fail(ctx->iter->sync->brain); - msg_save_callback(ctx); - break; - } -} - -static void -dsync_brain_sync_remove_guid_instance(struct dsync_brain_msg_iter *iter, - const struct dsync_brain_new_msg *msg) -{ - struct dsync_brain_guid_instance *inst; - void *orig_key, *orig_value; - - if (!hash_table_lookup_full(iter->guid_hash, msg->msg->guid, - &orig_key, &orig_value)) { - /* another failed copy already removed it */ - return; - } - inst = orig_value; - - if (inst->next == NULL) - hash_table_remove(iter->guid_hash, orig_key); - else - hash_table_update(iter->guid_hash, orig_key, inst->next); -} - -static void dsync_brain_copy_callback(bool success, void *context) -{ - struct dsync_brain_msg_copy_context *ctx = context; - struct dsync_brain_new_msg *msg; - - if (!success) { - /* mark the guid instance invalid and try again later */ - msg = array_idx_modifiable(&ctx->iter->new_msgs, ctx->msg_idx); - i_assert(msg->saved); - msg->saved = FALSE; - - if (ctx->iter->next_new_msg > ctx->msg_idx) - ctx->iter->next_new_msg = ctx->msg_idx; - - dsync_brain_sync_remove_guid_instance(ctx->iter, msg); - } - - if (--ctx->iter->copy_results_left == 0 && !ctx->iter->adding_msgs) - dsync_brain_msg_sync_add_new_msgs(ctx->iter); - i_free(ctx); -} - -static int -dsync_brain_msg_sync_add_new_msg(struct dsync_brain_msg_iter *dest_iter, - const mailbox_guid_t *src_mailbox, - unsigned int msg_idx, - struct dsync_brain_new_msg *msg) -{ - struct dsync_brain_msg_save_context *save_ctx; - struct dsync_brain_msg_copy_context *copy_ctx; - struct dsync_brain_msg_iter *src_iter; - const struct dsync_brain_guid_instance *inst; - const struct dsync_brain_mailbox *inst_box; - - msg->saved = TRUE; - - inst = hash_table_lookup(dest_iter->guid_hash, msg->msg->guid); - if (inst != NULL) { - /* we can save this by copying an existing message */ - inst_box = array_idx(&dest_iter->sync->mailboxes, - inst->mailbox_idx); - - copy_ctx = i_new(struct dsync_brain_msg_copy_context, 1); - copy_ctx->iter = dest_iter; - copy_ctx->msg_idx = msg_idx; - - dest_iter->copy_results_left++; - dest_iter->adding_msgs = TRUE; - dsync_worker_msg_copy(dest_iter->worker, - &inst_box->box.mailbox_guid, - inst->uid, msg->msg, - dsync_brain_copy_callback, copy_ctx); - dest_iter->adding_msgs = FALSE; - } else { - src_iter = dest_iter == dest_iter->sync->dest_msg_iter ? - dest_iter->sync->src_msg_iter : - dest_iter->sync->dest_msg_iter; - - save_ctx = i_new(struct dsync_brain_msg_save_context, 1); - save_ctx->iter = dest_iter; - save_ctx->msg = msg->msg; - save_ctx->mailbox_idx = dest_iter->mailbox_idx; - - dest_iter->save_results_left++; - dest_iter->adding_msgs = TRUE; - dsync_worker_msg_get(src_iter->worker, src_mailbox, - msg->orig_uid, msg_get_callback, save_ctx); - dest_iter->adding_msgs = FALSE; - if (dsync_worker_output_flush(src_iter->worker) < 0) - return -1; - if (dsync_worker_is_output_full(dest_iter->worker)) { - /* see if the output becomes less full by flushing */ - if (dsync_worker_output_flush(dest_iter->worker) < 0) - return -1; - } - } - return dsync_worker_is_output_full(dest_iter->worker) ? 0 : 1; -} - -static bool -dsync_brain_mailbox_add_new_msgs(struct dsync_brain_msg_iter *iter, - const mailbox_guid_t *mailbox_guid) -{ - struct dsync_brain_new_msg *msgs; - unsigned int msg_count; - bool ret = TRUE; - - msgs = array_get_modifiable(&iter->new_msgs, &msg_count); - while (iter->next_new_msg < msg_count) { - struct dsync_brain_new_msg *msg = &msgs[iter->next_new_msg]; - - if (msg->mailbox_idx != iter->mailbox_idx) { - i_assert(msg->mailbox_idx > iter->mailbox_idx); - ret = FALSE; - break; - } - iter->next_new_msg++; - - if (msg->saved) - continue; - if (dsync_brain_msg_sync_add_new_msg(iter, mailbox_guid, - iter->next_new_msg - 1, - msg) <= 0) { - /* failed / continue later */ - break; - } - } - if (iter->next_new_msg == msg_count) - ret = FALSE; - - /* flush copy commands */ - if (dsync_worker_output_flush(iter->worker) > 0 && ret) { - /* we have more space again, continue */ - return dsync_brain_mailbox_add_new_msgs(iter, mailbox_guid); - } else { - return ret; - } -} - -static void -dsync_brain_mailbox_save_conflicts(struct dsync_brain_msg_iter *iter) -{ - const struct dsync_brain_uid_conflict *conflicts; - unsigned int i, count; - - conflicts = array_get(&iter->uid_conflicts, &count); - for (i = iter->next_conflict; i < count; i++) { - if (conflicts[i].mailbox_idx != iter->mailbox_idx) - break; - - dsync_worker_msg_update_uid(iter->worker, conflicts[i].old_uid, - conflicts[i].new_uid); - } - iter->next_conflict = i; -} - -static void -dsync_brain_msg_sync_finish(struct dsync_brain_msg_iter *iter) -{ - struct dsync_brain_mailbox_sync *sync = iter->sync; - - i_assert(sync->brain->state == DSYNC_STATE_SYNC_MSGS); - - iter->msgs_sent = TRUE; - - /* done with all mailboxes from this iter */ - dsync_worker_set_input_callback(iter->worker, NULL, NULL); - - if (sync->src_msg_iter->msgs_sent && - sync->dest_msg_iter->msgs_sent && - sync->src_msg_iter->save_results_left == 0 && - sync->dest_msg_iter->save_results_left == 0 && - dsync_worker_output_flush(sync->dest_worker) > 0 && - dsync_worker_output_flush(sync->src_worker) > 0) { - dsync_worker_set_output_callback(sync->src_msg_iter->worker, - NULL, NULL); - dsync_worker_set_output_callback(sync->dest_msg_iter->worker, - NULL, NULL); - sync->brain->state++; - dsync_brain_sync(sync->brain); - } -} - -static bool -dsync_brain_msg_sync_select_mailbox(struct dsync_brain_msg_iter *iter) -{ - const struct dsync_brain_mailbox *mailbox; - - while (iter->mailbox_idx < array_count(&iter->sync->mailboxes)) { - if (array_count(&iter->new_msgs) == 0 && - array_count(&iter->uid_conflicts) == 0) { - /* optimization: don't even bother selecting this - mailbox */ - iter->mailbox_idx++; - continue; - } - - mailbox = array_idx(&iter->sync->mailboxes, iter->mailbox_idx); - dsync_worker_select_mailbox(iter->worker, &mailbox->box); - return TRUE; - } - dsync_brain_msg_sync_finish(iter); - return FALSE; -} - -static void -dsync_brain_msg_sync_add_new_msgs(struct dsync_brain_msg_iter *iter) -{ - const struct dsync_brain_mailbox *mailbox; - const mailbox_guid_t *mailbox_guid; - - if (iter->msgs_sent) { - dsync_brain_msg_sync_finish(iter); - return; - } - - do { - mailbox = array_idx(&iter->sync->mailboxes, iter->mailbox_idx); - mailbox_guid = &mailbox->box.mailbox_guid; - if (dsync_brain_mailbox_add_new_msgs(iter, mailbox_guid)) { - /* continue later */ - return; - } - - /* all messages saved for this mailbox. continue with saving - its conflicts and waiting for copies to finish. */ - dsync_brain_mailbox_save_conflicts(iter); - if (iter->save_results_left > 0 || - iter->copy_results_left > 0) { - /* wait for saves/copies to finish */ - return; - } - - /* done with this mailbox, try the next one */ - iter->mailbox_idx++; - } while (dsync_brain_msg_sync_select_mailbox(iter)); -} - -static void dsync_worker_new_msg_output(void *context) -{ - struct dsync_brain_msg_iter *iter = context; - - dsync_brain_msg_sync_add_new_msgs(iter); -} - -static int dsync_brain_new_msg_cmp(const struct dsync_brain_new_msg *m1, - const struct dsync_brain_new_msg *m2) -{ - if (m1->mailbox_idx < m2->mailbox_idx) - return -1; - if (m1->mailbox_idx > m2->mailbox_idx) - return 1; - - if (m1->msg->uid < m2->msg->uid) - return -1; - if (m1->msg->uid > m2->msg->uid) - return 1; - return 0; -} - -static int -dsync_brain_uid_conflict_cmp(const struct dsync_brain_uid_conflict *c1, - const struct dsync_brain_uid_conflict *c2) -{ - if (c1->mailbox_idx < c2->mailbox_idx) - return -1; - if (c1->mailbox_idx < c2->mailbox_idx) - return 1; - - if (c1->new_uid < c2->new_uid) - return -1; - if (c1->new_uid > c2->new_uid) - return 1; - return 0; -} - -static void -dsync_brain_msg_iter_sync_new_msgs(struct dsync_brain_msg_iter *iter) -{ - iter->mailbox_idx = 0; - - /* sort input by 1) mailbox, 2) new message UID */ - array_sort(&iter->new_msgs, dsync_brain_new_msg_cmp); - array_sort(&iter->uid_conflicts, dsync_brain_uid_conflict_cmp); - - dsync_worker_set_input_callback(iter->worker, NULL, iter); - dsync_worker_set_output_callback(iter->worker, - dsync_worker_new_msg_output, iter); - - if (dsync_brain_msg_sync_select_mailbox(iter)) - dsync_brain_msg_sync_add_new_msgs(iter); -} - -void dsync_brain_msg_sync_new_msgs(struct dsync_brain_mailbox_sync *sync) -{ - dsync_brain_msg_iter_sync_new_msgs(sync->src_msg_iter); - dsync_brain_msg_iter_sync_new_msgs(sync->dest_msg_iter); -}
--- a/src/doveadm/dsync/dsync-brain-msgs.c Fri May 04 05:35:36 2012 +0300 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,543 +0,0 @@ -/* Copyright (c) 2009-2012 Dovecot authors, see the included COPYING file */ - -/* This code synchronizes messages in all mailboxes between two workers. - The "src" and "dest" terms don't really have anything to do with reality, - they're both treated equal. - - 1. Iterate through all messages in all (wanted) mailboxes. The mailboxes - are iterated in the same order and messages in ascending order. - All of the expunged messages at the end of mailbox (i.e. - last_existing_uid+1 .. next_uid-1) are also returned with - DSYNC_MAIL_FLAG_EXPUNGED set. We only care about the end of the mailbox, - because we can detect UID conflicts for messages in the middle by looking - at the next existing message and seeing if it has UID conflict. - 2. For each seen non-expunged message, save it to GUID instance hash table: - message GUID => linked list of { uid, mailbox } - 3. Each message in a mailbox is matched between the two workers as long as - both have messages left (the last ones may be expunged). - The possibilities are: - - i) We don't know the GUIDs of both messages: - - a) Message is expunged in both. Do nothing. - b) Message is expunged in only one of them. If there have been no UID - conflicts seen so far, expunge the message in the other one. - Otherwise, give the existing a message a new UID (at step 6). - - ii) We know GUIDs of both messages (one/both of them may be expunged): - - a) Messages have conflicting GUIDs. Give new UIDs for the non-expunged - message(s) (at step 6). - b) Messages have matching GUIDs and one of them is expunged. - Expunge also the other one. (We don't need to care about previous - UID conflicts here, because we know this message is the same with - both workers, since they have the same GUID.) - c) Messages have matching GUIDs and both of them exist. Sync flags from - whichever has the higher modseq. If both modseqs equal but flags - don't, pick the one that has more flags. If even the flag count is - the same, just pick one of them. - 4. One of the workers may messages left in the mailbox. Copy these - (non-expunged) messages to the other worker (at step 6). - 5. If there are more mailboxes left, go to next one and goto 2. - - 6. Copy the new messages and give new UIDs to conflicting messages. - This code exists in dsync-brain-msgs-new.c -*/ - -#include "lib.h" -#include "array.h" -#include "hash.h" -#include "dsync-worker.h" -#include "dsync-brain-private.h" - -void dsync_brain_guid_add(struct dsync_brain_msg_iter *iter, - unsigned int mailbox_idx, - const struct dsync_message *msg) -{ - struct dsync_brain_guid_instance *inst, *prev_inst; - - if ((msg->flags & DSYNC_MAIL_FLAG_EXPUNGED) != 0) - return; - if (*msg->guid == '\0') - return; - - inst = p_new(iter->sync->pool, struct dsync_brain_guid_instance, 1); - inst->mailbox_idx = mailbox_idx; - inst->uid = msg->uid; - - prev_inst = hash_table_lookup(iter->guid_hash, msg->guid); - if (prev_inst == NULL) { - hash_table_insert(iter->guid_hash, - p_strdup(iter->sync->pool, msg->guid), - inst); - } else { - inst->next = prev_inst->next; - prev_inst->next = inst; - } -} - -static int dsync_brain_msg_iter_next(struct dsync_brain_msg_iter *iter) -{ - int ret = 1; - - if (iter->msg.guid == NULL) { - ret = dsync_worker_msg_iter_next(iter->iter, - &iter->mailbox_idx, - &iter->msg); - if (ret > 0) { - dsync_brain_guid_add(iter, iter->mailbox_idx, - &iter->msg); - } - } - - if (iter->sync->wanted_mailbox_idx != iter->mailbox_idx) { - /* finished with this mailbox */ - return -1; - } - return ret; -} - -static int -dsync_brain_msg_iter_skip_mailbox(struct dsync_brain_mailbox_sync *sync) -{ - int ret; - - while ((ret = dsync_brain_msg_iter_next(sync->src_msg_iter)) > 0) - sync->src_msg_iter->msg.guid = NULL; - if (ret == 0) - return 0; - - while ((ret = dsync_brain_msg_iter_next(sync->dest_msg_iter)) > 0) - sync->dest_msg_iter->msg.guid = NULL; - if (ret == 0) - return 0; - - sync->skip_mailbox = FALSE; - return -1; -} - -static int dsync_brain_msg_iter_next_pair(struct dsync_brain_mailbox_sync *sync) -{ - int ret1, ret2; - - if (sync->skip_mailbox) { - if (dsync_brain_msg_iter_skip_mailbox(sync) == 0) - return 0; - } - - ret1 = dsync_brain_msg_iter_next(sync->src_msg_iter); - ret2 = dsync_brain_msg_iter_next(sync->dest_msg_iter); - if (ret1 == 0 || ret2 == 0) { - /* make sure we iterate through everything in both iterators - (even if it might not seem necessary, because proxy - requires it) */ - return 0; - } - if (ret1 < 0 || ret2 < 0) - return -1; - return 1; -} - -static void -dsync_brain_msg_sync_save(struct dsync_brain_msg_iter *iter, - unsigned int mailbox_idx, - const struct dsync_message *msg) -{ - struct dsync_brain_new_msg *new_msg; - - if ((msg->flags & DSYNC_MAIL_FLAG_EXPUNGED) != 0) - return; - - new_msg = array_append_space(&iter->new_msgs); - new_msg->mailbox_idx = mailbox_idx; - new_msg->orig_uid = msg->uid; - new_msg->msg = dsync_message_dup(iter->sync->pool, msg); -} - -static void -dsync_brain_msg_sync_conflict(struct dsync_brain_msg_iter *conflict_iter, - struct dsync_brain_msg_iter *save_iter, - const struct dsync_message *msg) -{ - struct dsync_brain_uid_conflict *conflict; - struct dsync_brain_new_msg *new_msg; - struct dsync_brain_mailbox *brain_box; - uint32_t new_uid; - - brain_box = array_idx_modifiable(&save_iter->sync->mailboxes, - save_iter->mailbox_idx); - - if (save_iter->sync->brain->backup) { - i_warning("Destination mailbox %s has been modified, " - "need to recreate it before we can continue syncing", - brain_box->box.name); - dsync_worker_delete_mailbox(save_iter->sync->brain->dest_worker, - &brain_box->box); - save_iter->sync->brain->unexpected_changes = TRUE; - save_iter->sync->skip_mailbox = TRUE; - return; - } - - new_uid = brain_box->box.uid_next++; - - conflict = array_append_space(&conflict_iter->uid_conflicts); - conflict->mailbox_idx = conflict_iter->mailbox_idx; - conflict->old_uid = msg->uid; - conflict->new_uid = new_uid; - - new_msg = array_append_space(&save_iter->new_msgs); - new_msg->mailbox_idx = save_iter->mailbox_idx; - new_msg->orig_uid = msg->uid; - new_msg->msg = dsync_message_dup(save_iter->sync->pool, msg); - new_msg->msg->uid = new_uid; -} - -static int -dsync_message_flag_importance_cmp(const struct dsync_message *m1, - const struct dsync_message *m2) -{ - unsigned int i, count1, count2; - - if (m1->modseq > m2->modseq) - return -1; - else if (m1->modseq < m2->modseq) - return 1; - - if (m1->flags == m2->flags && - dsync_keyword_list_equals(m1->keywords, m2->keywords)) - return 0; - - /* modseqs match, but flags aren't the same. pick the one that - has more flags. */ - count1 = str_array_length(m1->keywords); - count2 = str_array_length(m2->keywords); - for (i = 1; i != MAIL_RECENT; i <<= 1) { - if ((m1->flags & i) != 0) - count1++; - if ((m2->flags & i) != 0) - count2++; - } - if (count1 > count2) - return -1; - else if (count1 < count2) - return 1; - - /* they even have the same number of flags. don't bother with further - guessing, just pick the first one. */ - return -1; -} - -static void dsync_brain_msg_sync_existing(struct dsync_brain_mailbox_sync *sync, - struct dsync_message *src_msg, - struct dsync_message *dest_msg) -{ - int ret; - - ret = dsync_message_flag_importance_cmp(src_msg, dest_msg); - if (ret < 0 || (sync->brain->backup && ret > 0)) - dsync_worker_msg_update_metadata(sync->dest_worker, src_msg); - else if (ret > 0) - dsync_worker_msg_update_metadata(sync->src_worker, dest_msg); -} - -static int dsync_brain_msg_sync_pair(struct dsync_brain_mailbox_sync *sync) -{ - struct dsync_message *src_msg = &sync->src_msg_iter->msg; - struct dsync_message *dest_msg = &sync->dest_msg_iter->msg; - const char *src_guid, *dest_guid; - unsigned char guid_128_data[GUID_128_SIZE * 2 + 1]; - bool src_expunged, dest_expunged; - - i_assert(sync->src_msg_iter->mailbox_idx == - sync->dest_msg_iter->mailbox_idx); - - src_expunged = (src_msg->flags & DSYNC_MAIL_FLAG_EXPUNGED) != 0; - dest_expunged = (dest_msg->flags & DSYNC_MAIL_FLAG_EXPUNGED) != 0; - - /* If a message is expunged, it's guaranteed to have a 128bit GUID. - If the other message isn't expunged, we'll need to convert its GUID - to the 128bit GUID form (if it's not already) so that we can compare - them. */ - if (src_expunged) { - src_guid = src_msg->guid; - dest_guid = dsync_get_guid_128_str(dest_msg->guid, - guid_128_data, - sizeof(guid_128_data)); - } else if (dest_expunged) { - src_guid = dsync_get_guid_128_str(src_msg->guid, guid_128_data, - sizeof(guid_128_data)); - dest_guid = dest_msg->guid; - } else { - src_guid = src_msg->guid; - dest_guid = dest_msg->guid; - } - - /* FIXME: checking for sync->uid_conflict isn't fully reliable here. - we should be checking if the next matching message pair has a - conflict, not if the previous pair had one. */ - if (src_msg->uid < dest_msg->uid) { - /* message has been expunged from dest. */ - if (src_expunged) { - /* expunged from source already */ - } else if (sync->uid_conflict || sync->brain->backup) { - /* update uid src, copy to dest */ - dsync_brain_msg_sync_conflict(sync->src_msg_iter, - sync->dest_msg_iter, - src_msg); - } else { - /* expunge from source */ - dsync_worker_msg_expunge(sync->src_worker, - src_msg->uid); - } - src_msg->guid = NULL; - return 0; - } else if (src_msg->uid > dest_msg->uid) { - /* message has been expunged from src. */ - if (dest_expunged) { - /* expunged from dest already */ - } else if (sync->uid_conflict && !sync->brain->backup) { - /* update uid in dest, copy to src */ - dsync_brain_msg_sync_conflict(sync->dest_msg_iter, - sync->src_msg_iter, - dest_msg); - } else { - /* expunge from dest */ - dsync_worker_msg_expunge(sync->dest_worker, - dest_msg->uid); - } - dest_msg->guid = NULL; - return 0; - } - - /* UIDs match, but do GUIDs? If either of the GUIDs aren't set, it - means that either the storage doesn't support GUIDs or we're - handling an old-style expunge record. In that case just assume - they match. */ - if (strcmp(src_guid, dest_guid) != 0 && - *src_guid != '\0' && *dest_guid != '\0') { - /* UID conflict. give new UIDs to messages in both src and - dest (if they're not expunged already) */ - sync->uid_conflict = TRUE; - if (!dest_expunged) { - dsync_brain_msg_sync_conflict(sync->dest_msg_iter, - sync->src_msg_iter, - dest_msg); - } - if (!src_expunged) { - dsync_brain_msg_sync_conflict(sync->src_msg_iter, - sync->dest_msg_iter, - src_msg); - } - } else if (dest_expunged) { - /* message expunged from destination */ - if (src_expunged) { - /* expunged from source already */ - } else if (sync->brain->backup) { - dsync_brain_msg_sync_conflict(sync->src_msg_iter, - sync->dest_msg_iter, - src_msg); - } else { - dsync_worker_msg_expunge(sync->src_worker, - src_msg->uid); - } - } else if (src_expunged) { - /* message expunged from source, expunge from destination too */ - dsync_worker_msg_expunge(sync->dest_worker, dest_msg->uid); - } else { - /* message exists in both source and dest, sync metadata */ - dsync_brain_msg_sync_existing(sync, src_msg, dest_msg); - } - src_msg->guid = NULL; - dest_msg->guid = NULL; - return 0; -} - -static bool dsync_brain_msg_sync_mailbox_end(struct dsync_brain_msg_iter *iter1, - struct dsync_brain_msg_iter *iter2) -{ - int ret; - - while ((ret = dsync_brain_msg_iter_next(iter1)) > 0) { - dsync_brain_msg_sync_save(iter2, iter1->mailbox_idx, - &iter1->msg); - iter1->msg.guid = NULL; - } - return ret < 0; -} - -static bool -dsync_brain_msg_sync_mailbox_more(struct dsync_brain_mailbox_sync *sync) -{ - int ret; - - while ((ret = dsync_brain_msg_iter_next_pair(sync)) > 0) { - if (dsync_brain_msg_sync_pair(sync) < 0) - break; - if (dsync_worker_is_output_full(sync->dest_worker)) { - if (dsync_worker_output_flush(sync->dest_worker) <= 0) - return FALSE; - } - } - if (ret == 0) - return FALSE; - - /* finished syncing messages in this mailbox that exist in both source - and destination. if there are messages left, we can't reliably know - if they should be expunged, so just copy them to the other side. */ - if (!sync->brain->backup) { - if (!dsync_brain_msg_sync_mailbox_end(sync->dest_msg_iter, - sync->src_msg_iter)) - return FALSE; - } - if (!dsync_brain_msg_sync_mailbox_end(sync->src_msg_iter, - sync->dest_msg_iter)) - return FALSE; - - /* done with this mailbox. the same iterator is still used for - getting messages from other mailboxes. */ - return TRUE; -} - -void dsync_brain_msg_sync_more(struct dsync_brain_mailbox_sync *sync) -{ - const struct dsync_brain_mailbox *mailboxes; - unsigned int count, mailbox_idx = 0; - - mailboxes = array_get(&sync->mailboxes, &count); - while (dsync_brain_msg_sync_mailbox_more(sync)) { - /* sync the next mailbox */ - sync->uid_conflict = FALSE; - mailbox_idx = ++sync->wanted_mailbox_idx; - if (mailbox_idx >= count) - break; - - dsync_worker_select_mailbox(sync->src_worker, - &mailboxes[mailbox_idx].box); - dsync_worker_select_mailbox(sync->dest_worker, - &mailboxes[mailbox_idx].box); - } - if (mailbox_idx < count) { - /* output buffer is full */ - return; - } - - /* finished with all mailboxes. */ - dsync_worker_set_input_callback(sync->src_msg_iter->worker, NULL, NULL); - dsync_worker_set_output_callback(sync->src_msg_iter->worker, NULL, NULL); - dsync_worker_set_input_callback(sync->dest_msg_iter->worker, NULL, NULL); - dsync_worker_set_output_callback(sync->dest_msg_iter->worker, NULL, NULL); - - if (dsync_worker_msg_iter_deinit(&sync->src_msg_iter->iter) < 0 || - dsync_worker_msg_iter_deinit(&sync->dest_msg_iter->iter) < 0) { - dsync_brain_fail(sync->brain); - return; - } - - dsync_brain_msg_sync_new_msgs(sync); -} - -static void dsync_worker_msg_callback(void *context) -{ - struct dsync_brain_mailbox_sync *sync = context; - - dsync_brain_msg_sync_more(sync); -} - -static struct dsync_brain_msg_iter * -dsync_brain_msg_iter_init(struct dsync_brain_mailbox_sync *sync, - struct dsync_worker *worker, - const mailbox_guid_t mailboxes[], - unsigned int mailbox_count) -{ - struct dsync_brain_msg_iter *iter; - - iter = p_new(sync->pool, struct dsync_brain_msg_iter, 1); - iter->sync = sync; - iter->worker = worker; - i_array_init(&iter->uid_conflicts, 128); - i_array_init(&iter->new_msgs, 128); - iter->guid_hash = hash_table_create(default_pool, sync->pool, 10000, - str_hash, - (hash_cmp_callback_t *)strcmp); - - iter->iter = dsync_worker_msg_iter_init(worker, mailboxes, - mailbox_count); - dsync_worker_set_input_callback(worker, - dsync_worker_msg_callback, sync); - dsync_worker_set_output_callback(worker, - dsync_worker_msg_callback, sync); - if (mailbox_count > 0) { - const struct dsync_brain_mailbox *first; - - first = array_idx(&sync->mailboxes, 0); - dsync_worker_select_mailbox(worker, &first->box); - } - return iter; -} - -static void dsync_brain_msg_iter_deinit(struct dsync_brain_msg_iter *iter) -{ - if (iter->iter != NULL) - (void)dsync_worker_msg_iter_deinit(&iter->iter); - - hash_table_destroy(&iter->guid_hash); - array_free(&iter->uid_conflicts); - array_free(&iter->new_msgs); -} - -static void -get_mailbox_guids(const ARRAY_TYPE(dsync_brain_mailbox) *mailboxes, - ARRAY_TYPE(mailbox_guid) *guids) -{ - const struct dsync_brain_mailbox *brain_box; - - t_array_init(guids, array_count(mailboxes)); - array_foreach(mailboxes, brain_box) - array_append(guids, &brain_box->box.mailbox_guid, 1); -} - -struct dsync_brain_mailbox_sync * -dsync_brain_msg_sync_init(struct dsync_brain *brain, - const ARRAY_TYPE(dsync_brain_mailbox) *mailboxes) -{ - struct dsync_brain_mailbox_sync *sync; - pool_t pool; - - pool = pool_alloconly_create("dsync brain mailbox sync", 1024*256); - sync = p_new(pool, struct dsync_brain_mailbox_sync, 1); - sync->pool = pool; - sync->brain = brain; - sync->src_worker = brain->src_worker; - sync->dest_worker = brain->dest_worker; - - p_array_init(&sync->mailboxes, pool, array_count(mailboxes)); - array_append_array(&sync->mailboxes, mailboxes); - T_BEGIN { - ARRAY_TYPE(mailbox_guid) guids_arr; - const mailbox_guid_t *guids; - unsigned int count; - - get_mailbox_guids(mailboxes, &guids_arr); - - /* initialize message iteration on both workers */ - guids = array_get(&guids_arr, &count); - sync->src_msg_iter = - dsync_brain_msg_iter_init(sync, brain->src_worker, - guids, count); - sync->dest_msg_iter = - dsync_brain_msg_iter_init(sync, brain->dest_worker, - guids, count); - } T_END; - return sync; -} - -void dsync_brain_msg_sync_deinit(struct dsync_brain_mailbox_sync **_sync) -{ - struct dsync_brain_mailbox_sync *sync = *_sync; - - *_sync = NULL; - - dsync_brain_msg_iter_deinit(sync->src_msg_iter); - dsync_brain_msg_iter_deinit(sync->dest_msg_iter); - pool_unref(&sync->pool); -}
--- a/src/doveadm/dsync/dsync-brain-private.h Fri May 04 05:35:36 2012 +0300 +++ b/src/doveadm/dsync/dsync-brain-private.h Tue May 22 23:17:31 2012 +0300 @@ -1,148 +1,99 @@ #ifndef DSYNC_BRAIN_PRIVATE_H #define DSYNC_BRAIN_PRIVATE_H -#include "dsync-data.h" #include "dsync-brain.h" +#include "dsync-mailbox.h" +#include "dsync-mailbox-state.h" + +struct dsync_mailbox_tree_sync_change; enum dsync_state { - DSYNC_STATE_GET_MAILBOXES = 0, - DSYNC_STATE_GET_SUBSCRIPTIONS, - DSYNC_STATE_SYNC_MAILBOXES, - DSYNC_STATE_SYNC_SUBSCRIPTIONS, - DSYNC_STATE_SYNC_MSGS, - DSYNC_STATE_SYNC_MSGS_FLUSH, - DSYNC_STATE_SYNC_MSGS_FLUSH2, - DSYNC_STATE_SYNC_UPDATE_MAILBOXES, - DSYNC_STATE_SYNC_FLUSH, - DSYNC_STATE_SYNC_FLUSH2, - DSYNC_STATE_SYNC_END + DSYNC_STATE_SLAVE_RECV_HANDSHAKE, + DSYNC_STATE_MASTER_SEND_LAST_COMMON, + DSYNC_STATE_SLAVE_RECV_LAST_COMMON, + DSYNC_STATE_SEND_MAILBOX_TREE, + DSYNC_STATE_SEND_MAILBOX_TREE_DELETES, + DSYNC_STATE_RECV_MAILBOX_TREE, + DSYNC_STATE_RECV_MAILBOX_TREE_DELETES, + DSYNC_STATE_MASTER_SEND_MAILBOX, + DSYNC_STATE_SLAVE_RECV_MAILBOX, + DSYNC_STATE_SYNC_MAILS, + DSYNC_STATE_DONE +}; + +enum dsync_box_state { + DSYNC_BOX_STATE_MAILBOX, + DSYNC_BOX_STATE_CHANGES, + DSYNC_BOX_STATE_MAIL_REQUESTS, + DSYNC_BOX_STATE_MAILS, + DSYNC_BOX_STATE_RECV_LAST_COMMON, + DSYNC_BOX_STATE_DONE }; -struct dsync_brain_mailbox_list { - pool_t pool; - struct dsync_brain *brain; - struct dsync_worker *worker; - struct dsync_worker_mailbox_iter *iter; - ARRAY_TYPE(dsync_mailbox) mailboxes; - ARRAY_TYPE(dsync_mailbox) dirs; -}; - -struct dsync_brain_subs_list { +struct dsync_brain { pool_t pool; - struct dsync_brain *brain; - struct dsync_worker *worker; - struct dsync_worker_subs_iter *iter; - ARRAY_DEFINE(subscriptions, struct dsync_worker_subscription); - ARRAY_DEFINE(unsubscriptions, struct dsync_worker_unsubscription); -}; + struct mail_user *user; + struct dsync_slave *slave; + struct mail_namespace *sync_ns; + enum dsync_brain_sync_type sync_type; + + char hierarchy_sep; + struct dsync_mailbox_tree *local_mailbox_tree; + struct dsync_mailbox_tree *remote_mailbox_tree; + struct dsync_mailbox_tree_iter *local_tree_iter; + + enum dsync_state state, pre_box_state; + enum dsync_box_state box_recv_state; + enum dsync_box_state box_send_state; + + struct dsync_transaction_log_scan *log_scan; + struct dsync_mailbox_importer *box_importer; + struct dsync_mailbox_exporter *box_exporter; -struct dsync_brain_guid_instance { - struct dsync_brain_guid_instance *next; - uint32_t uid; - /* mailbox index in dsync_brain_mailbox_list.mailboxes */ - unsigned int mailbox_idx:31; + struct mailbox *box; + struct dsync_mailbox local_dsync_box, remote_dsync_box; + /* list of mailbox states + for master brain: given to brain at init and + for slave brain: received from DSYNC_STATE_SLAVE_RECV_LAST_COMMON */ + ARRAY_TYPE(dsync_mailbox_state) mailbox_states; + /* DSYNC_STATE_MASTER_SEND_LAST_COMMON: current send position */ + unsigned int mailbox_state_idx; + /* state of the mailbox we're currently syncing, changed at + init and deinit */ + struct dsync_mailbox_state mailbox_state; + /* GUID -> dsync_mailbox_state for mailboxes that have already + been synced */ + struct hash_table *remote_mailbox_states; + + unsigned int master_brain:1; + unsigned int guid_requests:1; + unsigned int mails_have_guids:1; + unsigned int changes_during_sync:1; unsigned int failed:1; }; -struct dsync_brain_msg_iter { - struct dsync_brain_mailbox_sync *sync; - struct dsync_worker *worker; - - struct dsync_worker_msg_iter *iter; - struct dsync_message msg; - - unsigned int mailbox_idx; - - /* char *guid -> struct dsync_brain_guid_instance* */ - struct hash_table *guid_hash; - - ARRAY_DEFINE(new_msgs, struct dsync_brain_new_msg); - ARRAY_DEFINE(uid_conflicts, struct dsync_brain_uid_conflict); - unsigned int next_new_msg, next_conflict; - - /* copy operations that failed. indexes point to new_msgs array */ - unsigned int copy_results_left; - unsigned int save_results_left; - - unsigned int msgs_sent:1; - unsigned int adding_msgs:1; -}; - -struct dsync_brain_uid_conflict { - uint32_t mailbox_idx; - uint32_t old_uid, new_uid; -}; - -struct dsync_brain_new_msg { - unsigned int mailbox_idx:30; - /* TRUE if it currently looks like message has been saved/copied. - if copying fails, it sets this back to FALSE and updates - iter->next_new_msg. */ - unsigned int saved:1; - uint32_t orig_uid; - struct dsync_message *msg; -}; - -struct dsync_brain_mailbox { - struct dsync_mailbox box; - struct dsync_mailbox *src; - struct dsync_mailbox *dest; -}; -ARRAY_DEFINE_TYPE(dsync_brain_mailbox, struct dsync_brain_mailbox); - -struct dsync_brain_mailbox_sync { - struct dsync_brain *brain; - pool_t pool; +void dsync_brain_mailbox_trees_init(struct dsync_brain *brain); +void dsync_brain_send_mailbox_tree(struct dsync_brain *brain); +void dsync_brain_send_mailbox_tree_deletes(struct dsync_brain *brain); +bool dsync_brain_recv_mailbox_tree(struct dsync_brain *brain); +bool dsync_brain_recv_mailbox_tree_deletes(struct dsync_brain *brain); +int dsync_brain_mailbox_tree_sync_change(struct dsync_brain *brain, + const struct dsync_mailbox_tree_sync_change *change); - ARRAY_TYPE(dsync_brain_mailbox) mailboxes; - unsigned int wanted_mailbox_idx; - - struct dsync_worker *src_worker; - struct dsync_worker *dest_worker; - - struct dsync_brain_msg_iter *src_msg_iter; - struct dsync_brain_msg_iter *dest_msg_iter; - - unsigned int uid_conflict:1; - unsigned int skip_mailbox:1; -}; - -struct dsync_brain { - struct dsync_worker *src_worker; - struct dsync_worker *dest_worker; - char *mailbox; - enum dsync_brain_flags flags; - - enum dsync_state state; - - struct dsync_brain_mailbox_list *src_mailbox_list; - struct dsync_brain_mailbox_list *dest_mailbox_list; +void dsync_brain_sync_mailbox_deinit(struct dsync_brain *brain); +int dsync_brain_mailbox_alloc(struct dsync_brain *brain, const guid_128_t guid, + struct mailbox **box_r); +void dsync_brain_mailbox_update_pre(struct dsync_brain *brain, + struct mailbox *box, + const struct dsync_mailbox *local_box, + const struct dsync_mailbox *remote_box); +bool dsync_boxes_need_sync(const struct dsync_mailbox *box1, + const struct dsync_mailbox *box2); - struct dsync_brain_subs_list *src_subs_list; - struct dsync_brain_subs_list *dest_subs_list; - - struct dsync_brain_mailbox_sync *mailbox_sync; - struct timeout *to; - - unsigned int failed:1; - unsigned int verbose:1; - unsigned int backup:1; - unsigned int unexpected_changes:1; - unsigned int stdout_tty:1; -}; - -void dsync_brain_fail(struct dsync_brain *brain); - -struct dsync_brain_mailbox_sync * -dsync_brain_msg_sync_init(struct dsync_brain *brain, - const ARRAY_TYPE(dsync_brain_mailbox) *mailboxes); -void dsync_brain_msg_sync_more(struct dsync_brain_mailbox_sync *sync); -void dsync_brain_msg_sync_deinit(struct dsync_brain_mailbox_sync **_sync); - -void dsync_brain_msg_sync_new_msgs(struct dsync_brain_mailbox_sync *sync); - -void dsync_brain_guid_add(struct dsync_brain_msg_iter *iter, - unsigned int mailbox_idx, - const struct dsync_message *msg); +void dsync_brain_master_send_mailbox(struct dsync_brain *brain); +bool dsync_brain_slave_recv_mailbox(struct dsync_brain *brain); +void dsync_brain_sync_mailbox_init_remote(struct dsync_brain *brain, + const struct dsync_mailbox *remote_dsync_box); +bool dsync_brain_sync_mails(struct dsync_brain *brain); #endif
--- a/src/doveadm/dsync/dsync-brain.c Fri May 04 05:35:36 2012 +0300 +++ b/src/doveadm/dsync/dsync-brain.c Tue May 22 23:17:31 2012 +0300 @@ -1,918 +1,287 @@ -/* Copyright (c) 2009-2012 Dovecot authors, see the included COPYING file */ +/* Copyright (c) 2012 Dovecot authors, see the included COPYING file */ #include "lib.h" #include "array.h" #include "hash.h" -#include "dsync-worker.h" +#include "mail-namespace.h" +#include "dsync-mailbox-tree.h" +#include "dsync-slave.h" #include "dsync-brain-private.h" -#include <unistd.h> +static void dsync_brain_run_io(void *context) +{ + struct dsync_brain *brain = context; + bool changed, try_pending; + + if (dsync_slave_has_failed(brain->slave)) { + io_loop_stop(current_ioloop); + brain->failed = TRUE; + return; + } -#define DSYNC_WRONG_DIRECTION_ERROR_MSG \ - "dsync backup: " \ - "Looks like you're trying to run backup in wrong direction. " \ - "Source is empty and destination is not." + try_pending = TRUE; + do { + if (!dsync_brain_run(brain, &changed)) { + io_loop_stop(current_ioloop); + break; + } + if (changed) + try_pending = TRUE; + else if (try_pending) { + if (dsync_slave_has_pending_data(brain->slave)) + changed = TRUE; + try_pending = FALSE; + } + } while (changed); +} -static void -dsync_brain_mailbox_list_deinit(struct dsync_brain_mailbox_list **list); -static void -dsync_brain_subs_list_deinit(struct dsync_brain_subs_list **list); +static struct dsync_brain * +dsync_brain_common_init(struct mail_user *user, struct dsync_slave *slave) +{ + struct dsync_brain *brain; + pool_t pool; + + pool = pool_alloconly_create("dsync brain", 10240); + brain = p_new(pool, struct dsync_brain, 1); + brain->pool = pool; + brain->user = user; + brain->slave = slave; + brain->sync_type = DSYNC_BRAIN_SYNC_TYPE_UNKNOWN; + brain->remote_mailbox_states = + hash_table_create(default_pool, brain->pool, 0, + guid_128_hash, guid_128_cmp); + p_array_init(&brain->mailbox_states, pool, 64); + return brain; +} struct dsync_brain * -dsync_brain_init(struct dsync_worker *src_worker, - struct dsync_worker *dest_worker, - const char *mailbox, enum dsync_brain_flags flags) +dsync_brain_master_init(struct mail_user *user, struct dsync_slave *slave, + struct mail_namespace *sync_ns, + enum dsync_brain_sync_type sync_type, + enum dsync_brain_flags flags, + const char *state) +{ + struct dsync_slave_settings slave_set; + struct dsync_brain *brain; + const char *error; + + i_assert(sync_type != DSYNC_BRAIN_SYNC_TYPE_UNKNOWN); + i_assert(sync_type != DSYNC_BRAIN_SYNC_TYPE_STATE || state != NULL); + + brain = dsync_brain_common_init(user, slave); + brain->sync_type = sync_type; + if (sync_ns != NULL) + brain->sync_ns = sync_ns; + brain->master_brain = TRUE; + brain->mails_have_guids = + (flags & DSYNC_BRAIN_FLAG_MAILS_HAVE_GUIDS) != 0; + brain->guid_requests = + (flags & DSYNC_BRAIN_FLAG_SEND_REQUESTS) != 0; + + brain->state = DSYNC_STATE_SEND_MAILBOX_TREE; + if (sync_type == DSYNC_BRAIN_SYNC_TYPE_STATE) { + if (dsync_mailbox_states_import(&brain->mailbox_states, state, + &error) < 0) { + array_clear(&brain->mailbox_states); + i_error("Saved sync state is invalid, " + "falling back to full sync: %s", error); + brain->sync_type = DSYNC_BRAIN_SYNC_TYPE_FULL; + } else { + brain->state = DSYNC_STATE_MASTER_SEND_LAST_COMMON; + } + } + dsync_brain_mailbox_trees_init(brain); + + memset(&slave_set, 0, sizeof(slave_set)); + slave_set.sync_ns_prefix = sync_ns == NULL ? NULL : sync_ns->prefix; + slave_set.sync_type = sync_type; + slave_set.guid_requests = brain->guid_requests; + slave_set.mails_have_guids = brain->mails_have_guids; + dsync_slave_send_handshake(slave, &slave_set); + + dsync_slave_set_io_callback(slave, dsync_brain_run_io, brain); + return brain; +} + +struct dsync_brain * +dsync_brain_slave_init(struct mail_user *user, struct dsync_slave *slave) { struct dsync_brain *brain; - brain = i_new(struct dsync_brain, 1); - brain->src_worker = src_worker; - brain->dest_worker = dest_worker; - brain->mailbox = i_strdup(mailbox); - brain->flags = flags; - brain->verbose = (flags & DSYNC_BRAIN_FLAG_VERBOSE) != 0; - brain->backup = (flags & DSYNC_BRAIN_FLAG_BACKUP) != 0; - brain->stdout_tty = isatty(STDOUT_FILENO) > 0; + brain = dsync_brain_common_init(user, slave); + brain->state = DSYNC_STATE_SLAVE_RECV_HANDSHAKE; - if ((flags & DSYNC_BRAIN_FLAG_VERBOSE) != 0) { - dsync_worker_set_verbose(src_worker); - dsync_worker_set_verbose(dest_worker); - } + dsync_slave_set_io_callback(slave, dsync_brain_run_io, brain); return brain; } -void dsync_brain_fail(struct dsync_brain *brain) -{ - brain->failed = TRUE; - io_loop_stop(current_ioloop); -} - int dsync_brain_deinit(struct dsync_brain **_brain) { struct dsync_brain *brain = *_brain; - int ret = brain->failed ? -1 : 0; - - if (brain->state != DSYNC_STATE_SYNC_END) - ret = -1; - if (brain->to != NULL) - timeout_remove(&brain->to); - - if (ret < 0) { - /* make sure we unreference save input streams before workers - are deinitialized, so they can destroy the streams */ - dsync_worker_msg_save_cancel(brain->src_worker); - dsync_worker_msg_save_cancel(brain->dest_worker); - } - - if (brain->mailbox_sync != NULL) - dsync_brain_msg_sync_deinit(&brain->mailbox_sync); - - if (brain->src_mailbox_list != NULL) - dsync_brain_mailbox_list_deinit(&brain->src_mailbox_list); - if (brain->dest_mailbox_list != NULL) - dsync_brain_mailbox_list_deinit(&brain->dest_mailbox_list); - - if (brain->src_subs_list != NULL) - dsync_brain_subs_list_deinit(&brain->src_subs_list); - if (brain->dest_subs_list != NULL) - dsync_brain_subs_list_deinit(&brain->dest_subs_list); - - if (dsync_worker_has_failed(brain->src_worker) || - dsync_worker_has_failed(brain->dest_worker)) - ret = -1; + int ret; *_brain = NULL; - i_free(brain->mailbox); - i_free(brain); + + if (dsync_slave_has_failed(brain->slave) || + brain->state != DSYNC_STATE_DONE) + brain->failed = TRUE; + + if (brain->box != NULL) + dsync_brain_sync_mailbox_deinit(brain); + if (brain->local_tree_iter != NULL) + dsync_mailbox_tree_iter_deinit(&brain->local_tree_iter); + + hash_table_destroy(&brain->remote_mailbox_states); + + ret = brain->failed ? -1 : 0; + pool_unref(&brain->pool); return ret; } -static void dsync_brain_mailbox_list_finished(struct dsync_brain *brain) -{ - if (brain->src_mailbox_list->iter != NULL || - brain->dest_mailbox_list->iter != NULL) - return; - - /* both lists are finished */ - brain->state++; - dsync_brain_sync(brain); -} - -static void dsync_worker_mailbox_input(void *context) +static bool dsync_brain_slave_recv_handshake(struct dsync_brain *brain) { - struct dsync_brain_mailbox_list *list = context; - struct dsync_mailbox dsync_box, *dup_box; - int ret; - - while ((ret = dsync_worker_mailbox_iter_next(list->iter, - &dsync_box)) > 0) { - if (list->brain->mailbox != NULL && - strcmp(list->brain->mailbox, dsync_box.name) != 0) - continue; - - dup_box = dsync_mailbox_dup(list->pool, &dsync_box); - if (!dsync_mailbox_is_noselect(dup_box)) - array_append(&list->mailboxes, &dup_box, 1); - else - array_append(&list->dirs, &dup_box, 1); - } - if (ret < 0) { - /* finished listing mailboxes */ - if (dsync_worker_mailbox_iter_deinit(&list->iter) < 0) - dsync_brain_fail(list->brain); - array_sort(&list->mailboxes, dsync_mailbox_p_guid_cmp); - array_sort(&list->dirs, dsync_mailbox_p_name_sha1_cmp); - dsync_brain_mailbox_list_finished(list->brain); - } -} - -static struct dsync_brain_mailbox_list * -dsync_brain_mailbox_list_init(struct dsync_brain *brain, - struct dsync_worker *worker) -{ - struct dsync_brain_mailbox_list *list; - pool_t pool; + const struct dsync_slave_settings *slave_set; - pool = pool_alloconly_create("dsync brain mailbox list", 10240); - list = p_new(pool, struct dsync_brain_mailbox_list, 1); - list->pool = pool; - list->brain = brain; - list->worker = worker; - list->iter = dsync_worker_mailbox_iter_init(worker); - p_array_init(&list->mailboxes, pool, 128); - p_array_init(&list->dirs, pool, 32); - dsync_worker_set_input_callback(worker, dsync_worker_mailbox_input, - list); - return list; -} - -static void -dsync_brain_mailbox_list_deinit(struct dsync_brain_mailbox_list **_list) -{ - struct dsync_brain_mailbox_list *list = *_list; - - *_list = NULL; + i_assert(!brain->master_brain); - if (list->iter != NULL) - (void)dsync_worker_mailbox_iter_deinit(&list->iter); - pool_unref(&list->pool); -} - -static void dsync_brain_subs_list_finished(struct dsync_brain *brain) -{ - if (brain->src_subs_list->iter != NULL || - brain->dest_subs_list->iter != NULL) - return; - - /* both lists are finished */ - brain->state++; - dsync_brain_sync(brain); -} - -static int -dsync_worker_subscription_cmp(const struct dsync_worker_subscription *s1, - const struct dsync_worker_subscription *s2) -{ - return strcmp(s1->vname, s2->vname); -} - -static int -dsync_worker_unsubscription_cmp(const struct dsync_worker_unsubscription *u1, - const struct dsync_worker_unsubscription *u2) -{ - int ret; + if (dsync_slave_recv_handshake(brain->slave, &slave_set) == 0) + return FALSE; - ret = strcmp(u1->ns_prefix, u2->ns_prefix); - return ret != 0 ? ret : - dsync_guid_cmp(&u1->name_sha1, &u2->name_sha1); -} - -static void dsync_worker_subs_input(void *context) -{ - struct dsync_brain_subs_list *list = context; - struct dsync_worker_subscription subs; - struct dsync_worker_unsubscription unsubs; - int ret; - - memset(&subs, 0, sizeof(subs)); - while ((ret = dsync_worker_subs_iter_next(list->iter, &subs)) > 0) { - subs.vname = p_strdup(list->pool, subs.vname); - subs.storage_name = p_strdup(list->pool, subs.storage_name); - subs.ns_prefix = p_strdup(list->pool, subs.ns_prefix); - array_append(&list->subscriptions, &subs, 1); - } - if (ret == 0) - return; - - memset(&unsubs, 0, sizeof(unsubs)); - while ((ret = dsync_worker_subs_iter_next_un(list->iter, - &unsubs)) > 0) { - unsubs.ns_prefix = p_strdup(list->pool, unsubs.ns_prefix); - array_append(&list->unsubscriptions, &unsubs, 1); - } - - if (ret < 0) { - /* finished listing subscriptions */ - if (dsync_worker_subs_iter_deinit(&list->iter) < 0) - dsync_brain_fail(list->brain); - array_sort(&list->subscriptions, - dsync_worker_subscription_cmp); - array_sort(&list->unsubscriptions, - dsync_worker_unsubscription_cmp); - dsync_brain_subs_list_finished(list->brain); - } -} - -static struct dsync_brain_subs_list * -dsync_brain_subs_list_init(struct dsync_brain *brain, - struct dsync_worker *worker) -{ - struct dsync_brain_subs_list *list; - pool_t pool; - - pool = pool_alloconly_create(MEMPOOL_GROWING"dsync brain subs list", - 1024*4); - list = p_new(pool, struct dsync_brain_subs_list, 1); - list->pool = pool; - list->brain = brain; - list->worker = worker; - list->iter = dsync_worker_subs_iter_init(worker); - p_array_init(&list->subscriptions, pool, 128); - p_array_init(&list->unsubscriptions, pool, 64); - dsync_worker_set_input_callback(worker, dsync_worker_subs_input, list); - return list; -} - -static void -dsync_brain_subs_list_deinit(struct dsync_brain_subs_list **_list) -{ - struct dsync_brain_subs_list *list = *_list; - - *_list = NULL; - - if (list->iter != NULL) - (void)dsync_worker_subs_iter_deinit(&list->iter); - pool_unref(&list->pool); -} - -enum dsync_brain_mailbox_action { - DSYNC_BRAIN_MAILBOX_ACTION_NONE, - DSYNC_BRAIN_MAILBOX_ACTION_CREATE, - DSYNC_BRAIN_MAILBOX_ACTION_DELETE -}; - -static void -dsync_brain_mailbox_action(struct dsync_brain *brain, - enum dsync_brain_mailbox_action action, - struct dsync_worker *action_worker, - struct dsync_mailbox *action_box) -{ - struct dsync_mailbox new_box; - - if (brain->backup && action_worker == brain->src_worker) { - /* backup mode: switch actions */ - action_worker = brain->dest_worker; - switch (action) { - case DSYNC_BRAIN_MAILBOX_ACTION_NONE: - break; - case DSYNC_BRAIN_MAILBOX_ACTION_CREATE: - action = DSYNC_BRAIN_MAILBOX_ACTION_DELETE; - break; - case DSYNC_BRAIN_MAILBOX_ACTION_DELETE: - action = DSYNC_BRAIN_MAILBOX_ACTION_CREATE; - break; + if (slave_set->sync_ns_prefix != NULL) { + brain->sync_ns = mail_namespace_find(brain->user->namespaces, + slave_set->sync_ns_prefix); + if (brain->sync_ns == NULL) { + i_error("Requested sync namespace prefix=%s doesn't exist", + slave_set->sync_ns_prefix); + brain->failed = TRUE; + return TRUE; } } + i_assert(brain->sync_type == DSYNC_BRAIN_SYNC_TYPE_UNKNOWN); + brain->sync_type = slave_set->sync_type; + brain->guid_requests = slave_set->guid_requests; + brain->mails_have_guids = slave_set->mails_have_guids; - switch (action) { - case DSYNC_BRAIN_MAILBOX_ACTION_NONE: - break; - case DSYNC_BRAIN_MAILBOX_ACTION_CREATE: - new_box = *action_box; - new_box.uid_next = action_box->uid_validity == 0 ? 0 : 1; - new_box.first_recent_uid = 0; - new_box.highest_modseq = 0; - dsync_worker_create_mailbox(action_worker, &new_box); - break; - case DSYNC_BRAIN_MAILBOX_ACTION_DELETE: - if (!dsync_mailbox_is_noselect(action_box)) - dsync_worker_delete_mailbox(action_worker, action_box); - else - dsync_worker_delete_dir(action_worker, action_box); - break; - } -} + dsync_brain_mailbox_trees_init(brain); -static bool -dsync_mailbox_list_is_empty(const ARRAY_TYPE(dsync_mailbox) *boxes_arr) -{ - struct dsync_mailbox *const *boxes; - unsigned int count; - - boxes = array_get(boxes_arr, &count); - if (count == 0) - return TRUE; - if (count == 1 && strcasecmp(boxes[0]->name, "INBOX") == 0 && - boxes[0]->message_count == 0) - return TRUE; - return FALSE; + if (brain->sync_type == DSYNC_BRAIN_SYNC_TYPE_STATE) + brain->state = DSYNC_STATE_SLAVE_RECV_LAST_COMMON; + else + brain->state = DSYNC_STATE_SEND_MAILBOX_TREE; + return TRUE; } -static void dsync_brain_sync_mailboxes(struct dsync_brain *brain) +static void dsync_brain_master_send_last_common(struct dsync_brain *brain) { - struct dsync_mailbox *const *src_boxes, *const *dest_boxes; - struct dsync_mailbox *action_box = NULL; - struct dsync_worker *action_worker = NULL; - unsigned int src, dest, src_count, dest_count; - enum dsync_brain_mailbox_action action; - bool src_deleted, dest_deleted; - int ret; - - if (brain->backup && - dsync_mailbox_list_is_empty(&brain->src_mailbox_list->mailboxes) && - !dsync_mailbox_list_is_empty(&brain->dest_mailbox_list->mailboxes)) { - i_fatal(DSYNC_WRONG_DIRECTION_ERROR_MSG); - } + const struct dsync_mailbox_state *states; + unsigned int count; + enum dsync_slave_send_ret ret = DSYNC_SLAVE_SEND_RET_OK; - /* create/delete missing mailboxes. the mailboxes are sorted by - GUID, so we can do this quickly. */ - src_boxes = array_get(&brain->src_mailbox_list->mailboxes, &src_count); - dest_boxes = array_get(&brain->dest_mailbox_list->mailboxes, &dest_count); - for (src = dest = 0; src < src_count && dest < dest_count; ) { - action = DSYNC_BRAIN_MAILBOX_ACTION_NONE; - src_deleted = (src_boxes[src]->flags & - DSYNC_MAILBOX_FLAG_DELETED_MAILBOX) != 0; - dest_deleted = (dest_boxes[dest]->flags & - DSYNC_MAILBOX_FLAG_DELETED_MAILBOX) != 0; - ret = dsync_mailbox_guid_cmp(src_boxes[src], - dest_boxes[dest]); - if (ret < 0) { - /* exists only in source */ - if (!src_deleted) { - action = DSYNC_BRAIN_MAILBOX_ACTION_CREATE; - action_worker = brain->dest_worker; - action_box = src_boxes[src]; - } - src++; - } else if (ret > 0) { - /* exists only in dest */ - if (!dest_deleted) { - action = DSYNC_BRAIN_MAILBOX_ACTION_CREATE; - action_worker = brain->src_worker; - action_box = dest_boxes[dest]; - } - dest++; - } else if (src_deleted) { - /* delete from dest too */ - if (!dest_deleted) { - action = DSYNC_BRAIN_MAILBOX_ACTION_DELETE; - action_worker = brain->dest_worker; - action_box = dest_boxes[dest]; - } - src++; dest++; - } else if (dest_deleted) { - /* delete from src too */ - action = DSYNC_BRAIN_MAILBOX_ACTION_DELETE; - action_worker = brain->src_worker; - action_box = src_boxes[src]; - src++; dest++; - } else { - src++; dest++; - } - dsync_brain_mailbox_action(brain, action, - action_worker, action_box); - } - for (; src < src_count; src++) { - if ((src_boxes[src]->flags & - DSYNC_MAILBOX_FLAG_DELETED_MAILBOX) != 0) - continue; + i_assert(brain->master_brain); - dsync_brain_mailbox_action(brain, - DSYNC_BRAIN_MAILBOX_ACTION_CREATE, - brain->dest_worker, src_boxes[src]); - } - for (; dest < dest_count; dest++) { - if ((dest_boxes[dest]->flags & - DSYNC_MAILBOX_FLAG_DELETED_MAILBOX) != 0) - continue; - - dsync_brain_mailbox_action(brain, - DSYNC_BRAIN_MAILBOX_ACTION_CREATE, - brain->src_worker, dest_boxes[dest]); + states = array_get(&brain->mailbox_states, &count); + while (brain->mailbox_state_idx < count) { + if (ret == DSYNC_SLAVE_SEND_RET_FULL) + return; + ret = dsync_slave_send_mailbox_state(brain->slave, + &states[brain->mailbox_state_idx++]); } -} - -static void dsync_brain_sync_dirs(struct dsync_brain *brain) -{ - struct dsync_mailbox *const *src_boxes, *const *dest_boxes, *action_box; - unsigned int src, dest, src_count, dest_count; - enum dsync_brain_mailbox_action action; - struct dsync_worker *action_worker = NULL; - bool src_deleted, dest_deleted; - int ret; - - /* create/delete missing directories. */ - src_boxes = array_get(&brain->src_mailbox_list->dirs, &src_count); - dest_boxes = array_get(&brain->dest_mailbox_list->dirs, &dest_count); - for (src = dest = 0; src < src_count && dest < dest_count; ) { - action = DSYNC_BRAIN_MAILBOX_ACTION_NONE; - action_box = NULL; - - src_deleted = (src_boxes[src]->flags & - DSYNC_MAILBOX_FLAG_DELETED_DIR) != 0; - dest_deleted = (dest_boxes[dest]->flags & - DSYNC_MAILBOX_FLAG_DELETED_DIR) != 0; - ret = memcmp(src_boxes[src]->name_sha1.guid, - dest_boxes[dest]->name_sha1.guid, - sizeof(src_boxes[src]->name_sha1.guid)); - if (ret < 0) { - /* exists only in source */ - if (!src_deleted) { - action = DSYNC_BRAIN_MAILBOX_ACTION_CREATE; - action_worker = brain->dest_worker; - action_box = src_boxes[src]; - } - src++; - } else if (ret > 0) { - /* exists only in dest */ - if (!dest_deleted) { - action = DSYNC_BRAIN_MAILBOX_ACTION_CREATE; - action_worker = brain->src_worker; - action_box = dest_boxes[dest]; - } - dest++; - } else if (src_deleted) { - /* delete from dest too */ - if (!dest_deleted) { - action = DSYNC_BRAIN_MAILBOX_ACTION_DELETE; - action_worker = brain->dest_worker; - action_box = dest_boxes[dest]; - } - src++; dest++; - } else if (dest_deleted) { - /* delete from src too */ - action = DSYNC_BRAIN_MAILBOX_ACTION_DELETE; - action_worker = brain->src_worker; - action_box = src_boxes[src]; - src++; dest++; - } else { - src++; dest++; - } - i_assert(action_box == NULL || - dsync_mailbox_is_noselect(action_box)); - dsync_brain_mailbox_action(brain, action, - action_worker, action_box); - } - for (; src < src_count; src++) { - if ((src_boxes[src]->flags & - DSYNC_MAILBOX_FLAG_DELETED_DIR) != 0) - continue; - - dsync_brain_mailbox_action(brain, - DSYNC_BRAIN_MAILBOX_ACTION_CREATE, - brain->dest_worker, src_boxes[src]); - } - for (; dest < dest_count; dest++) { - if ((dest_boxes[dest]->flags & - DSYNC_MAILBOX_FLAG_DELETED_DIR) != 0) - continue; - - dsync_brain_mailbox_action(brain, - DSYNC_BRAIN_MAILBOX_ACTION_CREATE, - brain->src_worker, dest_boxes[dest]); - } + dsync_slave_send_end_of_list(brain->slave); + brain->state = DSYNC_STATE_SEND_MAILBOX_TREE; + brain->mailbox_state_idx = 0; } -static bool -dsync_brain_is_unsubscribed(struct dsync_brain_subs_list *list, - const struct dsync_worker_subscription *subs, - time_t *last_change_r) -{ - const struct dsync_worker_unsubscription *unsubs; - struct dsync_worker_unsubscription lookup; - - lookup.ns_prefix = subs->ns_prefix; - dsync_str_sha_to_guid(subs->storage_name, &lookup.name_sha1); - unsubs = array_bsearch(&list->unsubscriptions, &lookup, - dsync_worker_unsubscription_cmp); - if (unsubs == NULL) { - *last_change_r = 0; - return FALSE; - } else if (unsubs->last_change <= subs->last_change) { - *last_change_r = subs->last_change; - return FALSE; - } else { - *last_change_r = unsubs->last_change; - return TRUE; - } -} - -static void dsync_brain_sync_subscriptions(struct dsync_brain *brain) +static bool dsync_brain_slave_recv_last_common(struct dsync_brain *brain) { - const struct dsync_worker_subscription *src_subs, *dest_subs; - const struct dsync_worker_subscription *action_subs; - struct dsync_worker *action_worker; - unsigned int src, dest, src_count, dest_count; - time_t last_change; - bool subscribe; - int ret; + struct dsync_mailbox_state state; + enum dsync_slave_recv_ret ret; + bool changed = FALSE; - /* subscriptions are sorted by name. */ - src_subs = array_get(&brain->src_subs_list->subscriptions, &src_count); - dest_subs = array_get(&brain->dest_subs_list->subscriptions, &dest_count); - for (src = dest = 0;; ) { - if (src == src_count) { - if (dest == dest_count) - break; - ret = 1; - } else if (dest == dest_count) { - ret = -1; - } else { - ret = strcmp(src_subs[src].vname, - dest_subs[dest].vname); - if (ret == 0) { - src++; dest++; - continue; - } - } + i_assert(!brain->master_brain); - if (ret < 0) { - /* subscribed only in source */ - action_subs = &src_subs[src]; - if (dsync_brain_is_unsubscribed(brain->dest_subs_list, - &src_subs[src], - &last_change)) { - action_worker = brain->src_worker; - subscribe = FALSE; - } else { - action_worker = brain->dest_worker; - subscribe = TRUE; - } - src++; - } else { - /* subscribed only in dest */ - action_subs = &dest_subs[dest]; - if (dsync_brain_is_unsubscribed(brain->src_subs_list, - &dest_subs[dest], - &last_change)) { - action_worker = brain->dest_worker; - subscribe = FALSE; - } else { - action_worker = brain->src_worker; - subscribe = TRUE; - } - dest++; - } - - if (brain->backup && action_worker == brain->src_worker) { - /* backup mode: switch action */ - action_worker = brain->dest_worker; - subscribe = !subscribe; - last_change = ioloop_time; - } - dsync_worker_set_subscribed(action_worker, action_subs->vname, - last_change, subscribe); + while ((ret = dsync_slave_recv_mailbox_state(brain->slave, &state)) > 0) { + array_append(&brain->mailbox_states, &state, 1); + changed = TRUE; } -} - -static bool dsync_mailbox_has_changed_msgs(struct dsync_brain *brain, - const struct dsync_mailbox *box1, - const struct dsync_mailbox *box2) -{ - const char *name = *box1->name != '\0' ? box1->name : box2->name; - - if (box1->uid_validity != box2->uid_validity) { - if (brain->verbose) { - i_info("%s: uidvalidity changed: %u != %u", name, - box1->uid_validity, box2->uid_validity); - } - return TRUE; - } - if (box1->uid_next != box2->uid_next) { - if (brain->verbose) { - i_info("%s: uidnext changed: %u != %u", name, - box1->uid_next, box2->uid_next); - } - return TRUE; + if (ret == DSYNC_SLAVE_RECV_RET_FINISHED) { + brain->state = DSYNC_STATE_SEND_MAILBOX_TREE; + changed = TRUE; } - if (box1->highest_modseq != box2->highest_modseq) { - if (brain->verbose) { - i_info("%s: highest_modseq changed: %llu != %llu", name, - (unsigned long long)box1->highest_modseq, - (unsigned long long)box2->highest_modseq); - } - return TRUE; - } - if (box1->message_count != box2->message_count) { - if (brain->verbose) { - i_info("%s: message_count changed: %u != %u", name, - box1->message_count, box2->message_count); - } - return TRUE; - } - return FALSE; -} - -static bool dsync_mailbox_has_changes(struct dsync_brain *brain, - const struct dsync_mailbox *box1, - const struct dsync_mailbox *box2) -{ - if (strcmp(box1->name, box2->name) != 0) - return TRUE; - return dsync_mailbox_has_changed_msgs(brain, box1, box2); + return changed; } -static void -dsync_brain_get_changed_mailboxes(struct dsync_brain *brain, - ARRAY_TYPE(dsync_brain_mailbox) *brain_boxes, - bool full_sync) +static bool dsync_brain_run_real(struct dsync_brain *brain, bool *changed_r) { - struct dsync_mailbox *const *src_boxes, *const *dest_boxes; - struct dsync_brain_mailbox *brain_box; - unsigned int src, dest, src_count, dest_count; - bool src_deleted, dest_deleted; - int ret; - - src_boxes = array_get(&brain->src_mailbox_list->mailboxes, &src_count); - dest_boxes = array_get(&brain->dest_mailbox_list->mailboxes, &dest_count); + bool changed = FALSE, ret = TRUE; - for (src = dest = 0; src < src_count && dest < dest_count; ) { - src_deleted = (src_boxes[src]->flags & - DSYNC_MAILBOX_FLAG_DELETED_MAILBOX) != 0; - dest_deleted = (dest_boxes[dest]->flags & - DSYNC_MAILBOX_FLAG_DELETED_MAILBOX) != 0; - - ret = dsync_mailbox_guid_cmp(src_boxes[src], dest_boxes[dest]); - if (ret == 0) { - if ((full_sync || - dsync_mailbox_has_changes(brain, src_boxes[src], - dest_boxes[dest])) && - !src_deleted && !dest_deleted) { - brain_box = array_append_space(brain_boxes); - brain_box->box = *src_boxes[src]; + if (brain->failed) + return FALSE; - brain_box->box.highest_modseq = - I_MAX(src_boxes[src]->highest_modseq, - dest_boxes[dest]->highest_modseq); - brain_box->box.uid_next = - I_MAX(src_boxes[src]->uid_next, - dest_boxes[dest]->uid_next); - brain_box->src = src_boxes[src]; - brain_box->dest = dest_boxes[dest]; - } - src++; dest++; - } else if (ret < 0) { - /* exists only in source */ - if (!src_deleted) { - brain_box = array_append_space(brain_boxes); - brain_box->box = *src_boxes[src]; - brain_box->src = src_boxes[src]; - if (brain->verbose) { - i_info("%s: only in source (guid=%s)", - brain_box->box.name, - dsync_guid_to_str(&brain_box->box.mailbox_guid)); - } - } - src++; - } else { - /* exists only in dest */ - if (!dest_deleted) { - brain_box = array_append_space(brain_boxes); - brain_box->box = *dest_boxes[dest]; - brain_box->dest = dest_boxes[dest]; - if (brain->verbose) { - i_info("%s: only in dest (guid=%s)", - brain_box->box.name, - dsync_guid_to_str(&brain_box->box.mailbox_guid)); - } - } - dest++; - } + switch (brain->state) { + case DSYNC_STATE_SLAVE_RECV_HANDSHAKE: + changed = dsync_brain_slave_recv_handshake(brain); + break; + case DSYNC_STATE_MASTER_SEND_LAST_COMMON: + dsync_brain_master_send_last_common(brain); + changed = TRUE; + break; + case DSYNC_STATE_SLAVE_RECV_LAST_COMMON: + changed = dsync_brain_slave_recv_last_common(brain); + break; + case DSYNC_STATE_SEND_MAILBOX_TREE: + dsync_brain_send_mailbox_tree(brain); + changed = TRUE; + break; + case DSYNC_STATE_RECV_MAILBOX_TREE: + changed = dsync_brain_recv_mailbox_tree(brain); + break; + case DSYNC_STATE_SEND_MAILBOX_TREE_DELETES: + dsync_brain_send_mailbox_tree_deletes(brain); + changed = TRUE; + break; + case DSYNC_STATE_RECV_MAILBOX_TREE_DELETES: + changed = dsync_brain_recv_mailbox_tree_deletes(brain); + break; + case DSYNC_STATE_MASTER_SEND_MAILBOX: + dsync_brain_master_send_mailbox(brain); + changed = TRUE; + break; + case DSYNC_STATE_SLAVE_RECV_MAILBOX: + changed = dsync_brain_slave_recv_mailbox(brain); + break; + case DSYNC_STATE_SYNC_MAILS: + changed = dsync_brain_sync_mails(brain); + break; + case DSYNC_STATE_DONE: + changed = TRUE; + ret = FALSE; + break; } - for (; src < src_count; src++) { - if ((src_boxes[src]->flags & - DSYNC_MAILBOX_FLAG_DELETED_MAILBOX) != 0) - continue; - brain_box = array_append_space(brain_boxes); - brain_box->box = *src_boxes[src]; - brain_box->src = src_boxes[src]; - if (brain->verbose) { - i_info("%s: only in source (guid=%s)", - brain_box->box.name, - dsync_guid_to_str(&brain_box->box.mailbox_guid)); - } - } - for (; dest < dest_count; dest++) { - if ((dest_boxes[dest]->flags & - DSYNC_MAILBOX_FLAG_DELETED_MAILBOX) != 0) - continue; - - brain_box = array_append_space(brain_boxes); - brain_box->box = *dest_boxes[dest]; - brain_box->dest = dest_boxes[dest]; - if (brain->verbose) { - i_info("%s: only in dest (guid=%s)", - brain_box->box.name, - dsync_guid_to_str(&brain_box->box.mailbox_guid)); - } - } -} - -static bool dsync_brain_sync_msgs(struct dsync_brain *brain) -{ - ARRAY_TYPE(dsync_brain_mailbox) mailboxes; - pool_t pool; - bool ret; - - pool = pool_alloconly_create(MEMPOOL_GROWING"dsync changed mailboxes", - 10240); - p_array_init(&mailboxes, pool, 128); - dsync_brain_get_changed_mailboxes(brain, &mailboxes, - (brain->flags & DSYNC_BRAIN_FLAG_FULL_SYNC) != 0); - if (array_count(&mailboxes) > 0) { - brain->mailbox_sync = - dsync_brain_msg_sync_init(brain, &mailboxes); - dsync_brain_msg_sync_more(brain->mailbox_sync); - ret = TRUE; - } else { - ret = FALSE; - } - pool_unref(&pool); - return ret; -} - -static void -dsync_brain_sync_rename_mailbox(struct dsync_brain *brain, - const struct dsync_brain_mailbox *mailbox) -{ - if (mailbox->src->last_change > mailbox->dest->last_change || - brain->backup) { - dsync_worker_rename_mailbox(brain->dest_worker, - &mailbox->box.mailbox_guid, - mailbox->src); - } else { - dsync_worker_rename_mailbox(brain->src_worker, - &mailbox->box.mailbox_guid, - mailbox->dest); - } + *changed_r = changed; + return brain->failed ? FALSE : ret; } -static void -dsync_brain_sync_update_mailboxes(struct dsync_brain *brain) +bool dsync_brain_run(struct dsync_brain *brain, bool *changed_r) { - const struct dsync_brain_mailbox *mailbox; - bool failed_changes = dsync_brain_has_unexpected_changes(brain) || - dsync_worker_has_failed(brain->src_worker) || - dsync_worker_has_failed(brain->dest_worker); - - if (brain->mailbox_sync == NULL) { - /* no mailboxes changed */ - return; - } - - array_foreach(&brain->mailbox_sync->mailboxes, mailbox) { - /* don't update mailboxes if any changes had failed. - for example if some messages couldn't be saved, we don't - want to increase the next_uid to jump over them */ - if (!brain->backup && !failed_changes) { - dsync_worker_update_mailbox(brain->src_worker, - &mailbox->box); - } - if (!failed_changes) { - dsync_worker_update_mailbox(brain->dest_worker, - &mailbox->box); - } + bool ret; - if (mailbox->src != NULL && mailbox->dest != NULL && - strcmp(mailbox->src->name, mailbox->dest->name) != 0) - dsync_brain_sync_rename_mailbox(brain, mailbox); - } -} - -static void dsync_brain_worker_finished(bool success, void *context) -{ - struct dsync_brain *brain = context; + *changed_r = FALSE; - switch (brain->state) { - case DSYNC_STATE_SYNC_MSGS_FLUSH: - case DSYNC_STATE_SYNC_MSGS_FLUSH2: - case DSYNC_STATE_SYNC_FLUSH: - case DSYNC_STATE_SYNC_FLUSH2: - break; - default: - i_panic("dsync brain state=%d", brain->state); - } - - if (!success) - dsync_brain_fail(brain); - - brain->state++; - if (brain->to == NULL && (brain->flags & DSYNC_BRAIN_FLAG_LOCAL) == 0) - brain->to = timeout_add(0, dsync_brain_sync, brain); -} - -void dsync_brain_sync(struct dsync_brain *brain) -{ - if (dsync_worker_has_failed(brain->src_worker) || - dsync_worker_has_failed(brain->dest_worker)) { - /* we can't safely continue, especially with backup */ - return; + if (dsync_slave_has_failed(brain->slave)) { + brain->failed = TRUE; + return FALSE; } - if (brain->to != NULL) - timeout_remove(&brain->to); - switch (brain->state) { - case DSYNC_STATE_GET_MAILBOXES: - i_assert(brain->src_mailbox_list == NULL); - brain->src_mailbox_list = - dsync_brain_mailbox_list_init(brain, brain->src_worker); - brain->dest_mailbox_list = - dsync_brain_mailbox_list_init(brain, brain->dest_worker); - dsync_worker_mailbox_input(brain->src_mailbox_list); - dsync_worker_mailbox_input(brain->dest_mailbox_list); - break; - case DSYNC_STATE_GET_SUBSCRIPTIONS: - i_assert(brain->src_subs_list == NULL); - brain->src_subs_list = - dsync_brain_subs_list_init(brain, brain->src_worker); - brain->dest_subs_list = - dsync_brain_subs_list_init(brain, brain->dest_worker); - dsync_worker_subs_input(brain->src_subs_list); - dsync_worker_subs_input(brain->dest_subs_list); - break; - case DSYNC_STATE_SYNC_MAILBOXES: - dsync_worker_set_input_callback(brain->src_worker, NULL, NULL); - dsync_worker_set_input_callback(brain->dest_worker, NULL, NULL); - - dsync_brain_sync_mailboxes(brain); - dsync_brain_sync_dirs(brain); - brain->state++; - /* fall through */ - case DSYNC_STATE_SYNC_SUBSCRIPTIONS: - dsync_brain_sync_subscriptions(brain); - brain->state++; - /* fall through */ - case DSYNC_STATE_SYNC_MSGS: - if (dsync_brain_sync_msgs(brain)) - break; - brain->state++; - /* no mailboxes changed */ - case DSYNC_STATE_SYNC_MSGS_FLUSH: - /* wait until all saves are done, so we don't try to close - the mailbox too early */ - dsync_worker_finish(brain->src_worker, - dsync_brain_worker_finished, brain); - dsync_worker_finish(brain->dest_worker, - dsync_brain_worker_finished, brain); - break; - case DSYNC_STATE_SYNC_MSGS_FLUSH2: - break; - case DSYNC_STATE_SYNC_UPDATE_MAILBOXES: - dsync_brain_sync_update_mailboxes(brain); - brain->state++; - /* fall through */ - case DSYNC_STATE_SYNC_FLUSH: - dsync_worker_finish(brain->src_worker, - dsync_brain_worker_finished, brain); - dsync_worker_finish(brain->dest_worker, - dsync_brain_worker_finished, brain); - break; - case DSYNC_STATE_SYNC_FLUSH2: - break; - case DSYNC_STATE_SYNC_END: - io_loop_stop(current_ioloop); - break; - default: - i_unreached(); - } -} - -void dsync_brain_sync_all(struct dsync_brain *brain) -{ - enum dsync_state old_state; - - while (brain->state != DSYNC_STATE_SYNC_END) { - old_state = brain->state; - dsync_brain_sync(brain); - - if (dsync_brain_has_failed(brain)) - break; - - i_assert(brain->state != old_state); - } -} - -bool dsync_brain_has_unexpected_changes(struct dsync_brain *brain) -{ - return brain->unexpected_changes || - dsync_worker_has_unexpected_changes(brain->src_worker) || - dsync_worker_has_unexpected_changes(brain->dest_worker); + T_BEGIN { + ret = dsync_brain_run_real(brain, changed_r); + } T_END; + if (!brain->failed) + dsync_slave_flush(brain->slave); + return ret; } bool dsync_brain_has_failed(struct dsync_brain *brain) { - return brain->failed || - dsync_worker_has_failed(brain->src_worker) || - dsync_worker_has_failed(brain->dest_worker); + return brain->failed; }
--- a/src/doveadm/dsync/dsync-brain.h Fri May 04 05:35:36 2012 +0300 +++ b/src/doveadm/dsync/dsync-brain.h Tue May 22 23:17:31 2012 +0300 @@ -1,28 +1,42 @@ #ifndef DSYNC_BRAIN_H #define DSYNC_BRAIN_H +struct mail_namespace; +struct mail_user; +struct dsync_slave; + enum dsync_brain_flags { - DSYNC_BRAIN_FLAG_FULL_SYNC = 0x01, - DSYNC_BRAIN_FLAG_VERBOSE = 0x02, - /* Run in backup mode. All changes from src are forced into dest, - discarding any potential changes in dest. */ - DSYNC_BRAIN_FLAG_BACKUP = 0x04, - /* Run in "local mode". Don't use ioloop. */ - DSYNC_BRAIN_FLAG_LOCAL = 0x08 + DSYNC_BRAIN_FLAG_MAILS_HAVE_GUIDS = 0x01, + DSYNC_BRAIN_FLAG_SEND_REQUESTS = 0x02 }; -struct dsync_worker; +enum dsync_brain_sync_type { + DSYNC_BRAIN_SYNC_TYPE_UNKNOWN, + /* Go through all mailboxes to make sure everything is synced */ + DSYNC_BRAIN_SYNC_TYPE_FULL, + /* Go through all mailboxes that have changed (based on UIDVALIDITY, + UIDNEXT, HIGHESTMODSEQ). If both sides have had equal amount of + changes in some mailbox, it may get incorrectly skipped. */ + DSYNC_BRAIN_SYNC_TYPE_CHANGED, + /* Use saved state to find out what has changed. */ + DSYNC_BRAIN_SYNC_TYPE_STATE +}; struct dsync_brain * -dsync_brain_init(struct dsync_worker *src_worker, - struct dsync_worker *dest_worker, - const char *mailbox, enum dsync_brain_flags flags); +dsync_brain_master_init(struct mail_user *user, struct dsync_slave *slave, + struct mail_namespace *sync_ns, + enum dsync_brain_sync_type sync_type, + enum dsync_brain_flags flags, + const char *state); +struct dsync_brain * +dsync_brain_slave_init(struct mail_user *user, struct dsync_slave *slave); +/* Returns 0 if everything was successful, -1 if syncing failed in some way */ int dsync_brain_deinit(struct dsync_brain **brain); -void dsync_brain_sync(struct dsync_brain *brain); -void dsync_brain_sync_all(struct dsync_brain *brain); - -bool dsync_brain_has_unexpected_changes(struct dsync_brain *brain); +/* Returns TRUE if brain needs to run more, FALSE if it's finished. + changed_r is TRUE if anything happened during this run. */ +bool dsync_brain_run(struct dsync_brain *brain, bool *changed_r); +/* Returns TRUE if brain has failed, and there's no point in continuing. */ bool dsync_brain_has_failed(struct dsync_brain *brain); #endif
--- a/src/doveadm/dsync/dsync-data.c Fri May 04 05:35:36 2012 +0300 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,146 +0,0 @@ -/* Copyright (c) 2009-2012 Dovecot authors, see the included COPYING file */ - -#include "lib.h" -#include "array.h" -#include "hex-binary.h" -#include "sha1.h" -#include "dsync-data.h" - -struct dsync_mailbox * -dsync_mailbox_dup(pool_t pool, const struct dsync_mailbox *box) -{ - struct dsync_mailbox *dest; - const struct mailbox_cache_field *cache_fields = NULL; - struct mailbox_cache_field *dup; - unsigned int i, count = 0; - - dest = p_new(pool, struct dsync_mailbox, 1); - *dest = *box; - dest->name = p_strdup(pool, box->name); - - if (array_is_created(&box->cache_fields)) - cache_fields = array_get(&box->cache_fields, &count); - if (count == 0) - memset(&dest->cache_fields, 0, sizeof(dest->cache_fields)); - else { - p_array_init(&dest->cache_fields, pool, count); - for (i = 0; i < count; i++) { - dup = array_append_space(&dest->cache_fields); - *dup = cache_fields[i]; - dup->name = p_strdup(pool, dup->name); - } - } - return dest; -} - -struct dsync_message * -dsync_message_dup(pool_t pool, const struct dsync_message *msg) -{ - struct dsync_message *dest; - const char **keywords; - unsigned int i, count; - - dest = p_new(pool, struct dsync_message, 1); - *dest = *msg; - dest->guid = p_strdup(pool, msg->guid); - if (msg->keywords != NULL) { - count = str_array_length(msg->keywords); - keywords = p_new(pool, const char *, count+1); - for (i = 0; i < count; i++) - keywords[i] = p_strdup(pool, msg->keywords[i]); - dest->keywords = keywords; - } - return dest; -} - -int dsync_mailbox_guid_cmp(const struct dsync_mailbox *box1, - const struct dsync_mailbox *box2) -{ - return memcmp(box1->mailbox_guid.guid, box2->mailbox_guid.guid, - sizeof(box1->mailbox_guid.guid)); -} - -int dsync_mailbox_p_guid_cmp(struct dsync_mailbox *const *box1, - struct dsync_mailbox *const *box2) -{ - return dsync_mailbox_guid_cmp(*box1, *box2); -} - -int dsync_mailbox_name_sha1_cmp(const struct dsync_mailbox *box1, - const struct dsync_mailbox *box2) -{ - int ret; - - ret = memcmp(box1->name_sha1.guid, box2->name_sha1.guid, - sizeof(box1->name_sha1.guid)); - if (ret != 0) - return ret; - - return strcmp(box1->name, box2->name); -} - -int dsync_mailbox_p_name_sha1_cmp(struct dsync_mailbox *const *box1, - struct dsync_mailbox *const *box2) -{ - return dsync_mailbox_name_sha1_cmp(*box1, *box2); -} - -bool dsync_keyword_list_equals(const char *const *k1, const char *const *k2) -{ - unsigned int i; - - if (k1 == NULL) - return k2 == NULL || k2[0] == NULL; - if (k2 == NULL) - return k1[0] == NULL; - - for (i = 0;; i++) { - if (k1[i] == NULL) - return k2[i] == NULL; - if (k2[i] == NULL) - return FALSE; - - if (strcasecmp(k1[i], k2[i]) != 0) - return FALSE; - } -} - -bool dsync_guid_equals(const mailbox_guid_t *guid1, - const mailbox_guid_t *guid2) -{ - return memcmp(guid1->guid, guid2->guid, sizeof(guid1->guid)) == 0; -} - -int dsync_guid_cmp(const mailbox_guid_t *guid1, const mailbox_guid_t *guid2) -{ - return memcmp(guid1->guid, guid2->guid, sizeof(guid1->guid)); -} - -const char *dsync_guid_to_str(const mailbox_guid_t *guid) -{ - return guid_128_to_string(guid->guid); -} - -const char *dsync_get_guid_128_str(const char *guid, unsigned char *dest, - unsigned int dest_len) -{ - guid_128_t guid_128; - buffer_t guid_128_buf; - - i_assert(dest_len >= GUID_128_SIZE * 2 + 1); - buffer_create_data(&guid_128_buf, dest, dest_len); - mail_generate_guid_128_hash(guid, guid_128); - if (guid_128_is_empty(guid_128)) - return ""; - binary_to_hex_append(&guid_128_buf, guid_128, sizeof(guid_128)); - buffer_append_c(&guid_128_buf, '\0'); - return guid_128_buf.data; -} - -void dsync_str_sha_to_guid(const char *str, mailbox_guid_t *guid) -{ - unsigned char sha[SHA1_RESULTLEN]; - - sha1_get_digest(str, strlen(str), sha); - memcpy(guid->guid, sha, I_MIN(sizeof(guid->guid), sizeof(sha))); -}
--- a/src/doveadm/dsync/dsync-data.h Fri May 04 05:35:36 2012 +0300 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,85 +0,0 @@ -#ifndef DSYNC_DATA_H -#define DSYNC_DATA_H - -#include "mail-storage.h" - -typedef struct { - guid_128_t guid; -} mailbox_guid_t; -ARRAY_DEFINE_TYPE(mailbox_guid, mailbox_guid_t); - -enum dsync_mailbox_flags { - DSYNC_MAILBOX_FLAG_NOSELECT = 0x01, - DSYNC_MAILBOX_FLAG_DELETED_MAILBOX = 0x02, - DSYNC_MAILBOX_FLAG_DELETED_DIR = 0x04 -}; - -struct dsync_mailbox { - const char *name; - char name_sep; - /* 128bit SHA1 sum of mailbox name */ - mailbox_guid_t name_sha1; - /* Mailbox's GUID. Full of zero with \Noselect mailboxes. */ - mailbox_guid_t mailbox_guid; - - uint32_t uid_validity, uid_next, message_count, first_recent_uid; - uint64_t highest_modseq; - /* if mailbox is deleted, this is the deletion timestamp. - otherwise it's the last rename timestamp. */ - time_t last_change; - enum dsync_mailbox_flags flags; - ARRAY_TYPE(mailbox_cache_field) cache_fields; -}; -ARRAY_DEFINE_TYPE(dsync_mailbox, struct dsync_mailbox *); -#define dsync_mailbox_is_noselect(dsync_box) \ - (((dsync_box)->flags & DSYNC_MAILBOX_FLAG_NOSELECT) != 0) - -/* dsync_worker_msg_iter_next() returns also all expunged messages from - the end of mailbox with this flag set. The GUIDs are 128 bit GUIDs saved - to transaction log (mail_generate_guid_128_hash()). */ -#define DSYNC_MAIL_FLAG_EXPUNGED 0x10000000 - -struct dsync_message { - const char *guid; - uint32_t uid; - enum mail_flags flags; - /* keywords are sorted by name */ - const char *const *keywords; - uint64_t modseq; - time_t save_date; -}; - -struct dsync_msg_static_data { - const char *pop3_uidl; - unsigned int pop3_order; - time_t received_date; - struct istream *input; -}; - -struct dsync_mailbox * -dsync_mailbox_dup(pool_t pool, const struct dsync_mailbox *box); - -struct dsync_message * -dsync_message_dup(pool_t pool, const struct dsync_message *msg); - -int dsync_mailbox_guid_cmp(const struct dsync_mailbox *box1, - const struct dsync_mailbox *box2); -int dsync_mailbox_p_guid_cmp(struct dsync_mailbox *const *box1, - struct dsync_mailbox *const *box2); - -int dsync_mailbox_name_sha1_cmp(const struct dsync_mailbox *box1, - const struct dsync_mailbox *box2); -int dsync_mailbox_p_name_sha1_cmp(struct dsync_mailbox *const *box1, - struct dsync_mailbox *const *box2); - -bool dsync_keyword_list_equals(const char *const *k1, const char *const *k2); - -bool dsync_guid_equals(const mailbox_guid_t *guid1, - const mailbox_guid_t *guid2); -int dsync_guid_cmp(const mailbox_guid_t *guid1, const mailbox_guid_t *guid2); -const char *dsync_guid_to_str(const mailbox_guid_t *guid); -const char *dsync_get_guid_128_str(const char *guid, unsigned char *dest, - unsigned int dest_len); -void dsync_str_sha_to_guid(const char *str, mailbox_guid_t *guid); - -#endif
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/doveadm/dsync/dsync-deserializer.c Tue May 22 23:17:31 2012 +0300 @@ -0,0 +1,193 @@ +/* Copyright (c) 2012 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "str.h" +#include "strescape.h" +#include "dsync-serializer.h" +#include "dsync-deserializer.h" + +struct dsync_deserializer { + pool_t pool; + const char *name; + const char *const *required_fields; + const char *const *keys; + unsigned int *required_field_indexes; + unsigned int required_field_count; +}; + +struct dsync_deserializer_decoder { + pool_t pool; + struct dsync_deserializer *deserializer; + const char *const *values; + unsigned int values_count; +}; + +static bool field_find(const char *const *names, const char *name, + unsigned int *idx_r) +{ + unsigned int i; + + for (i = 0; names[i] != NULL; i++) { + if (strcmp(names[i], name) == 0) { + *idx_r = i; + return TRUE; + } + } + return FALSE; +} + +int dsync_deserializer_init(const char *name, const char *const *required_fields, + const char *header_line, + struct dsync_deserializer **deserializer_r, + const char **error_r) +{ + struct dsync_deserializer *deserializer; + const char **dup_required_fields; + unsigned int i, required_count; + pool_t pool; + + *deserializer_r = NULL; + + pool = pool_alloconly_create("dsync deserializer", 1024); + deserializer = p_new(pool, struct dsync_deserializer, 1); + deserializer->pool = pool; + deserializer->name = p_strdup(pool, name); + deserializer->keys = (void *)p_strsplit_tabescaped(pool, header_line); + + deserializer->required_field_count = required_count = + required_fields == NULL ? 0 : + str_array_length(required_fields); + dup_required_fields = p_new(pool, const char *, required_count + 1); + deserializer->required_field_indexes = + p_new(pool, unsigned int, required_count + 1); + for (i = 0; i < required_count; i++) { + dup_required_fields[i] = + p_strdup(pool, required_fields[i]); + if (!field_find(deserializer->keys, required_fields[i], + &deserializer->required_field_indexes[i])) { + *error_r = t_strdup_printf( + "Header missing required field %s", + required_fields[i]); + pool_unref(&pool); + return -1; + } + } + deserializer->required_fields = dup_required_fields; + + *deserializer_r = deserializer; + return 0; +} + +void dsync_deserializer_deinit(struct dsync_deserializer **_deserializer) +{ + struct dsync_deserializer *deserializer = *_deserializer; + + *_deserializer = NULL; + + pool_unref(&deserializer->pool); +} + +int dsync_deserializer_decode_begin(struct dsync_deserializer *deserializer, + const char *input, + struct dsync_deserializer_decoder **decoder_r, + const char **error_r) +{ + struct dsync_deserializer_decoder *decoder; + unsigned int i; + char **values; + pool_t pool; + + *decoder_r = NULL; + + pool = pool_alloconly_create("dsync deserializer decode", 1024); + decoder = p_new(pool, struct dsync_deserializer_decoder, 1); + decoder->pool = pool; + decoder->deserializer = deserializer; + values = p_strsplit_tabescaped(pool, input); + + /* fix NULLs */ + for (i = 0; values[i] != NULL; i++) { + if (values[i][0] == NULL_CHR) { + /* NULL? */ + if (values[i][1] == '\0') + values[i] = NULL; + else + values[i] += 1; + } + } + decoder->values_count = i; + + /* see if all required fields exist */ + for (i = 0; i < deserializer->required_field_count; i++) { + unsigned int ridx = deserializer->required_field_indexes[i]; + + if (ridx >= decoder->values_count || values[ridx] == NULL) { + *error_r = t_strdup_printf("Missing required field %s", + deserializer->required_fields[i]); + pool_unref(&pool); + return -1; + } + } + decoder->values = (void *)values; + + *decoder_r = decoder; + return 0; +} + +static bool +dsync_deserializer_find_field(struct dsync_deserializer *deserializer, + const char *key, unsigned int *idx_r) +{ + unsigned int i; + + for (i = 0; deserializer->keys[i] != NULL; i++) { + if (strcmp(deserializer->keys[i], key) == 0) { + *idx_r = i; + return TRUE; + } + } + return FALSE; +} + +bool dsync_deserializer_decode_try(struct dsync_deserializer_decoder *decoder, + const char *key, const char **value_r) +{ + unsigned int idx; + + if (!dsync_deserializer_find_field(decoder->deserializer, key, &idx) || + idx >= decoder->values_count) { + *value_r = NULL; + return FALSE; + } else { + *value_r = decoder->values[idx]; + return *value_r != NULL; + } +} + +const char * +dsync_deserializer_decode_get(struct dsync_deserializer_decoder *decoder, + const char *key) +{ + const char *value; + + if (!dsync_deserializer_decode_try(decoder, key, &value)) { + i_panic("dsync_deserializer_decode_get() " + "used for non-required key %s", key); + } + return value; +} + +const char * +dsync_deserializer_decoder_get_name(struct dsync_deserializer_decoder *decoder) +{ + return decoder->deserializer->name; +} + +void dsync_deserializer_decode_finish(struct dsync_deserializer_decoder **_decoder) +{ + struct dsync_deserializer_decoder *decoder = *_decoder; + + *_decoder = NULL; + + pool_unref(&decoder->pool); +}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/doveadm/dsync/dsync-deserializer.h Tue May 22 23:17:31 2012 +0300 @@ -0,0 +1,27 @@ +#ifndef DSYNC_DESERIALIZER_H +#define DSYNC_DESERIALIZER_H + +struct dsync_deserializer; +struct dsync_deserializer_decoder; + +int dsync_deserializer_init(const char *name, const char *const *required_fields, + const char *header_line, + struct dsync_deserializer **deserializer_r, + const char **error_r); +void dsync_deserializer_deinit(struct dsync_deserializer **deserializer); + +int dsync_deserializer_decode_begin(struct dsync_deserializer *deserializer, + const char *input, + struct dsync_deserializer_decoder **decoder_r, + const char **error_r); +bool dsync_deserializer_decode_try(struct dsync_deserializer_decoder *decoder, + const char *key, const char **value_r); +/* key must be in required fields. The return value is never NULL. */ +const char * +dsync_deserializer_decode_get(struct dsync_deserializer_decoder *decoder, + const char *key); +const char * +dsync_deserializer_decoder_get_name(struct dsync_deserializer_decoder *decoder); +void dsync_deserializer_decode_finish(struct dsync_deserializer_decoder **decoder); + +#endif
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/doveadm/dsync/dsync-mail.c Tue May 22 23:17:31 2012 +0300 @@ -0,0 +1,84 @@ +/* Copyright (c) 2012 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "hex-binary.h" +#include "md5.h" +#include "istream.h" +#include "message-size.h" +#include "mail-storage.h" +#include "dsync-mail.h" + +int dsync_mail_get_hdr_hash(struct mail *mail, const char **hdr_hash_r) +{ + struct message_size hdr_size; + struct istream *input, *hdr_input; + struct md5_context md5_ctx; + unsigned char md5_result[MD5_RESULTLEN]; + const unsigned char *data; + size_t size; + int ret = 0; + + if (mail_get_hdr_stream(mail, &hdr_size, &input) < 0) + return -1; + + md5_init(&md5_ctx); + hdr_input = i_stream_create_limit(input, hdr_size.physical_size); + while (!i_stream_is_eof(hdr_input)) { + if (i_stream_read_data(hdr_input, &data, &size, 0) == -1) + break; + if (size == 0) + break; + md5_update(&md5_ctx, data, size); + i_stream_skip(hdr_input, size); + } + if (hdr_input->stream_errno != 0) + ret = -1; + i_stream_unref(&hdr_input); + + md5_final(&md5_ctx, md5_result); + *hdr_hash_r = binary_to_hex(md5_result, sizeof(md5_result)); + return ret; +} + +static void +const_string_array_dup(pool_t pool, const ARRAY_TYPE(const_string) *src, + ARRAY_TYPE(const_string) *dest) +{ + const char *const *strings, *str; + unsigned int i, count; + + if (!array_is_created(src)) + return; + + strings = array_get(src, &count); + if (count == 0) + return; + + p_array_init(dest, pool, count); + for (i = 0; i < count; i++) { + str = p_strdup(pool, strings[i]); + array_append(dest, &str, 1); + } +} + +void dsync_mail_change_dup(pool_t pool, const struct dsync_mail_change *src, + struct dsync_mail_change *dest_r) +{ + dest_r->type = src->type; + dest_r->uid = src->uid; + if (src->guid != NULL) { + dest_r->guid = *src->guid == '\0' ? "" : + p_strdup(pool, src->guid); + } + dest_r->hdr_hash = p_strdup(pool, src->hdr_hash); + dest_r->modseq = src->modseq; + dest_r->save_timestamp = src->save_timestamp; + + dest_r->add_flags = src->add_flags; + dest_r->remove_flags = src->remove_flags; + dest_r->final_flags = src->final_flags; + dest_r->keywords_reset = src->keywords_reset; + const_string_array_dup(pool, &src->keyword_changes, + &dest_r->keyword_changes); +}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/doveadm/dsync/dsync-mail.h Tue May 22 23:17:31 2012 +0300 @@ -0,0 +1,74 @@ +#ifndef DSYNC_MAIL_H +#define DSYNC_MAIL_H + +#include "mail-types.h" + +struct mail; + +struct dsync_mail { + /* either GUID="" or uid=0 */ + const char *guid; + uint32_t uid; + + const char *pop3_uidl; + unsigned int pop3_order; + time_t received_date; + + /* Input stream containing the message text, or NULL if all instances + of the message were already expunged from this mailbox. */ + struct istream *input; +}; + +struct dsync_mail_request { + /* either GUID="" or uid=0 */ + const char *guid; + uint32_t uid; +}; + +enum dsync_mail_change_type { + DSYNC_MAIL_CHANGE_TYPE_SAVE, + DSYNC_MAIL_CHANGE_TYPE_EXPUNGE, + DSYNC_MAIL_CHANGE_TYPE_FLAG_CHANGE +}; + +#define KEYWORD_CHANGE_ADD '+' +#define KEYWORD_CHANGE_REMOVE '-' +#define KEYWORD_CHANGE_FINAL '=' + +struct dsync_mail_change { + enum dsync_mail_change_type type; + + uint32_t uid; + /* Message's GUID: + - for expunges either 128bit hex or NULL if unknown + - "" if backend doesn't support GUIDs */ + const char *guid; + /* If GUID is "", this contains hash of the message header, + otherwise NULL */ + const char *hdr_hash; + + /* Message's current modseq (saves, flag changes) */ + uint64_t modseq; + /* Message's save timestamp (saves) */ + time_t save_timestamp; + + /* List of flag/keyword changes: (saves, flag changes) */ + + /* Flags added/removed since last sync, and final flags containing + flags that exist now but haven't changed */ + uint8_t add_flags, remove_flags, final_flags; + /* Remove all keywords before applying changes. This is used only with + old transaction logs, new ones never reset keywords (just explicitly + remove unwanted keywords) */ + bool keywords_reset; + /* +add, -remove, =final. If the flag is both +added and in =final, + it's not not duplicated as =flag to avoid wasting space. */ + ARRAY_TYPE(const_string) keyword_changes; +}; + +int dsync_mail_get_hdr_hash(struct mail *mail, const char **hdr_hash_r); + +void dsync_mail_change_dup(pool_t pool, const struct dsync_mail_change *src, + struct dsync_mail_change *dest_r); + +#endif
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/doveadm/dsync/dsync-mailbox-export.c Tue May 22 23:17:31 2012 +0300 @@ -0,0 +1,692 @@ +/* Copyright (c) 2012 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "hash.h" +#include "mail-index-modseq.h" +#include "mail-storage-private.h" +#include "mail-search-build.h" +#include "dsync-transaction-log-scan.h" +#include "dsync-mail.h" +#include "dsync-mailbox-export.h" + +struct dsync_mail_guid_instances { + ARRAY_TYPE(seq_range) seqs; + bool requested; + bool searched; +}; + +struct dsync_mailbox_exporter { + pool_t pool; + struct mailbox *box; + struct dsync_transaction_log_scan *log_scan; + uint32_t last_common_uid; + uint64_t last_common_modseq; + + struct mailbox_transaction_context *trans; + struct mail_search_context *search_ctx; + + /* GUID => instances */ + struct hash_table *export_guids; + ARRAY_TYPE(seq_range) requested_uids; + unsigned int requested_uid_search_idx; + + ARRAY_TYPE(seq_range) expunged_seqs; + ARRAY_TYPE(const_string) expunged_guids; + unsigned int expunged_guid_idx; + + /* UID => struct dsync_mail_change */ + struct hash_table *changes; + /* changes sorted by UID */ + ARRAY_DEFINE(sorted_changes, struct dsync_mail_change *); + unsigned int change_idx; + uint32_t highest_changed_uid; + + struct dsync_mail_change change; + struct dsync_mail dsync_mail; + + const char *error; + unsigned int body_search_initialized:1; + unsigned int auto_export_mails:1; + unsigned int mails_have_guids:1; + unsigned int return_all_mails:1; +}; + +static int dsync_mail_error(struct dsync_mailbox_exporter *exporter, + struct mail *mail, const char *field) +{ + const char *errstr; + enum mail_error error; + + errstr = mailbox_get_last_error(exporter->box, &error); + if (error == MAIL_ERROR_EXPUNGED) + return 0; + + exporter->error = p_strdup_printf(exporter->pool, + "Can't lookup %s for UID=%u: %s", + field, mail->uid, errstr); + return -1; +} + +static bool +final_keyword_check(struct dsync_mail_change *change, const char *name) +{ + const char *const *changes; + unsigned int i, count; + + changes = array_get(&change->keyword_changes, &count); + for (i = 0; i < count; i++) { + if (strcmp(changes[i]+1, name) == 0) { + if (changes[i][0] == KEYWORD_CHANGE_REMOVE) { + /* a final keyword is marked as removed. + this shouldn't normally happen. */ + array_delete(&change->keyword_changes, i, 1); + break; + } + return TRUE; + } + } + return FALSE; +} + +static void +search_update_flag_changes(struct dsync_mailbox_exporter *exporter, + struct mail *mail, struct dsync_mail_change *change) +{ + const char *const *keywords; + unsigned int i; + + i_assert((change->add_flags & change->remove_flags) == 0); + + change->modseq = mail_get_modseq(mail); + change->final_flags = mail_get_flags(mail) & MAIL_FLAGS_NONRECENT; + + keywords = mail_get_keywords(mail); + if (!array_is_created(&change->keyword_changes) && + keywords[0] != NULL) { + p_array_init(&change->keyword_changes, exporter->pool, + str_array_length(keywords)); + } + for (i = 0; keywords[i] != NULL; i++) { + /* add the final keyword if it's not already there + as +keyword */ + if (!final_keyword_check(change, keywords[i])) { + const char *keyword_change = + p_strdup_printf(exporter->pool, "%c%s", + KEYWORD_CHANGE_FINAL, + keywords[i]); + array_append(&change->keyword_changes, + &keyword_change, 1); + } + } +} + +static int +exporter_get_guids(struct dsync_mailbox_exporter *exporter, + struct mail *mail, const char **guid_r, + const char **hdr_hash_r) +{ + *guid_r = ""; + *hdr_hash_r = NULL; + + /* always try to get GUID, even if we're also getting header hash */ + if (mail_get_special(mail, MAIL_FETCH_GUID, guid_r) < 0) + return dsync_mail_error(exporter, mail, "GUID"); + + if (!exporter->mails_have_guids) { + /* get header hash also */ + if (dsync_mail_get_hdr_hash(mail, hdr_hash_r) < 0) + return dsync_mail_error(exporter, mail, "hdr-stream"); + return 1; + } else if (**guid_r == '\0') { + exporter->error = "Backend doesn't support GUIDs, " + "sync with header hashes instead"; + return -1; + } else { + /* GUIDs are required, we don't need header hash */ + return 1; + } +} + +static int +search_update_flag_change_guid(struct dsync_mailbox_exporter *exporter, + struct mail *mail) +{ + struct dsync_mail_change *change, *log_change; + const char *guid, *hdr_hash; + int ret; + + change = hash_table_lookup(exporter->changes, + POINTER_CAST(mail->uid)); + i_assert(change != NULL); + i_assert(change->type == DSYNC_MAIL_CHANGE_TYPE_FLAG_CHANGE); + + if ((ret = exporter_get_guids(exporter, mail, &guid, &hdr_hash)) < 0) + return -1; + if (ret == 0) { + /* the message was expunged during export */ + memset(change, 0, sizeof(*change)); + change->type = DSYNC_MAIL_CHANGE_TYPE_EXPUNGE; + change->uid = mail->uid; + + /* find its GUID from log if possible */ + log_change = dsync_transaction_log_scan_find_new_expunge( + exporter->log_scan, mail->uid); + if (log_change != NULL) + change->guid = log_change->guid; + } else { + change->guid = *guid == '\0' ? "" : + p_strdup(exporter->pool, guid); + change->hdr_hash = p_strdup(exporter->pool, hdr_hash); + search_update_flag_changes(exporter, mail, change); + } + return 0; +} + +static struct dsync_mail_change * +export_save_change_get(struct dsync_mailbox_exporter *exporter, uint32_t uid) +{ + struct dsync_mail_change *change; + + change = hash_table_lookup(exporter->changes, POINTER_CAST(uid)); + if (change == NULL) { + change = p_new(exporter->pool, struct dsync_mail_change, 1); + change->uid = uid; + hash_table_insert(exporter->changes, POINTER_CAST(uid), change); + } else { + /* move flag changes into a save. this happens only when + last_common_uid isn't known */ + i_assert(change->type == DSYNC_MAIL_CHANGE_TYPE_FLAG_CHANGE); + i_assert(exporter->last_common_uid == 0); + } + + change->type = DSYNC_MAIL_CHANGE_TYPE_SAVE; + return change; +} + +static void +export_add_mail_instance(struct dsync_mailbox_exporter *exporter, + struct dsync_mail_change *change, uint32_t seq) +{ + struct dsync_mail_guid_instances *instances; + + if (exporter->auto_export_mails && !exporter->mails_have_guids) { + /* GUIDs not supported, mail is requested by UIDs */ + seq_range_array_add(&exporter->requested_uids, 0, change->uid); + return; + } + if (*change->guid == '\0') { + /* mail UIDs are manually requested */ + i_assert(!exporter->mails_have_guids); + return; + } + + instances = hash_table_lookup(exporter->export_guids, change->guid); + if (instances == NULL) { + instances = p_new(exporter->pool, + struct dsync_mail_guid_instances, 1); + p_array_init(&instances->seqs, exporter->pool, 2); + hash_table_insert(exporter->export_guids, + p_strdup(exporter->pool, change->guid), + instances); + if (exporter->auto_export_mails) + instances->requested = TRUE; + } + seq_range_array_add(&instances->seqs, 0, seq); +} + +static int +search_add_save(struct dsync_mailbox_exporter *exporter, struct mail *mail) +{ + struct dsync_mail_change *change; + const char *guid, *hdr_hash; + time_t save_timestamp; + int ret; + + /* If message is already expunged here, just skip it */ + if ((ret = exporter_get_guids(exporter, mail, &guid, &hdr_hash)) <= 0) + return ret; + if (mail_get_save_date(mail, &save_timestamp) < 0) + return dsync_mail_error(exporter, mail, "save-date"); + + change = export_save_change_get(exporter, mail->uid); + change->save_timestamp = save_timestamp; + + change->guid = *guid == '\0' ? "" : + p_strdup(exporter->pool, guid); + change->hdr_hash = p_strdup(exporter->pool, hdr_hash); + search_update_flag_changes(exporter, mail, change); + + export_add_mail_instance(exporter, change, mail->seq); + return 1; +} + +static void +dsync_mailbox_export_add_flagchange_uids(struct dsync_mailbox_exporter *exporter, + ARRAY_TYPE(seq_range) *uids) +{ + struct hash_iterate_context *iter; + void *key, *value; + + iter = hash_table_iterate_init(exporter->changes); + while (hash_table_iterate(iter, &key, &value)) { + const struct dsync_mail_change *change = value; + + if (change->type == DSYNC_MAIL_CHANGE_TYPE_FLAG_CHANGE) + seq_range_array_add(uids, 0, change->uid); + } + hash_table_iterate_deinit(&iter); +} + +static void +dsync_mailbox_export_drop_expunged_flag_changes(struct dsync_mailbox_exporter *exporter) +{ + struct hash_iterate_context *iter; + void *key, *value; + + /* any flag changes for UIDs above last_common_uid weren't found by + mailbox search, which means they were already expunged. for some + reason the log scanner found flag changes for the message, but not + the expunge. just remove these. */ + iter = hash_table_iterate_init(exporter->changes); + while (hash_table_iterate(iter, &key, &value)) { + const struct dsync_mail_change *change = value; + + if (change->type == DSYNC_MAIL_CHANGE_TYPE_FLAG_CHANGE && + change->uid > exporter->last_common_uid) + hash_table_remove(exporter->changes, key); + } + hash_table_iterate_deinit(&iter); +} + +static void +dsync_mailbox_export_search(struct dsync_mailbox_exporter *exporter) +{ + struct mail_search_context *search_ctx; + struct mail_search_args *search_args; + struct mail_search_arg *sarg; + struct mail *mail; + int ret; + + search_args = mail_search_build_init(); + sarg = mail_search_build_add(search_args, SEARCH_UIDSET); + p_array_init(&sarg->value.seqset, search_args->pool, 1); + + if (exporter->return_all_mails || exporter->last_common_uid <= 1) { + /* we want to know about all mails */ + seq_range_array_add_range(&sarg->value.seqset, 1, (uint32_t)-1); + } else { + /* lookup GUIDs for messages with flag changes */ + dsync_mailbox_export_add_flagchange_uids(exporter, + &sarg->value.seqset); + /* lookup new messages */ + seq_range_array_add_range(&sarg->value.seqset, + exporter->last_common_uid + 1, + (uint32_t)-1); + } + + exporter->trans = mailbox_transaction_begin(exporter->box, 0); + search_ctx = mailbox_search_init(exporter->trans, search_args, NULL, + 0, NULL); + mail_search_args_unref(&search_args); + + while (mailbox_search_next(search_ctx, &mail)) { + if (mail->uid <= exporter->last_common_uid) + ret = search_update_flag_change_guid(exporter, mail); + else + ret = search_add_save(exporter, mail); + if (ret < 0) + break; + } + + dsync_mailbox_export_drop_expunged_flag_changes(exporter); + + if (mailbox_search_deinit(&search_ctx) < 0 && + exporter->error == NULL) { + exporter->error = p_strdup_printf(exporter->pool, + "Mail search failed: %s", + mailbox_get_last_error(exporter->box, NULL)); + } +} + +static int dsync_mail_change_p_uid_cmp(struct dsync_mail_change *const *c1, + struct dsync_mail_change *const *c2) +{ + if ((*c1)->uid < (*c2)->uid) + return -1; + if ((*c1)->uid > (*c2)->uid) + return 1; + return 0; +} + +static void +dsync_mailbox_export_sort_changes(struct dsync_mailbox_exporter *exporter) +{ + struct hash_iterate_context *iter; + void *key, *value; + + p_array_init(&exporter->sorted_changes, exporter->pool, + hash_table_count(exporter->changes)); + + iter = hash_table_iterate_init(exporter->changes); + while (hash_table_iterate(iter, &key, &value)) { + struct dsync_mail_change *change = value; + array_append(&exporter->sorted_changes, &change, 1); + } + hash_table_iterate_deinit(&iter); + array_sort(&exporter->sorted_changes, dsync_mail_change_p_uid_cmp); +} + +static void +dsync_mailbox_export_log_scan(struct dsync_mailbox_exporter *exporter, + struct dsync_transaction_log_scan *log_scan) +{ + struct hash_table *log_changes; + struct hash_iterate_context *iter; + void *key, *value; + struct dsync_mail_change *dup_change; + + log_changes = dsync_transaction_log_scan_get_hash(log_scan); + if (dsync_transaction_log_scan_has_all_changes(log_scan)) + exporter->return_all_mails = TRUE; + + /* clone the hash table, since we're changing it. */ + exporter->changes = + hash_table_create(default_pool, exporter->pool, + hash_table_count(log_changes), NULL, NULL); + iter = hash_table_iterate_init(log_changes); + while (hash_table_iterate(iter, &key, &value)) { + const struct dsync_mail_change *change = value; + + dup_change = p_new(exporter->pool, struct dsync_mail_change, 1); + *dup_change = *change; + hash_table_insert(exporter->changes, key, dup_change); + if (exporter->highest_changed_uid < change->uid) + exporter->highest_changed_uid = change->uid; + } + hash_table_iterate_deinit(&iter); +} + +struct dsync_mailbox_exporter * +dsync_mailbox_export_init(struct mailbox *box, + struct dsync_transaction_log_scan *log_scan, + uint32_t last_common_uid, + uint64_t last_common_modseq, + enum dsync_mailbox_exporter_flags flags) +{ + struct dsync_mailbox_exporter *exporter; + pool_t pool; + + pool = pool_alloconly_create(MEMPOOL_GROWING"dsync mailbox export", + 4096); + exporter = p_new(pool, struct dsync_mailbox_exporter, 1); + exporter->pool = pool; + exporter->box = box; + exporter->log_scan = log_scan; + exporter->last_common_uid = last_common_uid; + exporter->last_common_modseq = last_common_modseq; + exporter->auto_export_mails = + (flags & DSYNC_MAILBOX_EXPORTER_FLAG_AUTO_EXPORT_MAILS) != 0; + exporter->mails_have_guids = + (flags & DSYNC_MAILBOX_EXPORTER_FLAG_MAILS_HAVE_GUIDS) != 0; + p_array_init(&exporter->requested_uids, pool, 16); + exporter->export_guids = + hash_table_create(default_pool, pool, 0, + str_hash, (hash_cmp_callback_t *)strcmp); + p_array_init(&exporter->expunged_seqs, pool, 16); + p_array_init(&exporter->expunged_guids, pool, 16); + + /* first scan transaction log and save any expunges and flag changes */ + dsync_mailbox_export_log_scan(exporter, log_scan); + /* get saves and also find GUIDs for flag changes */ + dsync_mailbox_export_search(exporter); + /* get the changes sorted by UID */ + dsync_mailbox_export_sort_changes(exporter); + return exporter; +} + +const struct dsync_mail_change * +dsync_mailbox_export_next(struct dsync_mailbox_exporter *exporter) +{ + struct dsync_mail_change *const *changes; + unsigned int count; + + if (exporter->error != NULL) + return NULL; + + changes = array_get(&exporter->sorted_changes, &count); + if (exporter->change_idx == count) + return NULL; + + return changes[exporter->change_idx++]; +} + +static int +dsync_mailbox_export_body_search_init(struct dsync_mailbox_exporter *exporter) +{ + struct mail_search_args *search_args; + struct mail_search_arg *sarg; + struct hash_iterate_context *iter; + const struct seq_range *uids; + void *key, *value; + const struct seq_range *range; + unsigned int i, count; + uint32_t seq, seq1, seq2; + + i_assert(exporter->search_ctx == NULL); + + search_args = mail_search_build_init(); + sarg = mail_search_build_add(search_args, SEARCH_SEQSET); + p_array_init(&sarg->value.seqset, search_args->pool, 128); + + /* get a list of messages we want to fetch. if there are more than one + instance for a GUID, use the first one. */ + iter = hash_table_iterate_init(exporter->export_guids); + while (hash_table_iterate(iter, &key, &value)) { + const char *guid = key; + struct dsync_mail_guid_instances *instances = value; + + if (!instances->requested || + array_count(&instances->seqs) == 0) + continue; + + uids = array_idx(&instances->seqs, 0); + seq = uids[0].seq1; + if (!instances->searched) { + instances->searched = TRUE; + seq_range_array_add(&sarg->value.seqset, 0, seq); + } else if (seq_range_exists(&exporter->expunged_seqs, seq)) { + /* we're on a second round, refetching expunged + messages */ + seq_range_array_remove(&instances->seqs, seq); + seq_range_array_remove(&exporter->expunged_seqs, seq); + if (array_count(&instances->seqs) == 0) { + /* no instances left */ + array_append(&exporter->expunged_guids, + &guid, 1); + continue; + } + uids = array_idx(&instances->seqs, 0); + seq = uids[0].seq1; + seq_range_array_add(&sarg->value.seqset, 0, seq); + } + } + hash_table_iterate_deinit(&iter); + + /* add requested UIDs */ + range = array_get(&exporter->requested_uids, &count); + for (i = exporter->requested_uid_search_idx; i < count; i++) { + mailbox_get_seq_range(exporter->box, + range->seq1, range->seq2, + &seq1, &seq2); + seq_range_array_add_range(&sarg->value.seqset, + seq1, seq2); + } + exporter->requested_uid_search_idx = count; + + exporter->search_ctx = + mailbox_search_init(exporter->trans, search_args, NULL, + MAIL_FETCH_GUID | + MAIL_FETCH_UIDL_BACKEND | + MAIL_FETCH_POP3_ORDER | + MAIL_FETCH_RECEIVED_DATE, NULL); + mail_search_args_unref(&search_args); + return array_count(&sarg->value.seqset) > 0 ? 1 : 0; +} + +static void +dsync_mailbox_export_body_search_deinit(struct dsync_mailbox_exporter *exporter) +{ + if (exporter->search_ctx == NULL) + return; + + if (mailbox_search_deinit(&exporter->search_ctx) < 0 && + exporter->error == NULL) { + exporter->error = p_strdup_printf(exporter->pool, + "Mail search failed: %s", + mailbox_get_last_error(exporter->box, NULL)); + } +} + +static int dsync_mailbox_export_mail(struct dsync_mailbox_exporter *exporter, + struct mail *mail) +{ + struct dsync_mail *dmail = &exporter->dsync_mail; + struct dsync_mail_guid_instances *instances; + const char *guid, *str; + + if (mail_get_special(mail, MAIL_FETCH_GUID, &guid) < 0) + return dsync_mail_error(exporter, mail, "GUID"); + + memset(dmail, 0, sizeof(*dmail)); + if (!seq_range_exists(&exporter->requested_uids, mail->uid)) + dmail->guid = guid; + else { + dmail->uid = mail->uid; + dmail->guid = ""; + } + + instances = *guid == '\0' ? NULL : + hash_table_lookup(exporter->export_guids, guid); + if (instances != NULL) { + /* GUID found */ + } else if (dmail->uid != 0) { + /* mail requested by UID */ + } else { + exporter->error = p_strdup_printf(exporter->pool, + "GUID unexpectedly changed for UID=%u GUID=%s", + mail->uid, guid); + return -1; + } + + if (mail_get_stream(mail, NULL, NULL, &dmail->input) < 0) + return dsync_mail_error(exporter, mail, "body"); + + if (mail_get_special(mail, MAIL_FETCH_UIDL_BACKEND, &dmail->pop3_uidl) < 0) + return dsync_mail_error(exporter, mail, "pop3-uidl"); + if (mail_get_special(mail, MAIL_FETCH_POP3_ORDER, &str) < 0) + return dsync_mail_error(exporter, mail, "pop3-order"); + if (*str != '\0') { + if (str_to_uint(str, &dmail->pop3_order) < 0) + i_unreached(); + } + if (mail_get_received_date(mail, &dmail->received_date) < 0) + return dsync_mail_error(exporter, mail, "received-date"); + + /* this message was successfully returned, don't try retrying it */ + if (instances != NULL) + array_clear(&instances->seqs); + return 1; +} + +void dsync_mailbox_export_want_mail(struct dsync_mailbox_exporter *exporter, + const struct dsync_mail_request *request) +{ + struct dsync_mail_guid_instances *instances; + + i_assert(!exporter->auto_export_mails); + + if (*request->guid == '\0') { + i_assert(request->uid > 0); + seq_range_array_add(&exporter->requested_uids, 0, request->uid); + return; + } + + instances = hash_table_lookup(exporter->export_guids, request->guid); + if (instances == NULL) { + exporter->error = p_strdup_printf(exporter->pool, + "Remote requested unexpected GUID %s", request->guid); + return; + } + instances->requested = TRUE; +} + +const struct dsync_mail * +dsync_mailbox_export_next_mail(struct dsync_mailbox_exporter *exporter) +{ + struct mail *mail; + const char *const *guids; + unsigned int count; + int ret; + + if (exporter->error != NULL) + return NULL; + if (!exporter->body_search_initialized) { + exporter->body_search_initialized = TRUE; + if (dsync_mailbox_export_body_search_init(exporter) < 0) { + i_assert(exporter->error != NULL); + return NULL; + } + } + + while (mailbox_search_next(exporter->search_ctx, &mail)) { + if ((ret = dsync_mailbox_export_mail(exporter, mail)) > 0) + return &exporter->dsync_mail; + if (ret < 0) { + i_assert(exporter->error != NULL); + return NULL; + } + /* the message was expunged. if the GUID has another instance, + try sending it later. */ + seq_range_array_add(&exporter->expunged_seqs, 0, mail->seq); + } + /* if some instances of messages were expunged, retry fetching them + with other instances */ + dsync_mailbox_export_body_search_deinit(exporter); + if ((ret = dsync_mailbox_export_body_search_init(exporter)) < 0) { + i_assert(exporter->error != NULL); + return NULL; + } + if (ret > 0) { + /* not finished yet */ + return dsync_mailbox_export_next_mail(exporter); + } + + /* finished with messages. if there are any expunged messages, + return them */ + guids = array_get(&exporter->expunged_guids, &count); + if (exporter->expunged_guid_idx < count) { + memset(&exporter->dsync_mail, 0, sizeof(exporter->dsync_mail)); + exporter->dsync_mail.guid = + guids[exporter->expunged_guid_idx++]; + return &exporter->dsync_mail; + } + return NULL; +} + +int dsync_mailbox_export_deinit(struct dsync_mailbox_exporter **_exporter, + const char **error_r) +{ + struct dsync_mailbox_exporter *exporter = *_exporter; + + *_exporter = NULL; + + dsync_mailbox_export_body_search_deinit(exporter); + (void)mailbox_transaction_commit(&exporter->trans); + + hash_table_destroy(&exporter->export_guids); + + *error_r = t_strdup(exporter->error); + pool_unref(&exporter->pool); + return *error_r != NULL ? -1 : 0; +}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/doveadm/dsync/dsync-mailbox-export.h Tue May 22 23:17:31 2012 +0300 @@ -0,0 +1,26 @@ +#ifndef DSYNC_MAILBOX_EXPORT_H +#define DSYNC_MAILBOX_EXPORT_H + +enum dsync_mailbox_exporter_flags { + DSYNC_MAILBOX_EXPORTER_FLAG_AUTO_EXPORT_MAILS = 0x01, + DSYNC_MAILBOX_EXPORTER_FLAG_MAILS_HAVE_GUIDS = 0x02 +}; + +struct dsync_mailbox_exporter * +dsync_mailbox_export_init(struct mailbox *box, + struct dsync_transaction_log_scan *log_scan, + uint32_t last_common_uid, + uint64_t last_common_modseq, + enum dsync_mailbox_exporter_flags flags); +const struct dsync_mail_change * +dsync_mailbox_export_next(struct dsync_mailbox_exporter *exporter); + +void dsync_mailbox_export_want_mail(struct dsync_mailbox_exporter *exporter, + const struct dsync_mail_request *request); +const struct dsync_mail * +dsync_mailbox_export_next_mail(struct dsync_mailbox_exporter *exporter); + +int dsync_mailbox_export_deinit(struct dsync_mailbox_exporter **exporter, + const char **error_r); + +#endif
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/doveadm/dsync/dsync-mailbox-import.c Tue May 22 23:17:31 2012 +0300 @@ -0,0 +1,1461 @@ +/* Copyright (c) 2012 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "hash.h" +#include "istream.h" +#include "seq-range-array.h" +#include "mail-storage-private.h" +#include "mail-search-build.h" +#include "dsync-transaction-log-scan.h" +#include "dsync-mail.h" +#include "dsync-mailbox-import.h" + +struct importer_mail { + const char *guid; + uint32_t uid; +}; + +struct importer_new_mail { + /* linked list of mails for this GUID */ + struct importer_new_mail *next; + /* if non-NULL, this mail exists in both local and remote. this link + points to the other side. */ + struct importer_new_mail *link; + + const char *guid; + struct dsync_mail_change *change; + + uint32_t uid; + unsigned int uid_in_local:1; + unsigned int uid_is_usable:1; + unsigned int skip:1; + unsigned int copy_failed:1; +}; + +struct dsync_mailbox_importer { + pool_t pool; + struct mailbox *box; + uint32_t last_common_uid; + uint64_t last_common_modseq; + uint32_t remote_uid_next; + uint32_t remote_first_recent_uid; + uint64_t remote_highest_modseq; + + struct mailbox_transaction_context *trans, *ext_trans; + struct mail_search_context *search_ctx; + struct mail *mail, *ext_mail; + + struct mail *cur_mail; + const char *cur_guid; + + /* UID => struct dsync_mail_change */ + const struct hash_table *local_changes; + + ARRAY_TYPE(seq_range) maybe_expunge_uids; + ARRAY_DEFINE(maybe_saves, struct dsync_mail_change *); + + /* GUID => struct importer_new_mail */ + struct hash_table *import_guids; + /* UID => struct importer_new_mail */ + struct hash_table *import_uids; + + ARRAY_DEFINE(newmails, struct importer_new_mail *); + ARRAY_TYPE(uint32_t) wanted_uids; + + ARRAY_DEFINE(mail_requests, struct dsync_mail_request); + unsigned int mail_request_idx; + + uint32_t prev_uid, next_local_seq, local_uid_next; + uint64_t local_initial_highestmodseq; + + unsigned int failed:1; + unsigned int last_common_uid_found:1; + unsigned int cur_uid_has_change:1; + unsigned int cur_mail_saved:1; + unsigned int local_expunged_guids_set:1; + unsigned int new_uids_assigned:1; + unsigned int want_mail_requests:1; + unsigned int mails_have_guids:1; + unsigned int master_brain:1; +}; + +static void +dsync_mailbox_import_search_init(struct dsync_mailbox_importer *importer) +{ + struct mail_search_args *search_args; + struct mail_search_arg *sarg; + + search_args = mail_search_build_init(); + sarg = mail_search_build_add(search_args, SEARCH_UIDSET); + p_array_init(&sarg->value.seqset, search_args->pool, 128); + seq_range_array_add_range(&sarg->value.seqset, + importer->last_common_uid+1, (uint32_t)-1); + + importer->search_ctx = + mailbox_search_init(importer->trans, search_args, NULL, + 0, NULL); + mail_search_args_unref(&search_args); + + if (mailbox_search_next(importer->search_ctx, &importer->cur_mail)) + importer->next_local_seq = importer->cur_mail->seq; + /* this flag causes cur_guid to be looked up later */ + importer->cur_mail_saved = TRUE; +} + +struct dsync_mailbox_importer * +dsync_mailbox_import_init(struct mailbox *box, + struct dsync_transaction_log_scan *log_scan, + uint32_t last_common_uid, + uint64_t last_common_modseq, + uint32_t remote_uid_next, + uint32_t remote_first_recent_uid, + uint64_t remote_highest_modseq, + enum dsync_mailbox_import_flags flags) +{ + const enum mailbox_transaction_flags ext_trans_flags = + MAILBOX_TRANSACTION_FLAG_SYNC | + MAILBOX_TRANSACTION_FLAG_EXTERNAL | + MAILBOX_TRANSACTION_FLAG_ASSIGN_UIDS; + struct dsync_mailbox_importer *importer; + struct mailbox_status status; + pool_t pool; + + pool = pool_alloconly_create(MEMPOOL_GROWING"dsync mailbox importer", + 10240); + importer = p_new(pool, struct dsync_mailbox_importer, 1); + importer->pool = pool; + importer->box = box; + importer->last_common_uid = last_common_uid; + importer->last_common_modseq = last_common_modseq; + importer->last_common_uid_found = + last_common_uid != 0 || last_common_modseq != 0; + importer->remote_uid_next = remote_uid_next; + importer->remote_first_recent_uid = remote_first_recent_uid; + importer->remote_highest_modseq = remote_highest_modseq; + + importer->import_guids = + hash_table_create(default_pool, pool, 0, + str_hash, (hash_cmp_callback_t *)strcmp); + importer->import_uids = + hash_table_create(default_pool, pool, 0, NULL, NULL); + i_array_init(&importer->maybe_expunge_uids, 16); + i_array_init(&importer->maybe_saves, 128); + i_array_init(&importer->newmails, 128); + i_array_init(&importer->wanted_uids, 128); + + importer->trans = mailbox_transaction_begin(importer->box, + MAILBOX_TRANSACTION_FLAG_SYNC); + importer->ext_trans = mailbox_transaction_begin(box, ext_trans_flags); + importer->mail = mail_alloc(importer->trans, 0, NULL); + importer->ext_mail = mail_alloc(importer->ext_trans, 0, NULL); + + if ((flags & DSYNC_MAILBOX_IMPORT_FLAG_WANT_MAIL_REQUESTS) != 0) { + i_array_init(&importer->mail_requests, 128); + importer->want_mail_requests = TRUE; + } + importer->mails_have_guids = + (flags & DSYNC_MAILBOX_IMPORT_FLAG_MAILS_HAVE_GUIDS) != 0; + importer->master_brain = + (flags & DSYNC_MAILBOX_IMPORT_FLAG_MASTER_BRAIN) != 0; + + mailbox_get_open_status(importer->box, + STATUS_UIDNEXT | STATUS_HIGHESTMODSEQ, + &status); + importer->local_uid_next = status.uidnext; + importer->local_initial_highestmodseq = status.highest_modseq; + dsync_mailbox_import_search_init(importer); + + importer->local_changes = dsync_transaction_log_scan_get_hash(log_scan); + return importer; +} + +static int dsync_mail_error(struct dsync_mailbox_importer *importer, + struct mail *mail, const char *field) +{ + const char *errstr; + enum mail_error error; + + errstr = mailbox_get_last_error(importer->box, &error); + if (error == MAIL_ERROR_EXPUNGED) + return 0; + + i_error("Can't lookup %s for UID=%u: %s", field, mail->uid, errstr); + importer->failed = TRUE; + return -1; +} + +static bool +importer_next_mail(struct dsync_mailbox_importer *importer, uint32_t wanted_uid) +{ + if (importer->cur_mail == NULL) { + /* end of search */ + return FALSE; + } + while (importer->cur_mail->seq < importer->next_local_seq || + importer->cur_mail->uid < wanted_uid) { + if (!importer->cur_uid_has_change && + !importer->last_common_uid_found) { + /* this message exists locally, but remote didn't send + expunge-change for it. if the message's + uid <= last-common-uid, it should be deleted */ + seq_range_array_add(&importer->maybe_expunge_uids, 0, + importer->cur_mail->uid); + } + + importer->cur_mail_saved = FALSE; + if (!mailbox_search_next(importer->search_ctx, + &importer->cur_mail)) { + importer->cur_mail = NULL; + importer->cur_guid = NULL; + return FALSE; + } + importer->cur_uid_has_change = FALSE; + } + importer->cur_uid_has_change = importer->cur_mail != NULL && + importer->cur_mail->uid == wanted_uid; + if (mail_get_special(importer->cur_mail, MAIL_FETCH_GUID, + &importer->cur_guid) < 0) { + dsync_mail_error(importer, importer->cur_mail, "GUID"); + return importer_next_mail(importer, wanted_uid); + } + /* make sure next_local_seq gets updated in case we came here + because of min_uid */ + importer->next_local_seq = importer->cur_mail->seq; + return TRUE; +} + +static int +importer_mail_cmp(const struct importer_mail *m1, + const struct importer_mail *m2) +{ + int ret; + + if (m1->guid == NULL) + return 1; + if (m2->guid == NULL) + return -1; + + ret = strcmp(m1->guid, m2->guid); + if (ret != 0) + return ret; + + if (m1->uid < m2->uid) + return -1; + if (m1->uid > m2->uid) + return 1; + return 0; +} + +static void importer_mail_request(struct dsync_mailbox_importer *importer, + struct importer_new_mail *newmail) +{ + struct dsync_mail_request *request; + + if (importer->want_mail_requests && !newmail->uid_in_local) { + request = array_append_space(&importer->mail_requests); + request->guid = newmail->guid; + request->uid = newmail->uid; + } +} + +static void newmail_link(struct dsync_mailbox_importer *importer, + struct importer_new_mail *newmail) +{ + struct importer_new_mail *first_mail, **last, *mail, *link = NULL; + + if (*newmail->guid != '\0') { + first_mail = hash_table_lookup(importer->import_guids, + newmail->guid); + if (first_mail == NULL) { + /* first mail for this GUID */ + hash_table_insert(importer->import_guids, + (void *)newmail->guid, newmail); + importer_mail_request(importer, newmail); + return; + } + } else { + if (!newmail->uid_in_local) { + /* FIXME: ? */ + return; + } + first_mail = hash_table_lookup(importer->import_uids, + POINTER_CAST(newmail->uid)); + if (first_mail == NULL) { + /* first mail for this UID */ + hash_table_insert(importer->import_uids, + POINTER_CAST(newmail->uid), newmail); + importer_mail_request(importer, newmail); + return; + } + } + /* 1) add the newmail to the end of the linked list + 2) find our link */ + last = &first_mail->next; + for (mail = first_mail; mail != NULL; mail = mail->next) { + if (mail->uid == newmail->uid) + mail->uid_is_usable = TRUE; + if (link == NULL && mail->link == NULL && + mail->uid_in_local != newmail->uid_in_local) + link = mail; + last = &mail->next; + } + *last = newmail; + if (link != NULL && newmail->link == NULL) { + link->link = newmail; + newmail->link = link; + } +} + +static bool dsync_mailbox_try_save_cur(struct dsync_mailbox_importer *importer, + struct dsync_mail_change *save_change) +{ + struct importer_mail m1, m2; + struct importer_new_mail *newmail; + int diff; + bool remote_saved; + + memset(&m1, 0, sizeof(m1)); + if (importer->cur_mail != NULL) { + m1.guid = importer->cur_guid; + m1.uid = importer->cur_mail->uid; + } + memset(&m2, 0, sizeof(m2)); + if (save_change != NULL) { + m2.guid = save_change->guid; + m2.uid = save_change->uid; + } + + newmail = p_new(importer->pool, struct importer_new_mail, 1); + + diff = importer_mail_cmp(&m1, &m2); + if (diff < 0) { + /* add a record for local mail */ + newmail->guid = p_strdup(importer->pool, importer->cur_guid); + newmail->uid = importer->cur_mail->uid; + newmail->uid_in_local = TRUE; + newmail->uid_is_usable = + newmail->uid >= importer->remote_uid_next; + remote_saved = FALSE; + } else if (diff > 0) { + newmail->guid = save_change->guid; + newmail->uid = save_change->uid; + newmail->uid_in_local = FALSE; + newmail->uid_is_usable = + newmail->uid >= importer->local_uid_next; + remote_saved = TRUE; + } else { + /* identical */ + newmail->guid = save_change->guid; + newmail->uid = importer->cur_mail->uid; + newmail->uid_in_local = TRUE; + newmail->uid_is_usable = TRUE; + newmail->link = newmail; + remote_saved = TRUE; + } + + if (newmail->uid_in_local) { + importer->cur_mail_saved = TRUE; + importer->next_local_seq++; + } else { + /* NOTE: assumes save_change is allocated from importer pool */ + newmail->change = save_change; + } + + array_append(&importer->newmails, &newmail, 1); + newmail_link(importer, newmail); + return remote_saved; +} + +static bool dsync_mailbox_try_save(struct dsync_mailbox_importer *importer, + struct dsync_mail_change *save_change) +{ + if (importer->cur_mail_saved) { + if (!importer_next_mail(importer, 0) && save_change == NULL) + return FALSE; + } + return dsync_mailbox_try_save_cur(importer, save_change); +} + +static void dsync_mailbox_save(struct dsync_mailbox_importer *importer, + struct dsync_mail_change *save_change) +{ + while (!dsync_mailbox_try_save(importer, save_change)) ; +} + +static bool +dsync_import_set_mail(struct dsync_mailbox_importer *importer, + const struct dsync_mail_change *change) +{ + const char *guid; + + if (!mail_set_uid(importer->mail, change->uid)) + return FALSE; + if (change->guid == NULL) { + /* GUID is unknown */ + return TRUE; + } + if (*change->guid == '\0') { + /* backend doesn't support GUIDs. if hdr_hash is set, we could + verify it, but since this message really is supposed to + match, it's probably too much trouble. */ + return TRUE; + } + + /* verify that GUID matches, just in case */ + if (mail_get_special(importer->mail, MAIL_FETCH_GUID, &guid) < 0) { + dsync_mail_error(importer, importer->mail, "GUID"); + return FALSE; + } + if (strcmp(guid, change->guid) != 0) { + i_error("Mailbox %s: Unexpected GUID mismatch for " + "UID=%u: %s != %s", mailbox_get_vname(importer->box), + change->uid, guid, change->guid); + importer->last_common_uid = 1; + importer->failed = TRUE; + return FALSE; + } + return TRUE; +} + +static void +merge_flags(uint32_t local_final, uint32_t local_add, uint32_t local_remove, + uint32_t remote_final, uint32_t remote_add, uint32_t remote_remove, + bool prefer_remote, + uint32_t *change_add_r, uint32_t *change_remove_r) +{ + uint32_t combined_add, combined_remove, conflict_flags; + uint32_t local_wanted, remote_wanted; + + /* resolve conflicts */ + conflict_flags = local_add & remote_remove; + if (conflict_flags != 0) { + if (prefer_remote) + local_add &= ~conflict_flags; + else + remote_remove &= ~conflict_flags; + } + conflict_flags = local_remove & remote_add; + if (conflict_flags != 0) { + if (prefer_remote) + local_remove &= ~conflict_flags; + else + remote_add &= ~conflict_flags; + } + combined_add = local_add|remote_add; + combined_remove = local_remove|remote_remove; + i_assert((combined_add & combined_remove) == 0); + + /* see if there are conflicting final flags */ + local_wanted = (local_final|combined_add) & ~combined_remove; + remote_wanted = (remote_final|combined_add) & ~combined_remove; + + conflict_flags = local_wanted ^ remote_wanted; + if (conflict_flags != 0) { + if (prefer_remote) + local_wanted = remote_wanted; + /*else + remote_wanted = local_wanted;*/ + } + + *change_add_r = local_wanted & ~local_final; + *change_remove_r = local_final & ~local_wanted; +} + +static bool +keyword_find(ARRAY_TYPE(const_string) *keywords, const char *name, + unsigned int *idx_r) +{ + const char *const *names; + unsigned int i, count; + + names = array_get(keywords, &count); + for (i = 0; i < count; i++) { + if (strcmp(names[i], name) == 0) { + *idx_r = i; + return TRUE; + } + } + return FALSE; +} + +static void keywords_append(ARRAY_TYPE(const_string) *dest, + const ARRAY_TYPE(const_string) *keywords, + uint32_t bits, unsigned int start_idx) +{ + const char *const *namep; + unsigned int i; + + for (i = 0; i < 32; i++) { + if ((bits & (1U << i)) == 0) + continue; + + namep = array_idx(keywords, start_idx+i); + array_append(dest, namep, 1); + } +} + +static void +merge_keywords(struct mail *mail, const ARRAY_TYPE(const_string) *local_changes, + const ARRAY_TYPE(const_string) *remote_changes, + bool prefer_remote) +{ + /* local_changes and remote_changes are assumed to have no + duplicates names */ + uint32_t *local_add, *local_remove, *local_final; + uint32_t *remote_add, *remote_remove, *remote_final; + uint32_t *change_add, *change_remove; + ARRAY_TYPE(const_string) all_keywords, add_keywords, remove_keywords; + const char *const *changes, *name, *const *local_keywords; + struct mail_keywords *kw; + unsigned int i, count, name_idx, array_size; + + local_keywords = mail_get_keywords(mail); + + /* we'll assign a common index for each keyword name and place + the changes to separate bit arrays. */ + if (array_is_created(remote_changes)) + changes = array_get(remote_changes, &count); + else { + changes = NULL; + count = 0; + } + + array_size = str_array_length(local_keywords) + count; + if (array_is_created(local_changes)) + array_size += array_count(local_changes); + if (array_size == 0) { + /* this message has no keywords */ + return; + } + t_array_init(&all_keywords, array_size); + t_array_init(&add_keywords, array_size); + t_array_init(&remove_keywords, array_size); + + /* @UNSAFE: create large enough arrays to fit all keyword indexes. */ + array_size = (array_size+31)/32; + local_add = t_new(uint32_t, array_size); + local_remove = t_new(uint32_t, array_size); + local_final = t_new(uint32_t, array_size); + remote_add = t_new(uint32_t, array_size); + remote_remove = t_new(uint32_t, array_size); + remote_final = t_new(uint32_t, array_size); + change_add = t_new(uint32_t, array_size); + change_remove = t_new(uint32_t, array_size); + + /* get remote changes */ + for (i = 0; i < count; i++) { + name = changes[i]+1; + name_idx = array_count(&all_keywords); + array_append(&all_keywords, &name, 1); + + switch (changes[i][0]) { + case KEYWORD_CHANGE_ADD: + remote_add[name_idx/32] |= 1U << (name_idx%32); + /* fall through */ + case KEYWORD_CHANGE_FINAL: + remote_final[name_idx/32] |= 1U << (name_idx%32); + break; + case KEYWORD_CHANGE_REMOVE: + remote_remove[name_idx/32] |= 1U << (name_idx%32); + break; + } + } + + /* get local changes. use existing indexes for names when they exist. */ + if (array_is_created(local_changes)) + changes = array_get(local_changes, &count); + else { + changes = NULL; + count = 0; + } + for (i = 0; i < count; i++) { + name = changes[i]+1; + if (!keyword_find(&all_keywords, name, &name_idx)) { + name_idx = array_count(&all_keywords); + array_append(&all_keywords, &name, 1); + } + + switch (changes[i][0]) { + case KEYWORD_CHANGE_ADD: + local_add[name_idx/32] |= 1U << (name_idx%32); + break; + case KEYWORD_CHANGE_REMOVE: + local_remove[name_idx/32] |= 1U << (name_idx%32); + break; + case KEYWORD_CHANGE_FINAL: + i_unreached(); + } + } + for (i = 0; local_keywords[i] != NULL; i++) { + name = local_keywords[i]; + if (!keyword_find(&all_keywords, name, &name_idx)) { + name_idx = array_count(&all_keywords); + array_append(&all_keywords, &name, 1); + } + local_final[name_idx/32] |= 1U << (name_idx%32); + } + i_assert(array_count(&all_keywords) <= array_size*32); + array_size = (array_count(&all_keywords)+31) / 32; + + /* merge keywords */ + for (i = 0; i < array_size; i++) { + merge_flags(local_final[i], local_add[i], local_remove[i], + remote_final[i], remote_add[i], remote_remove[i], + prefer_remote, &change_add[i], &change_remove[i]); + if (change_add[i] != 0) { + keywords_append(&add_keywords, &all_keywords, + change_add[i], i*32); + } + if (change_remove[i] != 0) { + keywords_append(&remove_keywords, &all_keywords, + change_add[i], i*32); + } + } + + /* apply changes */ + if (array_count(&add_keywords) > 0) { + (void)array_append_space(&add_keywords); + kw = mailbox_keywords_create_valid(mail->box, + array_idx(&add_keywords, 0)); + mail_update_keywords(mail, MODIFY_ADD, kw); + mailbox_keywords_unref(&kw); + } + if (array_count(&remove_keywords) > 0) { + (void)array_append_space(&remove_keywords); + kw = mailbox_keywords_create_valid(mail->box, + array_idx(&remove_keywords, 0)); + mail_update_keywords(mail, MODIFY_REMOVE, kw); + mailbox_keywords_unref(&kw); + } +} + +static void +dsync_mailbox_import_flag_change(struct dsync_mailbox_importer *importer, + const struct dsync_mail_change *change) +{ + const struct dsync_mail_change *local_change; + enum mail_flags local_add, local_remove; + uint32_t change_add, change_remove; + ARRAY_TYPE(const_string) local_keyword_changes = ARRAY_INIT; + struct mail *mail; + bool prefer_remote; + + i_assert((change->add_flags & change->remove_flags) == 0); + + if (importer->cur_mail != NULL && + importer->cur_mail->uid == change->uid) + mail = importer->cur_mail; + else { + if (!dsync_import_set_mail(importer, change)) + return; + mail = importer->mail; + } + + local_change = hash_table_lookup(importer->local_changes, + POINTER_CAST(change->uid)); + if (local_change == NULL) { + local_add = local_remove = 0; + } else { + local_add = local_change->add_flags; + local_remove = local_change->remove_flags; + local_keyword_changes = local_change->keyword_changes; + } + + if (mail_get_modseq(mail) < change->modseq) + prefer_remote = TRUE; + else if (mail_get_modseq(mail) > change->modseq) + prefer_remote = FALSE; + else { + /* identical modseq, we'll just have to pick one. + Note that both brains need to pick the same one, otherwise + they become unsynced. */ + prefer_remote = !importer->master_brain; + } + + /* merge flags */ + merge_flags(mail_get_flags(mail), local_add, local_remove, + change->final_flags, change->add_flags, change->remove_flags, + prefer_remote, &change_add, &change_remove); + + if (change_add != 0) + mail_update_flags(mail, MODIFY_ADD, change_add); + if (change_remove != 0) + mail_update_flags(mail, MODIFY_REMOVE, change_remove); + + /* merge keywords */ + merge_keywords(mail, &local_keyword_changes, &change->keyword_changes, + prefer_remote); + mail_update_modseq(mail, change->modseq); +} + +static void +dsync_mailbox_import_save(struct dsync_mailbox_importer *importer, + const struct dsync_mail_change *change) +{ + struct dsync_mail_change *save; + + i_assert(change->guid != NULL); + + if (change->uid == importer->last_common_uid) { + /* we've already verified that the GUID matches. + apply flag changes if there are any. */ + i_assert(!importer->last_common_uid_found); + dsync_mailbox_import_flag_change(importer, change); + return; + } + + save = p_new(importer->pool, struct dsync_mail_change, 1); + dsync_mail_change_dup(importer->pool, change, save); + + if (importer->last_common_uid_found) { + /* this is a new mail. its UID may or may not conflict with + an existing local mail, we'll figure it out later. */ + i_assert(change->uid > importer->last_common_uid); + dsync_mailbox_save(importer, save); + } else { + /* the local mail is expunged. we'll decide later if we want + to save this mail locally or expunge it form remote. */ + i_assert(change->uid > importer->last_common_uid); + i_assert(change->uid < importer->cur_mail->uid); + array_append(&importer->maybe_saves, &save, 1); + } +} + +static void +dsync_mailbox_import_expunge(struct dsync_mailbox_importer *importer, + const struct dsync_mail_change *change) +{ + + if (importer->last_common_uid_found) { + /* expunge the message, unless its GUID unexpectedly doesn't + match */ + i_assert(change->uid <= importer->last_common_uid); + if (dsync_import_set_mail(importer, change)) + mail_expunge(importer->mail); + } else if (change->uid < importer->cur_mail->uid) { + /* already expunged locally, we can ignore this. + uid=last_common_uid if we managed to verify from + transaction log that the GUIDs match */ + i_assert(change->uid >= importer->last_common_uid); + } else if (change->uid == importer->last_common_uid) { + /* already verified that the GUID matches */ + i_assert(importer->cur_mail->uid == change->uid); + mail_expunge(importer->cur_mail); + } else { + /* we don't know yet if we should expunge this + message or not. queue it until we do. */ + i_assert(change->uid > importer->last_common_uid); + seq_range_array_add(&importer->maybe_expunge_uids, 0, + change->uid); + } +} + +static void +dsync_mailbox_rewind_search(struct dsync_mailbox_importer *importer) +{ + /* If there are local mails after last_common_uid which we skipped + while trying to match the next message, we need to now go back */ + if (importer->cur_mail != NULL && + importer->cur_mail->uid <= importer->last_common_uid+1) + return; + + importer->cur_mail = NULL; + importer->cur_guid = NULL; + importer->next_local_seq = 0; + + mailbox_search_deinit(&importer->search_ctx); + dsync_mailbox_import_search_init(importer); +} + +static void +dsync_mailbox_common_uid_found(struct dsync_mailbox_importer *importer) +{ + struct dsync_mail_change *const *saves; + struct seq_range_iter iter; + unsigned int n, i, count; + uint32_t uid; + + importer->last_common_uid_found = TRUE; + dsync_mailbox_rewind_search(importer); + + /* expunge the messages whose expunge-decision we delayed previously */ + seq_range_array_iter_init(&iter, &importer->maybe_expunge_uids); n = 0; + while (seq_range_array_iter_nth(&iter, n++, &uid)) { + if (uid > importer->last_common_uid) { + /* we expunge messages only up to last_common_uid, + ignore the rest */ + break; + } + + if (mail_set_uid(importer->mail, uid)) + mail_expunge(importer->mail); + } + + /* handle pending saves */ + saves = array_get(&importer->maybe_saves, &count); + for (i = 0; i < count; i++) { + if (saves[i]->uid > importer->last_common_uid) + dsync_mailbox_save(importer, saves[i]); + } +} + +static int +dsync_mailbox_import_match_msg(struct dsync_mailbox_importer *importer, + const struct dsync_mail_change *change) +{ + const char *hdr_hash; + + if (*change->guid != '\0' && *importer->cur_guid != '\0') { + /* we have GUIDs, verify them */ + return strcmp(change->guid, importer->cur_guid) == 0 ? 1 : 0; + } + + /* verify hdr_hash if it exists */ + if (change->hdr_hash == NULL) { + i_assert(*importer->cur_guid == '\0'); + i_error("Mailbox %s: GUIDs not supported, " + "sync with header hashes instead", + mailbox_get_vname(importer->box)); + importer->failed = TRUE; + return -1; + } + + if (dsync_mail_get_hdr_hash(importer->cur_mail, &hdr_hash) < 0) { + dsync_mail_error(importer, importer->cur_mail, "hdr-stream"); + return -1; + } + return strcmp(change->hdr_hash, hdr_hash) == 0 ? 1 : 0; +} + +static void +dsync_mailbox_find_common_uid(struct dsync_mailbox_importer *importer, + const struct dsync_mail_change *change) +{ + const struct dsync_mail_change *local_change; + guid_128_t guid_128, change_guid_128; + int ret; + + /* try to find the matching local mail */ + if (!importer_next_mail(importer, change->uid)) { + /* no more local mails. use the last message with a matching + GUID as the last common UID. */ + dsync_mailbox_common_uid_found(importer); + return; + } + + if (change->guid == NULL) { + /* we can't know if this UID matches */ + return; + } + if (importer->cur_mail->uid == change->uid) { + /* we have a matching local UID. check GUID to see if it's + really the same mail or not */ + if ((ret = dsync_mailbox_import_match_msg(importer, change)) < 0) { + /* unknown */ + return; + } + if (ret == 0) { + /* mismatch - found the first non-common UID */ + dsync_mailbox_common_uid_found(importer); + } else { + importer->last_common_uid = change->uid; + } + return; + } + + if (*change->guid == '\0') { + /* remote doesn't support GUIDs, can't verify expunge */ + return; + } + + /* local message is expunged. see if we can find its GUID from + transaction log and check if the GUIDs match. The GUID in + log is a 128bit GUID, so we may need to convert the remote's + GUID string to 128bit GUID first. */ + local_change = hash_table_lookup(importer->local_changes, + POINTER_CAST(change->uid)); + if (local_change == NULL || local_change->guid == NULL) + return; + if (guid_128_from_string(local_change->guid, guid_128) < 0) + i_unreached(); + + mail_generate_guid_128_hash(change->guid, change_guid_128); + if (memcmp(change_guid_128, guid_128, GUID_128_SIZE) != 0) { + /* mismatch - found the first non-common UID */ + dsync_mailbox_common_uid_found(importer); + } else { + importer->last_common_uid = change->uid; + } + return; +} + +void dsync_mailbox_import_change(struct dsync_mailbox_importer *importer, + const struct dsync_mail_change *change) +{ + i_assert(!importer->new_uids_assigned); + i_assert(importer->prev_uid < change->uid); + + importer->prev_uid = change->uid; + + if (!importer->last_common_uid_found) + dsync_mailbox_find_common_uid(importer, change); + + if (importer->last_common_uid_found) { + /* a) uid <= last_common_uid for flag changes and expunges. + this happens only when last_common_uid was originally given + as parameter to importer. + + when we're finding the last_common_uid ourself, + uid>last_common_uid always in here, because + last_common_uid_found=TRUE only after we find the first + mismatch. + + b) uid > last_common_uid for i) new messages, ii) expunges + that were sent "just in case" */ + if (change->uid <= importer->last_common_uid) { + i_assert(change->type != DSYNC_MAIL_CHANGE_TYPE_SAVE); + } else if (change->type == DSYNC_MAIL_CHANGE_TYPE_EXPUNGE) { + /* ignore */ + return; + } else { + i_assert(change->type == DSYNC_MAIL_CHANGE_TYPE_SAVE); + } + } else { + /* a) uid < last_common_uid can never happen */ + i_assert(change->uid >= importer->last_common_uid); + /* b) uid = last_common_uid if we've verified that the + messages' GUIDs match so far. + + c) uid > last_common_uid: i) TYPE_EXPUNGE change has + GUID=NULL, so we couldn't verify yet if it matches our + local message, ii) local message is expunged and we couldn't + find its GUID */ + if (change->uid > importer->last_common_uid) { + i_assert(change->type == DSYNC_MAIL_CHANGE_TYPE_EXPUNGE || + change->uid < importer->cur_mail->uid); + } + } + + switch (change->type) { + case DSYNC_MAIL_CHANGE_TYPE_SAVE: + dsync_mailbox_import_save(importer, change); + break; + case DSYNC_MAIL_CHANGE_TYPE_EXPUNGE: + dsync_mailbox_import_expunge(importer, change); + break; + case DSYNC_MAIL_CHANGE_TYPE_FLAG_CHANGE: + i_assert(importer->last_common_uid_found); + dsync_mailbox_import_flag_change(importer, change); + break; + } +} + +static void +dsync_msg_update_uid(struct dsync_mailbox_importer *importer, + uint32_t old_uid, uint32_t new_uid) +{ + struct mail_save_context *save_ctx; + + if (!mail_set_uid(importer->mail, old_uid)) + return; + + save_ctx = mailbox_save_alloc(importer->ext_trans); + mailbox_save_copy_flags(save_ctx, importer->mail); + mailbox_save_set_uid(save_ctx, new_uid); + if (mailbox_copy(&save_ctx, importer->mail) == 0) { + array_append(&importer->wanted_uids, &new_uid, 1); + mail_expunge(importer->mail); + } +} + +static void +dsync_mailbox_import_assign_new_uids(struct dsync_mailbox_importer *importer) +{ + struct importer_new_mail *newmail, *const *newmailp; + uint32_t common_uid_next, new_uid; + + common_uid_next = I_MAX(importer->local_uid_next, + importer->remote_uid_next); + array_foreach_modifiable(&importer->newmails, newmailp) { + newmail = *newmailp; + if (newmail->skip) { + /* already assigned */ + if (newmail->uid_in_local) { + if (mail_set_uid(importer->mail, newmail->uid)) + mail_expunge(importer->mail); + } + continue; + } + + /* figure out what UID to use for the mail */ + if (newmail->uid_is_usable) { + /* keep the UID */ + new_uid = newmail->uid; + } else if (newmail->link != NULL && + newmail->link->uid_is_usable) + new_uid = newmail->link->uid; + else + new_uid = common_uid_next++; + + if (newmail->uid_in_local && newmail->uid != new_uid) { + /* local UID changed, reassign it by copying */ + dsync_msg_update_uid(importer, newmail->uid, new_uid); + } + newmail->uid = new_uid; + + if (newmail->link != NULL) { + /* skip the linked mail */ + newmail->link->skip = TRUE; + } + } + importer->last_common_uid = common_uid_next; + importer->new_uids_assigned = TRUE; +} + +void dsync_mailbox_import_changes_finish(struct dsync_mailbox_importer *importer) +{ + i_assert(!importer->new_uids_assigned); + + if (!importer->last_common_uid_found) { + /* handle pending expunges and flag updates */ + dsync_mailbox_common_uid_found(importer); + } + /* skip common local mails */ + importer_next_mail(importer, importer->last_common_uid+1); + /* if there are any local mails left, add them to newmails list */ + while (importer->cur_mail != NULL) + dsync_mailbox_try_save(importer, NULL); + + dsync_mailbox_import_assign_new_uids(importer); +} + +const struct dsync_mail_request * +dsync_mailbox_import_next_request(struct dsync_mailbox_importer *importer) +{ + const struct dsync_mail_request *requests; + unsigned int count; + + requests = array_get(&importer->mail_requests, &count); + if (importer->mail_request_idx == count) + return NULL; + return &requests[importer->mail_request_idx++]; +} + +static const char *const * +dsync_mailbox_get_final_keywords(const struct dsync_mail_change *change) +{ + ARRAY_TYPE(const_string) keywords; + const char *const *changes; + unsigned int i, count; + + if (!array_is_created(&change->keyword_changes)) + return NULL; + + changes = array_get(&change->keyword_changes, &count); + t_array_init(&keywords, count); + for (i = 0; i < count; i++) { + if (changes[i][0] == KEYWORD_CHANGE_ADD || + changes[i][0] == KEYWORD_CHANGE_FINAL) { + const char *name = changes[i]+1; + + array_append(&keywords, &name, 1); + } + } + if (array_count(&keywords) == 0) + return NULL; + + (void)array_append_space(&keywords); + return array_idx(&keywords, 0); +} + +static void +dsync_mailbox_save_set_metadata(struct dsync_mailbox_importer *importer, + struct mail_save_context *save_ctx, + const struct dsync_mail_change *change) +{ + const char *const *keyword_names; + struct mail_keywords *keywords; + + keyword_names = dsync_mailbox_get_final_keywords(change); + keywords = keyword_names == NULL ? NULL : + mailbox_keywords_create_valid(importer->box, + keyword_names); + mailbox_save_set_flags(save_ctx, change->final_flags, keywords); + if (keywords != NULL) + mailbox_keywords_unref(&keywords); + + mailbox_save_set_save_date(save_ctx, change->save_timestamp); + if (change->modseq > 1) { + (void)mailbox_enable(importer->box, MAILBOX_FEATURE_CONDSTORE); + mailbox_save_set_min_modseq(save_ctx, change->modseq); + } +} + +static int +dsync_msg_try_copy(struct dsync_mailbox_importer *importer, + struct mail_save_context **save_ctx_p, + struct importer_new_mail *all_newmails) +{ + struct importer_new_mail *inst; + + for (inst = all_newmails; inst != NULL; inst = inst->next) { + if (inst->uid_in_local && !inst->copy_failed && + mail_set_uid(importer->mail, inst->uid)) { + if (mailbox_copy(save_ctx_p, importer->mail) < 0) { + inst->copy_failed = TRUE; + return -1; + } + return 1; + } + } + return 0; +} + +static struct mail_save_context * +dsync_mailbox_save_init(struct dsync_mailbox_importer *importer, + const struct dsync_mail *mail, + struct importer_new_mail *newmail) +{ + struct mail_save_context *save_ctx; + + save_ctx = mailbox_save_alloc(importer->ext_trans); + mailbox_save_set_uid(save_ctx, newmail->uid); + if (*mail->guid != '\0') + mailbox_save_set_guid(save_ctx, mail->guid); + dsync_mailbox_save_set_metadata(importer, save_ctx, newmail->change); + if (*mail->pop3_uidl != '\0') + mailbox_save_set_pop3_uidl(save_ctx, mail->pop3_uidl); + if (mail->pop3_order > 0) + mailbox_save_set_pop3_order(save_ctx, mail->pop3_order); + mailbox_save_set_received_date(save_ctx, mail->received_date, 0); + return save_ctx; +} + +static void dsync_mailbox_save_body(struct dsync_mailbox_importer *importer, + const struct dsync_mail *mail, + struct importer_new_mail *newmail, + struct importer_new_mail *all_newmails) +{ + struct mail_save_context *save_ctx; + ssize_t ret; + bool save_failed = FALSE; + + /* try to save the mail by copying an existing mail */ + save_ctx = dsync_mailbox_save_init(importer, mail, newmail); + if ((ret = dsync_msg_try_copy(importer, &save_ctx, all_newmails)) < 0) { + if (save_ctx == NULL) + save_ctx = dsync_mailbox_save_init(importer, mail, newmail); + } + if (ret > 0) { + array_append(&importer->wanted_uids, &newmail->uid, 1); + return; + } + /* fallback to saving from remote stream */ + + if (mail->input == NULL) { + /* it was just expunged in remote, skip it */ + mailbox_save_cancel(&save_ctx); + return; + } + + i_stream_seek(mail->input, 0); + if (mailbox_save_begin(&save_ctx, mail->input) < 0) { + i_error("Can't save message to mailbox %s: %s", + mailbox_get_vname(importer->box), + mailbox_get_last_error(importer->box, NULL)); + importer->failed = TRUE; + return; + } + while ((ret = i_stream_read(mail->input)) > 0 || ret == -2) { + if (mailbox_save_continue(save_ctx) < 0) { + save_failed = TRUE; + ret = -1; + break; + } + } + i_assert(ret == -1); + + if (mail->input->stream_errno != 0) { + errno = mail->input->stream_errno; + i_error("read(msg input) failed: %m"); + mailbox_save_cancel(&save_ctx); + importer->failed = TRUE; + } else if (save_failed) { + mailbox_save_cancel(&save_ctx); + importer->failed = TRUE; + } else { + i_assert(mail->input->eof); + if (mailbox_save_finish(&save_ctx) < 0) { + i_error("Can't save message to mailbox %s: %s", + mailbox_get_vname(importer->box), + mailbox_get_last_error(importer->box, NULL)); + importer->failed = TRUE; + } else { + array_append(&importer->wanted_uids, &newmail->uid, 1); + } + } +} + +void dsync_mailbox_import_mail(struct dsync_mailbox_importer *importer, + const struct dsync_mail *mail) +{ + struct importer_new_mail *newmail, *allmails; + + i_assert(mail->input->seekable); + i_assert(importer->new_uids_assigned); + + newmail = *mail->guid != '\0' ? + hash_table_lookup(importer->import_guids, mail->guid) : + hash_table_lookup(importer->import_uids, POINTER_CAST(mail->uid)); + if (newmail == NULL) { + if (importer->want_mail_requests) { + i_error("%s: Remote sent unwanted message body for " + "GUID=%s UID=%u", + mailbox_get_vname(importer->box), + mail->guid, mail->uid); + } + return; + } + if (*mail->guid != '\0') + hash_table_remove(importer->import_guids, mail->guid); + else { + hash_table_remove(importer->import_uids, + POINTER_CAST(mail->uid)); + } + + /* save all instances of the message */ + allmails = newmail; + for (; newmail != NULL; newmail = newmail->next) { + if (newmail->skip) { + /* no need to do anything for this mail */ + continue; + } + if (newmail->uid_in_local) { + /* we already handled this by copying the mail */ + continue; + } + + T_BEGIN { + dsync_mailbox_save_body(importer, mail, newmail, + allmails); + } T_END; + } +} + +static int +reassign_uids_in_seq_range(struct mailbox *box, uint32_t seq1, uint32_t seq2) +{ + const enum mailbox_transaction_flags trans_flags = + MAILBOX_TRANSACTION_FLAG_EXTERNAL | + MAILBOX_TRANSACTION_FLAG_ASSIGN_UIDS; + struct mailbox_transaction_context *trans; + struct mail_save_context *save_ctx; + struct mail *mail; + uint32_t seq; + int ret = 0; + + trans = mailbox_transaction_begin(box, trans_flags); + mail = mail_alloc(trans, 0, NULL); + + for (seq = seq1; seq <= seq2; seq++) { + mail_set_seq(mail, seq); + + save_ctx = mailbox_save_alloc(trans); + mailbox_save_copy_flags(save_ctx, mail); + if (mailbox_copy(&save_ctx, mail) < 0) + ret = -1; + else + mail_expunge(mail); + } + mail_free(&mail); + + if (mailbox_transaction_commit(&trans) < 0) { + i_error("UID reassign commit failed to mailbox %s: %s", + mailbox_get_vname(box), + mailbox_get_last_error(box, NULL)); + ret = -1; + } + return ret; +} + +static bool +reassign_unwanted_uids(struct dsync_mailbox_importer *importer, + const struct mail_transaction_commit_changes *changes, + bool *changes_during_sync_r) +{ + struct seq_range_iter iter; + const uint32_t *wanted_uids; + uint32_t saved_uid, highest_unwanted_uid = 0; + uint32_t seq1, seq2, lowest_saved_uid = (uint32_t)-1; + unsigned int i, n, wanted_count; + int ret = 0; + + /* find the highest wanted UID that doesn't match what we got */ + wanted_uids = array_get(&importer->wanted_uids, &wanted_count); + seq_range_array_iter_init(&iter, &changes->saved_uids); i = n = 0; + while (seq_range_array_iter_nth(&iter, n++, &saved_uid)) { + i_assert(i < wanted_count); + if (lowest_saved_uid > saved_uid) + lowest_saved_uid = saved_uid; + if (saved_uid != wanted_uids[i]) { + if (highest_unwanted_uid < wanted_uids[i]) + highest_unwanted_uid = wanted_uids[i]; + } + i++; + } + + if (highest_unwanted_uid == 0 && i > 0 && + importer->local_uid_next <= lowest_saved_uid-1) { + /* we didn't see any unwanted UIDs, but we'll still need to + verify that messages didn't just get saved locally to a gap + that we left in local_uid_next..(lowest_saved_uid-1) */ + highest_unwanted_uid = lowest_saved_uid-1; + } + + if (highest_unwanted_uid == 0) + seq1 = seq2 = 0; + else { + mailbox_get_seq_range(importer->box, importer->local_uid_next, + highest_unwanted_uid, &seq1, &seq2); + } + if (seq1 > 0) { + ret = reassign_uids_in_seq_range(importer->box, seq1, seq2); + *changes_during_sync_r = TRUE; + } + return ret; +} + +static int dsync_mailbox_import_commit(struct dsync_mailbox_importer *importer, + bool *changes_during_sync_r) +{ + struct mail_transaction_commit_changes changes; + struct mailbox_update update; + int ret = 0; + + /* commit saves */ + if (mailbox_transaction_commit_get_changes(&importer->ext_trans, + &changes) < 0) { + i_error("Save commit failed to mailbox %s: %s", + mailbox_get_vname(importer->box), + mailbox_get_last_error(importer->box, NULL)); + mailbox_transaction_rollback(&importer->trans); + return -1; + } + + /* commit flag changes and expunges */ + if (mailbox_transaction_commit(&importer->trans) < 0) { + i_error("Commit failed to mailbox %s: %s", + mailbox_get_vname(importer->box), + mailbox_get_last_error(importer->box, NULL)); + pool_unref(&changes.pool); + return -1; + } + + /* update mailbox metadata. */ + memset(&update, 0, sizeof(update)); + update.min_next_uid = importer->remote_uid_next; + update.min_first_recent_uid = + I_MIN(importer->last_common_uid+1, + importer->remote_first_recent_uid); + update.min_highest_modseq = importer->remote_highest_modseq; + + if (mailbox_update(importer->box, &update) < 0) { + i_error("Mailbox update failed to mailbox %s: %s", + mailbox_get_vname(importer->box), + mailbox_get_last_error(importer->box, NULL)); + ret = -1; + } + + /* sync mailbox to finish flag changes and expunges. */ + if (mailbox_sync(importer->box, 0) < 0) { + i_error("Mailbox sync failed to mailbox %s: %s", + mailbox_get_vname(importer->box), + mailbox_get_last_error(importer->box, NULL)); + ret = -1; + } + + if (reassign_unwanted_uids(importer, &changes, + changes_during_sync_r) < 0) + ret = -1; + pool_unref(&changes.pool); + return ret; +} + +static unsigned int +dsync_mailbox_import_count_missing_imports(struct hash_table *imports) +{ + struct hash_iterate_context *iter; + void *key, *value; + unsigned int msgs_left = 0; + + iter = hash_table_iterate_init(imports); + while (hash_table_iterate(iter, &key, &value)) { + struct importer_new_mail *mail = value; + + for (; mail != NULL; mail = mail->next) { + if (!mail->uid_in_local) { + msgs_left++; + break; + } + } + } + hash_table_iterate_deinit(&iter); + return msgs_left; +} + +int dsync_mailbox_import_deinit(struct dsync_mailbox_importer **_importer, + uint32_t *last_common_uid_r, + uint64_t *last_common_modseq_r, + bool *changes_during_sync_r) +{ + struct dsync_mailbox_importer *importer = *_importer; + unsigned int msgs_left; + int ret; + + *_importer = NULL; + *changes_during_sync_r = FALSE; + + if (!importer->new_uids_assigned) + dsync_mailbox_import_assign_new_uids(importer); + + msgs_left = + dsync_mailbox_import_count_missing_imports(importer->import_guids) + + dsync_mailbox_import_count_missing_imports(importer->import_uids); + if (!importer->failed && msgs_left > 0) { + i_error("%s: Remote didn't send %u expected message bodies", + mailbox_get_vname(importer->box), msgs_left); + } + + if (importer->search_ctx != NULL) + mailbox_search_deinit(&importer->search_ctx); + mail_free(&importer->mail); + mail_free(&importer->ext_mail); + + if (dsync_mailbox_import_commit(importer, changes_during_sync_r) < 0) + importer->failed = TRUE; + + hash_table_destroy(&importer->import_guids); + hash_table_destroy(&importer->import_uids); + array_free(&importer->maybe_expunge_uids); + array_free(&importer->maybe_saves); + array_free(&importer->wanted_uids); + array_free(&importer->newmails); + if (array_is_created(&importer->mail_requests)) + array_free(&importer->mail_requests); + + *last_common_uid_r = importer->last_common_uid; + if (!*changes_during_sync_r) + *last_common_modseq_r = importer->last_common_modseq; + else { + /* local changes occurred during dsync. we exported changes up + to local_initial_highestmodseq, so all of the changes have + happened after it. we want the next run to see those changes, + so return it as the last common modseq */ + *last_common_modseq_r = importer->local_initial_highestmodseq; + } + + ret = importer->failed ? -1 : 0; + pool_unref(&importer->pool); + return ret; +}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/doveadm/dsync/dsync-mailbox-import.h Tue May 22 23:17:31 2012 +0300 @@ -0,0 +1,36 @@ +#ifndef DSYNC_MAILBOX_IMPORT_H +#define DSYNC_MAILBOX_IMPORT_H + +enum dsync_mailbox_import_flags { + DSYNC_MAILBOX_IMPORT_FLAG_MASTER_BRAIN = 0x01, + DSYNC_MAILBOX_IMPORT_FLAG_WANT_MAIL_REQUESTS = 0x02, + DSYNC_MAILBOX_IMPORT_FLAG_MAILS_HAVE_GUIDS = 0x04 +}; + +struct mailbox; +struct dsync_mail; +struct dsync_mail_change; +struct dsync_transaction_log_scan; + +struct dsync_mailbox_importer * +dsync_mailbox_import_init(struct mailbox *box, + struct dsync_transaction_log_scan *log_scan, + uint32_t last_common_uid, + uint64_t last_common_modseq, + uint32_t remote_uid_next, + uint32_t remote_first_recent_uid, + uint64_t remote_highest_modseq, + enum dsync_mailbox_import_flags flags); +void dsync_mailbox_import_change(struct dsync_mailbox_importer *importer, + const struct dsync_mail_change *change); +void dsync_mailbox_import_changes_finish(struct dsync_mailbox_importer *importer); +const struct dsync_mail_request * +dsync_mailbox_import_next_request(struct dsync_mailbox_importer *importer); +void dsync_mailbox_import_mail(struct dsync_mailbox_importer *importer, + const struct dsync_mail *mail); +int dsync_mailbox_import_deinit(struct dsync_mailbox_importer **importer, + uint32_t *last_common_uid_r, + uint64_t *last_common_modseq_r, + bool *changes_during_sync_r); + +#endif
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/doveadm/dsync/dsync-mailbox-state-export.h Tue May 22 23:17:31 2012 +0300 @@ -0,0 +1,14 @@ +#ifndef DSYNC_MAILBOX_STATE_EXPORT_H +#define DSYNC_MAILBOX_STATE_EXPORT_H + +struct dsync_mailbox_state_export * +dsync_mailbox_state_export_init(struct mailbox *box); +void dsync_mailbox_state_export_deinit(struct dsync_mailbox_state_export **exporter); + +void dsync_mailbox_state_export_more(struct dsync_mailbox_state_export **exporter); + +// vai + +int dsync_mailbox_state_export(struct mailbox *box, struct ostream *output); + +#endif
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/doveadm/dsync/dsync-mailbox-state.c Tue May 22 23:17:31 2012 +0300 @@ -0,0 +1,85 @@ +/* Copyright (c) 2012 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "base64.h" +#include "crc32.h" +#include "dsync-mailbox-state.h" + +#define MAILBOX_SIZE (GUID_128_SIZE + 4 + 4 + 8) + +static void put_uint32(buffer_t *output, uint32_t num) +{ + buffer_append_c(output, num & 0xff); + buffer_append_c(output, (num >> 8) & 0xff); + buffer_append_c(output, (num >> 16) & 0xff); + buffer_append_c(output, (num >> 24) & 0xff); +} + +static uint32_t get_uint32(const unsigned char *data) +{ + return data[0] | (data[1] << 8) | (data[2] << 16) | + ((unsigned int)data[3] << 24); +} + +void dsync_mailbox_states_export(const struct dsync_mailbox_state *states, + unsigned int states_count, string_t *output) +{ + buffer_t *buf = buffer_create_dynamic(pool_datastack_create(), 128); + unsigned int i; + + for (i = 0; i < states_count; i++) { + const struct dsync_mailbox_state *state = &states[i]; + + buffer_append(buf, state->mailbox_guid, + sizeof(state->mailbox_guid)); + put_uint32(buf, state->last_uidvalidity); + put_uint32(buf, state->last_common_uid); + put_uint32(buf, state->last_common_modseq & 0xffffffffU); + put_uint32(buf, state->last_common_modseq >> 32); + } + put_uint32(buf, crc32_data(buf->data, buf->used)); + base64_encode(buf->data, buf->used, output); +} + +int dsync_mailbox_states_import(ARRAY_TYPE(dsync_mailbox_state) *states, + const char *input, const char **error_r) +{ + struct dsync_mailbox_state *state; + buffer_t *buf; + const unsigned char *data; + size_t pos; + unsigned int i, count; + + buf = buffer_create_dynamic(pool_datastack_create(), strlen(input)); + if (base64_decode(input, strlen(input), &pos, buf) < 0) { + *error_r = "Invalid base64 data"; + return -1; + } + if (buf->used < 4) { + *error_r = "Input too small"; + return -1; + } + if ((buf->used-4) % MAILBOX_SIZE != 0) { + *error_r = "Invalid input size"; + return -1; + } + data = buf->data; + count = (buf->used-4) / MAILBOX_SIZE; + + if (get_uint32(data + buf->used-4) != crc32_data(data, buf->used-4)) { + *error_r = "CRC32 mismatch"; + return -1; + } + + for (i = 0; i < count; i++, data += MAILBOX_SIZE) { + state = array_append_space(states); + memcpy(state->mailbox_guid, data, GUID_128_SIZE); + state->last_uidvalidity = get_uint32(data + GUID_128_SIZE); + state->last_common_uid = get_uint32(data + GUID_128_SIZE + 4); + state->last_common_modseq = + get_uint32(data + GUID_128_SIZE + 8) | + (uint64_t)get_uint32(data + GUID_128_SIZE + 12) << 32; + } + return 0; +}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/doveadm/dsync/dsync-mailbox-state.h Tue May 22 23:17:31 2012 +0300 @@ -0,0 +1,19 @@ +#ifndef DSYNC_MAILBOX_STATE_H +#define DSYNC_MAILBOX_STATE_H + +#include "guid.h" + +struct dsync_mailbox_state { + guid_128_t mailbox_guid; + uint32_t last_uidvalidity; + uint32_t last_common_uid; + uint64_t last_common_modseq; +}; +ARRAY_DEFINE_TYPE(dsync_mailbox_state, struct dsync_mailbox_state); + +void dsync_mailbox_states_export(const struct dsync_mailbox_state *states, + unsigned int states_count, string_t *output); +int dsync_mailbox_states_import(ARRAY_TYPE(dsync_mailbox_state) *states, + const char *input, const char **error_r); + +#endif
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/doveadm/dsync/dsync-mailbox-tree-fill.c Tue May 22 23:17:31 2012 +0300 @@ -0,0 +1,211 @@ +/* Copyright (c) 2012 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "hash.h" +#include "guid.h" +#include "str.h" +#include "mailbox-log.h" +#include "mail-namespace.h" +#include "mail-storage.h" +#include "mailbox-list.h" +#include "dsync-mailbox-tree-private.h" + +static int +dsync_mailbox_tree_add_node(struct dsync_mailbox_tree *tree, + const struct mailbox_info *info, + struct dsync_mailbox_node **node_r) +{ + struct dsync_mailbox_node *node; + + node = dsync_mailbox_tree_get(tree, info->name); + if (node->ns != info->ns) { + i_assert(node->ns != NULL); + + i_error("Mailbox '%s' exists in two namespaces: '%s' and '%s'", + info->name, node->ns->prefix, info->ns->prefix); + return -1; + } + *node_r = node; + return 0; +} + +static int dsync_mailbox_tree_add(struct dsync_mailbox_tree *tree, + const struct mailbox_info *info) +{ + struct dsync_mailbox_node *node; + struct mailbox *box; + struct mailbox_metadata metadata; + struct mailbox_status status; + const char *errstr; + enum mail_error error; + + if ((info->flags & MAILBOX_NONEXISTENT) != 0) + return 0; + + if (dsync_mailbox_tree_add_node(tree, info, &node) < 0) + return -1; + node->existence = DSYNC_MAILBOX_NODE_EXISTS; + + if ((info->flags & MAILBOX_NOSELECT) != 0) + return 0; + + /* get GUID and UIDVALIDITY for selectable mailbox */ + box = mailbox_alloc(info->ns->list, info->name, 0); + if (mailbox_get_metadata(box, MAILBOX_METADATA_GUID, &metadata) < 0 || + mailbox_get_status(box, STATUS_UIDVALIDITY, &status) < 0) { + errstr = mailbox_get_last_error(box, &error); + switch (error) { + case MAIL_ERROR_NOTFOUND: + /* mailbox was just deleted? */ + break; + case MAIL_ERROR_NOTPOSSIBLE: + /* invalid mbox files? ignore */ + break; + default: + i_error("Failed to access mailbox %s: %s", + info->name, errstr); + mailbox_free(&box); + return -1; + } + } else { + memcpy(node->mailbox_guid, metadata.guid, + sizeof(node->mailbox_guid)); + node->uid_validity = status.uidvalidity; + } + mailbox_free(&box); + return 0; +} + +static struct dsync_mailbox_node * +dsync_mailbox_tree_find_sha(struct dsync_mailbox_tree *tree, + struct mail_namespace *ns, const guid_128_t sha128) +{ + struct dsync_mailbox_node *node; + + if (tree->name128_hash == NULL) + dsync_mailbox_tree_build_name128_hash(tree); + + node = hash_table_lookup(tree->name128_hash, sha128); + return node == NULL || node->ns != ns ? NULL : node; +} + +static int +dsync_mailbox_tree_add_change_timestamps(struct dsync_mailbox_tree *tree, + struct mail_namespace *ns) +{ + struct dsync_mailbox_node *node; + struct dsync_mailbox_delete *del; + struct mailbox_log *log; + struct mailbox_log_iter *iter; + const struct mailbox_log_record *rec; + time_t timestamp; + + log = mailbox_list_get_changelog(ns->list); + if (log == NULL) + return 0; + + iter = mailbox_log_iter_init(log); + while ((rec = mailbox_log_iter_next(iter)) != NULL) { + node = rec->type == MAILBOX_LOG_RECORD_DELETE_MAILBOX ? NULL : + dsync_mailbox_tree_find_sha(tree, ns, rec->mailbox_guid); + + switch (rec->type) { + case MAILBOX_LOG_RECORD_DELETE_MAILBOX: + if (hash_table_lookup(tree->guid_hash, + rec->mailbox_guid) != NULL) { + /* mailbox still exists. maybe it was restored + from backup or something. */ + break; + } + del = array_append_space(&tree->deletes); + del->delete_mailbox = TRUE; + memcpy(del->guid, rec->mailbox_guid, sizeof(del->guid)); + break; + case MAILBOX_LOG_RECORD_DELETE_DIR: + if (node != NULL) { + /* mailbox exists again, skip it */ + break; + } + del = array_append_space(&tree->deletes); + memcpy(del->guid, rec->mailbox_guid, sizeof(del->guid)); + break; + case MAILBOX_LOG_RECORD_RENAME: + case MAILBOX_LOG_RECORD_SUBSCRIBE: + case MAILBOX_LOG_RECORD_UNSUBSCRIBE: + if (node == NULL) + break; + + timestamp = mailbox_log_record_get_timestamp(rec); + if (rec->type == MAILBOX_LOG_RECORD_RENAME) + node->last_renamed = timestamp; + else + node->last_subscription_change = timestamp; + break; + } + } + if (mailbox_log_iter_deinit(&iter) < 0) { + i_error("Mailbox log iteration for namespace '%s' failed", + ns->prefix); + return -1; + } + return 0; +} + +int dsync_mailbox_tree_fill(struct dsync_mailbox_tree *tree, + struct mail_namespace *ns) +{ + const enum mailbox_list_iter_flags list_flags = + MAILBOX_LIST_ITER_NO_AUTO_BOXES; + const enum mailbox_list_iter_flags subs_list_flags = + MAILBOX_LIST_ITER_NO_AUTO_BOXES | + MAILBOX_LIST_ITER_SELECT_SUBSCRIBED | + MAILBOX_LIST_ITER_RETURN_NO_FLAGS; + struct mailbox_list_iterate_context *iter; + struct dsync_mailbox_node *node; + const struct mailbox_info *info; + int ret = 0; + + i_assert(mail_namespace_get_sep(ns) == tree->sep); + + /* assign namespace to its root, so it gets copied to children */ + if (ns->prefix_len > 0) { + node = dsync_mailbox_tree_get(tree, + t_strndup(ns->prefix, ns->prefix_len-1)); + node->ns = ns; + } else { + tree->root.ns = ns; + } + + /* first add all of the existing mailboxes */ + iter = mailbox_list_iter_init(ns->list, "*", list_flags); + while ((info = mailbox_list_iter_next(iter)) != NULL) { + if (dsync_mailbox_tree_add(tree, info) < 0) + ret = -1; + } + if (mailbox_list_iter_deinit(&iter) < 0) { + i_error("Mailbox listing for namespace '%s' failed", ns->prefix); + ret = -1; + } + + /* add subscriptions */ + iter = mailbox_list_iter_init(ns->list, "*", subs_list_flags); + while ((info = mailbox_list_iter_next(iter)) != NULL) { + if (dsync_mailbox_tree_add_node(tree, info, &node) < 0) + ret = -1; + else + node->subscribed = TRUE; + } + if (mailbox_list_iter_deinit(&iter) < 0) { + i_error("Mailbox listing for namespace '%s' failed", ns->prefix); + ret = -1; + } + + if (dsync_mailbox_tree_build_guid_hash(tree) < 0) + ret = -1; + + /* add timestamps */ + if (dsync_mailbox_tree_add_change_timestamps(tree, ns) < 0) + ret = -1; + return ret; +}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/doveadm/dsync/dsync-mailbox-tree-private.h Tue May 22 23:17:31 2012 +0300 @@ -0,0 +1,23 @@ +#ifndef DSYNC_MAILBOX_TREE_PRIVATE_H +#define DSYNC_MAILBOX_TREE_PRIVATE_H + +#include "dsync-mailbox-tree.h" + +struct dsync_mailbox_tree { + pool_t pool; + char sep, sep_str[2], remote_sep; + struct dsync_mailbox_node root; + + unsigned int iter_count; + + ARRAY_DEFINE(deletes, struct dsync_mailbox_delete); + + /* guid_128_t => struct dsync_mailbox_node */ + struct hash_table *name128_hash; + struct hash_table *name128_remotesep_hash; + struct hash_table *guid_hash; +}; + +void dsync_mailbox_tree_build_name128_hash(struct dsync_mailbox_tree *tree); + +#endif
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/doveadm/dsync/dsync-mailbox-tree-sync.c Tue May 22 23:17:31 2012 +0300 @@ -0,0 +1,603 @@ +/* Copyright (c) 2012 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "aqueue.h" +#include "hash.h" +#include "dsync-mailbox-tree-private.h" + +struct dsync_mailbox_tree_bfs_iter { + struct dsync_mailbox_tree *tree; + + ARRAY_DEFINE(queue_arr, struct dsync_mailbox_node *); + struct aqueue *queue; + struct dsync_mailbox_node *cur; +}; + +struct dsync_mailbox_tree_sync_ctx { + pool_t pool; + struct dsync_mailbox_tree *local_tree, *remote_tree; + + ARRAY_DEFINE(changes, struct dsync_mailbox_tree_sync_change); + unsigned int change_idx; +}; + +static struct dsync_mailbox_tree_bfs_iter * +dsync_mailbox_tree_bfs_iter_init(struct dsync_mailbox_tree *tree) +{ + struct dsync_mailbox_tree_bfs_iter *iter; + + iter = i_new(struct dsync_mailbox_tree_bfs_iter, 1); + iter->tree = tree; + i_array_init(&iter->queue_arr, 32); + iter->queue = aqueue_init(&iter->queue_arr.arr); + iter->cur = tree->root.first_child; + return iter; +} + +static bool +dsync_mailbox_tree_bfs_iter_next(struct dsync_mailbox_tree_bfs_iter *iter, + struct dsync_mailbox_node **node_r) +{ + struct dsync_mailbox_node *const *nodep; + + if (iter->cur == NULL) { + if (aqueue_count(iter->queue) == 0) + return FALSE; + nodep = array_idx(&iter->queue_arr, aqueue_idx(iter->queue, 0)); + iter->cur = *nodep; + aqueue_delete_tail(iter->queue); + } + *node_r = iter->cur; + + if (iter->cur->first_child != NULL) + aqueue_append(iter->queue, &iter->cur->first_child); + iter->cur = iter->cur->next; + return TRUE; +} + +static bool +dsync_mailbox_tree_bfs_iter_is_eob(struct dsync_mailbox_tree_bfs_iter *iter) +{ + return iter->cur == NULL; +} + +static void +dsync_mailbox_tree_bfs_iter_deinit(struct dsync_mailbox_tree_bfs_iter **_iter) +{ + struct dsync_mailbox_tree_bfs_iter *iter = *_iter; + + *_iter = NULL; + + aqueue_deinit(&iter->queue); + array_free(&iter->queue_arr); + i_free(iter); +} + +static int dsync_mailbox_node_sync_cmp(struct dsync_mailbox_node *const *n1, + struct dsync_mailbox_node *const *n2) +{ + return strcmp((*n1)->name, (*n2)->name); +} + +static void sort_siblings(ARRAY_TYPE(dsync_mailbox_node) *siblings) +{ + struct dsync_mailbox_node *const *nodes; + unsigned int i, count; + + array_sort(siblings, dsync_mailbox_node_sync_cmp); + + nodes = array_get(siblings, &count); + if (count == 0) + return; + + nodes[0]->parent->first_child = nodes[0]; + for (i = 1; i < count; i++) + nodes[i-1]->next = nodes[i]; + nodes[count-1]->next = NULL; +} + +static void +sync_delete_mailbox(struct dsync_mailbox_tree_sync_ctx *ctx, + struct dsync_mailbox_tree *tree, + struct dsync_mailbox_node *node) +{ + struct dsync_mailbox_tree *other_tree; + struct dsync_mailbox_node *other_node; + struct dsync_mailbox_tree_sync_change *change; + const char *name; + + other_tree = tree == ctx->local_tree ? + ctx->remote_tree : ctx->local_tree; + other_node = hash_table_lookup(other_tree->guid_hash, + node->mailbox_guid); + if (other_node == NULL) { + /* doesn't exist / already deleted */ + } else if (other_tree == ctx->local_tree) { + /* delete this mailbox locally */ + change = array_append_space(&ctx->changes); + change->type = DSYNC_MAILBOX_TREE_SYNC_TYPE_DELETE_BOX; + change->ns = other_node->ns; + name = dsync_mailbox_node_get_full_name(other_tree, other_node); + change->full_name = p_strdup(ctx->pool, name); + memcpy(change->mailbox_guid, node->mailbox_guid, + sizeof(change->mailbox_guid)); + } + + /* for the rest of this sync assume that the mailbox has + already been deleted */ + if (other_node != NULL) { + other_node->existence = DSYNC_MAILBOX_NODE_DELETED; + memset(other_node->mailbox_guid, 0, + sizeof(other_node->mailbox_guid)); + } + memset(node->mailbox_guid, 0, sizeof(node->mailbox_guid)); +} + +static void +sync_tree_sort_and_delete_mailboxes(struct dsync_mailbox_tree_sync_ctx *ctx, + struct dsync_mailbox_tree *tree) +{ + struct dsync_mailbox_tree_bfs_iter *iter; + struct dsync_mailbox_node *node, *parent = NULL; + ARRAY_TYPE(dsync_mailbox_node) siblings; + + t_array_init(&siblings, 64); + + iter = dsync_mailbox_tree_bfs_iter_init(tree); + while (dsync_mailbox_tree_bfs_iter_next(iter, &node)) { + if (node->parent != parent) { + sort_siblings(&siblings); + array_clear(&siblings); + parent = node->parent; + } + if (node->existence == DSYNC_MAILBOX_NODE_DELETED && + !guid_128_is_empty(node->mailbox_guid)) + sync_delete_mailbox(ctx, tree, node); + array_append(&siblings, &node, 1); + } + sort_siblings(&siblings); + dsync_mailbox_tree_bfs_iter_deinit(&iter); +} + +static struct dsync_mailbox_node * +sync_find_node(struct dsync_mailbox_tree *tree, + struct dsync_mailbox_node *other_node) +{ + struct dsync_mailbox_node *n1, *n2; + + if (!guid_128_is_empty(other_node->mailbox_guid)) { + return hash_table_lookup(tree->guid_hash, + other_node->mailbox_guid); + } + /* if we can find a node that has all of the same mailboxes as children, + return it. */ + for (n1 = other_node->first_child; n1 != NULL; n1 = n1->next) { + if (!guid_128_is_empty(n1->mailbox_guid)) + break; + } + if (n1 == NULL) + return NULL; + n2 = hash_table_lookup(tree->guid_hash, n1->mailbox_guid); + if (n2 == NULL) + return NULL; + + /* note that all of the nodes are sorted at this point. */ + n1 = n1->parent->first_child; + n2 = n2->parent->first_child; + for (; n1 != NULL && n2 != NULL; n1 = n1->next, n2 = n2->next) { + if (strcmp(n1->name, n2->name) != 0 || + memcmp(n1->mailbox_guid, n2->mailbox_guid, + sizeof(n1->mailbox_guid)) != 0) + break; + } + if (n1 != NULL || n2 != NULL) + return NULL; + return n2; +} + +static bool node_names_equal(const struct dsync_mailbox_node *n1, + const struct dsync_mailbox_node *n2) +{ + while (n1 != NULL && n2 != NULL) { + if (strcmp(n1->name, n2->name) != 0) + return FALSE; + n1 = n1->parent; + n2 = n2->parent; + } + return n1 == NULL && n2 == NULL; +} + +static void +dsync_mailbox_tree_node_detach(struct dsync_mailbox_node *node) +{ + struct dsync_mailbox_node **p; + for (p = &node->parent->first_child;; p = &(*p)->next) { + if (*p == node) { + *p = node->next; + break; + } + } +} + +static void +dsync_mailbox_tree_node_attach_sorted(struct dsync_mailbox_node *node, + struct dsync_mailbox_node *parent) +{ + struct dsync_mailbox_node **p; + + node->parent = parent; + for (p = &parent->first_child; *p != NULL; p = &(*p)->next) { + if (dsync_mailbox_node_sync_cmp(p, &node) > 0) + break; + } + node->next = *p; + *p = node; +} + +static void +dsync_mailbox_tree_node_move_sorted(struct dsync_mailbox_node *node, + struct dsync_mailbox_node *parent) +{ + if (node->parent != parent) { + /* detach from old parent */ + dsync_mailbox_tree_node_detach(node); + /* attach to new parent */ + dsync_mailbox_tree_node_attach_sorted(node, parent); + } +} + +static struct dsync_mailbox_node * +sorted_tree_get(struct dsync_mailbox_tree *tree, const char *name) +{ + struct dsync_mailbox_node *node, *parent, *ret; + + node = ret = dsync_mailbox_tree_get(tree, name); + while (node->parent != NULL && + node->existence == DSYNC_MAILBOX_NODE_NONEXISTENT) { + parent = node->parent; + dsync_mailbox_tree_node_detach(node); + dsync_mailbox_tree_node_attach_sorted(node, parent); + node = parent; + } + return ret; +} + +static struct dsync_mailbox_node * +sorted_tree_get_by_node_name(struct dsync_mailbox_tree *tree, + struct dsync_mailbox_tree *other_tree, + struct dsync_mailbox_node *other_node) +{ + const char *parent_name; + + if (other_node == &other_tree->root) + return &tree->root; + + parent_name = dsync_mailbox_node_get_full_name(other_tree, other_node); + return sorted_tree_get(tree, parent_name); +} + +static void +sync_rename_node(struct dsync_mailbox_tree_sync_ctx *ctx, + struct dsync_mailbox_tree *tree, + struct dsync_mailbox_node *node, + struct dsync_mailbox_node *other_node) +{ + struct dsync_mailbox_tree_sync_change *change; + struct dsync_mailbox_tree *other_tree; + struct dsync_mailbox_node *parent; + const char *name, *other_name; + bool use_node_name, local_tree_changed; + + other_tree = tree == ctx->local_tree ? + ctx->remote_tree : ctx->local_tree; + + name = dsync_mailbox_node_get_full_name(tree, node); + other_name = dsync_mailbox_node_get_full_name(other_tree, other_node); + + if (node->last_renamed > other_node->last_renamed || + (node->last_renamed == other_node->last_renamed && + strcmp(name, other_name) > 0)) { + /* use node's name */ + local_tree_changed = tree == ctx->remote_tree; + other_node->name = p_strdup(other_tree->pool, node->name); + + /* move node if necessary */ + parent = sorted_tree_get_by_node_name(other_tree, tree, + node->parent); + dsync_mailbox_tree_node_move_sorted(other_node, parent); + } else { + /* other other_node's name */ + use_node_name = FALSE; + local_tree_changed = other_tree == ctx->remote_tree; + node->name = p_strdup(tree->pool, other_node->name); + + /* move node if necessary */ + parent = sorted_tree_get_by_node_name(tree, other_tree, + other_node->parent); + dsync_mailbox_tree_node_move_sorted(node, parent); + } + + //FIXME: handle if destination name already exists + + if (local_tree_changed) { + change = array_append_space(&ctx->changes); + change->type = DSYNC_MAILBOX_TREE_SYNC_TYPE_RENAME; + change->ns = other_node->ns; + if (use_node_name) { + change->full_name = p_strdup(ctx->pool, other_name); + change->rename_dest_name = p_strdup(ctx->pool, name); + } else { + change->full_name = p_strdup(ctx->pool, name); + change->rename_dest_name = + p_strdup(ctx->pool, other_name); + } + } +} + +static void sync_rename_branch(struct dsync_mailbox_tree_sync_ctx *ctx, + struct dsync_mailbox_tree_bfs_iter *iter, + struct dsync_mailbox_tree *tree) +{ + struct dsync_mailbox_tree *other_tree; + struct dsync_mailbox_node *node, *other_node; + + other_tree = tree == ctx->local_tree ? + ctx->remote_tree : ctx->local_tree; + while (!dsync_mailbox_tree_bfs_iter_is_eob(iter)) { + if (!dsync_mailbox_tree_bfs_iter_next(iter, &node)) + i_unreached(); + + other_node = sync_find_node(other_tree, node); + if (other_node == NULL) { + /* this mailbox will be created later */ + } else if (node_names_equal(node, other_node)) { + /* mailbox name hasn't changed */ + } else { + /* mailbox name or tree branch has changed */ + sync_rename_node(ctx, tree, node, other_node); + } + } +} + +static void sync_rename_mailboxes(struct dsync_mailbox_tree_sync_ctx *ctx) +{ + struct dsync_mailbox_tree_bfs_iter *local_iter, *remote_iter; + struct dsync_mailbox_node *node; + + local_iter = dsync_mailbox_tree_bfs_iter_init(ctx->local_tree); + remote_iter = dsync_mailbox_tree_bfs_iter_init(ctx->remote_tree); + + do { + sync_rename_branch(ctx, local_iter, ctx->local_tree); + sync_rename_branch(ctx, remote_iter, ctx->remote_tree); + } while (dsync_mailbox_tree_bfs_iter_next(local_iter, &node) || + dsync_mailbox_tree_bfs_iter_next(remote_iter, &node)); + + dsync_mailbox_tree_bfs_iter_deinit(&local_iter); + dsync_mailbox_tree_bfs_iter_deinit(&remote_iter); +} + +static void +sync_add_create_change(struct dsync_mailbox_tree_sync_ctx *ctx, + const struct dsync_mailbox_node *node, const char *name) +{ + struct dsync_mailbox_tree_sync_change *change; + + change = array_append_space(&ctx->changes); + change->type = DSYNC_MAILBOX_TREE_SYNC_TYPE_CREATE_BOX; + change->ns = node->ns; + change->full_name = p_strdup(ctx->pool, name); + memcpy(change->mailbox_guid, node->mailbox_guid, + sizeof(change->mailbox_guid)); + change->uid_validity = node->uid_validity; +} + +static void sync_create_mailboxes(struct dsync_mailbox_tree_sync_ctx *ctx, + struct dsync_mailbox_tree *tree) +{ + struct dsync_mailbox_tree *other_tree; + struct dsync_mailbox_tree_iter *iter; + struct dsync_mailbox_node *node, *other_node; + const char *name; + + other_tree = tree == ctx->local_tree ? + ctx->remote_tree : ctx->local_tree; + + iter = dsync_mailbox_tree_iter_init(tree); + while (dsync_mailbox_tree_iter_next(iter, &name, &node)) { + if (guid_128_is_empty(node->mailbox_guid)) + continue; + + i_assert(node->existence == DSYNC_MAILBOX_NODE_EXISTS); + + other_node = hash_table_lookup(other_tree->guid_hash, + node->mailbox_guid); + if (other_node == NULL) + other_node = sorted_tree_get(other_tree, name); + if (!guid_128_is_empty(other_node->mailbox_guid)) { + /* already exists */ + i_assert(node->existence == DSYNC_MAILBOX_NODE_EXISTS); + // FIXME: remove this assert? for conflicting GUIDs + i_assert(memcmp(node->mailbox_guid, + other_node->mailbox_guid, + sizeof(node->mailbox_guid)) == 0); + } else { + other_node->existence = DSYNC_MAILBOX_NODE_EXISTS; + other_node->ns = node->ns; + other_node->uid_validity = node->uid_validity; + memcpy(other_node->mailbox_guid, node->mailbox_guid, + sizeof(other_node->mailbox_guid)); + if (other_tree == ctx->local_tree) + sync_add_create_change(ctx, other_node, name); + } + } + dsync_mailbox_tree_iter_deinit(&iter); +} + +static void +sync_add_dir_change(struct dsync_mailbox_tree_sync_ctx *ctx, + const struct dsync_mailbox_node *node, + enum dsync_mailbox_tree_sync_type type) +{ + struct dsync_mailbox_tree_sync_change *change; + const char *name; + + name = dsync_mailbox_node_get_full_name(ctx->local_tree, node); + + change = array_append_space(&ctx->changes); + change->type = type; + change->ns = node->ns; + change->full_name = p_strdup(ctx->pool, name); +} + +static struct dsync_mailbox_node * +sync_node_new(struct dsync_mailbox_tree *tree, + struct dsync_mailbox_node *parent, + const struct dsync_mailbox_node *src) +{ + struct dsync_mailbox_node *node; + + node = p_new(tree->pool, struct dsync_mailbox_node, 1); + node->name = p_strdup(tree->pool, src->name); + node->ns = src->ns; + dsync_mailbox_tree_node_attach_sorted(node, parent); + return node; +} + +static void +sync_subscription(struct dsync_mailbox_tree_sync_ctx *ctx, + struct dsync_mailbox_node *local_node, + struct dsync_mailbox_node *remote_node) +{ + if (local_node->last_subscription_change > + remote_node->last_subscription_change || + (local_node->last_subscription_change == + remote_node->last_subscription_change && local_node->subscribed)) { + /* use local subscription state */ + remote_node->subscribed = local_node->subscribed; + } else { + /* use remote subscription state */ + local_node->subscribed = remote_node->subscribed; + sync_add_dir_change(ctx, local_node, local_node->subscribed ? + DSYNC_MAILBOX_TREE_SYNC_TYPE_SUBSCRIBE : + DSYNC_MAILBOX_TREE_SYNC_TYPE_UNSUBSCRIBE); + } +} + +static void sync_mailbox_child_dirs(struct dsync_mailbox_tree_sync_ctx *ctx, + struct dsync_mailbox_node *local_parent, + struct dsync_mailbox_node *remote_parent) +{ + struct dsync_mailbox_node *local_node, *remote_node; + + /* NOTE: the nodes are always sorted */ + local_node = local_parent->first_child; + remote_node = remote_parent->first_child; + while (local_node != NULL || remote_node != NULL) { + /* add missing nodes, even if we don't really need to do + anything with them. */ + if (remote_node != NULL && + (local_node == NULL || + strcmp(local_node->name, remote_node->name) > 0)) { + local_node = sync_node_new(ctx->local_tree, local_parent, + remote_node); + } + if (local_node != NULL && + (remote_node == NULL || + strcmp(remote_node->name, local_node->name) > 0)) { + remote_node = sync_node_new(ctx->remote_tree, remote_parent, + local_node); + } + + if (local_node->existence == DSYNC_MAILBOX_NODE_EXISTS && + remote_node->existence == DSYNC_MAILBOX_NODE_NONEXISTENT) { + /* create to remote */ + remote_node->existence = DSYNC_MAILBOX_NODE_EXISTS; + } + if (remote_node->existence == DSYNC_MAILBOX_NODE_EXISTS && + local_node->existence == DSYNC_MAILBOX_NODE_NONEXISTENT) { + /* create to local */ + local_node->existence = DSYNC_MAILBOX_NODE_EXISTS; + sync_add_dir_change(ctx, local_node, + DSYNC_MAILBOX_TREE_SYNC_TYPE_CREATE_DIR); + } + + /* create/delete child dirs */ + sync_mailbox_child_dirs(ctx, local_node, remote_node); + + if (local_node->subscribed != remote_node->subscribed) + sync_subscription(ctx, local_node, remote_node); + + if (local_node->existence == DSYNC_MAILBOX_NODE_DELETED && + local_node->first_child == NULL && + remote_node->existence == DSYNC_MAILBOX_NODE_EXISTS) { + /* delete from remote */ + i_assert(remote_node->first_child == NULL); + remote_node->existence = DSYNC_MAILBOX_NODE_NONEXISTENT; + } + if (remote_node->existence == DSYNC_MAILBOX_NODE_DELETED && + remote_node->first_child == NULL && + local_node->existence == DSYNC_MAILBOX_NODE_EXISTS) { + /* delete from local */ + i_assert(local_node->first_child == NULL); + local_node->existence = DSYNC_MAILBOX_NODE_NONEXISTENT; + sync_add_dir_change(ctx, local_node, + DSYNC_MAILBOX_TREE_SYNC_TYPE_DELETE_DIR); + } + + local_node = local_node->next; + remote_node = remote_node->next; + } +} + +static void sync_mailbox_dirs(struct dsync_mailbox_tree_sync_ctx *ctx) +{ + sync_mailbox_child_dirs(ctx, &ctx->local_tree->root, + &ctx->remote_tree->root); +} + +struct dsync_mailbox_tree_sync_ctx * +dsync_mailbox_trees_sync_init(struct dsync_mailbox_tree *local_tree, + struct dsync_mailbox_tree *remote_tree) +{ + struct dsync_mailbox_tree_sync_ctx *ctx; + pool_t pool; + + i_assert(local_tree->guid_hash != NULL); + i_assert(remote_tree->guid_hash != NULL); + + pool = pool_alloconly_create(MEMPOOL_GROWING"dsync mailbox trees sync", + 1024*64); + ctx = p_new(pool, struct dsync_mailbox_tree_sync_ctx, 1); + ctx->pool = pool; + ctx->local_tree = local_tree; + ctx->remote_tree = remote_tree; + i_array_init(&ctx->changes, 128); + + sync_tree_sort_and_delete_mailboxes(ctx, remote_tree); + sync_tree_sort_and_delete_mailboxes(ctx, local_tree); + sync_rename_mailboxes(ctx); + sync_create_mailboxes(ctx, remote_tree); + sync_create_mailboxes(ctx, local_tree); + sync_mailbox_dirs(ctx); + return ctx; +} + +const struct dsync_mailbox_tree_sync_change * +dsync_mailbox_trees_sync_next(struct dsync_mailbox_tree_sync_ctx *ctx) +{ + if (ctx->change_idx == array_count(&ctx->changes)) + return NULL; + return array_idx(&ctx->changes, ctx->change_idx++); +} + +void dsync_mailbox_trees_sync_deinit(struct dsync_mailbox_tree_sync_ctx **_ctx) +{ + struct dsync_mailbox_tree_sync_ctx *ctx = *_ctx; + + *_ctx = NULL; + + array_free(&ctx->changes); + pool_unref(&ctx->pool); +}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/doveadm/dsync/dsync-mailbox-tree.c Tue May 22 23:17:31 2012 +0300 @@ -0,0 +1,349 @@ +/* Copyright (c) 2012 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "hash.h" +#include "str.h" +#include "doveadm-settings.h" +#include "mailbox-list-private.h" +#include "dsync-mailbox-tree-private.h" + +struct dsync_mailbox_tree_iter { + struct dsync_mailbox_tree *tree; + + struct dsync_mailbox_node *cur; + string_t *name; +}; + +struct dsync_mailbox_tree *dsync_mailbox_tree_init(char sep) +{ + struct dsync_mailbox_tree *tree; + pool_t pool; + + i_assert(sep != '\0'); + + pool = pool_alloconly_create("dsync mailbox tree", 4096); + tree = p_new(pool, struct dsync_mailbox_tree, 1); + tree->pool = pool; + tree->sep = tree->sep_str[0] = sep; + tree->root.name = ""; + i_array_init(&tree->deletes, 32); + return tree; +} + +void dsync_mailbox_tree_deinit(struct dsync_mailbox_tree **_tree) +{ + struct dsync_mailbox_tree *tree = *_tree; + + *_tree = NULL; + if (tree->name128_hash != NULL) + hash_table_destroy(&tree->name128_hash); + if (tree->guid_hash != NULL) + hash_table_destroy(&tree->guid_hash); + array_free(&tree->deletes); + pool_unref(&tree->pool); +} + +static struct dsync_mailbox_node * +dsync_mailbox_node_find(struct dsync_mailbox_node *nodes, const char *name) +{ + for (; nodes != NULL; nodes = nodes->next) { + if (strcmp(name, nodes->name) == 0) + return nodes; + } + return NULL; +} + +struct dsync_mailbox_node * +dsync_mailbox_tree_lookup(struct dsync_mailbox_tree *tree, + const char *full_name) +{ + struct dsync_mailbox_node *node = &tree->root; + + T_BEGIN { + const char *const *path; + + path = t_strsplit(full_name, tree->sep_str); + for (; *path != '\0' && node != NULL; path++) + node = dsync_mailbox_node_find(node->first_child, *path); + } T_END; + return node; +} + +struct dsync_mailbox_node * +dsync_mailbox_tree_get(struct dsync_mailbox_tree *tree, const char *full_name) +{ + struct dsync_mailbox_node *parent = NULL, *node = &tree->root; + + i_assert(tree->iter_count == 0); + + T_BEGIN { + const char *const *path; + + /* find the existing part */ + path = t_strsplit(full_name, tree->sep_str); + for (; *path != '\0'; path++) { + parent = node; + node = dsync_mailbox_node_find(node->first_child, *path); + if (node == NULL) + break; + } + /* create the rest */ + for (; *path != '\0'; path++) { + node = p_new(tree->pool, struct dsync_mailbox_node, 1); + node->name = p_strdup(tree->pool, *path); + node->ns = parent->ns; + node->parent = parent; + node->next = parent->first_child; + parent->first_child = node; + parent = node; + } + } T_END; + return node; +} + +static void +node_get_full_name_recurse(const struct dsync_mailbox_tree *tree, + const struct dsync_mailbox_node *node, string_t *str) +{ + if (node->parent != &tree->root) + node_get_full_name_recurse(tree, node->parent, str); + + str_append(str, node->name); + str_append_c(str, tree->sep); +} + +const char *dsync_mailbox_node_get_full_name(const struct dsync_mailbox_tree *tree, + const struct dsync_mailbox_node *node) +{ + string_t *str = t_str_new(128); + + i_assert(node->parent != NULL); + + node_get_full_name_recurse(tree, node, str); + /* remove the trailing separator */ + str_truncate(str, str_len(str)-1); + return str_c(str); +} + +void dsync_mailbox_node_copy_data(struct dsync_mailbox_node *dest, + const struct dsync_mailbox_node *src) +{ + memcpy(dest->mailbox_guid, src->mailbox_guid, + sizeof(dest->mailbox_guid)); + dest->uid_validity = src->uid_validity; + dest->existence = src->existence; + dest->last_renamed = src->last_renamed; + dest->subscribed = src->subscribed; + dest->last_subscription_change = src->last_subscription_change; +} + +struct dsync_mailbox_tree_iter * +dsync_mailbox_tree_iter_init(struct dsync_mailbox_tree *tree) +{ + struct dsync_mailbox_tree_iter *iter; + + iter = i_new(struct dsync_mailbox_tree_iter, 1); + iter->tree = tree; + iter->name = str_new(default_pool, 128); + iter->cur = &tree->root; + + tree->iter_count++; + return iter; +} + +static unsigned int node_get_full_name_length(struct dsync_mailbox_node *node) +{ + if (node->parent->parent == NULL) + return strlen(node->name); + else { + return strlen(node->name) + 1 + + node_get_full_name_length(node->parent); + } +} + +bool dsync_mailbox_tree_iter_next(struct dsync_mailbox_tree_iter *iter, + const char **full_name_r, + struct dsync_mailbox_node **node_r) +{ + unsigned int len; + + if (iter->cur->first_child != NULL) + iter->cur = iter->cur->first_child; + else { + while (iter->cur->next == NULL) { + if (iter->cur == &iter->tree->root) + return FALSE; + iter->cur = iter->cur->parent; + } + iter->cur = iter->cur->next; + len = iter->cur->parent == &iter->tree->root ? 0 : + node_get_full_name_length(iter->cur->parent); + str_truncate(iter->name, len); + } + if (str_len(iter->name) > 0) + str_append_c(iter->name, iter->tree->sep); + str_append(iter->name, iter->cur->name); + *full_name_r = str_c(iter->name); + *node_r = iter->cur; + return TRUE; +} + +void dsync_mailbox_tree_iter_deinit(struct dsync_mailbox_tree_iter **_iter) +{ + struct dsync_mailbox_tree_iter *iter = *_iter; + + *_iter = NULL; + + i_assert(iter->tree->iter_count > 0); + iter->tree->iter_count--; + + str_free(&iter->name); + i_free(iter); +} + +void dsync_mailbox_tree_build_name128_hash(struct dsync_mailbox_tree *tree) +{ + struct dsync_mailbox_tree_iter *iter; + struct dsync_mailbox_node *node; + const char *name; + guid_128_t *sha128; + + i_assert(tree->name128_hash == NULL); + + tree->name128_hash = hash_table_create(default_pool, tree->pool, 0, + guid_128_hash, guid_128_cmp); + iter = dsync_mailbox_tree_iter_init(tree); + while (dsync_mailbox_tree_iter_next(iter, &name, &node)) { + sha128 = p_new(tree->pool, guid_128_t, 1); + mailbox_name_get_sha128(name, *sha128); + hash_table_insert(tree->name128_hash, *sha128, node); + } + dsync_mailbox_tree_iter_deinit(&iter); +} + +static const char * +convert_name_to_remote_sep(struct dsync_mailbox_tree *tree, const char *name) +{ + string_t *str = t_str_new(128); + char alt_char = doveadm_settings->dsync_alt_char[0]; + + for (; *name != '\0'; name++) { + if (*name == tree->sep) + str_append_c(str, tree->remote_sep); + else if (*name == tree->remote_sep) + str_append_c(str, alt_char); + else + str_append_c(str, *name); + } + return str_c(str); +} + +static void +dsync_mailbox_tree_build_name128_remotesep_hash(struct dsync_mailbox_tree *tree) +{ + struct dsync_mailbox_tree_iter *iter; + struct dsync_mailbox_node *node; + const char *name; + guid_128_t *sha128; + + i_assert(tree->sep != tree->remote_sep); + i_assert(tree->name128_remotesep_hash == NULL); + + tree->name128_remotesep_hash = + hash_table_create(default_pool, tree->pool, 0, + guid_128_hash, guid_128_cmp); + iter = dsync_mailbox_tree_iter_init(tree); + while (dsync_mailbox_tree_iter_next(iter, &name, &node)) { + sha128 = p_new(tree->pool, guid_128_t, 1); + T_BEGIN { + const char *remote_name = + convert_name_to_remote_sep(tree, name); + mailbox_name_get_sha128(remote_name, *sha128); + } T_END; + hash_table_insert(tree->name128_remotesep_hash, *sha128, node); + } + dsync_mailbox_tree_iter_deinit(&iter); +} + +int dsync_mailbox_tree_guid_hash_add(struct dsync_mailbox_tree *tree, + struct dsync_mailbox_node *node) +{ + struct dsync_mailbox_node *old_node; + + if (guid_128_is_empty(node->mailbox_guid)) + return 0; + + old_node = hash_table_lookup(tree->guid_hash, node->mailbox_guid); + if (old_node != NULL) { + i_error("Duplicate mailbox GUID %s " + "for mailboxes %s and %s", + guid_128_to_string(node->mailbox_guid), + dsync_mailbox_node_get_full_name(tree, old_node), + dsync_mailbox_node_get_full_name(tree, node)); + return -1; + } + hash_table_insert(tree->guid_hash, node->mailbox_guid, node); + return 0; +} + +int dsync_mailbox_tree_build_guid_hash(struct dsync_mailbox_tree *tree) +{ + struct dsync_mailbox_tree_iter *iter; + struct dsync_mailbox_node *node; + const char *name; + int ret = 0; + + i_assert(tree->guid_hash == NULL); + + tree->guid_hash = hash_table_create(default_pool, tree->pool, 0, + guid_128_hash, guid_128_cmp); + iter = dsync_mailbox_tree_iter_init(tree); + while (dsync_mailbox_tree_iter_next(iter, &name, &node)) + dsync_mailbox_tree_guid_hash_add(tree, node); + dsync_mailbox_tree_iter_deinit(&iter); + return ret; +} + +const struct dsync_mailbox_delete * +dsync_mailbox_tree_get_deletes(struct dsync_mailbox_tree *tree, + unsigned int *count_r) +{ + return array_get(&tree->deletes, count_r); +} + +struct dsync_mailbox_node * +dsync_mailbox_tree_find_delete(struct dsync_mailbox_tree *tree, + const struct dsync_mailbox_delete *del) +{ + struct hash_table *hash; + + i_assert(tree->guid_hash != NULL); + i_assert(tree->remote_sep != '\0'); + + if (del->delete_mailbox) { + /* find node by GUID */ + return hash_table_lookup(tree->guid_hash, del->guid); + } + + /* find node by name. this is a bit tricky, since the hierarchy + separator may differ from ours. */ + if (tree->sep == tree->remote_sep) { + if (tree->name128_hash == NULL) + dsync_mailbox_tree_build_name128_hash(tree); + hash = tree->name128_hash; + } else { + if (tree->name128_remotesep_hash == NULL) + dsync_mailbox_tree_build_name128_remotesep_hash(tree); + hash = tree->name128_remotesep_hash; + } + return hash_table_lookup(hash, del->guid); +} + +void dsync_mailbox_tree_set_remote_sep(struct dsync_mailbox_tree *tree, + char remote_sep) +{ + i_assert(tree->remote_sep == '\0'); + + tree->remote_sep = remote_sep; +}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/doveadm/dsync/dsync-mailbox-tree.h Tue May 22 23:17:31 2012 +0300 @@ -0,0 +1,139 @@ +#ifndef DSYNC_MAILBOX_TREE_H +#define DSYNC_MAILBOX_TREE_H + +#include "guid.h" + +struct mail_namespace; + +enum dsync_mailbox_node_existence { + /* this is just a filler node for children or for + subscription deletion */ + DSYNC_MAILBOX_NODE_NONEXISTENT = 0, + /* if mailbox GUID is set, the mailbox exists. + otherwise the directory exists. */ + DSYNC_MAILBOX_NODE_EXISTS, + /* if mailbox GUID is set, the mailbox has been deleted. + otherwise the directory has been deleted. */ + DSYNC_MAILBOX_NODE_DELETED +}; + +struct dsync_mailbox_node { + struct dsync_mailbox_node *parent, *next, *first_child; + + /* namespace where this node belongs to */ + struct mail_namespace *ns; + /* this node's name (not including parents) */ + const char *name; + /* mailbox GUID, or full of zeros if this is about a directory name */ + guid_128_t mailbox_guid; + /* mailbox's UIDVALIDITY (may be 0 if not assigned yet) */ + uint32_t uid_validity; + + /* existence of this mailbox/directory. + doesn't affect subscription state. */ + enum dsync_mailbox_node_existence existence; + /* last time the mailbox was renamed, 0 if not known */ + time_t last_renamed; + + /* is this mailbox or directory subscribed? */ + bool subscribed; + /* last time the subscription state was changed, 0 if not known */ + time_t last_subscription_change; +}; +ARRAY_DEFINE_TYPE(dsync_mailbox_node, struct dsync_mailbox_node *); + +struct dsync_mailbox_delete { + /* true: guid = mailbox GUID + false: guid = sha1 of directory name */ + bool delete_mailbox; + guid_128_t guid; +}; + +enum dsync_mailbox_tree_sync_type { + DSYNC_MAILBOX_TREE_SYNC_TYPE_CREATE_BOX, + DSYNC_MAILBOX_TREE_SYNC_TYPE_CREATE_DIR, + DSYNC_MAILBOX_TREE_SYNC_TYPE_DELETE_BOX, + DSYNC_MAILBOX_TREE_SYNC_TYPE_DELETE_DIR, + /* Rename given mailbox name and its children */ + DSYNC_MAILBOX_TREE_SYNC_TYPE_RENAME, + DSYNC_MAILBOX_TREE_SYNC_TYPE_SUBSCRIBE, + DSYNC_MAILBOX_TREE_SYNC_TYPE_UNSUBSCRIBE +}; + +struct dsync_mailbox_tree_sync_change { + enum dsync_mailbox_tree_sync_type type; + + /* for all types: */ + struct mail_namespace *ns; + const char *full_name; + + /* for create_box and delete_box: */ + guid_128_t mailbox_guid; + /* for create_box: */ + uint32_t uid_validity; + /* for rename: */ + const char *rename_dest_name; +}; + +struct dsync_mailbox_tree *dsync_mailbox_tree_init(char sep); +void dsync_mailbox_tree_deinit(struct dsync_mailbox_tree **tree); + +/* Lookup a mailbox node by name. Returns NULL if not known. */ +struct dsync_mailbox_node * +dsync_mailbox_tree_lookup(struct dsync_mailbox_tree *tree, + const char *full_name); +/* Lookup or create a mailbox node by name. */ +struct dsync_mailbox_node * +dsync_mailbox_tree_get(struct dsync_mailbox_tree *tree, const char *full_name); + +/* Returns full name for the given mailbox node. */ +const char *dsync_mailbox_node_get_full_name(const struct dsync_mailbox_tree *tree, + const struct dsync_mailbox_node *node); + +/* Copy everything from src to dest, except name and hierarchy pointers */ +void dsync_mailbox_node_copy_data(struct dsync_mailbox_node *dest, + const struct dsync_mailbox_node *src); + +/* Add nodes to tree from the given namespace. */ +int dsync_mailbox_tree_fill(struct dsync_mailbox_tree *tree, + struct mail_namespace *ns); + +/* Return all known deleted mailboxes and directories. */ +const struct dsync_mailbox_delete * +dsync_mailbox_tree_get_deletes(struct dsync_mailbox_tree *tree, + unsigned int *count_r); +/* Return mailbox node for a given delete record, or NULL if it doesn't exist. + The delete record is intended to come from another tree, possibly with + a different hierarchy separator. dsync_mailbox_tree_build_guid_hash() must + have been called before this. */ +struct dsync_mailbox_node * +dsync_mailbox_tree_find_delete(struct dsync_mailbox_tree *tree, + const struct dsync_mailbox_delete *del); +/* Build GUID lookup hash, if it's not already built. */ +int dsync_mailbox_tree_build_guid_hash(struct dsync_mailbox_tree *tree); +/* Manually add a new node to hash. */ +int dsync_mailbox_tree_guid_hash_add(struct dsync_mailbox_tree *tree, + struct dsync_mailbox_node *node); +/* Set remote separator used for directory deletions in + dsync_mailbox_tree_find_delete() */ +void dsync_mailbox_tree_set_remote_sep(struct dsync_mailbox_tree *tree, + char remote_sep); + +/* Iterate through all nodes in a tree (depth-first) */ +struct dsync_mailbox_tree_iter * +dsync_mailbox_tree_iter_init(struct dsync_mailbox_tree *tree); +bool dsync_mailbox_tree_iter_next(struct dsync_mailbox_tree_iter *iter, + const char **full_name_r, + struct dsync_mailbox_node **node_r); +void dsync_mailbox_tree_iter_deinit(struct dsync_mailbox_tree_iter **iter); + +/* Sync local and remote trees so at the end they're exactly the same. + Return changes done to local tree. */ +struct dsync_mailbox_tree_sync_ctx * +dsync_mailbox_trees_sync_init(struct dsync_mailbox_tree *local_tree, + struct dsync_mailbox_tree *remote_tree); +const struct dsync_mailbox_tree_sync_change * +dsync_mailbox_trees_sync_next(struct dsync_mailbox_tree_sync_ctx *ctx); +void dsync_mailbox_trees_sync_deinit(struct dsync_mailbox_tree_sync_ctx **ctx); + +#endif
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/doveadm/dsync/dsync-mailbox.h Tue May 22 23:17:31 2012 +0300 @@ -0,0 +1,17 @@ +#ifndef DSYNC_MAILBOX_H +#define DSYNC_MAILBOX_H + +#include "mail-storage.h" + +/* Mailbox that is going to be synced. Its name was already sent in the + mailbox tree. */ +struct dsync_mailbox { + guid_128_t mailbox_guid; + bool mailbox_lost; + + uint32_t uid_validity, uid_next, messages_count, first_recent_uid; + uint64_t highest_modseq; + ARRAY_TYPE(mailbox_cache_field) cache_fields; +}; + +#endif
--- a/src/doveadm/dsync/dsync-proxy-client.c Fri May 04 05:35:36 2012 +0300 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,1191 +0,0 @@ -/* Copyright (c) 2009-2012 Dovecot authors, see the included COPYING file */ - -#include "lib.h" -#include "array.h" -#include "aqueue.h" -#include "fd-set-nonblock.h" -#include "istream.h" -#include "istream-dot.h" -#include "ostream.h" -#include "str.h" -#include "strescape.h" -#include "imap-util.h" -#include "dsync-proxy.h" -#include "dsync-worker-private.h" - -#include <stdlib.h> -#include <unistd.h> - -#define OUTBUF_THROTTLE_SIZE (1024*64) - -enum proxy_client_request_type { - PROXY_CLIENT_REQUEST_TYPE_COPY, - PROXY_CLIENT_REQUEST_TYPE_GET, - PROXY_CLIENT_REQUEST_TYPE_FINISH -}; - -struct proxy_client_request { - enum proxy_client_request_type type; - uint32_t uid; - union { - dsync_worker_msg_callback_t *get; - dsync_worker_copy_callback_t *copy; - dsync_worker_finish_callback_t *finish; - } callback; - void *context; -}; - -struct proxy_client_dsync_worker_mailbox_iter { - struct dsync_worker_mailbox_iter iter; - pool_t pool; -}; - -struct proxy_client_dsync_worker_subs_iter { - struct dsync_worker_subs_iter iter; - pool_t pool; -}; - -struct proxy_client_dsync_worker { - struct dsync_worker worker; - int fd_in, fd_out; - struct io *io; - struct istream *input; - struct ostream *output; - struct timeout *to, *to_input; - - mailbox_guid_t selected_box_guid; - - dsync_worker_save_callback_t *save_callback; - void *save_context; - struct istream *save_input; - struct io *save_io; - bool save_input_last_lf; - - pool_t msg_get_pool; - struct dsync_msg_static_data msg_get_data; - ARRAY_DEFINE(request_array, struct proxy_client_request); - struct aqueue *request_queue; - string_t *pending_commands; - - unsigned int handshake_received:1; - unsigned int finishing:1; - unsigned int finished:1; -}; - -extern struct dsync_worker_vfuncs proxy_client_dsync_worker; - -static void proxy_client_worker_input(struct proxy_client_dsync_worker *worker); -static void proxy_client_send_stream(struct proxy_client_dsync_worker *worker); - -static void proxy_client_fail(struct proxy_client_dsync_worker *worker) -{ - i_stream_close(worker->input); - dsync_worker_set_failure(&worker->worker); - io_loop_stop(current_ioloop); -} - -static int -proxy_client_worker_read_line(struct proxy_client_dsync_worker *worker, - const char **line_r) -{ - if (worker->worker.failed) - return -1; - - *line_r = i_stream_read_next_line(worker->input); - if (*line_r == NULL) { - if (worker->input->stream_errno != 0) { - errno = worker->input->stream_errno; - i_error("read() from worker server failed: %m"); - dsync_worker_set_failure(&worker->worker); - return -1; - } - if (worker->input->eof) { - if (!worker->finished) - i_error("read() from worker server failed: EOF"); - dsync_worker_set_failure(&worker->worker); - return -1; - } - } - if (*line_r == NULL) - return 0; - - if (!worker->handshake_received) { - if (strcmp(*line_r, DSYNC_PROXY_SERVER_GREETING_LINE) != 0) { - i_error("Invalid server handshake: %s", *line_r); - dsync_worker_set_failure(&worker->worker); - return -1; - } - worker->handshake_received = TRUE; - return proxy_client_worker_read_line(worker, line_r); - } - return 1; -} - -static void -proxy_client_worker_msg_get_finish(struct proxy_client_dsync_worker *worker) -{ - worker->msg_get_data.input = NULL; - worker->io = io_add(worker->fd_in, IO_READ, - proxy_client_worker_input, worker); - - /* some input may already be buffered. note that we may be coming here - from the input function itself, in which case this timeout must not - be called (we'll remove it later) */ - if (worker->to_input == NULL) { - worker->to_input = - timeout_add(0, proxy_client_worker_input, worker); - } -} - -static void -proxy_client_worker_read_to_eof(struct proxy_client_dsync_worker *worker) -{ - struct istream *input = worker->msg_get_data.input; - const unsigned char *data; - size_t size; - int ret; - - while ((ret = i_stream_read_data(input, &data, &size, 0)) > 0) - i_stream_skip(input, size); - if (ret == -1) { - i_stream_unref(&input); - io_remove(&worker->io); - proxy_client_worker_msg_get_finish(worker); - } - timeout_reset(worker->to); -} - -static void -proxy_client_worker_msg_get_done(struct proxy_client_dsync_worker *worker) -{ - struct istream *input = worker->msg_get_data.input; - - i_assert(worker->io == NULL); - - if (input->eof) - proxy_client_worker_msg_get_finish(worker); - else { - /* saving read the message only partially. we'll need to read - the input until EOF or we'll start treating the input as - commands. */ - worker->io = io_add(worker->fd_in, IO_READ, - proxy_client_worker_read_to_eof, worker); - worker->msg_get_data.input = - i_stream_create_dot(worker->input, FALSE); - } -} - -static bool -proxy_client_worker_next_copy(struct proxy_client_dsync_worker *worker, - const struct proxy_client_request *request, - const char *line) -{ - uint32_t uid; - bool success; - - if (line[0] == '1' && line[1] == '\t') - success = TRUE; - else if (line[0] == '0' && line[1] == '\t') - success = FALSE; - else { - i_error("msg-copy returned invalid input: %s", line); - proxy_client_fail(worker); - return FALSE; - } - uid = strtoul(line + 2, NULL, 10); - if (uid != request->uid) { - i_error("msg-copy returned invalid uid: %u != %u", - uid, request->uid); - proxy_client_fail(worker); - return FALSE; - } - - request->callback.copy(success, request->context); - return TRUE; -} - -static bool -proxy_client_worker_next_msg_get(struct proxy_client_dsync_worker *worker, - const struct proxy_client_request *request, - const char *line) -{ - enum dsync_msg_get_result result = DSYNC_MSG_GET_RESULT_FAILED; - const char *p, *error; - uint32_t uid; - - p_clear(worker->msg_get_pool); - switch (line[0]) { - case '1': - /* ok */ - if (line[1] != '\t') - break; - line += 2; - - if ((p = strchr(line, '\t')) == NULL) - break; - uid = strtoul(t_strcut(line, '\t'), NULL, 10); - line = p + 1; - - if (uid != request->uid) { - i_error("msg-get returned invalid uid: %u != %u", - uid, request->uid); - proxy_client_fail(worker); - return FALSE; - } - - if (dsync_proxy_msg_static_import(worker->msg_get_pool, - line, &worker->msg_get_data, - &error) < 0) { - i_error("Invalid msg-get static input: %s", error); - proxy_client_fail(worker); - return FALSE; - } - worker->msg_get_data.input = - i_stream_create_dot(worker->input, FALSE); - i_stream_set_destroy_callback(worker->msg_get_data.input, - proxy_client_worker_msg_get_done, - worker); - io_remove(&worker->io); - result = DSYNC_MSG_GET_RESULT_SUCCESS; - break; - case '0': - /* expunged */ - result = DSYNC_MSG_GET_RESULT_EXPUNGED; - break; - default: - /* failure */ - break; - } - - request->callback.get(result, &worker->msg_get_data, request->context); - return worker->io != NULL && worker->msg_get_data.input == NULL; -} - -static void -proxy_client_worker_next_finish(struct proxy_client_dsync_worker *worker, - const struct proxy_client_request *request, - const char *line) -{ - bool success = TRUE; - - i_assert(worker->finishing); - i_assert(!worker->finished); - - worker->finishing = FALSE; - worker->finished = TRUE; - - if (strcmp(line, "changes") == 0) - worker->worker.unexpected_changes = TRUE; - else if (strcmp(line, "fail") == 0) - success = FALSE; - else if (strcmp(line, "ok") != 0) { - i_error("Unexpected finish reply: %s", line); - success = FALSE; - } - - request->callback.finish(success, request->context); -} - -static bool -proxy_client_worker_next_reply(struct proxy_client_dsync_worker *worker, - const char *line) -{ - const struct proxy_client_request *requests; - struct proxy_client_request request; - bool ret = TRUE; - - i_assert(worker->msg_get_data.input == NULL); - - if (aqueue_count(worker->request_queue) == 0) { - i_error("Unexpected reply from server: %s", line); - proxy_client_fail(worker); - return FALSE; - } - - requests = array_idx(&worker->request_array, 0); - request = requests[aqueue_idx(worker->request_queue, 0)]; - aqueue_delete_tail(worker->request_queue); - - switch (request.type) { - case PROXY_CLIENT_REQUEST_TYPE_COPY: - ret = proxy_client_worker_next_copy(worker, &request, line); - break; - case PROXY_CLIENT_REQUEST_TYPE_GET: - ret = proxy_client_worker_next_msg_get(worker, &request, line); - break; - case PROXY_CLIENT_REQUEST_TYPE_FINISH: - proxy_client_worker_next_finish(worker, &request, line); - break; - } - return ret; -} - -static void proxy_client_worker_input(struct proxy_client_dsync_worker *worker) -{ - const char *line; - int ret; - - if (worker->to_input != NULL) - timeout_remove(&worker->to_input); - - if (worker->worker.input_callback != NULL) { - worker->worker.input_callback(worker->worker.input_context); - timeout_reset(worker->to); - return; - } - - while ((ret = proxy_client_worker_read_line(worker, &line)) > 0) { - if (!proxy_client_worker_next_reply(worker, line)) - break; - } - if (ret < 0) { - /* try to continue */ - proxy_client_worker_next_reply(worker, ""); - } - - if (worker->to_input != NULL) { - /* input stream's destroy callback was already called. - don't get back here. */ - timeout_remove(&worker->to_input); - } - timeout_reset(worker->to); -} - -static int -proxy_client_worker_output_real(struct proxy_client_dsync_worker *worker) -{ - int ret; - - if ((ret = o_stream_flush(worker->output)) < 0) - return 1; - - if (worker->save_input != NULL) { - /* proxy_client_worker_msg_save() hasn't finished yet. */ - o_stream_cork(worker->output); - proxy_client_send_stream(worker); - if (worker->save_input != NULL) { - /* still unfinished, make sure we get called again */ - return 0; - } - } - - if (worker->worker.output_callback != NULL) - worker->worker.output_callback(worker->worker.output_context); - return ret; -} - -static int proxy_client_worker_output(struct proxy_client_dsync_worker *worker) -{ - int ret; - - ret = proxy_client_worker_output_real(worker); - timeout_reset(worker->to); - return ret; -} - -static void -proxy_client_worker_timeout(struct proxy_client_dsync_worker *worker) -{ - const char *reason; - - if (worker->save_io != NULL) - reason = " (waiting for more input from mail being saved)"; - else if (worker->save_input != NULL) { - size_t bytes = o_stream_get_buffer_used_size(worker->output); - - reason = t_strdup_printf(" (waiting for output stream to flush, " - "%"PRIuSIZE_T" bytes left)", bytes); - } else if (worker->msg_get_data.input != NULL) { - reason = " (waiting for MSG-GET message from remote)"; - } else { - reason = ""; - } - i_error("proxy client timed out%s", reason); - proxy_client_fail(worker); -} - -struct dsync_worker *dsync_worker_init_proxy_client(int fd_in, int fd_out) -{ - struct proxy_client_dsync_worker *worker; - - worker = i_new(struct proxy_client_dsync_worker, 1); - worker->worker.v = proxy_client_dsync_worker; - worker->fd_in = fd_in; - worker->fd_out = fd_out; - worker->to = timeout_add(DSYNC_PROXY_CLIENT_TIMEOUT_MSECS, - proxy_client_worker_timeout, worker); - worker->io = io_add(fd_in, IO_READ, proxy_client_worker_input, worker); - worker->input = i_stream_create_fd(fd_in, (size_t)-1, FALSE); - worker->output = o_stream_create_fd(fd_out, (size_t)-1, FALSE); - o_stream_send_str(worker->output, DSYNC_PROXY_CLIENT_GREETING_LINE"\n"); - /* we'll keep the output corked until flush is needed */ - o_stream_cork(worker->output); - o_stream_set_flush_callback(worker->output, proxy_client_worker_output, - worker); - fd_set_nonblock(fd_in, TRUE); - fd_set_nonblock(fd_out, TRUE); - - worker->pending_commands = str_new(default_pool, 1024); - worker->msg_get_pool = pool_alloconly_create("dsync proxy msg", 128); - i_array_init(&worker->request_array, 64); - worker->request_queue = aqueue_init(&worker->request_array.arr); - - return &worker->worker; -} - -static void proxy_client_worker_deinit(struct dsync_worker *_worker) -{ - struct proxy_client_dsync_worker *worker = - (struct proxy_client_dsync_worker *)_worker; - - timeout_remove(&worker->to); - if (worker->to_input != NULL) - timeout_remove(&worker->to_input); - if (worker->io != NULL) - io_remove(&worker->io); - i_stream_destroy(&worker->input); - o_stream_destroy(&worker->output); - if (close(worker->fd_in) < 0) - i_error("close(worker input) failed: %m"); - if (worker->fd_in != worker->fd_out) { - if (close(worker->fd_out) < 0) - i_error("close(worker output) failed: %m"); - } - aqueue_deinit(&worker->request_queue); - array_free(&worker->request_array); - pool_unref(&worker->msg_get_pool); - str_free(&worker->pending_commands); - i_free(worker); -} - -static bool -worker_is_output_stream_full(struct proxy_client_dsync_worker *worker) -{ - return o_stream_get_buffer_used_size(worker->output) >= - OUTBUF_THROTTLE_SIZE; -} - -static bool proxy_client_worker_is_output_full(struct dsync_worker *_worker) -{ - struct proxy_client_dsync_worker *worker = - (struct proxy_client_dsync_worker *)_worker; - - if (worker->save_input != NULL) { - /* we haven't finished sending a message save, so we're full. */ - return TRUE; - } - return worker_is_output_stream_full(worker); -} - -static int proxy_client_worker_output_flush(struct dsync_worker *_worker) -{ - struct proxy_client_dsync_worker *worker = - (struct proxy_client_dsync_worker *)_worker; - int ret = 1; - - if (o_stream_flush(worker->output) < 0) - return -1; - - o_stream_uncork(worker->output); - if (o_stream_get_buffer_used_size(worker->output) > 0) - return 0; - - if (o_stream_send(worker->output, str_data(worker->pending_commands), - str_len(worker->pending_commands)) < 0) - ret = -1; - str_truncate(worker->pending_commands, 0); - o_stream_cork(worker->output); - return ret; -} - -static struct dsync_worker_mailbox_iter * -proxy_client_worker_mailbox_iter_init(struct dsync_worker *_worker) -{ - struct proxy_client_dsync_worker *worker = - (struct proxy_client_dsync_worker *)_worker; - struct proxy_client_dsync_worker_mailbox_iter *iter; - - iter = i_new(struct proxy_client_dsync_worker_mailbox_iter, 1); - iter->iter.worker = _worker; - iter->pool = pool_alloconly_create("proxy mailbox iter", 1024); - o_stream_send_str(worker->output, "BOX-LIST\n"); - (void)proxy_client_worker_output_flush(_worker); - return &iter->iter; -} - -static int -proxy_client_worker_mailbox_iter_next(struct dsync_worker_mailbox_iter *_iter, - struct dsync_mailbox *dsync_box_r) -{ - struct proxy_client_dsync_worker_mailbox_iter *iter = - (struct proxy_client_dsync_worker_mailbox_iter *)_iter; - struct proxy_client_dsync_worker *worker = - (struct proxy_client_dsync_worker *)_iter->worker; - const char *line, *error; - int ret; - - if ((ret = proxy_client_worker_read_line(worker, &line)) <= 0) { - if (ret < 0) - _iter->failed = TRUE; - return ret; - } - - if ((line[0] == '+' || line[0] == '-') && line[1] == '\0') { - /* end of mailboxes */ - if (line[0] == '-') { - i_error("Worker server's mailbox iteration failed"); - _iter->failed = TRUE; - } - return -1; - } - - p_clear(iter->pool); - if (dsync_proxy_mailbox_import(iter->pool, line, - dsync_box_r, &error) < 0) { - i_error("Invalid mailbox input from worker server: %s", error); - _iter->failed = TRUE; - return -1; - } - return 1; -} - -static int -proxy_client_worker_mailbox_iter_deinit(struct dsync_worker_mailbox_iter *_iter) -{ - struct proxy_client_dsync_worker_mailbox_iter *iter = - (struct proxy_client_dsync_worker_mailbox_iter *)_iter; - int ret = _iter->failed ? -1 : 0; - - pool_unref(&iter->pool); - i_free(iter); - return ret; -} - -static struct dsync_worker_subs_iter * -proxy_client_worker_subs_iter_init(struct dsync_worker *_worker) -{ - struct proxy_client_dsync_worker *worker = - (struct proxy_client_dsync_worker *)_worker; - struct proxy_client_dsync_worker_subs_iter *iter; - - iter = i_new(struct proxy_client_dsync_worker_subs_iter, 1); - iter->iter.worker = _worker; - iter->pool = pool_alloconly_create("proxy subscription iter", 1024); - o_stream_send_str(worker->output, "SUBS-LIST\n"); - (void)proxy_client_worker_output_flush(_worker); - return &iter->iter; -} - -static int -proxy_client_worker_subs_iter_next_line(struct proxy_client_dsync_worker_subs_iter *iter, - unsigned int wanted_arg_count, - char ***args_r) -{ - struct proxy_client_dsync_worker *worker = - (struct proxy_client_dsync_worker *)iter->iter.worker; - const char *line; - char **args; - int ret; - - if ((ret = proxy_client_worker_read_line(worker, &line)) <= 0) { - if (ret < 0) - iter->iter.failed = TRUE; - return ret; - } - - if ((line[0] == '+' || line[0] == '-') && line[1] == '\0') { - /* end of subscribed subscriptions */ - if (line[0] == '-') { - i_error("Worker server's subscription iteration failed"); - iter->iter.failed = TRUE; - } - return -1; - } - - p_clear(iter->pool); - args = p_strsplit(iter->pool, line, "\t"); - if (str_array_length((const char *const *)args) < wanted_arg_count) { - i_error("Invalid subscription input from worker server"); - iter->iter.failed = TRUE; - return -1; - } - *args_r = args; - return 1; -} - -static int -proxy_client_worker_subs_iter_next(struct dsync_worker_subs_iter *_iter, - struct dsync_worker_subscription *rec_r) -{ - struct proxy_client_dsync_worker_subs_iter *iter = - (struct proxy_client_dsync_worker_subs_iter *)_iter; - char **args; - int ret; - - ret = proxy_client_worker_subs_iter_next_line(iter, 4, &args); - if (ret <= 0) - return ret; - - rec_r->vname = str_tabunescape(args[0]); - rec_r->storage_name = str_tabunescape(args[1]); - rec_r->ns_prefix = str_tabunescape(args[2]); - rec_r->last_change = strtoul(args[3], NULL, 10); - return 1; -} - -static int -proxy_client_worker_subs_iter_next_un(struct dsync_worker_subs_iter *_iter, - struct dsync_worker_unsubscription *rec_r) -{ - struct proxy_client_dsync_worker_subs_iter *iter = - (struct proxy_client_dsync_worker_subs_iter *)_iter; - char **args; - int ret; - - ret = proxy_client_worker_subs_iter_next_line(iter, 3, &args); - if (ret <= 0) - return ret; - - memset(rec_r, 0, sizeof(*rec_r)); - if (dsync_proxy_mailbox_guid_import(args[0], &rec_r->name_sha1) < 0) { - i_error("Invalid subscription input from worker server: " - "Invalid unsubscription mailbox GUID"); - iter->iter.failed = TRUE; - return -1; - } - rec_r->ns_prefix = str_tabunescape(args[1]); - rec_r->last_change = strtoul(args[2], NULL, 10); - return 1; -} - -static int -proxy_client_worker_subs_iter_deinit(struct dsync_worker_subs_iter *_iter) -{ - struct proxy_client_dsync_worker_subs_iter *iter = - (struct proxy_client_dsync_worker_subs_iter *)_iter; - int ret = _iter->failed ? -1 : 0; - - pool_unref(&iter->pool); - i_free(iter); - return ret; -} - -static void -proxy_client_worker_set_subscribed(struct dsync_worker *_worker, - const char *name, time_t last_change, - bool set) -{ - struct proxy_client_dsync_worker *worker = - (struct proxy_client_dsync_worker *)_worker; - - T_BEGIN { - string_t *str = t_str_new(128); - - str_append(str, "SUBS-SET\t"); - str_tabescape_write(str, name); - str_printfa(str, "\t%s\t%d\n", dec2str(last_change), - set ? 1 : 0); - o_stream_send(worker->output, str_data(str), str_len(str)); - } T_END; -} - -struct proxy_client_dsync_worker_msg_iter { - struct dsync_worker_msg_iter iter; - pool_t pool; - bool done; -}; - -static struct dsync_worker_msg_iter * -proxy_client_worker_msg_iter_init(struct dsync_worker *_worker, - const mailbox_guid_t mailboxes[], - unsigned int mailbox_count) -{ - struct proxy_client_dsync_worker *worker = - (struct proxy_client_dsync_worker *)_worker; - struct proxy_client_dsync_worker_msg_iter *iter; - string_t *str; - unsigned int i; - - iter = i_new(struct proxy_client_dsync_worker_msg_iter, 1); - iter->iter.worker = _worker; - iter->pool = pool_alloconly_create("proxy message iter", 10240); - - str = str_new(iter->pool, 512); - str_append(str, "MSG-LIST"); - for (i = 0; i < mailbox_count; i++) T_BEGIN { - str_append_c(str, '\t'); - dsync_proxy_mailbox_guid_export(str, &mailboxes[i]); - } T_END; - str_append_c(str, '\n'); - o_stream_send(worker->output, str_data(str), str_len(str)); - p_clear(iter->pool); - - (void)proxy_client_worker_output_flush(_worker); - return &iter->iter; -} - -static int -proxy_client_worker_msg_iter_next(struct dsync_worker_msg_iter *_iter, - unsigned int *mailbox_idx_r, - struct dsync_message *msg_r) -{ - struct proxy_client_dsync_worker_msg_iter *iter = - (struct proxy_client_dsync_worker_msg_iter *)_iter; - struct proxy_client_dsync_worker *worker = - (struct proxy_client_dsync_worker *)_iter->worker; - const char *line, *error; - int ret; - - if (iter->done) - return -1; - - if ((ret = proxy_client_worker_read_line(worker, &line)) <= 0) { - if (ret < 0) - _iter->failed = TRUE; - return ret; - } - - if ((line[0] == '+' || line[0] == '-') && line[1] == '\0') { - /* end of messages */ - if (line[0] == '-') { - i_error("Worker server's message iteration failed"); - _iter->failed = TRUE; - } - iter->done = TRUE; - return -1; - } - - *mailbox_idx_r = 0; - while (*line >= '0' && *line <= '9') { - *mailbox_idx_r = *mailbox_idx_r * 10 + (*line - '0'); - line++; - } - if (*line != '\t') { - i_error("Invalid mailbox idx from worker server"); - _iter->failed = TRUE; - return -1; - } - line++; - - p_clear(iter->pool); - if (dsync_proxy_msg_import(iter->pool, line, msg_r, &error) < 0) { - i_error("Invalid message input from worker server: %s", error); - _iter->failed = TRUE; - return -1; - } - return 1; -} - -static int -proxy_client_worker_msg_iter_deinit(struct dsync_worker_msg_iter *_iter) -{ - struct proxy_client_dsync_worker_msg_iter *iter = - (struct proxy_client_dsync_worker_msg_iter *)_iter; - int ret = _iter->failed ? -1 : 0; - - pool_unref(&iter->pool); - i_free(iter); - return ret; -} - -static void -proxy_client_worker_cmd(struct proxy_client_dsync_worker *worker, string_t *str) -{ - if (worker->save_input == NULL) - o_stream_send(worker->output, str_data(str), str_len(str)); - else - str_append_str(worker->pending_commands, str); -} - -static void -proxy_client_worker_create_mailbox(struct dsync_worker *_worker, - const struct dsync_mailbox *dsync_box) -{ - struct proxy_client_dsync_worker *worker = - (struct proxy_client_dsync_worker *)_worker; - - T_BEGIN { - string_t *str = t_str_new(128); - - str_append(str, "BOX-CREATE\t"); - dsync_proxy_mailbox_export(str, dsync_box); - str_append_c(str, '\n'); - proxy_client_worker_cmd(worker, str); - } T_END; -} - -static void -proxy_client_worker_delete_mailbox(struct dsync_worker *_worker, - const struct dsync_mailbox *dsync_box) -{ - struct proxy_client_dsync_worker *worker = - (struct proxy_client_dsync_worker *)_worker; - - T_BEGIN { - string_t *str = t_str_new(128); - - str_append(str, "BOX-DELETE\t"); - dsync_proxy_mailbox_guid_export(str, &dsync_box->mailbox_guid); - str_printfa(str, "\t%s\n", dec2str(dsync_box->last_change)); - proxy_client_worker_cmd(worker, str); - } T_END; -} - -static void -proxy_client_worker_delete_dir(struct dsync_worker *_worker, - const struct dsync_mailbox *dsync_box) -{ - struct proxy_client_dsync_worker *worker = - (struct proxy_client_dsync_worker *)_worker; - - T_BEGIN { - string_t *str = t_str_new(128); - - str_append(str, "DIR-DELETE\t"); - str_tabescape_write(str, dsync_box->name); - str_printfa(str, "\t%s\n", dec2str(dsync_box->last_change)); - proxy_client_worker_cmd(worker, str); - } T_END; -} - -static void -proxy_client_worker_rename_mailbox(struct dsync_worker *_worker, - const mailbox_guid_t *mailbox, - const struct dsync_mailbox *dsync_box) -{ - struct proxy_client_dsync_worker *worker = - (struct proxy_client_dsync_worker *)_worker; - char sep[2]; - - T_BEGIN { - string_t *str = t_str_new(128); - - str_append(str, "BOX-RENAME\t"); - dsync_proxy_mailbox_guid_export(str, mailbox); - str_append_c(str, '\t'); - str_tabescape_write(str, dsync_box->name); - str_append_c(str, '\t'); - sep[0] = dsync_box->name_sep; sep[1] = '\0'; - str_tabescape_write(str, sep); - str_append_c(str, '\n'); - proxy_client_worker_cmd(worker, str); - } T_END; -} - -static void -proxy_client_worker_update_mailbox(struct dsync_worker *_worker, - const struct dsync_mailbox *dsync_box) -{ - struct proxy_client_dsync_worker *worker = - (struct proxy_client_dsync_worker *)_worker; - - T_BEGIN { - string_t *str = t_str_new(128); - - str_append(str, "BOX-UPDATE\t"); - dsync_proxy_mailbox_export(str, dsync_box); - str_append_c(str, '\n'); - proxy_client_worker_cmd(worker, str); - } T_END; -} - -static void -proxy_client_worker_select_mailbox(struct dsync_worker *_worker, - const mailbox_guid_t *mailbox, - const ARRAY_TYPE(mailbox_cache_field) *cache_fields) -{ - struct proxy_client_dsync_worker *worker = - (struct proxy_client_dsync_worker *)_worker; - - if (dsync_guid_equals(&worker->selected_box_guid, mailbox)) - return; - worker->selected_box_guid = *mailbox; - - T_BEGIN { - string_t *str = t_str_new(128); - - str_append(str, "BOX-SELECT\t"); - dsync_proxy_mailbox_guid_export(str, mailbox); - if (cache_fields != NULL) - dsync_proxy_cache_fields_export(str, cache_fields); - str_append_c(str, '\n'); - proxy_client_worker_cmd(worker, str); - } T_END; -} - -static void -proxy_client_worker_msg_update_metadata(struct dsync_worker *_worker, - const struct dsync_message *msg) -{ - struct proxy_client_dsync_worker *worker = - (struct proxy_client_dsync_worker *)_worker; - - T_BEGIN { - string_t *str = t_str_new(128); - - str_printfa(str, "MSG-UPDATE\t%u\t%llu\t", msg->uid, - (unsigned long long)msg->modseq); - imap_write_flags(str, msg->flags, msg->keywords); - str_append_c(str, '\n'); - proxy_client_worker_cmd(worker, str); - } T_END; -} - -static void -proxy_client_worker_msg_update_uid(struct dsync_worker *_worker, - uint32_t old_uid, uint32_t new_uid) -{ - struct proxy_client_dsync_worker *worker = - (struct proxy_client_dsync_worker *)_worker; - - T_BEGIN { - string_t *str = t_str_new(64); - str_printfa(str, "MSG-UID-CHANGE\t%u\t%u\n", old_uid, new_uid); - proxy_client_worker_cmd(worker, str); - } T_END; -} - -static void -proxy_client_worker_msg_expunge(struct dsync_worker *_worker, uint32_t uid) -{ - struct proxy_client_dsync_worker *worker = - (struct proxy_client_dsync_worker *)_worker; - - T_BEGIN { - string_t *str = t_str_new(64); - str_printfa(str, "MSG-EXPUNGE\t%u\n", uid); - proxy_client_worker_cmd(worker, str); - } T_END; -} - -static void -proxy_client_worker_msg_copy(struct dsync_worker *_worker, - const mailbox_guid_t *src_mailbox, - uint32_t src_uid, - const struct dsync_message *dest_msg, - dsync_worker_copy_callback_t *callback, - void *context) -{ - struct proxy_client_dsync_worker *worker = - (struct proxy_client_dsync_worker *)_worker; - struct proxy_client_request request; - - T_BEGIN { - string_t *str = t_str_new(128); - - str_append(str, "MSG-COPY\t"); - dsync_proxy_mailbox_guid_export(str, src_mailbox); - str_printfa(str, "\t%u\t", src_uid); - dsync_proxy_msg_export(str, dest_msg); - str_append_c(str, '\n'); - proxy_client_worker_cmd(worker, str); - } T_END; - - memset(&request, 0, sizeof(request)); - request.type = PROXY_CLIENT_REQUEST_TYPE_COPY; - request.callback.copy = callback; - request.context = context; - request.uid = src_uid; - aqueue_append(worker->request_queue, &request); -} - -static void -proxy_client_send_stream_real(struct proxy_client_dsync_worker *worker) -{ - dsync_worker_save_callback_t *callback; - void *context; - struct istream *input; - const unsigned char *data; - size_t size; - int ret; - - while ((ret = i_stream_read_data(worker->save_input, - &data, &size, 0)) > 0) { - dsync_proxy_send_dot_output(worker->output, - &worker->save_input_last_lf, - data, size); - i_stream_skip(worker->save_input, size); - - if (worker_is_output_stream_full(worker)) { - o_stream_uncork(worker->output); - if (worker_is_output_stream_full(worker)) - return; - o_stream_cork(worker->output); - } - } - if (ret == 0) { - /* waiting for more input */ - o_stream_uncork(worker->output); - if (worker->save_io == NULL) { - int fd = i_stream_get_fd(worker->save_input); - - worker->save_io = - io_add(fd, IO_READ, - proxy_client_send_stream, worker); - } - return; - } - if (worker->save_io != NULL) - io_remove(&worker->save_io); - if (worker->save_input->stream_errno != 0) { - errno = worker->save_input->stream_errno; - i_error("proxy: reading message input failed: %m"); - o_stream_close(worker->output); - } else { - i_assert(!i_stream_have_bytes_left(worker->save_input)); - o_stream_send(worker->output, "\n.\n", 3); - } - - callback = worker->save_callback; - context = worker->save_context; - worker->save_callback = NULL; - worker->save_context = NULL; - - /* a bit ugly way to free the stream. the problem is that local worker - has set a destroy callback, which in turn can call our msg_save() - again before the i_stream_unref() is finished. */ - input = worker->save_input; - worker->save_input = NULL; - i_stream_unref(&input); - - (void)proxy_client_worker_output_flush(&worker->worker); - - callback(context); -} - -static void proxy_client_send_stream(struct proxy_client_dsync_worker *worker) -{ - proxy_client_send_stream_real(worker); - timeout_reset(worker->to); -} - -static void -proxy_client_worker_msg_save(struct dsync_worker *_worker, - const struct dsync_message *msg, - const struct dsync_msg_static_data *data, - dsync_worker_save_callback_t *callback, - void *context) -{ - struct proxy_client_dsync_worker *worker = - (struct proxy_client_dsync_worker *)_worker; - - T_BEGIN { - string_t *str = t_str_new(128); - - str_append(str, "MSG-SAVE\t"); - dsync_proxy_msg_static_export(str, data); - str_append_c(str, '\t'); - dsync_proxy_msg_export(str, msg); - str_append_c(str, '\n'); - proxy_client_worker_cmd(worker, str); - } T_END; - - i_assert(worker->save_input == NULL); - worker->save_callback = callback; - worker->save_context = context; - worker->save_input = data->input; - worker->save_input_last_lf = TRUE; - i_stream_ref(worker->save_input); - proxy_client_send_stream(worker); -} - -static void -proxy_client_worker_msg_save_cancel(struct dsync_worker *_worker) -{ - struct proxy_client_dsync_worker *worker = - (struct proxy_client_dsync_worker *)_worker; - - if (worker->save_io != NULL) - io_remove(&worker->save_io); - if (worker->save_input != NULL) - i_stream_unref(&worker->save_input); -} - -static void -proxy_client_worker_msg_get(struct dsync_worker *_worker, - const mailbox_guid_t *mailbox, uint32_t uid, - dsync_worker_msg_callback_t *callback, - void *context) -{ - struct proxy_client_dsync_worker *worker = - (struct proxy_client_dsync_worker *)_worker; - struct proxy_client_request request; - - T_BEGIN { - string_t *str = t_str_new(128); - - str_append(str, "MSG-GET\t"); - dsync_proxy_mailbox_guid_export(str, mailbox); - str_printfa(str, "\t%u\n", uid); - proxy_client_worker_cmd(worker, str); - } T_END; - - memset(&request, 0, sizeof(request)); - request.type = PROXY_CLIENT_REQUEST_TYPE_GET; - request.callback.get = callback; - request.context = context; - request.uid = uid; - aqueue_append(worker->request_queue, &request); -} - -static void -proxy_client_worker_finish(struct dsync_worker *_worker, - dsync_worker_finish_callback_t *callback, - void *context) -{ - struct proxy_client_dsync_worker *worker = - (struct proxy_client_dsync_worker *)_worker; - struct proxy_client_request request; - - i_assert(worker->save_input == NULL); - i_assert(!worker->finishing); - - worker->finishing = TRUE; - worker->finished = FALSE; - - o_stream_send_str(worker->output, "FINISH\n"); - o_stream_uncork(worker->output); - - memset(&request, 0, sizeof(request)); - request.type = PROXY_CLIENT_REQUEST_TYPE_FINISH; - request.callback.finish = callback; - request.context = context; - aqueue_append(worker->request_queue, &request); -} - -struct dsync_worker_vfuncs proxy_client_dsync_worker = { - proxy_client_worker_deinit, - - proxy_client_worker_is_output_full, - proxy_client_worker_output_flush, - - proxy_client_worker_mailbox_iter_init, - proxy_client_worker_mailbox_iter_next, - proxy_client_worker_mailbox_iter_deinit, - - proxy_client_worker_subs_iter_init, - proxy_client_worker_subs_iter_next, - proxy_client_worker_subs_iter_next_un, - proxy_client_worker_subs_iter_deinit, - proxy_client_worker_set_subscribed, - - proxy_client_worker_msg_iter_init, - proxy_client_worker_msg_iter_next, - proxy_client_worker_msg_iter_deinit, - - proxy_client_worker_create_mailbox, - proxy_client_worker_delete_mailbox, - proxy_client_worker_delete_dir, - proxy_client_worker_rename_mailbox, - proxy_client_worker_update_mailbox, - - proxy_client_worker_select_mailbox, - proxy_client_worker_msg_update_metadata, - proxy_client_worker_msg_update_uid, - proxy_client_worker_msg_expunge, - proxy_client_worker_msg_copy, - proxy_client_worker_msg_save, - proxy_client_worker_msg_save_cancel, - proxy_client_worker_msg_get, - proxy_client_worker_finish -};
--- a/src/doveadm/dsync/dsync-proxy-server-cmd.c Fri May 04 05:35:36 2012 +0300 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,608 +0,0 @@ -/* Copyright (c) 2009-2012 Dovecot authors, see the included COPYING file */ - -#include "lib.h" -#include "array.h" -#include "str.h" -#include "strescape.h" -#include "network.h" -#include "istream.h" -#include "istream-dot.h" -#include "ostream.h" -#include "imap-util.h" -#include "dsync-worker.h" -#include "dsync-proxy.h" -#include "dsync-proxy-server.h" - -#include <stdlib.h> - -#define OUTBUF_THROTTLE_SIZE (1024*64) - -static bool -proxy_server_is_output_full(struct dsync_proxy_server *server) -{ - return o_stream_get_buffer_used_size(server->output) >= - OUTBUF_THROTTLE_SIZE; -} - -static int -cmd_box_list(struct dsync_proxy_server *server, - const char *const *args ATTR_UNUSED) -{ - struct dsync_mailbox dsync_box; - string_t *str; - int ret; - - if (server->mailbox_iter == NULL) { - server->mailbox_iter = - dsync_worker_mailbox_iter_init(server->worker); - } - - str = t_str_new(256); - while ((ret = dsync_worker_mailbox_iter_next(server->mailbox_iter, - &dsync_box)) > 0) { - str_truncate(str, 0); - dsync_proxy_mailbox_export(str, &dsync_box); - str_append_c(str, '\n'); - o_stream_send(server->output, str_data(str), str_len(str)); - if (proxy_server_is_output_full(server)) - break; - } - if (ret >= 0) { - /* continue later */ - o_stream_set_flush_pending(server->output, TRUE); - return 0; - } - if (dsync_worker_mailbox_iter_deinit(&server->mailbox_iter) < 0) { - o_stream_send(server->output, "-\n", 2); - return -1; - } else { - o_stream_send(server->output, "+\n", 2); - return 1; - } -} - -static bool cmd_subs_list_subscriptions(struct dsync_proxy_server *server) -{ - struct dsync_worker_subscription rec; - string_t *str; - int ret; - - str = t_str_new(256); - while ((ret = dsync_worker_subs_iter_next(server->subs_iter, - &rec)) > 0) { - str_truncate(str, 0); - str_tabescape_write(str, rec.vname); - str_append_c(str, '\t'); - str_tabescape_write(str, rec.storage_name); - str_append_c(str, '\t'); - str_tabescape_write(str, rec.ns_prefix); - str_printfa(str, "\t%ld\n", (long)rec.last_change); - o_stream_send(server->output, str_data(str), str_len(str)); - if (proxy_server_is_output_full(server)) - break; - } - if (ret >= 0) { - /* continue later */ - o_stream_set_flush_pending(server->output, TRUE); - return FALSE; - } - return TRUE; -} - -static bool cmd_subs_list_unsubscriptions(struct dsync_proxy_server *server) -{ - struct dsync_worker_unsubscription rec; - string_t *str; - int ret; - - str = t_str_new(256); - while ((ret = dsync_worker_subs_iter_next_un(server->subs_iter, - &rec)) > 0) { - str_truncate(str, 0); - dsync_proxy_mailbox_guid_export(str, &rec.name_sha1); - str_append_c(str, '\t'); - str_tabescape_write(str, rec.ns_prefix); - str_printfa(str, "\t%ld\n", (long)rec.last_change); - o_stream_send(server->output, str_data(str), str_len(str)); - if (proxy_server_is_output_full(server)) - break; - } - if (ret >= 0) { - /* continue later */ - o_stream_set_flush_pending(server->output, TRUE); - return FALSE; - } - return TRUE; -} - -static int -cmd_subs_list(struct dsync_proxy_server *server, - const char *const *args ATTR_UNUSED) -{ - if (server->subs_iter == NULL) { - server->subs_iter = - dsync_worker_subs_iter_init(server->worker); - } - - if (!server->subs_sending_unsubscriptions) { - if (!cmd_subs_list_subscriptions(server)) - return 0; - /* a bit hacky way to handle this. this assumes that caller - goes through all subscriptions first, and next starts - going through unsubscriptions */ - o_stream_send(server->output, "+\n", 2); - server->subs_sending_unsubscriptions = TRUE; - } - if (!cmd_subs_list_unsubscriptions(server)) - return 0; - - server->subs_sending_unsubscriptions = FALSE; - if (dsync_worker_subs_iter_deinit(&server->subs_iter) < 0) { - o_stream_send(server->output, "-\n", 2); - return -1; - } else { - o_stream_send(server->output, "+\n", 2); - return 1; - } -} - -static int -cmd_subs_set(struct dsync_proxy_server *server, const char *const *args) -{ - if (str_array_length(args) < 3) { - i_error("subs-set: Missing parameters"); - return -1; - } - - dsync_worker_set_subscribed(server->worker, args[0], - strtoul(args[1], NULL, 10), - strcmp(args[2], "1") == 0); - return 1; -} - -static int -cmd_msg_list_init(struct dsync_proxy_server *server, const char *const *args) -{ - mailbox_guid_t *mailboxes; - unsigned int i, count; - int ret; - - count = str_array_length(args); - mailboxes = count == 0 ? NULL : t_new(mailbox_guid_t, count); - for (i = 0; i < count; i++) { - T_BEGIN { - ret = dsync_proxy_mailbox_guid_import(args[i], - &mailboxes[i]); - } T_END; - - if (ret < 0) { - i_error("msg-list: Invalid mailbox GUID '%s'", args[i]); - return -1; - } - } - server->msg_iter = dsync_worker_msg_iter_init(server->worker, - mailboxes, count); - return 0; -} - -static int -cmd_msg_list(struct dsync_proxy_server *server, const char *const *args) -{ - unsigned int mailbox_idx; - struct dsync_message msg; - string_t *str; - int ret; - - if (server->msg_iter == NULL) { - if (cmd_msg_list_init(server, args) < 0) - return -1; - } - - str = t_str_new(256); - while ((ret = dsync_worker_msg_iter_next(server->msg_iter, - &mailbox_idx, &msg)) > 0) { - str_truncate(str, 0); - str_printfa(str, "%u\t", mailbox_idx); - dsync_proxy_msg_export(str, &msg); - str_append_c(str, '\n'); - o_stream_send(server->output, str_data(str), str_len(str)); - if (proxy_server_is_output_full(server)) - break; - } - if (ret >= 0) { - /* continue later */ - o_stream_set_flush_pending(server->output, TRUE); - return 0; - } - if (dsync_worker_msg_iter_deinit(&server->msg_iter) < 0) { - o_stream_send(server->output, "-\n", 2); - return -1; - } else { - o_stream_send(server->output, "+\n", 2); - return 1; - } -} - -static int -cmd_box_create(struct dsync_proxy_server *server, const char *const *args) -{ - struct dsync_mailbox dsync_box; - const char *error; - - if (dsync_proxy_mailbox_import_unescaped(pool_datastack_create(), - args, &dsync_box, - &error) < 0) { - i_error("Invalid mailbox input: %s", error); - return -1; - } - dsync_worker_create_mailbox(server->worker, &dsync_box); - return 1; -} - -static int -cmd_box_delete(struct dsync_proxy_server *server, const char *const *args) -{ - mailbox_guid_t guid; - struct dsync_mailbox dsync_box; - - if (str_array_length(args) < 2) - return -1; - if (dsync_proxy_mailbox_guid_import(args[0], &guid) < 0) { - i_error("box-delete: Invalid mailbox GUID '%s'", args[0]); - return -1; - } - - memset(&dsync_box, 0, sizeof(dsync_box)); - dsync_box.mailbox_guid = guid; - dsync_box.last_change = strtoul(args[1], NULL, 10); - dsync_worker_delete_mailbox(server->worker, &dsync_box); - return 1; -} - -static int -cmd_dir_delete(struct dsync_proxy_server *server, const char *const *args) -{ - struct dsync_mailbox dsync_box; - - if (str_array_length(args) < 2) - return -1; - - memset(&dsync_box, 0, sizeof(dsync_box)); - dsync_box.name = str_tabunescape(t_strdup_noconst(args[0])); - dsync_box.last_change = strtoul(args[1], NULL, 10); - dsync_worker_delete_dir(server->worker, &dsync_box); - return 1; -} - -static int -cmd_box_rename(struct dsync_proxy_server *server, const char *const *args) -{ - mailbox_guid_t guid; - struct dsync_mailbox dsync_box; - - if (str_array_length(args) < 3) - return -1; - if (dsync_proxy_mailbox_guid_import(args[0], &guid) < 0) { - i_error("box-delete: Invalid mailbox GUID '%s'", args[0]); - return -1; - } - - memset(&dsync_box, 0, sizeof(dsync_box)); - dsync_box.name = args[1]; - dsync_box.name_sep = args[2][0]; - dsync_worker_rename_mailbox(server->worker, &guid, &dsync_box); - return 1; -} - -static int -cmd_box_update(struct dsync_proxy_server *server, const char *const *args) -{ - struct dsync_mailbox dsync_box; - const char *error; - - if (dsync_proxy_mailbox_import_unescaped(pool_datastack_create(), - args, &dsync_box, - &error) < 0) { - i_error("Invalid mailbox input: %s", error); - return -1; - } - dsync_worker_update_mailbox(server->worker, &dsync_box); - return 1; -} - -static int -cmd_box_select(struct dsync_proxy_server *server, const char *const *args) -{ - struct dsync_mailbox box; - const char *error; - - memset(&box, 0, sizeof(box)); - if (args[0] == NULL || - dsync_proxy_mailbox_guid_import(args[0], &box.mailbox_guid) < 0) { - i_error("box-select: Invalid mailbox GUID '%s'", args[0]); - return -1; - } - args++; - - if (dsync_proxy_cache_fields_import(args, pool_datastack_create(), - &box.cache_fields, &error) < 0) { - i_error("box-select: %s", error); - return -1; - } - dsync_worker_select_mailbox(server->worker, &box); - return 1; -} - -static int -cmd_msg_update(struct dsync_proxy_server *server, const char *const *args) -{ - struct dsync_message msg; - - /* uid modseq flags */ - if (str_array_length(args) < 3) - return -1; - - memset(&msg, 0, sizeof(msg)); - msg.uid = strtoul(args[0], NULL, 10); - msg.modseq = strtoull(args[1], NULL, 10); - if (dsync_proxy_msg_parse_flags(pool_datastack_create(), - args[2], &msg) < 0) - return -1; - - dsync_worker_msg_update_metadata(server->worker, &msg); - return 1; -} - -static int -cmd_msg_uid_change(struct dsync_proxy_server *server, const char *const *args) -{ - if (args[0] == NULL || args[1] == NULL) - return -1; - - dsync_worker_msg_update_uid(server->worker, - strtoul(args[0], NULL, 10), - strtoul(args[1], NULL, 10)); - return 1; -} - -static int -cmd_msg_expunge(struct dsync_proxy_server *server, const char *const *args) -{ - if (args[0] == NULL) - return -1; - - dsync_worker_msg_expunge(server->worker, strtoul(args[0], NULL, 10)); - return 1; -} - -static void copy_callback(bool success, void *context) -{ - struct dsync_proxy_server *server = context; - const char *reply; - - i_assert(server->copy_uid != 0); - - reply = t_strdup_printf("%d\t%u\n", success ? 1 : 0, server->copy_uid); - o_stream_send_str(server->output, reply); -} - -static int -cmd_msg_copy(struct dsync_proxy_server *server, const char *const *args) -{ - mailbox_guid_t src_mailbox_guid; - uint32_t src_uid; - struct dsync_message msg; - const char *error; - - /* src_mailbox_guid src_uid <message> */ - if (str_array_length(args) < 3) - return -1; - - if (dsync_proxy_mailbox_guid_import(args[0], &src_mailbox_guid) < 0) { - i_error("msg-copy: Invalid mailbox GUID '%s'", args[0]); - return -1; - } - src_uid = strtoul(args[1], NULL, 10); - - if (dsync_proxy_msg_import_unescaped(pool_datastack_create(), - args + 2, &msg, &error) < 0) - i_error("Invalid message input: %s", error); - - server->copy_uid = src_uid; - dsync_worker_msg_copy(server->worker, &src_mailbox_guid, src_uid, &msg, - copy_callback, server); - server->copy_uid = 0; - return 1; -} - -static void cmd_msg_save_callback(void *context) -{ - struct dsync_proxy_server *server = context; - - server->save_finished = TRUE; -} - -static int -cmd_msg_save(struct dsync_proxy_server *server, const char *const *args) -{ - struct dsync_message msg; - struct dsync_msg_static_data data; - const char *error; - int ret; - - if (dsync_proxy_msg_static_import_unescaped(pool_datastack_create(), - args, &data, &error) < 0) { - i_error("Invalid message input: %s", error); - return -1; - } - data.input = i_stream_create_dot(server->input, FALSE); - - if (dsync_proxy_msg_import_unescaped(pool_datastack_create(), - args + 2, &msg, &error) < 0) { - i_error("Invalid message input: %s", error); - return -1; - } - - /* we rely on save reading the entire input */ - server->save_finished = FALSE; - net_set_nonblock(server->fd_in, FALSE); - dsync_worker_msg_save(server->worker, &msg, &data, - cmd_msg_save_callback, server); - net_set_nonblock(server->fd_in, TRUE); - ret = dsync_worker_has_failed(server->worker) ? -1 : 1; - i_assert(server->save_finished); - i_assert(data.input->eof || ret < 0); - i_stream_destroy(&data.input); - return ret; -} - -static void cmd_msg_get_send_more(struct dsync_proxy_server *server) -{ - const unsigned char *data; - size_t size; - int ret; - - while (!proxy_server_is_output_full(server)) { - ret = i_stream_read_data(server->get_input, &data, &size, 0); - if (ret == -1) { - /* done */ - o_stream_send(server->output, "\n.\n", 3); - i_stream_unref(&server->get_input); - return; - } else { - /* for now we assume input is blocking */ - i_assert(ret != 0); - } - - dsync_proxy_send_dot_output(server->output, - &server->get_input_last_lf, - data, size); - i_stream_skip(server->get_input, size); - } - o_stream_set_flush_pending(server->output, TRUE); -} - -static void -cmd_msg_get_callback(enum dsync_msg_get_result result, - const struct dsync_msg_static_data *data, void *context) -{ - struct dsync_proxy_server *server = context; - string_t *str; - - i_assert(server->get_uid != 0); - - switch (result) { - case DSYNC_MSG_GET_RESULT_SUCCESS: - break; - case DSYNC_MSG_GET_RESULT_EXPUNGED: - o_stream_send(server->output, "0\n", 3); - return; - case DSYNC_MSG_GET_RESULT_FAILED: - o_stream_send(server->output, "-\n", 3); - return; - } - - str = t_str_new(128); - str_printfa(str, "1\t%u\t", server->get_uid); - dsync_proxy_msg_static_export(str, data); - str_append_c(str, '\n'); - o_stream_send(server->output, str_data(str), str_len(str)); - - /* then we'll still have to send the message body. */ - server->get_input = data->input; - cmd_msg_get_send_more(server); - if (server->get_input == NULL) { - /* if we came here from ioloop, make sure the command gets - freed in the output flush callback */ - o_stream_set_flush_pending(server->output, TRUE); - } -} - -static int -cmd_msg_get(struct dsync_proxy_server *server, const char *const *args) -{ - mailbox_guid_t mailbox_guid; - uint32_t uid; - - if (str_array_length(args) < 2) - return -1; - - if (dsync_proxy_mailbox_guid_import(args[0], &mailbox_guid) < 0) { - i_error("msg-get: Invalid mailbox GUID '%s'", args[0]); - return -1; - } - - uid = strtoul(args[1], NULL, 10); - if (uid == 0) - return -1; - - if (server->get_input != NULL) { - i_assert(server->get_uid == uid); - cmd_msg_get_send_more(server); - } else { - server->get_uid = uid; - dsync_worker_msg_get(server->worker, &mailbox_guid, uid, - cmd_msg_get_callback, server); - } - if (server->get_input != NULL) - return 0; - server->get_uid = 0; - return 1; -} - -static void cmd_finish_callback(bool success, void *context) -{ - struct dsync_proxy_server *server = context; - const char *reply; - - if (!success) - reply = "fail\n"; - else if (dsync_worker_has_unexpected_changes(server->worker)) - reply = "changes\n"; - else - reply = "ok\n"; - - server->finished = TRUE; - o_stream_send_str(server->output, reply); -} - -static int -cmd_finish(struct dsync_proxy_server *server, - const char *const *args ATTR_UNUSED) -{ - dsync_worker_finish(server->worker, cmd_finish_callback, server); - return 1; -} - -static struct dsync_proxy_server_command commands[] = { - { "BOX-LIST", cmd_box_list }, - { "SUBS-LIST", cmd_subs_list }, - { "SUBS-SET", cmd_subs_set }, - { "MSG-LIST", cmd_msg_list }, - { "BOX-CREATE", cmd_box_create }, - { "BOX-DELETE", cmd_box_delete }, - { "DIR-DELETE", cmd_dir_delete }, - { "BOX-RENAME", cmd_box_rename }, - { "BOX-UPDATE", cmd_box_update }, - { "BOX-SELECT", cmd_box_select }, - { "MSG-UPDATE", cmd_msg_update }, - { "MSG-UID-CHANGE", cmd_msg_uid_change }, - { "MSG-EXPUNGE", cmd_msg_expunge }, - { "MSG-COPY", cmd_msg_copy }, - { "MSG-SAVE", cmd_msg_save }, - { "MSG-GET", cmd_msg_get }, - { "FINISH", cmd_finish }, - { NULL, NULL } -}; - -struct dsync_proxy_server_command * -dsync_proxy_server_command_find(const char *name) -{ - unsigned int i; - - for (i = 0; commands[i].name != NULL; i++) { - if (strcasecmp(commands[i].name, name) == 0) - return &commands[i]; - } - return NULL; -}
--- a/src/doveadm/dsync/dsync-proxy-server.c Fri May 04 05:35:36 2012 +0300 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,205 +0,0 @@ -/* Copyright (c) 2009-2012 Dovecot authors, see the included COPYING file */ - -#include "lib.h" -#include "strescape.h" -#include "fd-set-nonblock.h" -#include "istream.h" -#include "ostream.h" -#include "dsync-worker.h" -#include "dsync-proxy.h" -#include "dsync-proxy-server.h" - -#include <stdlib.h> - -static int -proxy_server_read_line(struct dsync_proxy_server *server, - const char **line_r) -{ - *line_r = i_stream_read_next_line(server->input); - if (*line_r == NULL) { - if (server->input->stream_errno != 0) { - errno = server->input->stream_errno; - i_error("read() from proxy client failed: %m"); - io_loop_stop(current_ioloop); - return -1; - } - if (server->input->eof) { - if (!server->finished) - i_error("read() from proxy client failed: EOF"); - io_loop_stop(current_ioloop); - return -1; - } - } - if (*line_r == NULL) - return 0; - - if (!server->handshake_received) { - if (strcmp(*line_r, DSYNC_PROXY_CLIENT_GREETING_LINE) != 0) { - i_error("Invalid client handshake: %s", *line_r); - io_loop_stop(current_ioloop); - return -1; - } - server->handshake_received = TRUE; - return proxy_server_read_line(server, line_r); - } - return 1; -} - -static int proxy_server_run_cmd(struct dsync_proxy_server *server) -{ - int ret; - - if ((ret = server->cur_cmd->func(server, server->cur_args)) == 0) - return 0; - if (ret < 0) { - i_error("command %s failed", server->cur_cmd->name); - return -1; - } - - server->cur_cmd = NULL; - server->cur_args = NULL; - return 1; -} - -static int -proxy_server_input_line(struct dsync_proxy_server *server, const char *line) -{ - const char *const *args; - const char **cmd_args; - unsigned int i, count; - - i_assert(server->cur_cmd == NULL); - - p_clear(server->cmd_pool); - args = (const char *const *)p_strsplit(server->cmd_pool, line, "\t"); - if (args[0] == NULL) { - i_error("proxy client sent invalid input: %s", line); - return -1; - } - - server->cur_cmd = dsync_proxy_server_command_find(args[0]); - if (server->cur_cmd == NULL) { - i_error("proxy client sent unknown command: %s", args[0]); - return -1; - } else { - args++; - count = str_array_length(args); - - cmd_args = p_new(server->cmd_pool, const char *, count + 1); - for (i = 0; i < count; i++) { - cmd_args[i] = str_tabunescape(p_strdup(server->cmd_pool, - args[i])); - } - - server->cur_args = cmd_args; - return proxy_server_run_cmd(server); - } -} - -static void proxy_server_input(struct dsync_proxy_server *server) -{ - const char *line; - int ret = 0; - - if (server->cur_cmd != NULL) { - /* wait until command handling is finished */ - io_remove(&server->io); - return; - } - - o_stream_cork(server->output); - while (proxy_server_read_line(server, &line) > 0) { - T_BEGIN { - ret = proxy_server_input_line(server, line); - } T_END; - if (ret <= 0) - break; - } - o_stream_uncork(server->output); - if (server->output->closed) - ret = -1; - - if (ret < 0) - io_loop_stop(current_ioloop); - timeout_reset(server->to); -} - -static int proxy_server_output(struct dsync_proxy_server *server) -{ - struct ostream *output = server->output; - int ret; - - if ((ret = o_stream_flush(output)) < 0) - ret = 1; - else if (server->cur_cmd != NULL) { - o_stream_cork(output); - (void)proxy_server_run_cmd(server); - o_stream_uncork(output); - - if (server->cur_cmd == NULL) { - if (server->io == NULL) { - server->io = io_add(server->fd_in, IO_READ, - proxy_server_input, server); - } - /* handle pending input */ - proxy_server_input(server); - } - } - if (output->closed) - io_loop_stop(current_ioloop); - timeout_reset(server->to); - return ret; -} - -static void dsync_proxy_server_timeout(void *context ATTR_UNUSED) -{ - i_error("proxy server timed out"); - io_loop_stop(current_ioloop); -} - -struct dsync_proxy_server * -dsync_proxy_server_init(int fd_in, int fd_out, struct dsync_worker *worker) -{ - struct dsync_proxy_server *server; - - server = i_new(struct dsync_proxy_server, 1); - server->worker = worker; - server->fd_in = fd_in; - server->fd_out = fd_out; - - server->cmd_pool = pool_alloconly_create("worker server cmd", 2048); - server->io = io_add(fd_in, IO_READ, proxy_server_input, server); - server->input = i_stream_create_fd(fd_in, (size_t)-1, FALSE); - server->output = o_stream_create_fd(fd_out, (size_t)-1, FALSE); - server->to = timeout_add(DSYNC_PROXY_SERVER_TIMEOUT_MSECS, - dsync_proxy_server_timeout, NULL); - o_stream_set_flush_callback(server->output, proxy_server_output, - server); - o_stream_send_str(server->output, DSYNC_PROXY_SERVER_GREETING_LINE"\n"); - fd_set_nonblock(fd_in, TRUE); - fd_set_nonblock(fd_out, TRUE); - return server; -} - -void dsync_proxy_server_deinit(struct dsync_proxy_server **_server) -{ - struct dsync_proxy_server *server = *_server; - - *_server = NULL; - - if (server->get_input != NULL) - i_stream_unref(&server->get_input); - pool_unref(&server->cmd_pool); - timeout_remove(&server->to); - if (server->io != NULL) - io_remove(&server->io); - i_stream_destroy(&server->input); - o_stream_destroy(&server->output); - if (close(server->fd_in) < 0) - i_error("close(proxy input) failed: %m"); - if (server->fd_in != server->fd_out) { - if (close(server->fd_out) < 0) - i_error("close(proxy output) failed: %m"); - } - i_free(server); -}
--- a/src/doveadm/dsync/dsync-proxy-server.h Fri May 04 05:35:36 2012 +0300 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,46 +0,0 @@ -#ifndef DSYNC_PROXY_SERVER_H -#define DSYNC_PROXY_SERVER_H - -struct dsync_proxy_server; - -struct dsync_proxy_server_command { - const char *name; - int (*func)(struct dsync_proxy_server *server, - const char *const *args); -}; - -struct dsync_proxy_server { - int fd_in, fd_out; - struct io *io; - struct istream *input; - struct ostream *output; - struct timeout *to; - - struct dsync_worker *worker; - - pool_t cmd_pool; - struct dsync_proxy_server_command *cur_cmd; - const char *const *cur_args; - - struct dsync_worker_mailbox_iter *mailbox_iter; - struct dsync_worker_subs_iter *subs_iter; - struct dsync_worker_msg_iter *msg_iter; - - struct istream *get_input; - bool get_input_last_lf; - uint32_t get_uid, copy_uid; - - unsigned int handshake_received:1; - unsigned int subs_sending_unsubscriptions:1; - unsigned int save_finished:1; - unsigned int finished:1; -}; - -struct dsync_proxy_server * -dsync_proxy_server_init(int fd_in, int fd_out, struct dsync_worker *worker); -void dsync_proxy_server_deinit(struct dsync_proxy_server **server); - -struct dsync_proxy_server_command * -dsync_proxy_server_command_find(const char *name); - -#endif
--- a/src/doveadm/dsync/dsync-proxy.c Fri May 04 05:35:36 2012 +0300 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,410 +0,0 @@ -/* Copyright (c) 2009-2012 Dovecot authors, see the included COPYING file */ - -#include "lib.h" -#include "array.h" -#include "str.h" -#include "strescape.h" -#include "ostream.h" -#include "hex-binary.h" -#include "mail-types.h" -#include "imap-util.h" -#include "mail-cache.h" -#include "dsync-data.h" -#include "dsync-proxy.h" - -#include <stdlib.h> - -#define DSYNC_CACHE_DECISION_NO 'n' -#define DSYNC_CACHE_DECISION_YES 'y' -#define DSYNC_CACHE_DECISION_TEMP 't' -#define DSYNC_CACHE_DECISION_FORCED 'f' - -void dsync_proxy_cache_fields_export(string_t *str, - const ARRAY_TYPE(mailbox_cache_field) *_fields) -{ - const struct mailbox_cache_field *fields; - unsigned int i, count; - - if (!array_is_created(_fields)) - return; - - fields = array_get(_fields, &count); - for (i = 0; i < count; i++) { - str_append_c(str, '\t'); - str_tabescape_write(str, fields[i].name); - str_append_c(str, '\t'); - switch (fields[i].decision & ~MAIL_CACHE_DECISION_FORCED) { - case MAIL_CACHE_DECISION_NO: - str_append_c(str, DSYNC_CACHE_DECISION_NO); - break; - case MAIL_CACHE_DECISION_YES: - str_append_c(str, DSYNC_CACHE_DECISION_YES); - break; - case MAIL_CACHE_DECISION_TEMP: - str_append_c(str, DSYNC_CACHE_DECISION_TEMP); - break; - } - if ((fields[i].decision & MAIL_CACHE_DECISION_FORCED) != 0) - str_append_c(str, DSYNC_CACHE_DECISION_FORCED); - str_printfa(str, "\t%ld", fields[i].last_used); - } -} - -static int dsync_proxy_cache_dec_parse(const char *str, - enum mail_cache_decision_type *dec_r) -{ - switch (*str++) { - case DSYNC_CACHE_DECISION_NO: - *dec_r = MAIL_CACHE_DECISION_NO; - break; - case DSYNC_CACHE_DECISION_YES: - *dec_r = MAIL_CACHE_DECISION_YES; - break; - case DSYNC_CACHE_DECISION_TEMP: - *dec_r = MAIL_CACHE_DECISION_TEMP; - break; - default: - return -1; - } - if (*str == DSYNC_CACHE_DECISION_FORCED) { - *dec_r |= MAIL_CACHE_DECISION_FORCED; - str++; - } - if (*str != '\0') - return -1; - return 0; -} - -int dsync_proxy_cache_fields_import(const char *const *args, pool_t pool, - ARRAY_TYPE(mailbox_cache_field) *fields, - const char **error_r) -{ - struct mailbox_cache_field *cache_field; - enum mail_cache_decision_type dec; - unsigned int i, count = str_array_length(args); - - if (count % 3 != 0) { - *error_r = "Invalid mailbox cache fields"; - return -1; - } - if (!array_is_created(fields)) - p_array_init(fields, pool, (count/3) + 1); - for (i = 0; i < count; i += 3) { - cache_field = array_append_space(fields); - cache_field->name = p_strdup(pool, args[i]); - - if (dsync_proxy_cache_dec_parse(args[i+1], &dec) < 0) { - *error_r = "Invalid cache decision"; - return -1; - } - cache_field->decision = dec; - if (str_to_time(args[i+2], &cache_field->last_used) < 0) { - *error_r = "Invalid cache last_used"; - return -1; - } - } - return 0; -} - -void dsync_proxy_msg_export(string_t *str, - const struct dsync_message *msg) -{ - str_tabescape_write(str, msg->guid); - str_printfa(str, "\t%u\t%llu\t", msg->uid, - (unsigned long long)msg->modseq); - if ((msg->flags & DSYNC_MAIL_FLAG_EXPUNGED) != 0) - str_append(str, "\\dsync-expunged "); - imap_write_flags(str, msg->flags & MAIL_FLAGS_MASK, msg->keywords); - str_printfa(str, "\t%ld", (long)msg->save_date); -} - -int dsync_proxy_msg_parse_flags(pool_t pool, const char *str, - struct dsync_message *msg_r) -{ - ARRAY_TYPE(const_string) keywords; - const char *const *args, *kw; - enum mail_flags flag; - - msg_r->flags = 0; - p_array_init(&keywords, pool, 16); - for (args = t_strsplit_spaces(str, " "); *args != NULL; args++) { - if (**args != '\\') { - kw = p_strdup(pool, *args); - array_append(&keywords, &kw, 1); - } else if (strcasecmp(*args, "\\dsync-expunged") == 0) { - msg_r->flags |= DSYNC_MAIL_FLAG_EXPUNGED; - } else { - flag = imap_parse_system_flag(*args); - if (flag == 0) - return -1; - msg_r->flags |= flag; - } - } - (void)array_append_space(&keywords); - - msg_r->keywords = array_idx(&keywords, 0); - return 0; -} - -int dsync_proxy_msg_import_unescaped(pool_t pool, const char *const *args, - struct dsync_message *msg_r, - const char **error_r) -{ - /* guid uid modseq flags save_date */ - if (str_array_length(args) < 5) { - *error_r = "Missing parameters"; - return -1; - } - - memset(msg_r, 0, sizeof(*msg_r)); - msg_r->guid = p_strdup(pool, args[0]); - msg_r->uid = strtoul(args[1], NULL, 10); - msg_r->modseq = strtoull(args[2], NULL, 10); - if (dsync_proxy_msg_parse_flags(pool, args[3], msg_r) < 0) { - *error_r = "Invalid system flags"; - return -1; - } - msg_r->save_date = strtoul(args[4], NULL, 10); - return 0; -} - -int dsync_proxy_msg_import(pool_t pool, const char *str, - struct dsync_message *msg_r, const char **error_r) -{ - char **args; - unsigned int i; - int ret; - - T_BEGIN { - args = p_strsplit(pool_datastack_create(), str, "\t"); - for (i = 0; args[i] != NULL; i++) - args[i] = str_tabunescape(args[i]); - ret = dsync_proxy_msg_import_unescaped(pool, - (const char *const *)args, - msg_r, error_r); - } T_END; - return ret; -} - -void dsync_proxy_msg_static_export(string_t *str, - const struct dsync_msg_static_data *msg) -{ - str_printfa(str, "%ld\t", (long)msg->received_date); - str_tabescape_write(str, msg->pop3_uidl); -} - -int dsync_proxy_msg_static_import_unescaped(pool_t pool, - const char *const *args, - struct dsync_msg_static_data *msg_r, - const char **error_r) -{ - /* received_date pop3_uidl */ - if (str_array_length(args) < 2) { - *error_r = "Missing parameters"; - return -1; - } - - memset(msg_r, 0, sizeof(*msg_r)); - msg_r->received_date = strtoul(args[0], NULL, 10); - msg_r->pop3_uidl = p_strdup(pool, args[1]); - return 0; -} - -int dsync_proxy_msg_static_import(pool_t pool, const char *str, - struct dsync_msg_static_data *msg_r, - const char **error_r) -{ - char **args; - unsigned int i; - int ret; - - T_BEGIN { - args = p_strsplit(pool_datastack_create(), str, "\t"); - for (i = 0; args[i] != NULL; i++) - args[i] = str_tabunescape(args[i]); - ret = dsync_proxy_msg_static_import_unescaped(pool, - (const char *const *)args, - msg_r, error_r); - } T_END; - return ret; -} - -void dsync_proxy_mailbox_export(string_t *str, - const struct dsync_mailbox *box) -{ - char s[2]; - - str_tabescape_write(str, box->name); - str_append_c(str, '\t'); - s[0] = box->name_sep; s[1] = '\0'; - str_tabescape_write(str, s); - str_printfa(str, "\t%lu\t%u", (unsigned long)box->last_change, - box->flags); - - if (dsync_mailbox_is_noselect(box)) { - i_assert(box->uid_validity == 0); - return; - } - i_assert(box->uid_validity != 0 || - (box->flags & DSYNC_MAILBOX_FLAG_DELETED_MAILBOX) != 0); - i_assert(box->uid_validity == 0 || box->uid_next != 0); - - str_append_c(str, '\t'); - dsync_proxy_mailbox_guid_export(str, &box->mailbox_guid); - str_printfa(str, "\t%u\t%u\t%u\t%llu\t%u", - box->uid_validity, box->uid_next, box->message_count, - (unsigned long long)box->highest_modseq, - box->first_recent_uid); - dsync_proxy_cache_fields_export(str, &box->cache_fields); -} - -int dsync_proxy_mailbox_import_unescaped(pool_t pool, const char *const *args, - struct dsync_mailbox *box_r, - const char **error_r) -{ - unsigned int i = 0, count; - bool box_deleted; - char *p; - - memset(box_r, 0, sizeof(*box_r)); - - count = str_array_length(args); - if (count != 4 && count < 8) { - *error_r = "Mailbox missing parameters"; - return -1; - } - - /* name dir_guid mailbox_guid uid_validity uid_next - message_count highest_modseq */ - box_r->name = p_strdup(pool, args[i++]); - dsync_str_sha_to_guid(box_r->name, &box_r->name_sha1); - - if (strlen(args[i]) > 1) { - *error_r = "Invalid mailbox name hierarchy separator"; - return -1; - } - box_r->name_sep = args[i++][0]; - - box_r->last_change = strtoul(args[i++], &p, 10); - if (*p != '\0') { - *error_r = "Invalid mailbox last_change"; - return -1; - } - box_r->flags = strtoul(args[i++], &p, 10); - if (*p != '\0' || - (dsync_mailbox_is_noselect(box_r) != (args[i] == NULL))) { - *error_r = "Invalid mailbox flags"; - return -1; - } - box_deleted = (box_r->flags & (DSYNC_MAILBOX_FLAG_DELETED_MAILBOX | - DSYNC_MAILBOX_FLAG_DELETED_DIR)) != 0; - if (box_r->name_sep == '\0' && !box_deleted) { - *error_r = "Missing mailbox name hierarchy separator"; - return -1; - } - - if (args[i] == NULL) { - /* \noselect mailbox */ - return 0; - } - - if (dsync_proxy_mailbox_guid_import(args[i++], - &box_r->mailbox_guid) < 0) { - *error_r = "Invalid mailbox GUID"; - return -1; - } - - box_r->uid_validity = strtoul(args[i++], &p, 10); - if (*p != '\0' || (box_r->uid_validity == 0 && !box_deleted)) { - abort(); - *error_r = "Invalid mailbox uid_validity"; - return -1; - } - - box_r->uid_next = strtoul(args[i++], &p, 10); - if (*p != '\0' || (box_r->uid_next == 0 && !box_deleted)) { - *error_r = "Invalid mailbox uid_next"; - return -1; - } - - box_r->message_count = strtoul(args[i++], &p, 10); - if (*p != '\0') { - *error_r = "Invalid mailbox message_count"; - return -1; - } - - box_r->highest_modseq = strtoull(args[i++], &p, 10); - if (*p != '\0') { - *error_r = "Invalid mailbox highest_modseq"; - return -1; - } - - box_r->first_recent_uid = strtoul(args[i++], &p, 10); - if (*p != '\0') { - *error_r = "Invalid mailbox first_recent_uid"; - return -1; - } - - if (dsync_proxy_cache_fields_import(args + i, pool, - &box_r->cache_fields, error_r) < 0) - return -1; - return 0; -} - -int dsync_proxy_mailbox_import(pool_t pool, const char *str, - struct dsync_mailbox *box_r, - const char **error_r) -{ - char **args; - int ret; - - T_BEGIN { - args = p_strsplit(pool_datastack_create(), str, "\t"); - if (args[0] != NULL) - args[0] = str_tabunescape(args[0]); - ret = dsync_proxy_mailbox_import_unescaped(pool, - (const char *const *)args, - box_r, error_r); - } T_END; - return ret; -} - -void dsync_proxy_mailbox_guid_export(string_t *str, - const mailbox_guid_t *mailbox) -{ - str_append(str, dsync_guid_to_str(mailbox)); -} - -int dsync_proxy_mailbox_guid_import(const char *str, mailbox_guid_t *guid_r) -{ - buffer_t *buf; - - buf = buffer_create_dynamic(pool_datastack_create(), - sizeof(guid_r->guid)); - if (hex_to_binary(str, buf) < 0 || buf->used != sizeof(guid_r->guid)) - return -1; - memcpy(guid_r->guid, buf->data, sizeof(guid_r->guid)); - return 0; -} - -void dsync_proxy_send_dot_output(struct ostream *output, bool *last_lf, - const unsigned char *data, size_t size) -{ - size_t i, start; - - i_assert(size > 0); - - if (*last_lf && data[0] == '.') - o_stream_send(output, ".", 1); - - for (i = 1, start = 0; i < size; i++) { - if (data[i-1] == '\n' && data[i] == '.') { - o_stream_send(output, data + start, i - start); - o_stream_send(output, ".", 1); - start = i; - } - } - o_stream_send(output, data + start, i - start); - *last_lf = data[i-1] == '\n'; - i_assert(i == size); -}
--- a/src/doveadm/dsync/dsync-proxy.h Fri May 04 05:35:36 2012 +0300 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,55 +0,0 @@ -#ifndef DSYNC_PROXY_H -#define DSYNC_PROXY_H - -#include "dsync-data.h" - -#define DSYNC_PROXY_CLIENT_TIMEOUT_MSECS (14*60*1000) -#define DSYNC_PROXY_SERVER_TIMEOUT_MSECS (15*60*1000) - -#define DSYNC_PROXY_CLIENT_GREETING_LINE "dsync-client\t2" -#define DSYNC_PROXY_SERVER_GREETING_LINE "dsync-server\t2" - -struct dsync_message; -struct dsync_mailbox; - -void dsync_proxy_cache_fields_export(string_t *str, - const ARRAY_TYPE(mailbox_cache_field) *fields); -int dsync_proxy_cache_fields_import(const char *const *args, pool_t pool, - ARRAY_TYPE(mailbox_cache_field) *fields, - const char **error_r); - -void dsync_proxy_msg_export(string_t *str, const struct dsync_message *msg); -int dsync_proxy_msg_parse_flags(pool_t pool, const char *str, - struct dsync_message *msg_r); -int dsync_proxy_msg_import_unescaped(pool_t pool, const char *const *args, - struct dsync_message *msg_r, - const char **error_r); -int dsync_proxy_msg_import(pool_t pool, const char *str, - struct dsync_message *msg_r, const char **error_r); - -void dsync_proxy_msg_static_export(string_t *str, - const struct dsync_msg_static_data *msg); -int dsync_proxy_msg_static_import(pool_t pool, const char *str, - struct dsync_msg_static_data *msg_r, - const char **error_r); -int dsync_proxy_msg_static_import_unescaped(pool_t pool, - const char *const *args, - struct dsync_msg_static_data *msg_r, - const char **error_r); - -void dsync_proxy_mailbox_export(string_t *str, const struct dsync_mailbox *box); -int dsync_proxy_mailbox_import(pool_t pool, const char *str, - struct dsync_mailbox *box_r, - const char **error_r); -int dsync_proxy_mailbox_import_unescaped(pool_t pool, const char *const *args, - struct dsync_mailbox *box_r, - const char **error_r); - -void dsync_proxy_mailbox_guid_export(string_t *str, - const mailbox_guid_t *mailbox); -int dsync_proxy_mailbox_guid_import(const char *str, mailbox_guid_t *guid_r); - -void dsync_proxy_send_dot_output(struct ostream *output, bool *last_lf, - const unsigned char *data, size_t size); - -#endif
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/doveadm/dsync/dsync-serializer.c Tue May 22 23:17:31 2012 +0300 @@ -0,0 +1,117 @@ +/* Copyright (c) 2012 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "str.h" +#include "strescape.h" +#include "dsync-serializer.h" + +struct dsync_serializer { + pool_t pool; + const char *const *keys; + unsigned int keys_count; +}; + +struct dsync_serializer_encoder { + pool_t pool; + struct dsync_serializer *serializer; + ARRAY_TYPE(const_string) values; +}; + +struct dsync_serializer *dsync_serializer_init(const char *const keys[]) +{ + struct dsync_serializer *serializer; + pool_t pool; + const char **dup_keys; + unsigned int i, count; + + pool = pool_alloconly_create("dsync serializer", 512); + serializer = p_new(pool, struct dsync_serializer, 1); + serializer->pool = pool; + + count = str_array_length(keys); + dup_keys = p_new(pool, const char *, count + 1); + for (i = 0; i < count; i++) + dup_keys[i] = p_strdup(pool, keys[i]); + serializer->keys = dup_keys; + serializer->keys_count = count; + return serializer; +} + +void dsync_serializer_deinit(struct dsync_serializer **_serializer) +{ + struct dsync_serializer *serializer = *_serializer; + + *_serializer = NULL; + + pool_unref(&serializer->pool); +} + +const char * +dsync_serializer_encode_header_line(struct dsync_serializer *serializer) +{ + string_t *str = t_str_new(128); + unsigned int i; + + for (i = 0; serializer->keys[i] != NULL; i++) { + if (i > 0) + str_append_c(str, '\t'); + str_tabescape_write(str, serializer->keys[i]); + } + str_append_c(str, '\n'); + return str_c(str); +} + +struct dsync_serializer_encoder * +dsync_serializer_encode_begin(struct dsync_serializer *serializer) +{ + struct dsync_serializer_encoder *encoder; + pool_t pool = pool_alloconly_create("dsync serializer encode", 1024); + + encoder = p_new(pool, struct dsync_serializer_encoder, 1); + encoder->pool = pool; + encoder->serializer = serializer; + p_array_init(&encoder->values, pool, serializer->keys_count); + return encoder; +} + +void dsync_serializer_encode_add(struct dsync_serializer_encoder *encoder, + const char *key, const char *value) +{ + unsigned int i; + + for (i = 0; encoder->serializer->keys[i] != NULL; i++) { + if (strcmp(encoder->serializer->keys[i], key) == 0) { + value = p_strdup(encoder->pool, value); + array_idx_set(&encoder->values, i, &value); + return; + } + } + i_panic("Unknown key: %s", key); +} + +void dsync_serializer_encode_finish(struct dsync_serializer_encoder **_encoder, + string_t *output) +{ + struct dsync_serializer_encoder *encoder = *_encoder; + const char *const *values; + unsigned int i, count; + + *_encoder = NULL; + + values = array_get(&encoder->values, &count); + for (i = 0; i < count; i++) { + if (i > 0) + str_append_c(output, '\t'); + if (values[i] == NULL) + str_append_c(output, NULL_CHR); + else { + if (values[i][0] == NULL_CHR) + str_append_c(output, NULL_CHR); + str_tabescape_write(output, values[i]); + } + } + str_append_c(output, '\n'); + + pool_unref(&encoder->pool); +}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/doveadm/dsync/dsync-serializer.h Tue May 22 23:17:31 2012 +0300 @@ -0,0 +1,18 @@ +#ifndef DSYNC_SERIALIZER_H +#define DSYNC_SERIALIZER_H + +#define NULL_CHR '\002' + +struct dsync_serializer *dsync_serializer_init(const char *const keys[]); +void dsync_serializer_deinit(struct dsync_serializer **serializer); + +const char * +dsync_serializer_encode_header_line(struct dsync_serializer *serializer); +struct dsync_serializer_encoder * +dsync_serializer_encode_begin(struct dsync_serializer *serializer); +void dsync_serializer_encode_add(struct dsync_serializer_encoder *encoder, + const char *key, const char *value); +void dsync_serializer_encode_finish(struct dsync_serializer_encoder **encoder, + string_t *output); + +#endif
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/doveadm/dsync/dsync-slave-io.c Tue May 22 23:17:31 2012 +0300 @@ -0,0 +1,1517 @@ +/* Copyright (c) 2012 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "close-keep-errno.h" +#include "fd-set-nonblock.h" +#include "safe-mkstemp.h" +#include "ioloop.h" +#include "istream.h" +#include "istream-seekable.h" +#include "istream-dot.h" +#include "ostream.h" +#include "str.h" +#include "strescape.h" +#include "master-service.h" +#include "mail-cache.h" +#include "mail-storage-private.h" +#include "dsync-serializer.h" +#include "dsync-deserializer.h" +#include "dsync-mail.h" +#include "dsync-mailbox.h" +#include "dsync-mailbox-state.h" +#include "dsync-mailbox-tree.h" +#include "dsync-slave-private.h" + +#include <stdlib.h> + +#define DSYNC_SLAVE_IO_TIMEOUT_MSECS (60*10*1000) +#define DSYNC_SLAVE_IO_OUTBUF_THROTTLE_SIZE (1024*128) + +#define DSYNC_PROTOCOL_VERSION_MAJOR 3 +#define DSYNC_HANDSHAKE_VERSION "VERSION\tdsync\t3\t0\n" + +enum item_type { + ITEM_NONE, + + ITEM_HANDSHAKE, + ITEM_MAILBOX_STATE, + ITEM_MAILBOX_TREE_NODE, + ITEM_MAILBOX_DELETE, + ITEM_MAILBOX, + + ITEM_MAIL_CHANGE, + ITEM_MAIL_REQUEST, + ITEM_MAIL, + + ITEM_MAILBOX_CACHE_FIELD, + + ITEM_END_OF_LIST +}; + +#define END_OF_LIST_LINE "." +static const struct { + /* full human readable name of the item */ + const char *name; + /* unique character identifying the item */ + char chr; + const char *required_keys; + const char *optional_keys; +} items[ITEM_END_OF_LIST+1] = { + { NULL, '\0', NULL, NULL }, + { .name = "handshake", + .chr = 'H', + .optional_keys = "sync_ns_prefix sync_type " + "guid_requests mails_have_guids" + }, + { .name = "mailbox_state", + .chr = 'S', + .required_keys = "mailbox_guid last_uidvalidity last_common_uid " + "last_common_modseq" + }, + { .name = "mailbox_tree_node", + .chr = 'N', + .required_keys = "name existence", + .optional_keys = "mailbox_guid uid_validity " + "last_renamed subscribed last_subscription_change" + }, + { .name = "mailbox_delete", + .chr = 'D', + .required_keys = "hierarchy_sep", + .optional_keys = "mailboxes dirs" + }, + { .name = "mailbox", + .chr = 'B', + .required_keys = "mailbox_guid uid_validity uid_next " + "messages_count first_recent_uid highest_modseq", + .optional_keys = "cache_fields" + }, + { .name = "mail_change", + .chr = 'C', + .required_keys = "type uid", + .optional_keys = "guid hdr_hash modseq save_timestamp " + "add_flags remove_flags final_flags " + "keywords_reset keyword_changes" + }, + { .name = "mail_request", + .chr = 'R', + .optional_keys = "guid uid" + }, + { .name = "mail", + .chr = 'M', + .optional_keys = "guid uid pop3_uidl pop3_order received_date stream" + }, + { .name = "mailbox_cache_field", + .chr = 'c', + .required_keys = "name decision", + .optional_keys = "last_used" + }, + + { "end_of_list", '\0', NULL, NULL } +}; + +struct dsync_slave_io { + struct dsync_slave slave; + + char *name, *temp_path_prefix; + int fd_in, fd_out; + struct istream *input; + struct ostream *output; + struct io *io; + struct timeout *to; + + struct dsync_serializer *serializers[ITEM_END_OF_LIST]; + struct dsync_deserializer *deserializers[ITEM_END_OF_LIST]; + + pool_t ret_pool; + struct dsync_deserializer_decoder *cur_decoder; + + struct istream *mail_output, *mail_input; + struct dsync_mail *cur_mail; + char mail_output_last; + + unsigned int version_received:1; + unsigned int handshake_received:1; + unsigned int has_pending_data:1; +}; + +static void dsync_slave_io_stop(struct dsync_slave_io *slave) +{ + i_stream_close(slave->input); + o_stream_close(slave->output); + io_loop_stop(current_ioloop); +} + +static int dsync_slave_io_read_mail_stream(struct dsync_slave_io *slave) +{ + size_t size; + + if (i_stream_read(slave->mail_input) < 0) { + if (slave->mail_input->stream_errno != 0) { + errno = slave->mail_input->stream_errno; + i_error("dsync(%s): read() failed: %m", slave->name); + dsync_slave_io_stop(slave); + return -1; + } + /* finished reading the mail stream */ + i_assert(slave->mail_input->eof); + i_stream_seek(slave->mail_input, 0); + slave->mail_input = NULL; + return 1; + } + (void)i_stream_get_data(slave->mail_input, &size); + i_stream_skip(slave->mail_input, size); + return 0; +} + +static void dsync_slave_io_input(struct dsync_slave_io *slave) +{ + if (slave->mail_input != NULL) { + if (dsync_slave_io_read_mail_stream(slave) == 0) + return; + } + slave->slave.io_callback(slave->slave.io_context); +} + +static int dsync_slave_io_send_mail_stream(struct dsync_slave_io *slave) +{ + const unsigned char *data; + unsigned char add; + size_t i, size; + int ret; + + while ((ret = i_stream_read_data(slave->mail_output, + &data, &size, 0)) > 0) { + add = '\0'; + for (i = 0; i < size; i++) { + if (data[i] == '\n') { + if ((i == 0 && slave->mail_output_last != '\r') || + (i > 0 && data[i-1] != '\r')) { + /* missing CR */ + add = '\r'; + break; + } + } else if (data[i] == '.' && + ((i == 0 && slave->mail_output_last == '\n') || + (i > 0 && data[i-1] == '\n'))) { + /* escape the dot */ + add = '.'; + break; + } + } + + if (i > 0) { + (void)o_stream_send(slave->output, data, i); + slave->mail_output_last = data[i-1]; + i_stream_skip(slave->mail_output, i); + } + + if (o_stream_get_buffer_used_size(slave->output) >= 4096) { + if ((ret = o_stream_flush(slave->output)) < 0) { + dsync_slave_io_stop(slave); + return -1; + } + if (ret == 0) { + /* continue later */ + o_stream_set_flush_pending(slave->output, TRUE); + return 0; + } + } + + if (add != '\0') { + (void)o_stream_send(slave->output, &add, 1); + slave->mail_output_last = add; + } + } + i_assert(ret == -1); + + if (slave->mail_output->stream_errno != 0) { + i_error("dsync(%s): read(%s) failed: %m", + slave->name, i_stream_get_name(slave->mail_output)); + dsync_slave_io_stop(slave); + return -1; + } + + /* finished sending the stream */ + (void)o_stream_send_str(slave->output, "\r\n.\r\n"); + i_stream_unref(&slave->mail_output); + return 1; +} + +static int dsync_slave_io_output(struct dsync_slave_io *slave) +{ + struct ostream *output = slave->output; + int ret; + + if ((ret = o_stream_flush(output)) < 0) + ret = 1; + else if (slave->mail_output != NULL) { + if (dsync_slave_io_send_mail_stream(slave) < 0) + ret = 1; + } + timeout_reset(slave->to); + + if (!dsync_slave_is_send_queue_full(&slave->slave)) + slave->slave.io_callback(slave->slave.io_context); + return ret; +} + +static void dsync_slave_io_timeout(struct dsync_slave_io *slave) +{ + i_error("dsync(%s): I/O has stalled, no activity for %u seconds", + slave->name, DSYNC_SLAVE_IO_TIMEOUT_MSECS/1000); + dsync_slave_io_stop(slave); +} + +static void dsync_slave_io_init(struct dsync_slave_io *slave) +{ + unsigned int i; + + fd_set_nonblock(slave->fd_in, TRUE); + fd_set_nonblock(slave->fd_out, TRUE); + + slave->input = i_stream_create_fd(slave->fd_in, (size_t)-1, FALSE); + slave->output = o_stream_create_fd(slave->fd_out, (size_t)-1, FALSE); + slave->io = io_add(slave->fd_in, IO_READ, dsync_slave_io_input, slave); + o_stream_set_flush_callback(slave->output, dsync_slave_io_output, slave); + slave->to = timeout_add(DSYNC_SLAVE_IO_TIMEOUT_MSECS, + dsync_slave_io_timeout, NULL); + o_stream_cork(slave->output); + o_stream_send_str(slave->output, DSYNC_HANDSHAKE_VERSION); + + /* initialize serializers and send their headers to remote */ + for (i = 1; i < ITEM_END_OF_LIST; i++) T_BEGIN { + const char *keys; + + keys = items[i].required_keys == NULL ? items[i].optional_keys : + t_strconcat(items[i].required_keys, " ", + items[i].optional_keys, NULL); + if (keys != NULL) { + i_assert(items[i].chr != '\0'); + + slave->serializers[i] = + dsync_serializer_init(t_strsplit_spaces(keys, " ")); + o_stream_send(slave->output, &items[i].chr, 1); + o_stream_send_str(slave->output, + dsync_serializer_encode_header_line(slave->serializers[i])); + } + } T_END; + o_stream_send_str(slave->output, ".\n"); + + dsync_slave_flush(&slave->slave); +} + +static void dsync_slave_io_deinit(struct dsync_slave *_slave) +{ + struct dsync_slave_io *slave = (struct dsync_slave_io *)_slave; + + if (slave->cur_decoder != NULL) + dsync_deserializer_decode_finish(&slave->cur_decoder); + if (slave->mail_output != NULL) + i_stream_unref(&slave->mail_output); + + timeout_remove(&slave->to); + if (slave->io != NULL) + io_remove(&slave->io); + i_stream_destroy(&slave->input); + o_stream_destroy(&slave->output); + if (close(slave->fd_in) < 0) + i_error("close(%s) failed: %m", slave->name); + if (slave->fd_in != slave->fd_out) { + if (close(slave->fd_out) < 0) + i_error("close(%s) failed: %m", slave->name); + } + pool_unref(&slave->ret_pool); + i_free(slave->temp_path_prefix); + i_free(slave->name); + i_free(slave); +} + +static int dsync_slave_io_next_line(struct dsync_slave_io *slave, + const char **line_r) +{ + const char *line; + + line = i_stream_next_line(slave->input); + if (line != NULL) { + *line_r = line; + return 1; + } + + /* try reading some */ + switch (i_stream_read(slave->input)) { + case -1: + if (slave->input->stream_errno != 0) { + errno = slave->input->stream_errno; + i_error("read(%s) failed: %m", slave->name); + } else { + i_assert(slave->input->eof); + i_error("read(%s) failed: EOF", slave->name); + } + dsync_slave_io_stop(slave); + return -1; + case 0: + return 0; + } + *line_r = i_stream_next_line(slave->input); + if (*line_r == NULL) { + slave->has_pending_data = FALSE; + return 0; + } + slave->has_pending_data = TRUE; + return 1; +} + +static void ATTR_FORMAT(3, 4) +dsync_slave_input_error(struct dsync_slave_io *slave, + struct dsync_deserializer_decoder *decoder, + const char *fmt, ...) +{ + va_list args; + const char *error; + + va_start(args, fmt); + error = t_strdup_vprintf(fmt, args); + if (decoder == NULL) + i_error("dsync(%s): %s", slave->name, error); + else { + i_error("dsync(%s): %s: %s", slave->name, + dsync_deserializer_decoder_get_name(decoder), error); + } + va_end(args); + + dsync_slave_io_stop(slave); +} + +static void +dsync_slave_io_send_string(struct dsync_slave_io *slave, const string_t *str) +{ + i_assert(slave->mail_output == NULL); + o_stream_send(slave->output, str_data(str), str_len(str)); +} + +static int dsync_slave_check_missing_deserializers(struct dsync_slave_io *slave) +{ + unsigned int i; + int ret = 0; + + for (i = 1; i < ITEM_END_OF_LIST; i++) { + if (slave->deserializers[i] == NULL && + (items[i].required_keys != NULL || + items[i].optional_keys != NULL)) { + dsync_slave_input_error(slave, NULL, + "Remote didn't handshake deserializer for %s", + items[i].name); + ret = -1; + } + } + return ret; +} + +static bool +dsync_slave_io_handshake(struct dsync_slave_io *slave, const char *line) +{ + enum item_type item = ITEM_NONE; + const char *const *required_keys, *error; + unsigned int i; + + if (slave->handshake_received) + return TRUE; + + if (!slave->version_received) { + if (!version_string_verify(line, "dsync", + DSYNC_PROTOCOL_VERSION_MAJOR)) { + dsync_slave_input_error(slave, NULL, + "Remote dsync doesn't use compatible protocol"); + return DSYNC_SLAVE_RECV_RET_TRYAGAIN; + } + slave->version_received = TRUE; + return FALSE; + } + + if (strcmp(line, END_OF_LIST_LINE) == 0) { + /* finished handshaking */ + if (dsync_slave_check_missing_deserializers(slave) < 0) + return FALSE; + slave->handshake_received = TRUE; + return FALSE; + } + + for (i = 1; i < ITEM_END_OF_LIST; i++) { + if (items[i].chr == line[0]) { + item = i; + break; + } + } + if (item == ITEM_NONE) { + /* unknown deserializer, ignore */ + return FALSE; + } + + required_keys = items[item].required_keys == NULL ? NULL : + t_strsplit(items[item].required_keys, " "); + if (dsync_deserializer_init(items[item].name, + required_keys, line + 1, + &slave->deserializers[item], &error) < 0) { + dsync_slave_input_error(slave, NULL, + "Remote sent invalid handshake for %s: %s", + items[item].name, error); + } + return FALSE; +} + +static enum dsync_slave_recv_ret +dsync_slave_io_input_next(struct dsync_slave_io *slave, enum item_type item, + struct dsync_deserializer_decoder **decoder_r) +{ + enum item_type line_item = ITEM_NONE; + const char *line, *error; + unsigned int i; + + i_assert(slave->mail_input == NULL); + + timeout_reset(slave->to); + + do { + if (dsync_slave_io_next_line(slave, &line) <= 0) + return DSYNC_SLAVE_RECV_RET_TRYAGAIN; + } while (!dsync_slave_io_handshake(slave, line)); + + if (strcmp(line, END_OF_LIST_LINE) == 0) { + /* end of this list */ + return DSYNC_SLAVE_RECV_RET_FINISHED; + } + for (i = 1; i < ITEM_END_OF_LIST; i++) { + if (*line == items[i].chr) { + line_item = i; + break; + } + } + if (line_item != item) { + dsync_slave_input_error(slave, NULL, + "Received unexpected input %c != %c", + *line, items[item].chr); + return DSYNC_SLAVE_RECV_RET_TRYAGAIN; + } + + if (slave->cur_decoder != NULL) + dsync_deserializer_decode_finish(&slave->cur_decoder); + if (dsync_deserializer_decode_begin(slave->deserializers[item], + line+1, &slave->cur_decoder, + &error) < 0) { + dsync_slave_input_error(slave, NULL, "Invalid input to %s: %s", + items[item].name, error); + return DSYNC_SLAVE_RECV_RET_TRYAGAIN; + } + *decoder_r = slave->cur_decoder; + return DSYNC_SLAVE_RECV_RET_OK; +} + +static void +dsync_slave_io_send_handshake(struct dsync_slave *_slave, + const struct dsync_slave_settings *set) +{ + struct dsync_slave_io *slave = (struct dsync_slave_io *)_slave; + struct dsync_serializer_encoder *encoder; + string_t *str = t_str_new(128); + char sync_type[2]; + + str_append_c(str, items[ITEM_HANDSHAKE].chr); + encoder = dsync_serializer_encode_begin(slave->serializers[ITEM_HANDSHAKE]); + if (set->sync_ns_prefix != NULL) { + dsync_serializer_encode_add(encoder, "sync_ns_prefix", + set->sync_ns_prefix); + } + + sync_type[0] = sync_type[1] = '\0'; + switch (set->sync_type) { + case DSYNC_BRAIN_SYNC_TYPE_UNKNOWN: + break; + case DSYNC_BRAIN_SYNC_TYPE_FULL: + sync_type[0] = 'f'; + break; + case DSYNC_BRAIN_SYNC_TYPE_CHANGED: + sync_type[0] = 'c'; + break; + case DSYNC_BRAIN_SYNC_TYPE_STATE: + sync_type[0] = 's'; + break; + } + i_assert(sync_type[0] != '\0'); + dsync_serializer_encode_add(encoder, "sync_type", sync_type); + if (set->guid_requests) + dsync_serializer_encode_add(encoder, "guid_requests", ""); + if (set->mails_have_guids) + dsync_serializer_encode_add(encoder, "mails_have_guids", ""); + + dsync_serializer_encode_finish(&encoder, str); + dsync_slave_io_send_string(slave, str); +} + +static enum dsync_slave_recv_ret +dsync_slave_io_recv_handshake(struct dsync_slave *_slave, + const struct dsync_slave_settings **set_r) +{ + struct dsync_slave_io *slave = (struct dsync_slave_io *)_slave; + struct dsync_deserializer_decoder *decoder; + struct dsync_slave_settings *set; + const char *value; + pool_t pool = slave->ret_pool; + enum dsync_slave_recv_ret ret; + + ret = dsync_slave_io_input_next(slave, ITEM_HANDSHAKE, &decoder); + if (ret != DSYNC_SLAVE_RECV_RET_OK) { + if (ret != DSYNC_SLAVE_RECV_RET_TRYAGAIN) { + i_error("dsync(%s): Unexpected input in handshake", + slave->name); + dsync_slave_io_stop(slave); + } + return DSYNC_SLAVE_RECV_RET_TRYAGAIN; + } + + p_clear(pool); + set = p_new(pool, struct dsync_slave_settings, 1); + + if (dsync_deserializer_decode_try(decoder, "sync_ns_prefix", &value)) + set->sync_ns_prefix = p_strdup(pool, value); + if (dsync_deserializer_decode_try(decoder, "sync_type", &value)) { + switch (value[0]) { + case 'f': + set->sync_type = DSYNC_BRAIN_SYNC_TYPE_FULL; + break; + case 'c': + set->sync_type = DSYNC_BRAIN_SYNC_TYPE_CHANGED; + break; + case 's': + set->sync_type = DSYNC_BRAIN_SYNC_TYPE_STATE; + break; + default: + dsync_slave_input_error(slave, decoder, + "Unknown sync_type: %s", value); + return DSYNC_SLAVE_RECV_RET_TRYAGAIN; + } + } + if (dsync_deserializer_decode_try(decoder, "guid_requests", &value)) + set->guid_requests = TRUE; + if (dsync_deserializer_decode_try(decoder, "mails_have_guids", &value)) + set->mails_have_guids = TRUE; + + *set_r = set; + return DSYNC_SLAVE_RECV_RET_OK; +} + +static void +dsync_slave_io_send_end_of_list(struct dsync_slave *_slave) +{ + struct dsync_slave_io *slave = (struct dsync_slave_io *)_slave; + + i_assert(slave->mail_output == NULL); + + o_stream_send_str(slave->output, END_OF_LIST_LINE"\n"); +} + +static void +dsync_slave_io_send_mailbox_state(struct dsync_slave *_slave, + const struct dsync_mailbox_state *state) +{ + struct dsync_slave_io *slave = (struct dsync_slave_io *)_slave; + struct dsync_serializer_encoder *encoder; + string_t *str = t_str_new(128); + + str_append_c(str, items[ITEM_MAILBOX_STATE].chr); + encoder = dsync_serializer_encode_begin(slave->serializers[ITEM_MAILBOX_STATE]); + dsync_serializer_encode_add(encoder, "mailbox_guid", + guid_128_to_string(state->mailbox_guid)); + dsync_serializer_encode_add(encoder, "last_uidvalidity", + dec2str(state->last_uidvalidity)); + dsync_serializer_encode_add(encoder, "last_common_uid", + dec2str(state->last_common_uid)); + dsync_serializer_encode_add(encoder, "last_common_modseq", + dec2str(state->last_common_modseq)); + + dsync_serializer_encode_finish(&encoder, str); + dsync_slave_io_send_string(slave, str); +} + +static enum dsync_slave_recv_ret +dsync_slave_io_recv_mailbox_state(struct dsync_slave *_slave, + struct dsync_mailbox_state *state_r) +{ + struct dsync_slave_io *slave = (struct dsync_slave_io *)_slave; + struct dsync_deserializer_decoder *decoder; + const char *value; + enum dsync_slave_recv_ret ret; + + memset(state_r, 0, sizeof(*state_r)); + + ret = dsync_slave_io_input_next(slave, ITEM_MAILBOX_STATE, &decoder); + if (ret != DSYNC_SLAVE_RECV_RET_OK) + return ret; + + value = dsync_deserializer_decode_get(decoder, "mailbox_guid"); + if (guid_128_from_string(value, state_r->mailbox_guid) < 0) { + dsync_slave_input_error(slave, decoder, "Invalid mailbox_guid"); + return DSYNC_SLAVE_RECV_RET_TRYAGAIN; + } + value = dsync_deserializer_decode_get(decoder, "last_uidvalidity"); + if (str_to_uint32(value, &state_r->last_uidvalidity) < 0) { + dsync_slave_input_error(slave, decoder, "Invalid last_uidvalidity"); + return DSYNC_SLAVE_RECV_RET_TRYAGAIN; + } + value = dsync_deserializer_decode_get(decoder, "last_common_uid"); + if (str_to_uint32(value, &state_r->last_uidvalidity) < 0) { + dsync_slave_input_error(slave, decoder, "Invalid last_common_uid"); + return DSYNC_SLAVE_RECV_RET_TRYAGAIN; + } + value = dsync_deserializer_decode_get(decoder, "last_common_modseq"); + if (str_to_uint32(value, &state_r->last_uidvalidity) < 0) { + dsync_slave_input_error(slave, decoder, "Invalid last_common_modseq"); + return DSYNC_SLAVE_RECV_RET_TRYAGAIN; + } + return DSYNC_SLAVE_RECV_RET_OK; +} + +static void +dsync_slave_io_send_mailbox_tree_node(struct dsync_slave *_slave, + const char *const *name, + const struct dsync_mailbox_node *node) +{ + struct dsync_slave_io *slave = (struct dsync_slave_io *)_slave; + struct dsync_serializer_encoder *encoder; + string_t *str, *namestr; + + i_assert(*name != NULL); + + str = t_str_new(128); + str_append_c(str, items[ITEM_MAILBOX_TREE_NODE].chr); + + /* convert all hierarchy separators to tabs. mailbox names really + aren't supposed to have any tabs, but escape them anyway if there + are. */ + namestr = t_str_new(128); + for (; *name != NULL; name++) { + str_tabescape_write(namestr, *name); + str_append_c(namestr, '\t'); + } + str_truncate(namestr, str_len(namestr)-1); + + encoder = dsync_serializer_encode_begin(slave->serializers[ITEM_MAILBOX_TREE_NODE]); + dsync_serializer_encode_add(encoder, "name", str_c(namestr)); + switch (node->existence) { + case DSYNC_MAILBOX_NODE_NONEXISTENT: + dsync_serializer_encode_add(encoder, "existence", "n"); + break; + case DSYNC_MAILBOX_NODE_EXISTS: + dsync_serializer_encode_add(encoder, "existence", "y"); + break; + case DSYNC_MAILBOX_NODE_DELETED: + dsync_serializer_encode_add(encoder, "existence", "d"); + break; + } + + if (!guid_128_is_empty(node->mailbox_guid)) { + dsync_serializer_encode_add(encoder, "mailbox_guid", + guid_128_to_string(node->mailbox_guid)); + } + if (node->uid_validity != 0) { + dsync_serializer_encode_add(encoder, "uid_validity", + dec2str(node->uid_validity)); + } + if (node->last_renamed != 0) { + dsync_serializer_encode_add(encoder, "last_renamed", + dec2str(node->last_renamed)); + } + if (node->last_subscription_change != 0) { + dsync_serializer_encode_add(encoder, "last_subscription_change", + dec2str(node->last_subscription_change)); + } + if (node->subscribed) + dsync_serializer_encode_add(encoder, "subscribed", ""); + dsync_serializer_encode_finish(&encoder, str); + dsync_slave_io_send_string(slave, str); +} + +static enum dsync_slave_recv_ret +dsync_slave_io_recv_mailbox_tree_node(struct dsync_slave *_slave, + const char *const **name_r, + const struct dsync_mailbox_node **node_r) +{ + struct dsync_slave_io *slave = (struct dsync_slave_io *)_slave; + struct dsync_deserializer_decoder *decoder; + struct dsync_mailbox_node *node; + const char *value; + enum dsync_slave_recv_ret ret; + + ret = dsync_slave_io_input_next(slave, ITEM_MAILBOX_TREE_NODE, &decoder); + if (ret != DSYNC_SLAVE_RECV_RET_OK) + return ret; + + p_clear(slave->ret_pool); + node = p_new(slave->ret_pool, struct dsync_mailbox_node, 1); + + value = dsync_deserializer_decode_get(decoder, "name"); + if (*value == '\0') { + dsync_slave_input_error(slave, decoder, "Empty name"); + return DSYNC_SLAVE_RECV_RET_TRYAGAIN; + } + *name_r = (void *)p_strsplit_tabescaped(slave->ret_pool, value); + + value = dsync_deserializer_decode_get(decoder, "existence"); + switch (*value) { + case 'n': + node->existence = DSYNC_MAILBOX_NODE_NONEXISTENT; + break; + case 'y': + node->existence = DSYNC_MAILBOX_NODE_EXISTS; + break; + case 'd': + node->existence = DSYNC_MAILBOX_NODE_DELETED; + break; + } + + if (dsync_deserializer_decode_try(decoder, "mailbox_guid", &value) && + guid_128_from_string(value, node->mailbox_guid) < 0) { + dsync_slave_input_error(slave, decoder, "Invalid mailbox_guid"); + return DSYNC_SLAVE_RECV_RET_TRYAGAIN; + } + if (dsync_deserializer_decode_try(decoder, "uid_validity", &value) && + str_to_uint32(value, &node->uid_validity) < 0) { + dsync_slave_input_error(slave, decoder, "Invalid uid_validity"); + return DSYNC_SLAVE_RECV_RET_TRYAGAIN; + } + if (dsync_deserializer_decode_try(decoder, "last_renamed", &value) && + str_to_time(value, &node->last_renamed) < 0) { + dsync_slave_input_error(slave, decoder, "Invalid last_renamed"); + return DSYNC_SLAVE_RECV_RET_TRYAGAIN; + } + if (dsync_deserializer_decode_try(decoder, "last_subscription_change", &value) && + str_to_time(value, &node->last_subscription_change) < 0) { + dsync_slave_input_error(slave, decoder, "Invalid last_subscription_change"); + return DSYNC_SLAVE_RECV_RET_TRYAGAIN; + } + if (dsync_deserializer_decode_try(decoder, "subscribed", &value)) + node->subscribed = TRUE; + + *node_r = node; + return DSYNC_SLAVE_RECV_RET_OK; +} + +static void +dsync_slave_io_send_mailbox_deletes(struct dsync_slave *_slave, + const struct dsync_mailbox_delete *deletes, + unsigned int count, char hierarchy_sep) +{ + struct dsync_slave_io *slave = (struct dsync_slave_io *)_slave; + struct dsync_serializer_encoder *encoder; + string_t *str, *guidstr; + char sep[2]; + unsigned int i; + + str = t_str_new(128); + str_append_c(str, items[ITEM_MAILBOX_DELETE].chr); + + encoder = dsync_serializer_encode_begin(slave->serializers[ITEM_MAILBOX_DELETE]); + sep[0] = hierarchy_sep; sep[1] = '\0'; + dsync_serializer_encode_add(encoder, "hierarchy_sep", sep); + + guidstr = t_str_new(128); + for (i = 0; i < count; i++) { + if (deletes[i].delete_mailbox) { + str_append(guidstr, guid_128_to_string(deletes[i].guid)); + str_append_c(guidstr, ' '); + } + } + if (str_len(guidstr) > 0) { + str_truncate(guidstr, str_len(guidstr)-1); + dsync_serializer_encode_add(encoder, "mailboxes", + str_c(guidstr)); + } + + str_truncate(guidstr, 0); + for (i = 0; i < count; i++) { + if (!deletes[i].delete_mailbox) { + str_append(guidstr, guid_128_to_string(deletes[i].guid)); + str_append_c(guidstr, ' '); + } + } + if (str_len(guidstr) > 0) { + str_truncate(guidstr, str_len(guidstr)-1); + dsync_serializer_encode_add(encoder, "dirs", str_c(guidstr)); + } + dsync_serializer_encode_finish(&encoder, str); + dsync_slave_io_send_string(slave, str); +} + +ARRAY_DEFINE_TYPE(dsync_mailbox_delete, struct dsync_mailbox_delete); +static int +decode_mailbox_deletes(ARRAY_TYPE(dsync_mailbox_delete) *deletes, + const char *value, bool delete_mailbox) +{ + struct dsync_mailbox_delete *del; + const char *const *guid_strings; + unsigned int i; + + guid_strings = t_strsplit(value, " "); + for (i = 0; guid_strings[i] != NULL; i++) { + del = array_append_space(deletes); + del->delete_mailbox = delete_mailbox; + if (guid_128_from_string(guid_strings[i], del->guid) < 0) + return -1; + } + return 0; +} + +static enum dsync_slave_recv_ret +dsync_slave_io_recv_mailbox_deletes(struct dsync_slave *_slave, + const struct dsync_mailbox_delete **deletes_r, + unsigned int *count_r, char *hierarchy_sep_r) +{ + struct dsync_slave_io *slave = (struct dsync_slave_io *)_slave; + struct dsync_deserializer_decoder *decoder; + ARRAY_TYPE(dsync_mailbox_delete) deletes; + const char *value; + enum dsync_slave_recv_ret ret; + + ret = dsync_slave_io_input_next(slave, ITEM_MAILBOX_DELETE, &decoder); + if (ret != DSYNC_SLAVE_RECV_RET_OK) + return ret; + + p_clear(slave->ret_pool); + p_array_init(&deletes, slave->ret_pool, 16); + + value = dsync_deserializer_decode_get(decoder, "hierarchy_sep"); + if (strlen(value) != 1) { + dsync_slave_input_error(slave, decoder, "Invalid hierarchy_sep"); + return DSYNC_SLAVE_RECV_RET_TRYAGAIN; + } + *hierarchy_sep_r = value[0]; + + if (dsync_deserializer_decode_try(decoder, "mailboxes", &value) && + decode_mailbox_deletes(&deletes, value, TRUE) < 0) { + dsync_slave_input_error(slave, decoder, "Invalid mailboxes"); + return DSYNC_SLAVE_RECV_RET_TRYAGAIN; + } + if (dsync_deserializer_decode_try(decoder, "dirs", &value) && + decode_mailbox_deletes(&deletes, value, FALSE) < 0) { + dsync_slave_input_error(slave, decoder, "Invalid dirs"); + return DSYNC_SLAVE_RECV_RET_TRYAGAIN; + } + *deletes_r = array_get(&deletes, count_r); + return DSYNC_SLAVE_RECV_RET_OK; +} + +static const char * +get_cache_fields(struct dsync_slave_io *slave, + const struct dsync_mailbox *dsync_box) +{ + struct dsync_serializer_encoder *encoder; + string_t *str; + const struct mailbox_cache_field *cache_fields; + unsigned int i, count; + char decision[3]; + + cache_fields = array_get(&dsync_box->cache_fields, &count); + if (count == 0) + return ""; + + str = t_str_new(128); + for (i = 0; i < count; i++) { + const struct mailbox_cache_field *field = &cache_fields[i]; + + encoder = dsync_serializer_encode_begin(slave->serializers[ITEM_MAILBOX_CACHE_FIELD]); + dsync_serializer_encode_add(encoder, "name", field->name); + + memset(decision, 0, sizeof(decision)); + switch (field->decision & ~MAIL_CACHE_DECISION_FORCED) { + case MAIL_CACHE_DECISION_NO: + decision[0] = 'n'; + break; + case MAIL_CACHE_DECISION_TEMP: + decision[0] = 't'; + break; + case MAIL_CACHE_DECISION_YES: + decision[0] = 'y'; + break; + } + i_assert(decision[0] != '\0'); + if ((field->decision & MAIL_CACHE_DECISION_FORCED) != 0) + decision[1] = 'F'; + dsync_serializer_encode_add(encoder, "decision", decision); + if (field->last_used != 0) { + dsync_serializer_encode_add(encoder, "last_used", + dec2str(field->last_used)); + } + dsync_serializer_encode_finish(&encoder, str); + } + if (i > 0) { + /* remove the trailing LF */ + str_truncate(str, str_len(str)-1); + } + return str_c(str); +} + +static void +dsync_slave_io_send_mailbox(struct dsync_slave *_slave, + const struct dsync_mailbox *dsync_box) +{ + struct dsync_slave_io *slave = (struct dsync_slave_io *)_slave; + struct dsync_serializer_encoder *encoder; + string_t *str = t_str_new(128); + const char *value; + + str_append_c(str, items[ITEM_MAILBOX].chr); + encoder = dsync_serializer_encode_begin(slave->serializers[ITEM_MAILBOX]); + dsync_serializer_encode_add(encoder, "mailbox_guid", + guid_128_to_string(dsync_box->mailbox_guid)); + + if (dsync_box->mailbox_lost) + dsync_serializer_encode_add(encoder, "mailbox_lost", ""); + dsync_serializer_encode_add(encoder, "uid_validity", + dec2str(dsync_box->uid_validity)); + dsync_serializer_encode_add(encoder, "uid_next", + dec2str(dsync_box->uid_next)); + dsync_serializer_encode_add(encoder, "messages_count", + dec2str(dsync_box->messages_count)); + dsync_serializer_encode_add(encoder, "first_recent_uid", + dec2str(dsync_box->first_recent_uid)); + dsync_serializer_encode_add(encoder, "highest_modseq", + dec2str(dsync_box->highest_modseq)); + + value = get_cache_fields(slave, dsync_box); + if (value != NULL) + dsync_serializer_encode_add(encoder, "cache_fields", value); + + dsync_serializer_encode_finish(&encoder, str); + dsync_slave_io_send_string(slave, str); +} + +static int +parse_cache_field(struct dsync_slave_io *slave, struct dsync_mailbox *box, + const char *value) +{ + struct dsync_deserializer_decoder *decoder; + struct mailbox_cache_field field; + const char *error; + int ret = 0; + + if (dsync_deserializer_decode_begin(slave->deserializers[ITEM_MAILBOX_CACHE_FIELD], + value, &decoder, &error) < 0) { + dsync_slave_input_error(slave, NULL, + "cache_field: Invalid input: %s", error); + return -1; + } + + memset(&field, 0, sizeof(field)); + value = dsync_deserializer_decode_get(decoder, "name"); + field.name = p_strdup(slave->ret_pool, value); + + value = dsync_deserializer_decode_get(decoder, "decision"); + switch (*value) { + case 'n': + field.decision = MAIL_CACHE_DECISION_NO; + break; + case 't': + field.decision = MAIL_CACHE_DECISION_TEMP; + break; + case 'y': + field.decision = MAIL_CACHE_DECISION_YES; + break; + default: + dsync_slave_input_error(slave, decoder, "Invalid decision: %s", + value); + ret = -1; + break; + } + if (value[1] == 'F') + field.decision |= MAIL_CACHE_DECISION_FORCED; + + if (dsync_deserializer_decode_try(decoder, "last_used", &value) && + str_to_time(value, &field.last_used) < 0) { + dsync_slave_input_error(slave, decoder, "Invalid last_used"); + ret = -1; + } + array_append(&box->cache_fields, &field, 1); + + dsync_deserializer_decode_finish(&decoder); + return ret; +} + +static enum dsync_slave_recv_ret +dsync_slave_io_recv_mailbox(struct dsync_slave *_slave, + const struct dsync_mailbox **dsync_box_r) +{ + struct dsync_slave_io *slave = (struct dsync_slave_io *)_slave; + pool_t pool = slave->ret_pool; + struct dsync_deserializer_decoder *decoder; + struct dsync_mailbox *box; + const char *value; + enum dsync_slave_recv_ret ret; + + p_clear(pool); + box = p_new(pool, struct dsync_mailbox, 1); + + ret = dsync_slave_io_input_next(slave, ITEM_MAILBOX, &decoder); + if (ret != DSYNC_SLAVE_RECV_RET_OK) + return ret; + + value = dsync_deserializer_decode_get(decoder, "mailbox_guid"); + if (guid_128_from_string(value, box->mailbox_guid) < 0) { + dsync_slave_input_error(slave, decoder, "Invalid mailbox_guid"); + return DSYNC_SLAVE_RECV_RET_TRYAGAIN; + } + + if (dsync_deserializer_decode_try(decoder, "mailbox_lost", &value)) + box->mailbox_lost = TRUE; + value = dsync_deserializer_decode_get(decoder, "uid_validity"); + if (str_to_uint32(value, &box->uid_validity) < 0) { + dsync_slave_input_error(slave, decoder, "Invalid uid_validity"); + return DSYNC_SLAVE_RECV_RET_TRYAGAIN; + } + value = dsync_deserializer_decode_get(decoder, "uid_next"); + if (str_to_uint32(value, &box->uid_next) < 0) { + dsync_slave_input_error(slave, decoder, "Invalid uid_next"); + return DSYNC_SLAVE_RECV_RET_TRYAGAIN; + } + value = dsync_deserializer_decode_get(decoder, "messages_count"); + if (str_to_uint32(value, &box->messages_count) < 0) { + dsync_slave_input_error(slave, decoder, "Invalid messages_count"); + return DSYNC_SLAVE_RECV_RET_TRYAGAIN; + } + value = dsync_deserializer_decode_get(decoder, "first_recent_uid"); + if (str_to_uint32(value, &box->first_recent_uid) < 0) { + dsync_slave_input_error(slave, decoder, "Invalid first_recent_uid"); + return DSYNC_SLAVE_RECV_RET_TRYAGAIN; + } + value = dsync_deserializer_decode_get(decoder, "highest_modseq"); + if (str_to_uint64(value, &box->highest_modseq) < 0) { + dsync_slave_input_error(slave, decoder, "Invalid highest_modseq"); + return DSYNC_SLAVE_RECV_RET_TRYAGAIN; + } + + p_array_init(&box->cache_fields, pool, 32); + if (dsync_deserializer_decode_try(decoder, "cache_fields", &value)) { + const char *const *fields = t_strsplit(value, "\n"); + for (; *fields != NULL; fields++) { + if (parse_cache_field(slave, box, *fields) < 0) + return DSYNC_SLAVE_RECV_RET_TRYAGAIN; + } + } + + *dsync_box_r = box; + return DSYNC_SLAVE_RECV_RET_OK; +} + +static void +dsync_slave_io_send_change(struct dsync_slave *_slave, + const struct dsync_mail_change *change) +{ + struct dsync_slave_io *slave = (struct dsync_slave_io *)_slave; + struct dsync_serializer_encoder *encoder; + string_t *str = t_str_new(128); + char type[2]; + + str_append_c(str, items[ITEM_MAIL_CHANGE].chr); + encoder = dsync_serializer_encode_begin(slave->serializers[ITEM_MAIL_CHANGE]); + + type[0] = type[1] = '\0'; + switch (change->type) { + case DSYNC_MAIL_CHANGE_TYPE_SAVE: + type[0] = 's'; + break; + case DSYNC_MAIL_CHANGE_TYPE_EXPUNGE: + type[0] = 'e'; + break; + case DSYNC_MAIL_CHANGE_TYPE_FLAG_CHANGE: + type[0] = 'f'; + break; + } + i_assert(type[0] != '\0'); + dsync_serializer_encode_add(encoder, "type", type); + dsync_serializer_encode_add(encoder, "uid", dec2str(change->uid)); + if (change->guid != NULL) + dsync_serializer_encode_add(encoder, "guid", change->guid); + if (change->hdr_hash != NULL) { + dsync_serializer_encode_add(encoder, "hdr_hash", + change->hdr_hash); + } + if (change->modseq != 0) { + dsync_serializer_encode_add(encoder, "modseq", + dec2str(change->modseq)); + } + if (change->save_timestamp != 0) { + dsync_serializer_encode_add(encoder, "save_timestamp", + dec2str(change->save_timestamp)); + } + if (change->add_flags != 0) { + dsync_serializer_encode_add(encoder, "add_flags", + t_strdup_printf("%x", change->add_flags)); + } + if (change->remove_flags != 0) { + dsync_serializer_encode_add(encoder, "remove_flags", + t_strdup_printf("%x", change->remove_flags)); + } + if (change->final_flags != 0) { + dsync_serializer_encode_add(encoder, "final_flags", + t_strdup_printf("%x", change->final_flags)); + } + if (change->keywords_reset) + dsync_serializer_encode_add(encoder, "keywords_reset", ""); + + if (array_is_created(&change->keyword_changes) && + array_count(&change->keyword_changes) > 0) { + string_t *kw_str = t_str_new(128); + const char *const *changes; + unsigned int i, count; + + changes = array_get(&change->keyword_changes, &count); + str_tabescape_write(kw_str, changes[0]); + for (i = 1; i < count; i++) { + str_append_c(kw_str, '\t'); + str_tabescape_write(kw_str, changes[i]); + } + dsync_serializer_encode_add(encoder, "keyword_changes", + str_c(kw_str)); + } + + dsync_serializer_encode_finish(&encoder, str); + dsync_slave_io_send_string(slave, str); +} + +static enum dsync_slave_recv_ret +dsync_slave_io_recv_change(struct dsync_slave *_slave, + const struct dsync_mail_change **change_r) +{ + struct dsync_slave_io *slave = (struct dsync_slave_io *)_slave; + pool_t pool = slave->ret_pool; + struct dsync_deserializer_decoder *decoder; + struct dsync_mail_change *change; + const char *value; + enum dsync_slave_recv_ret ret; + + p_clear(pool); + change = p_new(pool, struct dsync_mail_change, 1); + + ret = dsync_slave_io_input_next(slave, ITEM_MAIL_CHANGE, &decoder); + if (ret != DSYNC_SLAVE_RECV_RET_OK) + return ret; + + value = dsync_deserializer_decode_get(decoder, "type"); + switch (*value) { + case 's': + change->type = DSYNC_MAIL_CHANGE_TYPE_SAVE; + break; + case 'e': + change->type = DSYNC_MAIL_CHANGE_TYPE_EXPUNGE; + break; + case 'f': + change->type = DSYNC_MAIL_CHANGE_TYPE_FLAG_CHANGE; + break; + default: + dsync_slave_input_error(slave, decoder, + "Invalid type: %s", value); + return DSYNC_SLAVE_RECV_RET_TRYAGAIN; + } + + value = dsync_deserializer_decode_get(decoder, "uid"); + if (str_to_uint32(value, &change->uid) < 0) { + dsync_slave_input_error(slave, decoder, "Invalid uid"); + return DSYNC_SLAVE_RECV_RET_TRYAGAIN; + } + + if (dsync_deserializer_decode_try(decoder, "guid", &value)) + change->guid = p_strdup(pool, value); + if (dsync_deserializer_decode_try(decoder, "hdr_hash", &value)) + change->hdr_hash = p_strdup(pool, value); + if (dsync_deserializer_decode_try(decoder, "modseq", &value) && + str_to_uint64(value, &change->modseq) < 0) { + dsync_slave_input_error(slave, decoder, "Invalid modseq"); + return DSYNC_SLAVE_RECV_RET_TRYAGAIN; + } + if (dsync_deserializer_decode_try(decoder, "save_timestamp", &value) && + str_to_time(value, &change->save_timestamp) < 0) { + dsync_slave_input_error(slave, decoder, "Invalid save_timestamp"); + return DSYNC_SLAVE_RECV_RET_TRYAGAIN; + } + + if (dsync_deserializer_decode_try(decoder, "add_flags", &value)) + change->add_flags = strtoul(value, NULL, 16); + if (dsync_deserializer_decode_try(decoder, "remove_flags", &value)) + change->remove_flags = strtoul(value, NULL, 16); + if (dsync_deserializer_decode_try(decoder, "final_flags", &value)) + change->final_flags = strtoul(value, NULL, 16); + if (dsync_deserializer_decode_try(decoder, "keywords_reset", &value)) + change->keywords_reset = TRUE; + + if (dsync_deserializer_decode_try(decoder, "keyword_changes", &value) && + *value != '\0') { + const char *const *changes = t_strsplit_tab(value); + unsigned int i, count = str_array_length(changes); + + p_array_init(&change->keyword_changes, pool, count); + for (i = 0; i < count; i++) { + value = p_strdup(pool, changes[i]); + array_append(&change->keyword_changes, &value, 1); + } + } + + *change_r = change; + return DSYNC_SLAVE_RECV_RET_OK; +} + +static void +dsync_slave_io_send_mail_request(struct dsync_slave *_slave, + const struct dsync_mail_request *request) +{ + struct dsync_slave_io *slave = (struct dsync_slave_io *)_slave; + struct dsync_serializer_encoder *encoder; + string_t *str = t_str_new(128); + + str_append_c(str, items[ITEM_MAIL_REQUEST].chr); + encoder = dsync_serializer_encode_begin(slave->serializers[ITEM_MAIL_REQUEST]); + if (request->guid != NULL) + dsync_serializer_encode_add(encoder, "guid", request->guid); + if (request->uid != 0) { + dsync_serializer_encode_add(encoder, "uid", + dec2str(request->uid)); + } + dsync_serializer_encode_finish(&encoder, str); + dsync_slave_io_send_string(slave, str); +} + +static enum dsync_slave_recv_ret +dsync_slave_io_recv_mail_request(struct dsync_slave *_slave, + const struct dsync_mail_request **request_r) +{ + struct dsync_slave_io *slave = (struct dsync_slave_io *)_slave; + struct dsync_deserializer_decoder *decoder; + struct dsync_mail_request *request; + const char *value; + enum dsync_slave_recv_ret ret; + + p_clear(slave->ret_pool); + request = p_new(slave->ret_pool, struct dsync_mail_request, 1); + + ret = dsync_slave_io_input_next(slave, ITEM_MAIL_REQUEST, &decoder); + if (ret != DSYNC_SLAVE_RECV_RET_OK) + return ret; + + if (dsync_deserializer_decode_try(decoder, "guid", &value)) + request->guid = p_strdup(slave->ret_pool, value); + if (dsync_deserializer_decode_try(decoder, "uid", &value) && + str_to_uint32(value, &request->uid) < 0) { + dsync_slave_input_error(slave, decoder, "Invalid uid"); + return DSYNC_SLAVE_RECV_RET_TRYAGAIN; + } + + *request_r = request; + return DSYNC_SLAVE_RECV_RET_OK; +} + +static void +dsync_slave_io_send_mail(struct dsync_slave *_slave, + const struct dsync_mail *mail) +{ + struct dsync_slave_io *slave = (struct dsync_slave_io *)_slave; + struct dsync_serializer_encoder *encoder; + string_t *str = t_str_new(128); + + i_assert(slave->mail_output == NULL); + + str_append_c(str, items[ITEM_MAIL].chr); + encoder = dsync_serializer_encode_begin(slave->serializers[ITEM_MAIL]); + if (mail->guid != NULL) + dsync_serializer_encode_add(encoder, "guid", mail->guid); + if (mail->uid != 0) + dsync_serializer_encode_add(encoder, "uid", dec2str(mail->uid)); + if (mail->pop3_uidl != NULL) { + dsync_serializer_encode_add(encoder, "pop3_uidl", + mail->pop3_uidl); + } + if (mail->pop3_order > 0) { + dsync_serializer_encode_add(encoder, "pop3_order", + dec2str(mail->pop3_order)); + } + if (mail->received_date > 0) { + dsync_serializer_encode_add(encoder, "received_date", + dec2str(mail->received_date)); + } + if (mail->input != NULL) + dsync_serializer_encode_add(encoder, "stream", ""); + + dsync_serializer_encode_finish(&encoder, str); + dsync_slave_io_send_string(slave, str); + + if (mail->input != NULL) { + slave->mail_output_last = '\0'; + slave->mail_output = mail->input; + i_stream_ref(slave->mail_output); + (void)dsync_slave_io_send_mail_stream(slave); + } +} + +static int seekable_fd_callback(const char **path_r, void *context) +{ + struct dsync_slave_io *slave = context; + string_t *path; + int fd; + + path = t_str_new(128); + str_append(path, slave->temp_path_prefix); + fd = safe_mkstemp(path, 0600, (uid_t)-1, (gid_t)-1); + if (fd == -1) { + i_error("safe_mkstemp(%s) failed: %m", str_c(path)); + return -1; + } + + /* we just want the fd, unlink it */ + if (unlink(str_c(path)) < 0) { + /* shouldn't happen.. */ + i_error("unlink(%s) failed: %m", str_c(path)); + close_keep_errno(fd); + return -1; + } + + *path_r = str_c(path); + return fd; +} + +static enum dsync_slave_recv_ret +dsync_slave_io_recv_mail(struct dsync_slave *_slave, + struct dsync_mail **mail_r) +{ + struct dsync_slave_io *slave = (struct dsync_slave_io *)_slave; + pool_t pool = slave->ret_pool; + struct dsync_deserializer_decoder *decoder; + struct dsync_mail *mail; + struct istream *inputs[2]; + const char *value; + enum dsync_slave_recv_ret ret; + + if (slave->mail_input != NULL) { + /* wait until the mail's stream has been read */ + return DSYNC_SLAVE_RECV_RET_TRYAGAIN; + } + if (slave->cur_mail != NULL) { + /* finished reading the stream, return the mail now */ + *mail_r = slave->cur_mail; + slave->cur_mail = NULL; + return DSYNC_SLAVE_RECV_RET_OK; + } + + p_clear(pool); + mail = p_new(pool, struct dsync_mail, 1); + + ret = dsync_slave_io_input_next(slave, ITEM_MAIL, &decoder); + if (ret != DSYNC_SLAVE_RECV_RET_OK) + return ret; + + if (dsync_deserializer_decode_try(decoder, "guid", &value)) + mail->guid = p_strdup(pool, value); + if (dsync_deserializer_decode_try(decoder, "uid", &value) && + str_to_uint32(value, &mail->uid) < 0) { + dsync_slave_input_error(slave, decoder, "Invalid uid"); + return DSYNC_SLAVE_RECV_RET_TRYAGAIN; + } + if (dsync_deserializer_decode_try(decoder, "pop3_uidl", &value)) + mail->pop3_uidl = p_strdup(pool, value); + if (dsync_deserializer_decode_try(decoder, "pop3_order", &value) && + str_to_uint(value, &mail->pop3_order) < 0) { + dsync_slave_input_error(slave, decoder, "Invalid pop3_order"); + return DSYNC_SLAVE_RECV_RET_TRYAGAIN; + } + if (dsync_deserializer_decode_try(decoder, "received_date", &value) && + str_to_time(value, &mail->received_date) < 0) { + dsync_slave_input_error(slave, decoder, "Invalid received_date"); + return DSYNC_SLAVE_RECV_RET_TRYAGAIN; + } + if (dsync_deserializer_decode_try(decoder, "stream", &value)) { + inputs[0] = i_stream_create_dot(slave->input, FALSE); + inputs[1] = NULL; + mail->input = i_stream_create_seekable(inputs, + MAIL_READ_FULL_BLOCK_SIZE, seekable_fd_callback, slave); + i_stream_unref(&inputs[0]); + + slave->mail_input = mail->input; + if (dsync_slave_io_read_mail_stream(slave) <= 0) { + slave->cur_mail = mail; + return DSYNC_SLAVE_RECV_RET_TRYAGAIN; + } + /* already finished reading the stream */ + i_assert(slave->mail_input == NULL); + } + + *mail_r = mail; + return DSYNC_SLAVE_RECV_RET_OK; +} + +static void dsync_slave_io_flush(struct dsync_slave *_slave) +{ + struct dsync_slave_io *slave = (struct dsync_slave_io *)_slave; + + o_stream_uncork(slave->output); + o_stream_cork(slave->output); +} + +static bool dsync_slave_io_is_send_queue_full(struct dsync_slave *_slave) +{ + struct dsync_slave_io *slave = (struct dsync_slave_io *)_slave; + size_t bytes; + + if (slave->mail_output != NULL) + return TRUE; + + bytes = o_stream_get_buffer_used_size(slave->output); + if (bytes < DSYNC_SLAVE_IO_OUTBUF_THROTTLE_SIZE) + return FALSE; + + o_stream_set_flush_pending(slave->output, TRUE); + return TRUE; +} + +static bool dsync_slave_io_has_pending_data(struct dsync_slave *_slave) +{ + struct dsync_slave_io *slave = (struct dsync_slave_io *)_slave; + + return slave->has_pending_data; +} + +static const struct dsync_slave_vfuncs dsync_slave_io_vfuncs = { + dsync_slave_io_deinit, + dsync_slave_io_send_handshake, + dsync_slave_io_recv_handshake, + dsync_slave_io_send_end_of_list, + dsync_slave_io_send_mailbox_state, + dsync_slave_io_recv_mailbox_state, + dsync_slave_io_send_mailbox_tree_node, + dsync_slave_io_recv_mailbox_tree_node, + dsync_slave_io_send_mailbox_deletes, + dsync_slave_io_recv_mailbox_deletes, + dsync_slave_io_send_mailbox, + dsync_slave_io_recv_mailbox, + dsync_slave_io_send_change, + dsync_slave_io_recv_change, + dsync_slave_io_send_mail_request, + dsync_slave_io_recv_mail_request, + dsync_slave_io_send_mail, + dsync_slave_io_recv_mail, + dsync_slave_io_flush, + dsync_slave_io_is_send_queue_full, + dsync_slave_io_has_pending_data +}; + +struct dsync_slave * +dsync_slave_init_io(int fd_in, int fd_out, const char *name, + const char *temp_path_prefix) +{ + struct dsync_slave_io *slave; + + slave = i_new(struct dsync_slave_io, 1); + slave->slave.v = dsync_slave_io_vfuncs; + slave->fd_in = fd_in; + slave->fd_out = fd_out; + slave->name = i_strdup(name); + slave->temp_path_prefix = i_strdup(temp_path_prefix); + slave->ret_pool = pool_alloconly_create("slave io data", 2048); + dsync_slave_io_init(slave); + return &slave->slave; +}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/doveadm/dsync/dsync-slave-pipe.c Tue May 22 23:17:31 2012 +0300 @@ -0,0 +1,485 @@ +/* Copyright (c) 2012 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "istream.h" +#include "dsync-mail.h" +#include "dsync-mailbox.h" +#include "dsync-mailbox-state.h" +#include "dsync-mailbox-tree.h" +#include "dsync-slave-private.h" + +enum item_type { + ITEM_END_OF_LIST, + ITEM_HANDSHAKE, + ITEM_MAILBOX_STATE, + ITEM_MAILBOX_TREE_NODE, + ITEM_MAILBOX_DELETE, + ITEM_MAILBOX, + ITEM_MAIL_CHANGE, + ITEM_MAIL_REQUEST, + ITEM_MAIL +}; + +struct item { + enum item_type type; + pool_t pool; + + union { + struct dsync_slave_settings set; + struct dsync_mailbox_state state; + struct dsync_mailbox_node node; + guid_128_t mailbox_guid; + struct dsync_mailbox dsync_box; + struct dsync_mail_change change; + struct dsync_mail_request request; + struct dsync_mail mail; + struct { + const struct dsync_mailbox_delete *deletes; + unsigned int count; + char hierarchy_sep; + } mailbox_delete; + } u; +}; + +struct dsync_slave_pipe { + struct dsync_slave slave; + + ARRAY_DEFINE(pools, pool_t); + ARRAY_DEFINE(item_queue, struct item); + struct dsync_slave_pipe *remote; + + pool_t pop_pool; + struct item pop_item; +}; + +static pool_t dsync_slave_pipe_get_pool(struct dsync_slave_pipe *pipe) +{ + pool_t *pools, ret; + unsigned int count; + + pools = array_get_modifiable(&pipe->pools, &count); + if (count == 0) + return pool_alloconly_create("pipe item pool", 128); + + ret = pools[count-1]; + array_delete(&pipe->pools, count-1, 1); + p_clear(ret); + return ret; +} + +static struct item * +dsync_slave_pipe_push_item(struct dsync_slave_pipe *pipe, enum item_type type) +{ + struct item *item; + + item = array_append_space(&pipe->item_queue); + item->type = type; + + switch (type) { + case ITEM_END_OF_LIST: + case ITEM_MAILBOX_STATE: + case ITEM_MAILBOX_DELETE: + break; + case ITEM_HANDSHAKE: + case ITEM_MAILBOX: + case ITEM_MAILBOX_TREE_NODE: + case ITEM_MAIL_CHANGE: + case ITEM_MAIL_REQUEST: + case ITEM_MAIL: + item->pool = dsync_slave_pipe_get_pool(pipe); + break; + } + return item; +} + +static struct item * +dsync_slave_pipe_pop_item(struct dsync_slave_pipe *pipe, enum item_type type) +{ + struct item *item; + + if (array_count(&pipe->item_queue) == 0) + return NULL; + + item = array_idx_modifiable(&pipe->item_queue, 0); + i_assert(item->type == type); + pipe->pop_item = *item; + array_delete(&pipe->item_queue, 0, 1); + item = NULL; + + if (pipe->pop_pool != NULL) + pool_unref(&pipe->pop_pool); + pipe->pop_pool = pipe->pop_item.pool; + return &pipe->pop_item; +} + +static bool dsync_slave_pipe_try_pop_eol(struct dsync_slave_pipe *pipe) +{ + const struct item *item; + + if (array_count(&pipe->item_queue) == 0) + return FALSE; + + item = array_idx(&pipe->item_queue, 0); + if (item->type != ITEM_END_OF_LIST) + return FALSE; + + array_delete(&pipe->item_queue, 0, 1); + return TRUE; +} + +static void dsync_slave_pipe_deinit(struct dsync_slave *slave) +{ + struct dsync_slave_pipe *pipe = (struct dsync_slave_pipe *)slave; + pool_t *poolp; + + if (pipe->remote != NULL) { + i_assert(pipe->remote->remote == pipe); + pipe->remote->remote = NULL; + } + + if (pipe->pop_pool != NULL) + pool_unref(&pipe->pop_pool); + array_foreach_modifiable(&pipe->pools, poolp) + pool_unref(poolp); + array_free(&pipe->pools); + array_free(&pipe->item_queue); + i_free(pipe); +} + +static void +dsync_slave_pipe_send_handshake(struct dsync_slave *slave, + const struct dsync_slave_settings *set) +{ + struct dsync_slave_pipe *pipe = (struct dsync_slave_pipe *)slave; + struct item *item; + + item = dsync_slave_pipe_push_item(pipe->remote, ITEM_HANDSHAKE); + item->u.set = *set; + item->u.set.sync_ns_prefix = p_strdup(item->pool, set->sync_ns_prefix); +} + +static enum dsync_slave_recv_ret +dsync_slave_pipe_recv_handshake(struct dsync_slave *slave, + const struct dsync_slave_settings **set_r) +{ + struct dsync_slave_pipe *pipe = (struct dsync_slave_pipe *)slave; + struct item *item; + + item = dsync_slave_pipe_pop_item(pipe, ITEM_HANDSHAKE); + if (item == NULL) + return DSYNC_SLAVE_RECV_RET_TRYAGAIN; + + *set_r = &item->u.set; + return DSYNC_SLAVE_RECV_RET_OK; +} + +static bool dsync_slave_pipe_is_send_queue_full(struct dsync_slave *slave) +{ + struct dsync_slave_pipe *pipe = (struct dsync_slave_pipe *)slave; + + return array_count(&pipe->remote->item_queue) > 0; +} + +static bool dsync_slave_pipe_has_pending_data(struct dsync_slave *slave) +{ + struct dsync_slave_pipe *pipe = (struct dsync_slave_pipe *)slave; + + return array_count(&pipe->item_queue) > 0; +} + +static void +dsync_slave_pipe_send_end_of_list(struct dsync_slave *slave) +{ + struct dsync_slave_pipe *pipe = (struct dsync_slave_pipe *)slave; + + (void)dsync_slave_pipe_push_item(pipe->remote, ITEM_END_OF_LIST); +} + +static void +dsync_slave_pipe_send_mailbox_state(struct dsync_slave *slave, + const struct dsync_mailbox_state *state) +{ + struct dsync_slave_pipe *pipe = (struct dsync_slave_pipe *)slave; + struct item *item; + + item = dsync_slave_pipe_push_item(pipe->remote, ITEM_MAILBOX_STATE); + item->u.state = *state; +} + +static enum dsync_slave_recv_ret +dsync_slave_pipe_recv_mailbox_state(struct dsync_slave *slave, + struct dsync_mailbox_state *state_r) +{ + struct dsync_slave_pipe *pipe = (struct dsync_slave_pipe *)slave; + struct item *item; + + if (dsync_slave_pipe_try_pop_eol(pipe)) + return DSYNC_SLAVE_RECV_RET_FINISHED; + + item = dsync_slave_pipe_pop_item(pipe, ITEM_MAILBOX_STATE); + if (item == NULL) + return DSYNC_SLAVE_RECV_RET_TRYAGAIN; + + *state_r = item->u.state; + return DSYNC_SLAVE_RECV_RET_OK; +} + +static void +dsync_slave_pipe_send_mailbox_tree_node(struct dsync_slave *slave, + const char *const *name, + const struct dsync_mailbox_node *node) +{ + struct dsync_slave_pipe *pipe = (struct dsync_slave_pipe *)slave; + struct item *item; + + item = dsync_slave_pipe_push_item(pipe->remote, ITEM_MAILBOX_TREE_NODE); + + /* a little bit kludgy way to send it */ + item->u.node.name = (void *)p_strarray_dup(item->pool, name); + dsync_mailbox_node_copy_data(&item->u.node, node); +} + +static enum dsync_slave_recv_ret +dsync_slave_pipe_recv_mailbox_tree_node(struct dsync_slave *slave, + const char *const **name_r, + const struct dsync_mailbox_node **node_r) +{ + struct dsync_slave_pipe *pipe = (struct dsync_slave_pipe *)slave; + struct item *item; + + if (dsync_slave_pipe_try_pop_eol(pipe)) + return DSYNC_SLAVE_RECV_RET_FINISHED; + + item = dsync_slave_pipe_pop_item(pipe, ITEM_MAILBOX_TREE_NODE); + if (item == NULL) + return DSYNC_SLAVE_RECV_RET_TRYAGAIN; + + *name_r = (void *)item->u.node.name; + item->u.node.name = NULL; + + *node_r = &item->u.node; + return DSYNC_SLAVE_RECV_RET_OK; +} + +static void +dsync_slave_pipe_send_mailbox_deletes(struct dsync_slave *slave, + const struct dsync_mailbox_delete *deletes, + unsigned int count, char hierarchy_sep) +{ + struct dsync_slave_pipe *pipe = (struct dsync_slave_pipe *)slave; + struct item *item; + + item = dsync_slave_pipe_push_item(pipe->remote, ITEM_MAILBOX_DELETE); + + /* we'll assume that the deletes are permanent. this works for now.. */ + /* a little bit kludgy way to send it */ + item->u.mailbox_delete.deletes = deletes; + item->u.mailbox_delete.count = count; + item->u.mailbox_delete.hierarchy_sep = hierarchy_sep; +} + +static enum dsync_slave_recv_ret +dsync_slave_pipe_recv_mailbox_deletes(struct dsync_slave *slave, + const struct dsync_mailbox_delete **deletes_r, + unsigned int *count_r, + char *hierarchy_sep_r) +{ + struct dsync_slave_pipe *pipe = (struct dsync_slave_pipe *)slave; + struct item *item; + + if (dsync_slave_pipe_try_pop_eol(pipe)) + return DSYNC_SLAVE_RECV_RET_FINISHED; + + item = dsync_slave_pipe_pop_item(pipe, ITEM_MAILBOX_DELETE); + if (item == NULL) + return DSYNC_SLAVE_RECV_RET_TRYAGAIN; + + *deletes_r = item->u.mailbox_delete.deletes; + *count_r = item->u.mailbox_delete.count; + *hierarchy_sep_r = item->u.mailbox_delete.hierarchy_sep; + return DSYNC_SLAVE_RECV_RET_OK; +} + +static void +dsync_slave_pipe_send_mailbox(struct dsync_slave *slave, + const struct dsync_mailbox *dsync_box) +{ + struct dsync_slave_pipe *pipe = (struct dsync_slave_pipe *)slave; + struct item *item; + const struct mailbox_cache_field *cf; + struct mailbox_cache_field *ncf; + + item = dsync_slave_pipe_push_item(pipe->remote, ITEM_MAILBOX); + item->u.dsync_box = *dsync_box; + p_array_init(&item->u.dsync_box.cache_fields, item->pool, + array_count(&dsync_box->cache_fields)); + array_foreach(&dsync_box->cache_fields, cf) { + ncf = array_append_space(&item->u.dsync_box.cache_fields); + ncf->name = p_strdup(item->pool, cf->name); + ncf->decision = cf->decision; + ncf->last_used = cf->last_used; + } +} + +static enum dsync_slave_recv_ret +dsync_slave_pipe_recv_mailbox(struct dsync_slave *slave, + const struct dsync_mailbox **dsync_box_r) +{ + struct dsync_slave_pipe *pipe = (struct dsync_slave_pipe *)slave; + struct item *item; + + if (dsync_slave_pipe_try_pop_eol(pipe)) + return DSYNC_SLAVE_RECV_RET_FINISHED; + + item = dsync_slave_pipe_pop_item(pipe, ITEM_MAILBOX); + if (item == NULL) + return DSYNC_SLAVE_RECV_RET_TRYAGAIN; + + *dsync_box_r = &item->u.dsync_box; + return DSYNC_SLAVE_RECV_RET_OK; +} + +static void +dsync_slave_pipe_send_change(struct dsync_slave *slave, + const struct dsync_mail_change *change) +{ + struct dsync_slave_pipe *pipe = (struct dsync_slave_pipe *)slave; + struct item *item; + + item = dsync_slave_pipe_push_item(pipe->remote, ITEM_MAIL_CHANGE); + dsync_mail_change_dup(item->pool, change, &item->u.change); +} + +static enum dsync_slave_recv_ret +dsync_slave_pipe_recv_change(struct dsync_slave *slave, + const struct dsync_mail_change **change_r) +{ + struct dsync_slave_pipe *pipe = (struct dsync_slave_pipe *)slave; + struct item *item; + + if (dsync_slave_pipe_try_pop_eol(pipe)) + return DSYNC_SLAVE_RECV_RET_FINISHED; + + item = dsync_slave_pipe_pop_item(pipe, ITEM_MAIL_CHANGE); + if (item == NULL) + return DSYNC_SLAVE_RECV_RET_TRYAGAIN; + + *change_r = &item->u.change; + return DSYNC_SLAVE_RECV_RET_OK; +} + +static void +dsync_slave_pipe_send_mail_request(struct dsync_slave *slave, + const struct dsync_mail_request *request) +{ + struct dsync_slave_pipe *pipe = (struct dsync_slave_pipe *)slave; + struct item *item; + + item = dsync_slave_pipe_push_item(pipe->remote, ITEM_MAIL_REQUEST); + item->u.request.guid = p_strdup(item->pool, request->guid); + item->u.request.uid = request->uid; +} + +static enum dsync_slave_recv_ret +dsync_slave_pipe_recv_mail_request(struct dsync_slave *slave, + const struct dsync_mail_request **request_r) +{ + struct dsync_slave_pipe *pipe = (struct dsync_slave_pipe *)slave; + struct item *item; + + if (dsync_slave_pipe_try_pop_eol(pipe)) + return DSYNC_SLAVE_RECV_RET_FINISHED; + + item = dsync_slave_pipe_pop_item(pipe, ITEM_MAIL_REQUEST); + if (item == NULL) + return DSYNC_SLAVE_RECV_RET_TRYAGAIN; + + *request_r = &item->u.request; + return DSYNC_SLAVE_RECV_RET_OK; +} + +static void +dsync_slave_pipe_send_mail(struct dsync_slave *slave, + const struct dsync_mail *mail) +{ + struct dsync_slave_pipe *pipe = (struct dsync_slave_pipe *)slave; + struct item *item; + + item = dsync_slave_pipe_push_item(pipe->remote, ITEM_MAIL); + item->u.mail.guid = p_strdup(item->pool, mail->guid); + item->u.mail.pop3_uidl = p_strdup(item->pool, mail->pop3_uidl); + item->u.mail.pop3_order = mail->pop3_order; + item->u.mail.received_date = mail->received_date; + if (mail->input != NULL) { + item->u.mail.input = mail->input; + i_stream_ref(mail->input); + } +} + +static enum dsync_slave_recv_ret +dsync_slave_pipe_recv_mail(struct dsync_slave *slave, + struct dsync_mail **mail_r) +{ + struct dsync_slave_pipe *pipe = (struct dsync_slave_pipe *)slave; + struct item *item; + + if (dsync_slave_pipe_try_pop_eol(pipe)) + return DSYNC_SLAVE_RECV_RET_FINISHED; + + item = dsync_slave_pipe_pop_item(pipe, ITEM_MAIL); + if (item == NULL) + return DSYNC_SLAVE_RECV_RET_TRYAGAIN; + + *mail_r = &item->u.mail; + return DSYNC_SLAVE_RECV_RET_OK; +} + +static const struct dsync_slave_vfuncs dsync_slave_pipe_vfuncs = { + dsync_slave_pipe_deinit, + dsync_slave_pipe_send_handshake, + dsync_slave_pipe_recv_handshake, + dsync_slave_pipe_send_end_of_list, + dsync_slave_pipe_send_mailbox_state, + dsync_slave_pipe_recv_mailbox_state, + dsync_slave_pipe_send_mailbox_tree_node, + dsync_slave_pipe_recv_mailbox_tree_node, + dsync_slave_pipe_send_mailbox_deletes, + dsync_slave_pipe_recv_mailbox_deletes, + dsync_slave_pipe_send_mailbox, + dsync_slave_pipe_recv_mailbox, + dsync_slave_pipe_send_change, + dsync_slave_pipe_recv_change, + dsync_slave_pipe_send_mail_request, + dsync_slave_pipe_recv_mail_request, + dsync_slave_pipe_send_mail, + dsync_slave_pipe_recv_mail, + NULL, + dsync_slave_pipe_is_send_queue_full, + dsync_slave_pipe_has_pending_data +}; + +static struct dsync_slave_pipe * +dsync_slave_pipe_alloc(void) +{ + struct dsync_slave_pipe *pipe; + + pipe = i_new(struct dsync_slave_pipe, 1); + pipe->slave.v = dsync_slave_pipe_vfuncs; + i_array_init(&pipe->pools, 4); + i_array_init(&pipe->item_queue, 4); + return pipe; +} + +void dsync_slave_init_pipe(struct dsync_slave **slave1_r, + struct dsync_slave **slave2_r) +{ + struct dsync_slave_pipe *pipe1, *pipe2; + + pipe1 = dsync_slave_pipe_alloc(); + pipe2 = dsync_slave_pipe_alloc(); + pipe1->remote = pipe2; + pipe2->remote = pipe1; + *slave1_r = &pipe1->slave; + *slave2_r = &pipe2->slave; +}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/doveadm/dsync/dsync-slave-private.h Tue May 22 23:17:31 2012 +0300 @@ -0,0 +1,78 @@ +#ifndef DSYNC_SLAVE_PRIVATE_H +#define DSYNC_SLAVE_PRIVATE_H + +#include "dsync-slave.h" + +struct dsync_slave_vfuncs { + void (*deinit)(struct dsync_slave *slave); + + void (*send_handshake)(struct dsync_slave *slave, + const struct dsync_slave_settings *set); + enum dsync_slave_recv_ret + (*recv_handshake)(struct dsync_slave *slave, + const struct dsync_slave_settings **set_r); + + void (*send_end_of_list)(struct dsync_slave *slave); + + void (*send_mailbox_state)(struct dsync_slave *slave, + const struct dsync_mailbox_state *state); + enum dsync_slave_recv_ret + (*recv_mailbox_state)(struct dsync_slave *slave, + struct dsync_mailbox_state *state_r); + + void (*send_mailbox_tree_node)(struct dsync_slave *slave, + const char *const *name, + const struct dsync_mailbox_node *node); + enum dsync_slave_recv_ret + (*recv_mailbox_tree_node)(struct dsync_slave *slave, + const char *const **name_r, + const struct dsync_mailbox_node **node_r); + + void (*send_mailbox_deletes)(struct dsync_slave *slave, + const struct dsync_mailbox_delete *deletes, + unsigned int count, char hierarchy_sep); + enum dsync_slave_recv_ret + (*recv_mailbox_deletes)(struct dsync_slave *slave, + const struct dsync_mailbox_delete **deletes_r, + unsigned int *count_r, + char *hierarchy_sep_r); + + void (*send_mailbox)(struct dsync_slave *slave, + const struct dsync_mailbox *dsync_box); + enum dsync_slave_recv_ret + (*recv_mailbox)(struct dsync_slave *slave, + const struct dsync_mailbox **dsync_box_r); + + void (*send_change)(struct dsync_slave *slave, + const struct dsync_mail_change *change); + enum dsync_slave_recv_ret + (*recv_change)(struct dsync_slave *slave, + const struct dsync_mail_change **change_r); + + void (*send_mail_request)(struct dsync_slave *slave, + const struct dsync_mail_request *request); + enum dsync_slave_recv_ret + (*recv_mail_request)(struct dsync_slave *slave, + const struct dsync_mail_request **request_r); + + void (*send_mail)(struct dsync_slave *slave, + const struct dsync_mail *mail); + enum dsync_slave_recv_ret + (*recv_mail)(struct dsync_slave *slave, + struct dsync_mail **mail_r); + + void (*flush)(struct dsync_slave *slave); + bool (*is_send_queue_full)(struct dsync_slave *slave); + bool (*has_pending_data)(struct dsync_slave *slave); +}; + +struct dsync_slave { + struct dsync_slave_vfuncs v; + + io_callback_t *io_callback; + void *io_context; + + unsigned int failed:1; +}; + +#endif
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/doveadm/dsync/dsync-slave.c Tue May 22 23:17:31 2012 +0300 @@ -0,0 +1,202 @@ +/* Copyright (c) 2012 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "dsync-mail.h" +#include "dsync-slave-private.h" + +void dsync_slave_deinit(struct dsync_slave **_slave) +{ + struct dsync_slave *slave = *_slave; + + *_slave = NULL; + slave->v.deinit(slave); +} + +void dsync_slave_set_io_callback(struct dsync_slave *slave, + io_callback_t *callback, void *context) +{ + slave->io_callback = callback; + slave->io_context = context; +} + +void dsync_slave_send_handshake(struct dsync_slave *slave, + const struct dsync_slave_settings *set) +{ + slave->v.send_handshake(slave, set); +} + +enum dsync_slave_recv_ret +dsync_slave_recv_handshake(struct dsync_slave *slave, + const struct dsync_slave_settings **set_r) +{ + return slave->v.recv_handshake(slave, set_r); +} + +static enum dsync_slave_send_ret +dsync_slave_send_ret(struct dsync_slave *slave) +{ + return slave->v.is_send_queue_full(slave) ? + DSYNC_SLAVE_SEND_RET_FULL : + DSYNC_SLAVE_SEND_RET_OK; +} + +enum dsync_slave_send_ret +dsync_slave_send_end_of_list(struct dsync_slave *slave) +{ + slave->v.send_end_of_list(slave); + return dsync_slave_send_ret(slave); +} + +enum dsync_slave_send_ret +dsync_slave_send_mailbox_state(struct dsync_slave *slave, + const struct dsync_mailbox_state *state) +{ + T_BEGIN { + slave->v.send_mailbox_state(slave, state); + } T_END; + return dsync_slave_send_ret(slave); +} + +enum dsync_slave_recv_ret +dsync_slave_recv_mailbox_state(struct dsync_slave *slave, + struct dsync_mailbox_state *state_r) +{ + return slave->v.recv_mailbox_state(slave, state_r); +} + +enum dsync_slave_send_ret +dsync_slave_send_mailbox_tree_node(struct dsync_slave *slave, + const char *const *name, + const struct dsync_mailbox_node *node) +{ + i_assert(*name != NULL); + + T_BEGIN { + slave->v.send_mailbox_tree_node(slave, name, node); + } T_END; + return dsync_slave_send_ret(slave); +} + +enum dsync_slave_recv_ret +dsync_slave_recv_mailbox_tree_node(struct dsync_slave *slave, + const char *const **name_r, + const struct dsync_mailbox_node **node_r) +{ + return slave->v.recv_mailbox_tree_node(slave, name_r, node_r); +} + +enum dsync_slave_send_ret +dsync_slave_send_mailbox_deletes(struct dsync_slave *slave, + const struct dsync_mailbox_delete *deletes, + unsigned int count, char hierarchy_sep) +{ + T_BEGIN { + slave->v.send_mailbox_deletes(slave, deletes, count, + hierarchy_sep); + } T_END; + return dsync_slave_send_ret(slave); +} + +enum dsync_slave_recv_ret +dsync_slave_recv_mailbox_deletes(struct dsync_slave *slave, + const struct dsync_mailbox_delete **deletes_r, + unsigned int *count_r, char *hierarchy_sep_r) +{ + return slave->v.recv_mailbox_deletes(slave, deletes_r, count_r, + hierarchy_sep_r); +} + +enum dsync_slave_send_ret +dsync_slave_send_mailbox(struct dsync_slave *slave, + const struct dsync_mailbox *dsync_box) +{ + T_BEGIN { + slave->v.send_mailbox(slave, dsync_box); + } T_END; + return dsync_slave_send_ret(slave); +} + +enum dsync_slave_recv_ret +dsync_slave_recv_mailbox(struct dsync_slave *slave, + const struct dsync_mailbox **dsync_box_r) +{ + return slave->v.recv_mailbox(slave, dsync_box_r); +} + +enum dsync_slave_send_ret +dsync_slave_send_change(struct dsync_slave *slave, + const struct dsync_mail_change *change) +{ + i_assert(change->uid > 0); + + T_BEGIN { + slave->v.send_change(slave, change); + } T_END; + return dsync_slave_send_ret(slave); +} + +enum dsync_slave_recv_ret +dsync_slave_recv_change(struct dsync_slave *slave, + const struct dsync_mail_change **change_r) +{ + return slave->v.recv_change(slave, change_r); +} + +enum dsync_slave_send_ret +dsync_slave_send_mail_request(struct dsync_slave *slave, + const struct dsync_mail_request *request) +{ + i_assert(*request->guid != '\0' || request->uid != 0); + + T_BEGIN { + slave->v.send_mail_request(slave, request); + } T_END; + return dsync_slave_send_ret(slave); +} + +enum dsync_slave_recv_ret +dsync_slave_recv_mail_request(struct dsync_slave *slave, + const struct dsync_mail_request **request_r) +{ + return slave->v.recv_mail_request(slave, request_r); +} + +enum dsync_slave_send_ret +dsync_slave_send_mail(struct dsync_slave *slave, + const struct dsync_mail *mail) +{ + i_assert(*mail->guid != '\0' || mail->uid != 0); + + T_BEGIN { + slave->v.send_mail(slave, mail); + } T_END; + return dsync_slave_send_ret(slave); +} + +enum dsync_slave_recv_ret +dsync_slave_recv_mail(struct dsync_slave *slave, + struct dsync_mail **mail_r) +{ + return slave->v.recv_mail(slave, mail_r); +} + +void dsync_slave_flush(struct dsync_slave *slave) +{ + if (slave->v.flush != NULL) + slave->v.flush(slave); +} + +bool dsync_slave_has_failed(struct dsync_slave *slave) +{ + return slave->failed; +} + +bool dsync_slave_is_send_queue_full(struct dsync_slave *slave) +{ + return slave->v.is_send_queue_full(slave); +} + +bool dsync_slave_has_pending_data(struct dsync_slave *slave) +{ + return slave->v.has_pending_data(slave); +}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/doveadm/dsync/dsync-slave.h Tue May 22 23:17:31 2012 +0300 @@ -0,0 +1,118 @@ +#ifndef DSYNC_SLAVE_H +#define DSYNC_SLAVE_H + +#include "ioloop.h" +#include "guid.h" +#include "dsync-brain.h" + +struct dsync_mailbox; +struct dsync_mailbox_state; +struct dsync_mailbox_node; +struct dsync_mailbox_delete; +struct dsync_mail; +struct dsync_mail_change; +struct dsync_mail_request; +struct dsync_slave_settings; + +enum dsync_slave_send_ret { + DSYNC_SLAVE_SEND_RET_OK = 1, + /* send queue is full, stop sending more */ + DSYNC_SLAVE_SEND_RET_FULL = 0 +}; + +enum dsync_slave_recv_ret { + DSYNC_SLAVE_RECV_RET_FINISHED = -1, + /* try again / error (the error handling delayed until io callback) */ + DSYNC_SLAVE_RECV_RET_TRYAGAIN = 0, + DSYNC_SLAVE_RECV_RET_OK = 1 +}; + +struct dsync_slave_settings { + /* if non-NULL, sync only this namespace */ + const char *sync_ns_prefix; + + enum dsync_brain_sync_type sync_type; + bool guid_requests; + bool mails_have_guids; +}; + +void dsync_slave_init_pipe(struct dsync_slave **slave1_r, + struct dsync_slave **slave2_r); +struct dsync_slave * +dsync_slave_init_io(int fd_in, int fd_out, const char *name, + const char *temp_path_prefix); +void dsync_slave_deinit(struct dsync_slave **slave); + +/* I/O callback is called whenever new data is available. It's also called on + errors, so check first the error status. */ +void dsync_slave_set_io_callback(struct dsync_slave *slave, + io_callback_t *callback, void *context); + +void dsync_slave_send_handshake(struct dsync_slave *slave, + const struct dsync_slave_settings *set); +enum dsync_slave_recv_ret +dsync_slave_recv_handshake(struct dsync_slave *slave, + const struct dsync_slave_settings **set_r); + +enum dsync_slave_send_ret +dsync_slave_send_end_of_list(struct dsync_slave *slave); + +enum dsync_slave_send_ret +dsync_slave_send_mailbox_state(struct dsync_slave *slave, + const struct dsync_mailbox_state *state); +enum dsync_slave_recv_ret +dsync_slave_recv_mailbox_state(struct dsync_slave *slave, + struct dsync_mailbox_state *state_r); + +enum dsync_slave_send_ret +dsync_slave_send_mailbox_tree_node(struct dsync_slave *slave, + const char *const *name, + const struct dsync_mailbox_node *node); +enum dsync_slave_recv_ret +dsync_slave_recv_mailbox_tree_node(struct dsync_slave *slave, + const char *const **name_r, + const struct dsync_mailbox_node **node_r); + +enum dsync_slave_send_ret +dsync_slave_send_mailbox_deletes(struct dsync_slave *slave, + const struct dsync_mailbox_delete *deletes, + unsigned int count, char hierarchy_sep); +enum dsync_slave_recv_ret +dsync_slave_recv_mailbox_deletes(struct dsync_slave *slave, + const struct dsync_mailbox_delete **deletes_r, + unsigned int *count_r, char *hierarchy_sep_r); + +enum dsync_slave_send_ret +dsync_slave_send_mailbox(struct dsync_slave *slave, + const struct dsync_mailbox *dsync_box); +enum dsync_slave_recv_ret +dsync_slave_recv_mailbox(struct dsync_slave *slave, + const struct dsync_mailbox **dsync_box_r); + +enum dsync_slave_send_ret +dsync_slave_send_change(struct dsync_slave *slave, + const struct dsync_mail_change *change); +enum dsync_slave_recv_ret +dsync_slave_recv_change(struct dsync_slave *slave, + const struct dsync_mail_change **change_r); + +enum dsync_slave_send_ret +dsync_slave_send_mail_request(struct dsync_slave *slave, + const struct dsync_mail_request *request); +enum dsync_slave_recv_ret +dsync_slave_recv_mail_request(struct dsync_slave *slave, + const struct dsync_mail_request **request_r); + +enum dsync_slave_send_ret +dsync_slave_send_mail(struct dsync_slave *slave, + const struct dsync_mail *mail); +enum dsync_slave_recv_ret +dsync_slave_recv_mail(struct dsync_slave *slave, + struct dsync_mail **mail_r); + +void dsync_slave_flush(struct dsync_slave *slave); +bool dsync_slave_has_failed(struct dsync_slave *slave); +bool dsync_slave_is_send_queue_full(struct dsync_slave *slave); +bool dsync_slave_has_pending_data(struct dsync_slave *slave); + +#endif
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/doveadm/dsync/dsync-transaction-log-scan.c Tue May 22 23:17:31 2012 +0300 @@ -0,0 +1,467 @@ +/* Copyright (c) 2012 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "hash.h" +#include "mail-index-modseq.h" +#include "mail-storage-private.h" +#include "dsync-mail.h" +#include "dsync-transaction-log-scan.h" + +struct dsync_transaction_log_scan { + pool_t pool; + struct hash_table *changes; + struct mail_index_view *view; + uint32_t highest_wanted_uid; + + uint32_t last_log_seq; + uoff_t last_log_offset; + + bool returned_all_changes; +}; + +static bool +export_change_get(struct dsync_transaction_log_scan *ctx, uint32_t uid, + enum dsync_mail_change_type type, + struct dsync_mail_change **change_r) +{ + struct dsync_mail_change *change; + const char *orig_guid; + + i_assert(uid > 0); + i_assert(type != DSYNC_MAIL_CHANGE_TYPE_SAVE); + + *change_r = NULL; + + if (uid > ctx->highest_wanted_uid) + return FALSE; + + change = hash_table_lookup(ctx->changes, POINTER_CAST(uid)); + if (change == NULL) { + /* first change for this UID */ + change = p_new(ctx->pool, struct dsync_mail_change, 1); + change->uid = uid; + change->type = type; + hash_table_insert(ctx->changes, POINTER_CAST(uid), change); + } else if (type == DSYNC_MAIL_CHANGE_TYPE_EXPUNGE) { + /* expunge overrides flag changes */ + orig_guid = change->guid; + memset(change, 0, sizeof(*change)); + change->type = type; + change->uid = uid; + change->guid = orig_guid; + } else if (change->type == DSYNC_MAIL_CHANGE_TYPE_EXPUNGE) { + /* already expunged, this change doesn't matter */ + return FALSE; + } else { + /* another flag update */ + } + *change_r = change; + return TRUE; +} + +static void +log_add_expunge(struct dsync_transaction_log_scan *ctx, const void *data, + const struct mail_transaction_header *hdr) +{ + const struct mail_transaction_expunge *rec = data, *end; + struct dsync_mail_change *change; + uint32_t uid; + + if ((hdr->type & MAIL_TRANSACTION_EXTERNAL) == 0) { + /* this is simply a request for expunge */ + return; + } + end = CONST_PTR_OFFSET(data, hdr->size); + for (; rec != end; rec++) { + for (uid = rec->uid1; uid <= rec->uid2; uid++) { + (void)export_change_get(ctx, uid, + DSYNC_MAIL_CHANGE_TYPE_EXPUNGE, + &change); + } + } +} + +static bool +log_add_expunge_uid(struct dsync_transaction_log_scan *ctx, const void *data, + const struct mail_transaction_header *hdr, uint32_t uid) +{ + const struct mail_transaction_expunge *rec = data, *end; + struct dsync_mail_change *change; + + if ((hdr->type & MAIL_TRANSACTION_EXTERNAL) == 0) { + /* this is simply a request for expunge */ + return FALSE; + } + end = CONST_PTR_OFFSET(data, hdr->size); + for (; rec != end; rec++) { + if (uid >= rec->uid1 && uid <= rec->uid2) { + (void)export_change_get(ctx, uid, + DSYNC_MAIL_CHANGE_TYPE_EXPUNGE, + &change); + return TRUE; + } + } + return FALSE; +} + +static void +log_add_expunge_guid(struct dsync_transaction_log_scan *ctx, const void *data, + const struct mail_transaction_header *hdr) +{ + const struct mail_transaction_expunge_guid *rec = data, *end; + struct mail_index_view *view = ctx->view; + struct dsync_mail_change *change; + uint32_t seq; + bool external; + + external = (hdr->type & MAIL_TRANSACTION_EXTERNAL) != 0; + + end = CONST_PTR_OFFSET(data, hdr->size); + for (; rec != end; rec++) { + if (!external && !mail_index_lookup_seq(view, rec->uid, &seq)) { + /* expunge request that hasn't been actually done yet. + we check non-external ones because they might have + the GUID while external ones don't. */ + continue; + } + if (export_change_get(ctx, rec->uid, + DSYNC_MAIL_CHANGE_TYPE_EXPUNGE, + &change)) T_BEGIN { + change->guid = p_strdup(ctx->pool, + guid_128_to_string(rec->guid_128)); + } T_END; + } +} + +static bool +log_add_expunge_guid_uid(struct dsync_transaction_log_scan *ctx, const void *data, + const struct mail_transaction_header *hdr, uint32_t uid) +{ + const struct mail_transaction_expunge_guid *rec = data, *end; + struct dsync_mail_change *change; + + /* we're assuming UID is already known to be expunged */ + end = CONST_PTR_OFFSET(data, hdr->size); + for (; rec != end; rec++) { + if (rec->uid != uid) + continue; + + if (!export_change_get(ctx, rec->uid, + DSYNC_MAIL_CHANGE_TYPE_EXPUNGE, + &change)) + i_unreached(); + T_BEGIN { + change->guid = p_strdup(ctx->pool, + guid_128_to_string(rec->guid_128)); + } T_END; + return TRUE; + } + return FALSE; +} + +static void +log_add_flag_update(struct dsync_transaction_log_scan *ctx, const void *data, + const struct mail_transaction_header *hdr) +{ + const struct mail_transaction_flag_update *rec = data, *end; + struct dsync_mail_change *change; + uint32_t uid; + + end = CONST_PTR_OFFSET(data, hdr->size); + for (; rec != end; rec++) { + for (uid = rec->uid1; uid <= rec->uid2; uid++) { + if (export_change_get(ctx, uid, + DSYNC_MAIL_CHANGE_TYPE_FLAG_CHANGE, + &change)) { + change->add_flags |= rec->add_flags; + change->remove_flags &= ~rec->add_flags; + change->remove_flags |= rec->remove_flags; + change->add_flags &= ~rec->remove_flags; + } + } + } +} + +static void +log_add_keyword_reset(struct dsync_transaction_log_scan *ctx, const void *data, + const struct mail_transaction_header *hdr) +{ + const struct mail_transaction_keyword_reset *rec = data, *end; + struct dsync_mail_change *change; + uint32_t uid; + + end = CONST_PTR_OFFSET(data, hdr->size); + for (; rec != end; rec++) { + for (uid = rec->uid1; uid <= rec->uid2; uid++) { + if (!export_change_get(ctx, uid, + DSYNC_MAIL_CHANGE_TYPE_FLAG_CHANGE, + &change)) + continue; + + change->keywords_reset = TRUE; + if (array_is_created(&change->keyword_changes)) + array_clear(&change->keyword_changes); + } + } +} + +static void +keywords_change_remove(struct dsync_mail_change *change, const char *name) +{ + const char *const *changes; + unsigned int i, count; + + changes = array_get(&change->keyword_changes, &count); + for (i = 0; i < count; i++) { + if (strcmp(changes[i]+1, name) == 0) { + array_delete(&change->keyword_changes, i, 1); + break; + } + } +} + +static void +log_add_keyword_update(struct dsync_transaction_log_scan *ctx, const void *data, + const struct mail_transaction_header *hdr) +{ + const struct mail_transaction_keyword_update *rec = data; + struct dsync_mail_change *change; + const char *kw_name, *change_str; + const uint32_t *uids, *end; + unsigned int uids_offset; + uint32_t uid; + + uids_offset = sizeof(*rec) + rec->name_size; + if ((uids_offset % 4) != 0) + uids_offset += 4 - (uids_offset % 4); + + kw_name = t_strndup((const void *)(rec+1), rec->name_size); + switch (rec->modify_type) { + case MODIFY_ADD: + change_str = p_strdup_printf(ctx->pool, "%c%s", + KEYWORD_CHANGE_ADD, kw_name); + break; + case MODIFY_REMOVE: + change_str = p_strdup_printf(ctx->pool, "%c%s", + KEYWORD_CHANGE_REMOVE, kw_name); + break; + default: + i_unreached(); + } + + uids = CONST_PTR_OFFSET(rec, uids_offset); + end = CONST_PTR_OFFSET(rec, hdr->size); + + for (; uids <= end; uids += 2) { + for (uid = uids[0]; uid <= uids[1]; uid++) { + if (!export_change_get(ctx, uid, + DSYNC_MAIL_CHANGE_TYPE_FLAG_CHANGE, + &change)) + continue; + if (!array_is_created(&change->keyword_changes)) { + p_array_init(&change->keyword_changes, + ctx->pool, 4); + } else { + keywords_change_remove(change, kw_name); + } + array_append(&change->keyword_changes, &change_str, 1); + } + } +} + +static void +log_add_modseq_update(struct dsync_transaction_log_scan *ctx, const void *data, + const struct mail_transaction_header *hdr) +{ + const struct mail_transaction_modseq_update *rec = data, *end; + struct dsync_mail_change *change; + uint64_t modseq; + + /* update message's modseq, possibly by creating an empty flag change */ + end = CONST_PTR_OFFSET(rec, hdr->size); + for (; rec != end; rec++) { + if (rec->uid == 0) { + /* highestmodseq update */ + continue; + } + + if (!export_change_get(ctx, rec->uid, + DSYNC_MAIL_CHANGE_TYPE_FLAG_CHANGE, + &change)) + continue; + + modseq = rec->modseq_low32 | + ((uint64_t)rec->modseq_high32 << 32); + if (change->modseq < modseq) + change->modseq = modseq; + } +} + +static int +dsync_log_set(struct dsync_transaction_log_scan *ctx, + struct mail_transaction_log_view *log_view, uint64_t modseq) +{ + struct mail_index_view *view = ctx->view; + uint32_t log_seq; + uoff_t log_offset; + bool reset; + int ret; + + if (modseq == 0 || + !mail_index_modseq_get_next_log_offset(view, modseq, + &log_seq, &log_offset)) + ret = 0; + else { + ret = mail_transaction_log_view_set(log_view, + log_seq, log_offset, + view->log_file_head_seq, + view->log_file_head_offset, + &reset); + } + if (ret == 0) { + /* return everything we've got */ + ctx->returned_all_changes = TRUE; + return mail_transaction_log_view_set_all(log_view); + } + return ret < 0 ? -1 : 0; +} + +int dsync_transaction_log_scan_init(struct mail_index_view *view, + uint32_t highest_wanted_uid, + uint64_t modseq, + struct dsync_transaction_log_scan **scan_r) +{ + struct dsync_transaction_log_scan *ctx; + struct mail_transaction_log_view *log_view; + const struct mail_transaction_header *hdr; + const void *data; + uint32_t file_seq, max_seq; + uoff_t file_offset, max_offset; + pool_t pool; + + pool = pool_alloconly_create(MEMPOOL_GROWING"dsync transaction log scan", + 10240); + ctx = p_new(pool, struct dsync_transaction_log_scan, 1); + ctx->pool = pool; + ctx->changes = + hash_table_create(default_pool, pool, 0, NULL, NULL); + ctx->view = view; + ctx->highest_wanted_uid = highest_wanted_uid; + + log_view = mail_transaction_log_view_open(view->index->log); + if (dsync_log_set(ctx, log_view, modseq) < 0) { + mail_transaction_log_view_close(&log_view); + return -1; + } + + /* read the log only up to current position in view */ + max_seq = view->log_file_expunge_seq; + max_offset = view->log_file_expunge_offset; + + while (mail_transaction_log_view_next(log_view, &hdr, &data) > 0) { + mail_transaction_log_view_get_prev_pos(log_view, &file_seq, + &file_offset); + if (file_offset >= max_offset && file_seq == max_seq) + break; + + if ((hdr->type & MAIL_TRANSACTION_SYNC) != 0) { + /* ignore changes done by dsync, unless we can get + expunged message's GUID from it */ + if ((hdr->type & MAIL_TRANSACTION_TYPE_MASK) != + MAIL_TRANSACTION_EXPUNGE_GUID) + continue; + } + + switch (hdr->type & MAIL_TRANSACTION_TYPE_MASK) { + case MAIL_TRANSACTION_EXPUNGE: + log_add_expunge(ctx, data, hdr); + break; + case MAIL_TRANSACTION_EXPUNGE_GUID: + log_add_expunge_guid(ctx, data, hdr); + break; + case MAIL_TRANSACTION_FLAG_UPDATE: + log_add_flag_update(ctx, data, hdr); + break; + case MAIL_TRANSACTION_KEYWORD_RESET: + log_add_keyword_reset(ctx, data, hdr); + break; + case MAIL_TRANSACTION_KEYWORD_UPDATE: + T_BEGIN { + log_add_keyword_update(ctx, data, hdr); + } T_END; + break; + case MAIL_TRANSACTION_MODSEQ_UPDATE: + log_add_modseq_update(ctx, data, hdr); + break; + } + } + + ctx->last_log_seq = file_seq; + ctx->last_log_offset = file_offset; + + mail_transaction_log_view_close(&log_view); + *scan_r = ctx; + return 0; +} + +struct hash_table * +dsync_transaction_log_scan_get_hash(struct dsync_transaction_log_scan *scan) +{ + return scan->changes; +} + +bool +dsync_transaction_log_scan_has_all_changes(struct dsync_transaction_log_scan *scan) +{ + return scan->returned_all_changes; +} + +struct dsync_mail_change * +dsync_transaction_log_scan_find_new_expunge(struct dsync_transaction_log_scan *scan, + uint32_t uid) +{ + struct mail_transaction_log_view *log_view; + const struct mail_transaction_header *hdr; + const void *data; + bool reset, found = FALSE; + + i_assert(uid > 0); + + if (scan->highest_wanted_uid < uid) + scan->highest_wanted_uid = uid; + + log_view = mail_transaction_log_view_open(scan->view->index->log); + if (mail_transaction_log_view_set(log_view, + scan->last_log_seq, + scan->last_log_offset, + (uint32_t)-1, (uoff_t)-1, + &reset) > 0) { + while (!found && + mail_transaction_log_view_next(log_view, &hdr, &data) > 0) { + switch (hdr->type & MAIL_TRANSACTION_TYPE_MASK) { + case MAIL_TRANSACTION_EXPUNGE: + if (log_add_expunge_uid(scan, data, hdr, uid)) + found = TRUE; + break; + case MAIL_TRANSACTION_EXPUNGE_GUID: + if (log_add_expunge_guid_uid(scan, data, hdr, uid)) + found = TRUE; + break; + } + } + } + mail_transaction_log_view_close(&log_view); + + return !found ? NULL : + hash_table_lookup(scan->changes, POINTER_CAST(uid)); +} + +void dsync_transaction_log_scan_deinit(struct dsync_transaction_log_scan **_scan) +{ + struct dsync_transaction_log_scan *scan = *_scan; + + *_scan = NULL; + + hash_table_destroy(&scan->changes); + pool_unref(&scan->pool); +}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/doveadm/dsync/dsync-transaction-log-scan.h Tue May 22 23:17:31 2012 +0300 @@ -0,0 +1,22 @@ +#ifndef DSYNC_TRANSACTION_LOG_SCAN_H +#define DSYNC_TRANSACTION_LOG_SCAN_H + +struct mail_index_view; +struct dsync_transaction_log_scan; + +int dsync_transaction_log_scan_init(struct mail_index_view *view, + uint32_t highest_wanted_uid, + uint64_t modseq, + struct dsync_transaction_log_scan **scan_r); +struct hash_table * +dsync_transaction_log_scan_get_hash(struct dsync_transaction_log_scan *scan); +/* Returns TRUE if the entire transaction log was scanned */ +bool dsync_transaction_log_scan_has_all_changes(struct dsync_transaction_log_scan *scan); +/* If the given UID has been expunged after the initial log scan, create/update + a change record for it and return it. */ +struct dsync_mail_change * +dsync_transaction_log_scan_find_new_expunge(struct dsync_transaction_log_scan *scan, + uint32_t uid); +void dsync_transaction_log_scan_deinit(struct dsync_transaction_log_scan **scan); + +#endif
--- a/src/doveadm/dsync/dsync-worker-local.c Fri May 04 05:35:36 2012 +0300 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,1922 +0,0 @@ -/* Copyright (c) 2009-2012 Dovecot authors, see the included COPYING file */ - -#include "lib.h" -#include "array.h" -#include "hash.h" -#include "str.h" -#include "hex-binary.h" -#include "network.h" -#include "istream.h" -#include "settings-parser.h" -#include "mailbox-log.h" -#include "mail-user.h" -#include "mail-namespace.h" -#include "mail-storage.h" -#include "mail-search-build.h" -#include "mailbox-list-private.h" -#include "dsync-worker-private.h" - -#include <ctype.h> - -struct local_dsync_worker_mailbox_iter { - struct dsync_worker_mailbox_iter iter; - pool_t ret_pool; - struct mailbox_list_iterate_context *list_iter; - struct hash_iterate_context *deleted_iter; - struct hash_iterate_context *deleted_dir_iter; -}; - -struct local_dsync_worker_subs_iter { - struct dsync_worker_subs_iter iter; - struct mailbox_list_iterate_context *list_iter; - struct hash_iterate_context *deleted_iter; -}; - -struct local_dsync_worker_msg_iter { - struct dsync_worker_msg_iter iter; - mailbox_guid_t *mailboxes; - unsigned int mailbox_idx, mailbox_count; - - struct mail_search_context *search_ctx; - struct mailbox *box; - struct mailbox_transaction_context *trans; - uint32_t prev_uid; - - string_t *tmp_guid_str; - ARRAY_TYPE(mailbox_expunge_rec) expunges; - unsigned int expunge_idx; - unsigned int expunges_set:1; -}; - -struct local_dsync_mailbox { - struct mail_namespace *ns; - mailbox_guid_t guid; - const char *name; - bool deleted; -}; - -struct local_dsync_mailbox_change { - mailbox_guid_t guid; - time_t last_delete; - - unsigned int deleted_mailbox:1; -}; - -struct local_dsync_dir_change { - mailbox_guid_t name_sha1; - struct mailbox_list *list; - - time_t last_rename; - time_t last_delete; - time_t last_subs_change; - - unsigned int unsubscribed:1; - unsigned int deleted_dir:1; -}; - -struct local_dsync_worker_msg_get { - mailbox_guid_t mailbox; - uint32_t uid; - dsync_worker_msg_callback_t *callback; - void *context; -}; - -struct local_dsync_worker { - struct dsync_worker worker; - struct mail_user *user; - - pool_t pool; - /* mailbox_guid_t -> struct local_dsync_mailbox* */ - struct hash_table *mailbox_hash; - /* mailbox_guid_t -> struct local_dsync_mailbox_change* */ - struct hash_table *mailbox_changes_hash; - /* <-> struct local_dsync_dir_change */ - struct hash_table *dir_changes_hash; - - char alt_char; - const char *namespace_prefix; - - mailbox_guid_t selected_box_guid; - struct mailbox *selected_box; - struct mail *mail, *ext_mail; - - ARRAY_TYPE(uint32_t) saved_uids; - - mailbox_guid_t get_mailbox; - struct mail *get_mail; - ARRAY_DEFINE(msg_get_queue, struct local_dsync_worker_msg_get); - - struct io *save_io; - struct mail_save_context *save_ctx; - struct istream *save_input; - dsync_worker_save_callback_t *save_callback; - void *save_context; - - dsync_worker_finish_callback_t *finish_callback; - void *finish_context; - - unsigned int reading_mail:1; - unsigned int finishing:1; - unsigned int finished:1; -}; - -extern struct dsync_worker_vfuncs local_dsync_worker; - -static void local_worker_mailbox_close(struct local_dsync_worker *worker); -static void local_worker_msg_box_close(struct local_dsync_worker *worker); -static void -local_worker_msg_get_next(struct local_dsync_worker *worker, - const struct local_dsync_worker_msg_get *get); - -static int mailbox_guid_cmp(const void *p1, const void *p2) -{ - const mailbox_guid_t *g1 = p1, *g2 = p2; - - return memcmp(g1->guid, g2->guid, sizeof(g1->guid)); -} - -static unsigned int mailbox_guid_hash(const void *p) -{ - const mailbox_guid_t *guid = p; - const uint8_t *s = guid->guid; - unsigned int i, g, h = 0; - - for (i = 0; i < sizeof(guid->guid); i++) { - h = (h << 4) + s[i]; - if ((g = h & 0xf0000000UL)) { - h = h ^ (g >> 24); - h = h ^ g; - } - } - return h; -} - -static bool local_worker_want_namespace(struct local_dsync_worker *worker, - struct mail_namespace *ns) -{ - if (worker->namespace_prefix == NULL) { - return strcmp(ns->unexpanded_set->location, - SETTING_STRVAR_UNEXPANDED) == 0; - } else { - return strcmp(ns->prefix, worker->namespace_prefix) == 0; - } -} - -static void dsync_check_namespaces(struct local_dsync_worker *worker) -{ - struct mail_namespace *ns; - - if (worker->namespace_prefix != NULL) { - ns = mail_namespace_find_prefix(worker->user->namespaces, - worker->namespace_prefix); - if (ns == NULL) { - i_fatal("Namespace prefix '%s' not found", - worker->namespace_prefix); - } - return; - } - - for (ns = worker->user->namespaces; ns != NULL; ns = ns->next) { - if (local_worker_want_namespace(worker, ns)) - return; - } - i_fatal("All your namespaces have a location setting. " - "It should be empty (default mail_location) in the " - "namespace to be converted."); -} - -struct dsync_worker * -dsync_worker_init_local(struct mail_user *user, const char *namespace_prefix, - char alt_char) -{ - struct local_dsync_worker *worker; - pool_t pool; - - pool = pool_alloconly_create("local dsync worker", 10240); - worker = p_new(pool, struct local_dsync_worker, 1); - worker->worker.v = local_dsync_worker; - worker->user = user; - worker->pool = pool; - worker->namespace_prefix = p_strdup(pool, namespace_prefix); - worker->alt_char = alt_char; - worker->mailbox_hash = - hash_table_create(default_pool, pool, 0, - mailbox_guid_hash, mailbox_guid_cmp); - i_array_init(&worker->saved_uids, 128); - i_array_init(&worker->msg_get_queue, 32); - dsync_check_namespaces(worker); - - mail_user_ref(worker->user); - return &worker->worker; -} - -static void local_worker_deinit(struct dsync_worker *_worker) -{ - struct local_dsync_worker *worker = - (struct local_dsync_worker *)_worker; - - i_assert(worker->save_input == NULL); - - local_worker_msg_box_close(worker); - local_worker_mailbox_close(worker); - mail_user_unref(&worker->user); - - hash_table_destroy(&worker->mailbox_hash); - if (worker->mailbox_changes_hash != NULL) - hash_table_destroy(&worker->mailbox_changes_hash); - if (worker->dir_changes_hash != NULL) - hash_table_destroy(&worker->dir_changes_hash); - array_free(&worker->msg_get_queue); - array_free(&worker->saved_uids); - pool_unref(&worker->pool); -} - -static bool local_worker_is_output_full(struct dsync_worker *worker ATTR_UNUSED) -{ - return FALSE; -} - -static int local_worker_output_flush(struct dsync_worker *worker ATTR_UNUSED) -{ - return 1; -} - -static void -dsync_worker_save_mailbox_change(struct local_dsync_worker *worker, - const struct mailbox_log_record *rec) -{ - struct local_dsync_mailbox_change *change; - time_t stamp; - - change = hash_table_lookup(worker->mailbox_changes_hash, - rec->mailbox_guid); - if (change == NULL) { - change = i_new(struct local_dsync_mailbox_change, 1); - memcpy(change->guid.guid, rec->mailbox_guid, - sizeof(change->guid.guid)); - hash_table_insert(worker->mailbox_changes_hash, - change->guid.guid, change); - } - - stamp = mailbox_log_record_get_timestamp(rec); - switch (rec->type) { - case MAILBOX_LOG_RECORD_DELETE_MAILBOX: - change->deleted_mailbox = TRUE; - if (change->last_delete < stamp) - change->last_delete = stamp; - break; - case MAILBOX_LOG_RECORD_DELETE_DIR: - case MAILBOX_LOG_RECORD_RENAME: - case MAILBOX_LOG_RECORD_SUBSCRIBE: - case MAILBOX_LOG_RECORD_UNSUBSCRIBE: - i_unreached(); - } -} - -static void -dsync_worker_save_dir_change(struct local_dsync_worker *worker, - struct mailbox_list *list, - const struct mailbox_log_record *rec) -{ - struct local_dsync_dir_change *change, new_change; - time_t stamp; - - memset(&new_change, 0, sizeof(new_change)); - new_change.list = list; - memcpy(new_change.name_sha1.guid, rec->mailbox_guid, - sizeof(new_change.name_sha1.guid)); - - stamp = mailbox_log_record_get_timestamp(rec); - change = hash_table_lookup(worker->dir_changes_hash, &new_change); - if (change == NULL) { - change = i_new(struct local_dsync_dir_change, 1); - *change = new_change; - hash_table_insert(worker->dir_changes_hash, change, change); - } - - switch (rec->type) { - case MAILBOX_LOG_RECORD_DELETE_MAILBOX: - i_unreached(); - case MAILBOX_LOG_RECORD_DELETE_DIR: - change->deleted_dir = TRUE; - if (change->last_delete < stamp) - change->last_delete = stamp; - break; - case MAILBOX_LOG_RECORD_RENAME: - if (change->last_rename < stamp) - change->last_rename = stamp; - break; - case MAILBOX_LOG_RECORD_SUBSCRIBE: - case MAILBOX_LOG_RECORD_UNSUBSCRIBE: - if (change->last_subs_change > stamp) { - /* we've already seen a newer subscriptions state. this - is probably a stale record created by dsync */ - } else { - change->last_subs_change = stamp; - change->unsubscribed = - rec->type == MAILBOX_LOG_RECORD_UNSUBSCRIBE; - } - break; - } -} - -static int -dsync_worker_get_list_mailbox_log(struct local_dsync_worker *worker, - struct mailbox_list *list) -{ - struct mailbox_log *log; - struct mailbox_log_iter *iter; - const struct mailbox_log_record *rec; - - log = mailbox_list_get_changelog(list); - if (log == NULL) - return 0; - iter = mailbox_log_iter_init(log); - while ((rec = mailbox_log_iter_next(iter)) != NULL) { - switch (rec->type) { - case MAILBOX_LOG_RECORD_DELETE_MAILBOX: - dsync_worker_save_mailbox_change(worker, rec); - break; - case MAILBOX_LOG_RECORD_DELETE_DIR: - case MAILBOX_LOG_RECORD_RENAME: - case MAILBOX_LOG_RECORD_SUBSCRIBE: - case MAILBOX_LOG_RECORD_UNSUBSCRIBE: - dsync_worker_save_dir_change(worker, list, rec); - break; - } - } - return mailbox_log_iter_deinit(&iter); -} - -static unsigned int mailbox_log_record_hash(const void *p) -{ - const uint8_t *guid = p; - - return ((unsigned int)guid[0] << 24) | - ((unsigned int)guid[1] << 16) | - ((unsigned int)guid[2] << 8) | - (unsigned int)guid[3]; -} - -static int mailbox_log_record_cmp(const void *p1, const void *p2) -{ - return memcmp(p1, p2, GUID_128_SIZE); -} - -static unsigned int dir_change_hash(const void *p) -{ - const struct local_dsync_dir_change *change = p; - - return mailbox_log_record_hash(change->name_sha1.guid) ^ - POINTER_CAST_TO(change->list, unsigned int); -} - -static int dir_change_cmp(const void *p1, const void *p2) -{ - const struct local_dsync_dir_change *c1 = p1, *c2 = p2; - - if (c1->list != c2->list) - return 1; - - return memcmp(c1->name_sha1.guid, c2->name_sha1.guid, - GUID_128_SIZE); -} - -static int dsync_worker_get_mailbox_log(struct local_dsync_worker *worker) -{ - struct mail_namespace *ns; - int ret = 0; - - if (worker->mailbox_changes_hash != NULL) - return 0; - - worker->mailbox_changes_hash = - hash_table_create(default_pool, worker->pool, 0, - mailbox_log_record_hash, - mailbox_log_record_cmp); - worker->dir_changes_hash = - hash_table_create(default_pool, worker->pool, 0, - dir_change_hash, dir_change_cmp); - for (ns = worker->user->namespaces; ns != NULL; ns = ns->next) { - if (ns->alias_for != NULL || - !local_worker_want_namespace(worker, ns)) - continue; - - if (dsync_worker_get_list_mailbox_log(worker, ns->list) < 0) - ret = -1; - } - return ret; -} - -static struct dsync_worker_mailbox_iter * -local_worker_mailbox_iter_init(struct dsync_worker *_worker) -{ - struct local_dsync_worker *worker = - (struct local_dsync_worker *)_worker; - struct local_dsync_worker_mailbox_iter *iter; - enum mailbox_list_iter_flags list_flags = - MAILBOX_LIST_ITER_SKIP_ALIASES | - MAILBOX_LIST_ITER_NO_AUTO_BOXES; - static const char *patterns[] = { "*", NULL }; - - iter = i_new(struct local_dsync_worker_mailbox_iter, 1); - iter->iter.worker = _worker; - iter->ret_pool = pool_alloconly_create("local mailbox iter", 1024); - iter->list_iter = - mailbox_list_iter_init_namespaces(worker->user->namespaces, - patterns, NAMESPACE_PRIVATE, - list_flags); - (void)dsync_worker_get_mailbox_log(worker); - return &iter->iter; -} - -static void -local_dsync_worker_add_mailbox(struct local_dsync_worker *worker, - struct mail_namespace *ns, const char *name, - const mailbox_guid_t *guid) -{ - struct local_dsync_mailbox *lbox; - - lbox = p_new(worker->pool, struct local_dsync_mailbox, 1); - lbox->ns = ns; - memcpy(lbox->guid.guid, guid->guid, sizeof(lbox->guid.guid)); - lbox->name = p_strdup(worker->pool, name); - - hash_table_insert(worker->mailbox_hash, &lbox->guid, lbox); -} - -static int -iter_next_deleted(struct local_dsync_worker_mailbox_iter *iter, - struct local_dsync_worker *worker, - struct dsync_mailbox *dsync_box_r) -{ - void *key, *value; - - if (iter->deleted_iter == NULL) { - iter->deleted_iter = - hash_table_iterate_init(worker->mailbox_changes_hash); - } - while (hash_table_iterate(iter->deleted_iter, &key, &value)) { - const struct local_dsync_mailbox_change *change = value; - - if (change->deleted_mailbox) { - /* the name doesn't matter */ - dsync_box_r->name = ""; - dsync_box_r->mailbox_guid = change->guid; - dsync_box_r->last_change = change->last_delete; - dsync_box_r->flags |= - DSYNC_MAILBOX_FLAG_DELETED_MAILBOX; - return 1; - } - } - - if (iter->deleted_dir_iter == NULL) { - iter->deleted_dir_iter = - hash_table_iterate_init(worker->dir_changes_hash); - } - while (hash_table_iterate(iter->deleted_dir_iter, &key, &value)) { - const struct local_dsync_dir_change *change = value; - - if (change->deleted_dir) { - /* the name doesn't matter */ - dsync_box_r->name = ""; - dsync_box_r->name_sha1 = change->name_sha1; - dsync_box_r->last_change = change->last_delete; - dsync_box_r->flags |= DSYNC_MAILBOX_FLAG_NOSELECT | - DSYNC_MAILBOX_FLAG_DELETED_DIR; - return 1; - } - } - hash_table_iterate_deinit(&iter->deleted_iter); - return -1; -} - -static int -local_worker_mailbox_iter_next(struct dsync_worker_mailbox_iter *_iter, - struct dsync_mailbox *dsync_box_r) -{ - struct local_dsync_worker_mailbox_iter *iter = - (struct local_dsync_worker_mailbox_iter *)_iter; - struct local_dsync_worker *worker = - (struct local_dsync_worker *)_iter->worker; - const enum mailbox_flags flags = MAILBOX_FLAG_READONLY; - const enum mailbox_status_items status_items = - STATUS_UIDNEXT | STATUS_UIDVALIDITY | - STATUS_HIGHESTMODSEQ | STATUS_FIRST_RECENT_UID; - const enum mailbox_metadata_items metadata_items = - MAILBOX_METADATA_CACHE_FIELDS | MAILBOX_METADATA_GUID; - const struct mailbox_info *info; - const char *storage_name; - struct mailbox *box; - struct mailbox_status status; - struct mailbox_metadata metadata; - struct local_dsync_mailbox_change *change; - struct local_dsync_dir_change *dir_change, change_lookup; - struct local_dsync_mailbox *old_lbox; - enum mail_error error; - struct mailbox_cache_field *cache_fields; - unsigned int i, cache_field_count; - - memset(dsync_box_r, 0, sizeof(*dsync_box_r)); - - while ((info = mailbox_list_iter_next(iter->list_iter)) != NULL) { - if (local_worker_want_namespace(worker, info->ns)) - break; - } - if (info == NULL) - return iter_next_deleted(iter, worker, dsync_box_r); - - dsync_box_r->name = info->name; - dsync_box_r->name_sep = mail_namespace_get_sep(info->ns); - - storage_name = mailbox_list_get_storage_name(info->ns->list, info->name); - dsync_str_sha_to_guid(storage_name, &dsync_box_r->name_sha1); - - /* get last change timestamp */ - change_lookup.list = info->ns->list; - change_lookup.name_sha1 = dsync_box_r->name_sha1; - dir_change = hash_table_lookup(worker->dir_changes_hash, - &change_lookup); - if (dir_change != NULL) { - /* it shouldn't be marked as deleted, but drop it to be sure */ - dir_change->deleted_dir = FALSE; - dsync_box_r->last_change = dir_change->last_rename; - } - - if ((info->flags & (MAILBOX_NOSELECT | MAILBOX_NONEXISTENT)) != 0) { - dsync_box_r->flags |= DSYNC_MAILBOX_FLAG_NOSELECT; - local_dsync_worker_add_mailbox(worker, info->ns, info->name, - &dsync_box_r->name_sha1); - return 1; - } - - box = mailbox_alloc(info->ns->list, info->name, flags); - if (mailbox_get_status(box, status_items, &status) < 0 || - mailbox_get_metadata(box, metadata_items, &metadata) < 0) { - i_error("Failed to sync mailbox %s: %s", info->name, - mailbox_get_last_error(box, &error)); - mailbox_free(&box); - if (error == MAIL_ERROR_NOTFOUND || - error == MAIL_ERROR_NOTPOSSIBLE) { - /* Mailbox isn't selectable, try the next one. We - should have already caught \Noselect mailboxes, but - check them anyway here. The NOTPOSSIBLE check is - mainly for invalid mbox files. */ - return local_worker_mailbox_iter_next(_iter, - dsync_box_r); - } - _iter->failed = TRUE; - return -1; - } - - change = hash_table_lookup(worker->mailbox_changes_hash, metadata.guid); - if (change != NULL) { - /* it shouldn't be marked as deleted, but drop it to be sure */ - change->deleted_mailbox = FALSE; - } - - memcpy(dsync_box_r->mailbox_guid.guid, metadata.guid, - sizeof(dsync_box_r->mailbox_guid.guid)); - dsync_box_r->uid_validity = status.uidvalidity; - dsync_box_r->uid_next = status.uidnext; - dsync_box_r->message_count = status.messages; - dsync_box_r->first_recent_uid = status.first_recent_uid; - dsync_box_r->highest_modseq = status.highest_modseq; - - p_clear(iter->ret_pool); - p_array_init(&dsync_box_r->cache_fields, iter->ret_pool, - array_count(metadata.cache_fields)); - array_append_array(&dsync_box_r->cache_fields, metadata.cache_fields); - cache_fields = array_get_modifiable(&dsync_box_r->cache_fields, - &cache_field_count); - for (i = 0; i < cache_field_count; i++) { - cache_fields[i].name = - p_strdup(iter->ret_pool, cache_fields[i].name); - } - - old_lbox = hash_table_lookup(worker->mailbox_hash, - &dsync_box_r->mailbox_guid); - if (old_lbox != NULL) { - i_error("Mailboxes don't have unique GUIDs: " - "%s is shared by %s and %s", - dsync_guid_to_str(&dsync_box_r->mailbox_guid), - old_lbox->name, info->name); - mailbox_free(&box); - _iter->failed = TRUE; - return -1; - } - local_dsync_worker_add_mailbox(worker, info->ns, info->name, - &dsync_box_r->mailbox_guid); - mailbox_free(&box); - return 1; -} - -static int -local_worker_mailbox_iter_deinit(struct dsync_worker_mailbox_iter *_iter) -{ - struct local_dsync_worker_mailbox_iter *iter = - (struct local_dsync_worker_mailbox_iter *)_iter; - int ret = _iter->failed ? -1 : 0; - - if (mailbox_list_iter_deinit(&iter->list_iter) < 0) - ret = -1; - pool_unref(&iter->ret_pool); - i_free(iter); - return ret; -} - -static struct dsync_worker_subs_iter * -local_worker_subs_iter_init(struct dsync_worker *_worker) -{ - struct local_dsync_worker *worker = - (struct local_dsync_worker *)_worker; - struct local_dsync_worker_subs_iter *iter; - const enum mailbox_list_iter_flags list_flags = - MAILBOX_LIST_ITER_SKIP_ALIASES | - MAILBOX_LIST_ITER_SELECT_SUBSCRIBED; - const enum namespace_type namespace_mask = - NAMESPACE_PRIVATE | NAMESPACE_SHARED | NAMESPACE_PUBLIC; - static const char *patterns[] = { "*", NULL }; - - iter = i_new(struct local_dsync_worker_subs_iter, 1); - iter->iter.worker = _worker; - iter->list_iter = - mailbox_list_iter_init_namespaces(worker->user->namespaces, - patterns, namespace_mask, - list_flags); - (void)dsync_worker_get_mailbox_log(worker); - return &iter->iter; -} - -static int -local_worker_subs_iter_next(struct dsync_worker_subs_iter *_iter, - struct dsync_worker_subscription *rec_r) -{ - struct local_dsync_worker_subs_iter *iter = - (struct local_dsync_worker_subs_iter *)_iter; - struct local_dsync_worker *worker = - (struct local_dsync_worker *)_iter->worker; - struct local_dsync_dir_change *change, change_lookup; - const struct mailbox_info *info; - const char *storage_name; - - memset(rec_r, 0, sizeof(*rec_r)); - - while ((info = mailbox_list_iter_next(iter->list_iter)) != NULL) { - if (local_worker_want_namespace(worker, info->ns) || - (info->ns->flags & NAMESPACE_FLAG_SUBSCRIPTIONS) == 0) - break; - } - if (info == NULL) - return -1; - - storage_name = mailbox_list_get_storage_name(info->ns->list, info->name); - if ((info->ns->flags & NAMESPACE_FLAG_SUBSCRIPTIONS) == 0) - storage_name = t_strconcat(info->ns->prefix, storage_name, NULL); - - dsync_str_sha_to_guid(storage_name, &change_lookup.name_sha1); - change_lookup.list = info->ns->list; - - change = hash_table_lookup(worker->dir_changes_hash, - &change_lookup); - if (change != NULL) { - /* it shouldn't be marked as unsubscribed, but drop it to - be sure */ - change->unsubscribed = FALSE; - rec_r->last_change = change->last_subs_change; - } - if ((info->ns->flags & NAMESPACE_FLAG_SUBSCRIPTIONS) == 0) - rec_r->ns_prefix = ""; - else - rec_r->ns_prefix = info->ns->prefix; - rec_r->vname = info->name; - rec_r->storage_name = storage_name; - return 1; -} - -static int -local_worker_subs_iter_next_un(struct dsync_worker_subs_iter *_iter, - struct dsync_worker_unsubscription *rec_r) -{ - struct local_dsync_worker_subs_iter *iter = - (struct local_dsync_worker_subs_iter *)_iter; - struct local_dsync_worker *worker = - (struct local_dsync_worker *)_iter->worker; - void *key, *value; - - if (iter->deleted_iter == NULL) { - iter->deleted_iter = - hash_table_iterate_init(worker->dir_changes_hash); - } - while (hash_table_iterate(iter->deleted_iter, &key, &value)) { - const struct local_dsync_dir_change *change = value; - - if (change->unsubscribed) { - /* the name doesn't matter */ - struct mail_namespace *ns = - mailbox_list_get_namespace(change->list); - memset(rec_r, 0, sizeof(*rec_r)); - rec_r->name_sha1 = change->name_sha1; - rec_r->ns_prefix = ns->prefix; - rec_r->last_change = change->last_subs_change; - return 1; - } - } - hash_table_iterate_deinit(&iter->deleted_iter); - return -1; -} - -static int -local_worker_subs_iter_deinit(struct dsync_worker_subs_iter *_iter) -{ - struct local_dsync_worker_subs_iter *iter = - (struct local_dsync_worker_subs_iter *)_iter; - int ret = _iter->failed ? -1 : 0; - - if (mailbox_list_iter_deinit(&iter->list_iter) < 0) - ret = -1; - i_free(iter); - return ret; -} - -static void -local_worker_set_subscribed(struct dsync_worker *_worker, - const char *name, time_t last_change, bool set) -{ - struct local_dsync_worker *worker = - (struct local_dsync_worker *)_worker; - struct mail_namespace *ns; - struct mailbox *box; - - ns = mail_namespace_find(worker->user->namespaces, name); - if (ns == NULL) { - i_error("Can't find namespace for mailbox %s", name); - return; - } - - box = mailbox_alloc(ns->list, name, 0); - ns = mailbox_get_namespace(box); - - mailbox_list_set_changelog_timestamp(ns->list, last_change); - if (mailbox_set_subscribed(box, set) < 0) { - dsync_worker_set_failure(_worker); - i_error("Can't update subscription %s: %s", name, - mail_storage_get_last_error(mailbox_get_storage(box), - NULL)); - } - mailbox_list_set_changelog_timestamp(ns->list, (time_t)-1); - mailbox_free(&box); -} - -static int local_mailbox_open(struct local_dsync_worker *worker, - const mailbox_guid_t *guid, - struct mailbox **box_r) -{ - struct local_dsync_mailbox *lbox; - struct mailbox *box; - struct mailbox_metadata metadata; - - lbox = hash_table_lookup(worker->mailbox_hash, guid); - if (lbox == NULL) { - i_error("Trying to open a non-listed mailbox with guid=%s", - dsync_guid_to_str(guid)); - return -1; - } - if (lbox->deleted) { - *box_r = NULL; - return 0; - } - - box = mailbox_alloc(lbox->ns->list, lbox->name, 0); - if (mailbox_sync(box, 0) < 0 || - mailbox_get_metadata(box, MAILBOX_METADATA_GUID, &metadata) < 0) { - i_error("Failed to sync mailbox %s: %s", lbox->name, - mailbox_get_last_error(box, NULL)); - mailbox_free(&box); - return -1; - } - - if (memcmp(metadata.guid, guid->guid, sizeof(guid->guid)) != 0) { - i_error("Mailbox %s changed its GUID (%s -> %s)", - lbox->name, dsync_guid_to_str(guid), - guid_128_to_string(metadata.guid)); - mailbox_free(&box); - return -1; - } - *box_r = box; - return 1; -} - -static int iter_local_mailbox_open(struct local_dsync_worker_msg_iter *iter) -{ - struct local_dsync_worker *worker = - (struct local_dsync_worker *)iter->iter.worker; - mailbox_guid_t *guid; - struct mailbox *box; - struct mail_search_args *search_args; - int ret; - - for (;;) { - if (iter->mailbox_idx == iter->mailbox_count) { - /* no more mailboxes */ - return -1; - } - - guid = &iter->mailboxes[iter->mailbox_idx]; - ret = local_mailbox_open(worker, guid, &box); - if (ret != 0) - break; - /* mailbox was deleted. try next one. */ - iter->mailbox_idx++; - } - if (ret < 0) { - i_error("msg iteration failed: Couldn't open mailbox %s", - dsync_guid_to_str(guid)); - iter->iter.failed = TRUE; - return -1; - } - - search_args = mail_search_build_init(); - mail_search_build_add_all(search_args); - - iter->box = box; - iter->trans = mailbox_transaction_begin(box, 0); - iter->search_ctx = - mailbox_search_init(iter->trans, search_args, NULL, - MAIL_FETCH_FLAGS | MAIL_FETCH_GUID, NULL); - return 0; -} - -static void -iter_local_mailbox_close(struct local_dsync_worker_msg_iter *iter) -{ - iter->prev_uid = 0; - iter->expunges_set = FALSE; - if (mailbox_search_deinit(&iter->search_ctx) < 0) { - i_error("msg search failed: %s", - mailbox_get_last_error(iter->box, NULL)); - iter->iter.failed = TRUE; - } - (void)mailbox_transaction_commit(&iter->trans); - mailbox_free(&iter->box); -} - -static struct dsync_worker_msg_iter * -local_worker_msg_iter_init(struct dsync_worker *worker, - const mailbox_guid_t mailboxes[], - unsigned int mailbox_count) -{ - struct local_dsync_worker_msg_iter *iter; - unsigned int i; - - iter = i_new(struct local_dsync_worker_msg_iter, 1); - iter->iter.worker = worker; - iter->mailboxes = mailbox_count == 0 ? NULL : - i_new(mailbox_guid_t, mailbox_count); - iter->mailbox_count = mailbox_count; - for (i = 0; i < mailbox_count; i++) { - memcpy(iter->mailboxes[i].guid, &mailboxes[i], - sizeof(iter->mailboxes[i].guid)); - } - i_array_init(&iter->expunges, 32); - iter->tmp_guid_str = str_new(default_pool, GUID_128_SIZE * 2 + 1); - (void)iter_local_mailbox_open(iter); - return &iter->iter; -} - -static int mailbox_expunge_rec_cmp(const struct mailbox_expunge_rec *e1, - const struct mailbox_expunge_rec *e2) -{ - if (e1->uid < e2->uid) - return -1; - else if (e1->uid > e2->uid) - return 1; - else - return 0; -} - -static bool -iter_local_mailbox_next_expunge(struct local_dsync_worker_msg_iter *iter, - uint32_t prev_uid, struct dsync_message *msg_r) -{ - struct mailbox *box = iter->box; - struct mailbox_status status; - const uint8_t *guid_128; - const struct mailbox_expunge_rec *expunges; - unsigned int count; - - if (iter->expunges_set) { - expunges = array_get(&iter->expunges, &count); - if (iter->expunge_idx == count) - return FALSE; - - memset(msg_r, 0, sizeof(*msg_r)); - str_truncate(iter->tmp_guid_str, 0); - guid_128 = expunges[iter->expunge_idx].guid_128; - if (!guid_128_is_empty(guid_128)) { - binary_to_hex_append(iter->tmp_guid_str, guid_128, - GUID_128_SIZE); - } - msg_r->guid = str_c(iter->tmp_guid_str); - msg_r->uid = expunges[iter->expunge_idx].uid; - msg_r->flags = DSYNC_MAIL_FLAG_EXPUNGED; - iter->expunge_idx++; - return TRUE; - } - - /* initialize list of expunged messages at the end of mailbox */ - iter->expunge_idx = 0; - array_clear(&iter->expunges); - iter->expunges_set = TRUE; - - mailbox_get_open_status(box, STATUS_UIDNEXT, &status); - if (prev_uid + 1 >= status.uidnext) { - /* no expunged messages at the end of mailbox */ - return FALSE; - } - - T_BEGIN { - ARRAY_TYPE(seq_range) uids_filter; - - t_array_init(&uids_filter, 1); - seq_range_array_add_range(&uids_filter, prev_uid + 1, - status.uidnext - 1); - (void)mailbox_get_expunges(box, 0, &uids_filter, - &iter->expunges); - array_sort(&iter->expunges, mailbox_expunge_rec_cmp); - } T_END; - return iter_local_mailbox_next_expunge(iter, prev_uid, msg_r); -} - -static int -local_worker_msg_iter_next(struct dsync_worker_msg_iter *_iter, - unsigned int *mailbox_idx_r, - struct dsync_message *msg_r) -{ - struct local_dsync_worker_msg_iter *iter = - (struct local_dsync_worker_msg_iter *)_iter; - struct mail *mail; - const char *guid; - - if (_iter->failed || iter->search_ctx == NULL) - return -1; - - if (!mailbox_search_next(iter->search_ctx, &mail)) { - if (iter_local_mailbox_next_expunge(iter, iter->prev_uid, - msg_r)) { - *mailbox_idx_r = iter->mailbox_idx; - return 1; - } - iter_local_mailbox_close(iter); - iter->mailbox_idx++; - if (iter_local_mailbox_open(iter) < 0) - return -1; - return local_worker_msg_iter_next(_iter, mailbox_idx_r, msg_r); - } - *mailbox_idx_r = iter->mailbox_idx; - iter->prev_uid = mail->uid; - - if (mail_get_special(mail, MAIL_FETCH_GUID, &guid) < 0) { - if (!mail->expunged) { - i_error("msg guid lookup failed: %s", - mailbox_get_last_error(mail->box, NULL)); - _iter->failed = TRUE; - return -1; - } - return local_worker_msg_iter_next(_iter, mailbox_idx_r, msg_r); - } - - memset(msg_r, 0, sizeof(*msg_r)); - msg_r->guid = guid; - msg_r->uid = mail->uid; - msg_r->flags = mail_get_flags(mail); - msg_r->keywords = mail_get_keywords(mail); - msg_r->modseq = mail_get_modseq(mail); - if (mail_get_save_date(mail, &msg_r->save_date) < 0) - msg_r->save_date = (time_t)-1; - return 1; -} - -static int -local_worker_msg_iter_deinit(struct dsync_worker_msg_iter *_iter) -{ - struct local_dsync_worker_msg_iter *iter = - (struct local_dsync_worker_msg_iter *)_iter; - int ret = _iter->failed ? -1 : 0; - - if (iter->box != NULL) - iter_local_mailbox_close(iter); - array_free(&iter->expunges); - str_free(&iter->tmp_guid_str); - i_free(iter->mailboxes); - i_free(iter); - return ret; -} - -static void -local_worker_copy_mailbox_update(const struct dsync_mailbox *dsync_box, - struct mailbox_update *update_r) -{ - memset(update_r, 0, sizeof(*update_r)); - memcpy(update_r->mailbox_guid, dsync_box->mailbox_guid.guid, - sizeof(update_r->mailbox_guid)); - update_r->uid_validity = dsync_box->uid_validity; - update_r->min_next_uid = dsync_box->uid_next; - update_r->min_first_recent_uid = dsync_box->first_recent_uid; - update_r->min_highest_modseq = dsync_box->highest_modseq; -} - -static const char * -mailbox_name_convert(struct local_dsync_worker *worker, - const char *name, char src_sep, char dest_sep) -{ - char *dest_name, *p; - - dest_name = t_strdup_noconst(name); - for (p = dest_name; *p != '\0'; p++) { - if (*p == dest_sep && worker->alt_char != '\0') - *p = worker->alt_char; - else if (*p == src_sep) - *p = dest_sep; - } - return dest_name; -} - -static const char * -mailbox_name_cleanup(const char *input, char real_sep, char alt_char) -{ - char *output, *p; - - output = t_strdup_noconst(input); - for (p = output; *p != '\0'; p++) { - if (*p == real_sep || (uint8_t)*input < 32 || - (uint8_t)*input >= 0x80) - *p = alt_char; - } - return output; -} - -static const char *mailbox_name_force_cleanup(const char *input, char alt_char) -{ - char *output, *p; - - output = t_strdup_noconst(input); - for (p = output; *p != '\0'; p++) { - if (!i_isalnum(*p)) - *p = alt_char; - } - return output; -} - -static const char * -local_worker_convert_mailbox_name(struct local_dsync_worker *worker, - const char *vname, struct mail_namespace *ns, - const struct dsync_mailbox *dsync_box, - bool creating) -{ - const char *name = vname; - char list_sep, ns_sep = mail_namespace_get_sep(ns); - - if (dsync_box->name_sep != ns_sep) { - /* mailbox names use different separators. convert them. */ - name = mailbox_name_convert(worker, name, - dsync_box->name_sep, ns_sep); - } - name = mailbox_list_get_storage_name(ns->list, name); - - if (creating) { - list_sep = mailbox_list_get_hierarchy_sep(ns->list); - if (!mailbox_list_is_valid_create_name(ns->list, name)) { - /* change any real separators to alt separators, - drop any potentially invalid characters */ - name = mailbox_name_cleanup(name, list_sep, - worker->alt_char); - } - if (!mailbox_list_is_valid_create_name(ns->list, name)) { - /* still not working, apparently it's not valid mUTF-7. - just drop all non-alphanumeric characters. */ - name = mailbox_name_force_cleanup(name, - worker->alt_char); - } - if (!mailbox_list_is_valid_create_name(ns->list, name)) { - /* probably some reserved name (e.g. dbox-Mails) */ - name = t_strconcat("_", name, NULL); - } - if (!mailbox_list_is_valid_create_name(ns->list, name)) { - /* name is too long? just give up and generate a - unique name */ - guid_128_t guid; - - guid_128_generate(guid); - name = guid_128_to_string(guid); - } - i_assert(mailbox_list_is_valid_create_name(ns->list, name)); - } - return mailbox_list_get_vname(ns->list, name); -} - -static struct mailbox * -local_worker_mailbox_alloc(struct local_dsync_worker *worker, - const struct dsync_mailbox *dsync_box, bool creating) -{ - struct mail_namespace *ns; - struct local_dsync_mailbox *lbox; - const char *name; - - lbox = dsync_mailbox_is_noselect(dsync_box) ? NULL : - hash_table_lookup(worker->mailbox_hash, - &dsync_box->mailbox_guid); - if (lbox != NULL) { - /* use the existing known mailbox name */ - return mailbox_alloc(lbox->ns->list, lbox->name, 0); - } - - ns = mail_namespace_find(worker->user->namespaces, dsync_box->name); - if (ns == NULL) { - i_error("Can't find namespace for mailbox %s", dsync_box->name); - return NULL; - } - - name = local_worker_convert_mailbox_name(worker, dsync_box->name, ns, - dsync_box, creating); - if (!dsync_mailbox_is_noselect(dsync_box)) { - local_dsync_worker_add_mailbox(worker, ns, name, - &dsync_box->mailbox_guid); - } - return mailbox_alloc(ns->list, name, 0); -} - -static int local_worker_create_dir(struct mailbox *box, - const struct dsync_mailbox *dsync_box) -{ - struct mailbox_list *list = mailbox_get_namespace(box)->list; - const char *errstr; - enum mail_error error; - - if (mailbox_list_create_dir(list, mailbox_get_name(box)) == 0) - return 0; - - errstr = mailbox_list_get_last_error(list, &error); - switch (error) { - case MAIL_ERROR_EXISTS: - /* directory already exists - that's ok */ - return 0; - case MAIL_ERROR_NOTPOSSIBLE: - /* \noselect mailboxes not supported - just ignore them - (we don't want to create a selectable mailbox if the other - side of the sync doesn't support dual-use mailboxes, - e.g. mbox) */ - return 0; - default: - i_error("Can't create mailbox %s: %s", dsync_box->name, errstr); - return -1; - } -} - -static int -local_worker_create_allocated_mailbox(struct local_dsync_worker *worker, - struct mailbox *box, - const struct dsync_mailbox *dsync_box) -{ - struct mailbox_update update; - const char *errstr; - enum mail_error error; - - local_worker_copy_mailbox_update(dsync_box, &update); - - if (dsync_mailbox_is_noselect(dsync_box)) { - if (local_worker_create_dir(box, dsync_box) < 0) { - dsync_worker_set_failure(&worker->worker); - return -1; - } - return 1; - } - - if (mailbox_create(box, &update, FALSE) < 0) { - errstr = mailbox_get_last_error(box, &error); - if (error == MAIL_ERROR_EXISTS) { - /* mailbox already exists */ - return 0; - } - - dsync_worker_set_failure(&worker->worker); - i_error("Can't create mailbox %s: %s", dsync_box->name, errstr); - return -1; - } - - local_dsync_worker_add_mailbox(worker, - mailbox_get_namespace(box), - mailbox_get_vname(box), - &dsync_box->mailbox_guid); - return 1; -} - -static void -local_worker_create_mailbox(struct dsync_worker *_worker, - const struct dsync_mailbox *dsync_box) -{ - struct local_dsync_worker *worker = - (struct local_dsync_worker *)_worker; - struct mailbox *box; - struct mail_namespace *ns; - const char *new_name; - int ret; - - box = local_worker_mailbox_alloc(worker, dsync_box, TRUE); - if (box == NULL) { - dsync_worker_set_failure(_worker); - return; - } - - ret = local_worker_create_allocated_mailbox(worker, box, dsync_box); - if (ret != 0) { - mailbox_free(&box); - return; - } - - /* mailbox name already exists. add mailbox guid to the name, - that shouldn't exist. */ - new_name = t_strconcat(mailbox_get_vname(box), "_", - dsync_guid_to_str(&dsync_box->mailbox_guid), - NULL); - ns = mailbox_get_namespace(box); - mailbox_free(&box); - - local_dsync_worker_add_mailbox(worker, ns, new_name, - &dsync_box->mailbox_guid); - box = mailbox_alloc(ns->list, new_name, 0); - (void)local_worker_create_allocated_mailbox(worker, box, dsync_box); - mailbox_free(&box); -} - -static void -local_worker_delete_mailbox(struct dsync_worker *_worker, - const struct dsync_mailbox *dsync_box) -{ - struct local_dsync_worker *worker = - (struct local_dsync_worker *)_worker; - struct local_dsync_mailbox *lbox; - const mailbox_guid_t *mailbox = &dsync_box->mailbox_guid; - struct mailbox *box; - - lbox = hash_table_lookup(worker->mailbox_hash, mailbox); - if (lbox == NULL) { - i_error("Trying to delete a non-listed mailbox with guid=%s", - dsync_guid_to_str(mailbox)); - dsync_worker_set_failure(_worker); - return; - } - - mailbox_list_set_changelog_timestamp(lbox->ns->list, - dsync_box->last_change); - box = mailbox_alloc(lbox->ns->list, lbox->name, 0); - if (mailbox_delete(box) < 0) { - i_error("Can't delete mailbox %s: %s", lbox->name, - mailbox_get_last_error(box, NULL)); - dsync_worker_set_failure(_worker); - } else { - lbox->deleted = TRUE; - } - mailbox_free(&box); - mailbox_list_set_changelog_timestamp(lbox->ns->list, (time_t)-1); -} - -static void -local_worker_delete_dir(struct dsync_worker *_worker, - const struct dsync_mailbox *dsync_box) -{ - struct local_dsync_worker *worker = - (struct local_dsync_worker *)_worker; - struct mail_namespace *ns; - const char *storage_name; - enum mail_error error; - - ns = mail_namespace_find(worker->user->namespaces, dsync_box->name); - storage_name = mailbox_list_get_storage_name(ns->list, dsync_box->name); - - mailbox_list_set_changelog_timestamp(ns->list, dsync_box->last_change); - if (mailbox_list_delete_dir(ns->list, storage_name) < 0) { - (void)mailbox_list_get_last_error(ns->list, &error); - if (error == MAIL_ERROR_EXISTS) { - /* we're probably doing Maildir++ -> FS layout sync, - where a nonexistent Maildir++ mailbox had to be - created as \Noselect FS directory. - just ignore this. */ - } else { - i_error("Can't delete mailbox directory %s: %s", - dsync_box->name, - mailbox_list_get_last_error(ns->list, NULL)); - } - } - mailbox_list_set_changelog_timestamp(ns->list, (time_t)-1); -} - -static void -local_worker_rename_mailbox(struct dsync_worker *_worker, - const mailbox_guid_t *mailbox, - const struct dsync_mailbox *dsync_box) -{ - struct local_dsync_worker *worker = - (struct local_dsync_worker *)_worker; - struct local_dsync_mailbox *lbox; - struct mailbox_list *list; - struct mailbox *old_box, *new_box; - const char *newname; - - lbox = hash_table_lookup(worker->mailbox_hash, mailbox); - if (lbox == NULL) { - i_error("Trying to rename a non-listed mailbox with guid=%s", - dsync_guid_to_str(mailbox)); - dsync_worker_set_failure(_worker); - return; - } - - list = lbox->ns->list; - newname = local_worker_convert_mailbox_name(worker, dsync_box->name, - lbox->ns, dsync_box, TRUE); - if (strcmp(lbox->name, newname) == 0) { - /* nothing changed after all. probably because some characters - in mailbox name weren't valid. */ - return; - } - - mailbox_list_set_changelog_timestamp(list, dsync_box->last_change); - old_box = mailbox_alloc(list, lbox->name, 0); - new_box = mailbox_alloc(list, newname, 0); - if (mailbox_rename(old_box, new_box, FALSE) < 0) { - i_error("Can't rename mailbox %s to %s: %s", lbox->name, - newname, mailbox_get_last_error(old_box, NULL)); - dsync_worker_set_failure(_worker); - } else { - lbox->name = p_strdup(worker->pool, newname); - } - mailbox_free(&old_box); - mailbox_free(&new_box); - mailbox_list_set_changelog_timestamp(list, (time_t)-1); -} - -static bool -has_expected_save_uids(struct local_dsync_worker *worker, - const struct mail_transaction_commit_changes *changes) -{ - struct seq_range_iter iter; - const uint32_t *expected_uids; - uint32_t uid; - unsigned int i, n, expected_count; - - expected_uids = array_get(&worker->saved_uids, &expected_count); - seq_range_array_iter_init(&iter, &changes->saved_uids); i = n = 0; - while (seq_range_array_iter_nth(&iter, n++, &uid)) { - if (i == expected_count || uid != expected_uids[i++]) - return FALSE; - } - return i == expected_count; -} - -static void local_worker_mailbox_close(struct local_dsync_worker *worker) -{ - struct mailbox_transaction_context *trans, *ext_trans; - struct mail_transaction_commit_changes changes; - - i_assert(worker->save_input == NULL); - - memset(&worker->selected_box_guid, 0, - sizeof(worker->selected_box_guid)); - - if (worker->selected_box == NULL) - return; - - trans = worker->mail->transaction; - ext_trans = worker->ext_mail->transaction; - mail_free(&worker->mail); - mail_free(&worker->ext_mail); - - /* all saves and copies go to ext_trans */ - if (mailbox_transaction_commit_get_changes(&ext_trans, &changes) < 0) - dsync_worker_set_failure(&worker->worker); - else { - if (changes.ignored_modseq_changes != 0) { - if (worker->worker.verbose) { - i_info("%s: Ignored %u modseq changes", - mailbox_get_vname(worker->selected_box), - changes.ignored_modseq_changes); - } - worker->worker.unexpected_changes = TRUE; - } - if (!has_expected_save_uids(worker, &changes)) { - if (worker->worker.verbose) { - i_info("%s: Couldn't keep all uids", - mailbox_get_vname(worker->selected_box)); - } - worker->worker.unexpected_changes = TRUE; - } - pool_unref(&changes.pool); - } - array_clear(&worker->saved_uids); - - if (mailbox_transaction_commit(&trans) < 0 || - mailbox_sync(worker->selected_box, MAILBOX_SYNC_FLAG_FULL_WRITE) < 0) - dsync_worker_set_failure(&worker->worker); - - mailbox_free(&worker->selected_box); -} - -static void -local_worker_update_mailbox(struct dsync_worker *_worker, - const struct dsync_mailbox *dsync_box) -{ - struct local_dsync_worker *worker = - (struct local_dsync_worker *)_worker; - struct mailbox *box; - struct mailbox_update update; - bool selected = FALSE; - - /* if we're updating a selected mailbox, close it first so that all - pending changes get committed. */ - selected = worker->selected_box != NULL && - dsync_guid_equals(&dsync_box->mailbox_guid, - &worker->selected_box_guid); - if (selected) - local_worker_mailbox_close(worker); - - box = local_worker_mailbox_alloc(worker, dsync_box, FALSE); - if (box == NULL) { - dsync_worker_set_failure(_worker); - return; - } - - local_worker_copy_mailbox_update(dsync_box, &update); - if (mailbox_update(box, &update) < 0) { - dsync_worker_set_failure(_worker); - i_error("Can't update mailbox %s: %s", dsync_box->name, - mailbox_get_last_error(box, NULL)); - } - mailbox_free(&box); - - if (selected) - dsync_worker_select_mailbox(_worker, dsync_box); -} - -static void -local_worker_set_cache_fields(struct local_dsync_worker *worker, - const ARRAY_TYPE(mailbox_cache_field) *cache_fields) -{ - struct mailbox_update update; - const struct mailbox_cache_field *fields; - struct mailbox_cache_field *new_fields; - unsigned int count; - - fields = array_get(cache_fields, &count); - new_fields = t_new(struct mailbox_cache_field, count + 1); - memcpy(new_fields, fields, sizeof(struct mailbox_cache_field) * count); - - memset(&update, 0, sizeof(update)); - update.cache_updates = new_fields; - mailbox_update(worker->selected_box, &update); -} - -static void -local_worker_select_mailbox(struct dsync_worker *_worker, - const mailbox_guid_t *mailbox, - const ARRAY_TYPE(mailbox_cache_field) *cache_fields) -{ - struct local_dsync_worker *worker = - (struct local_dsync_worker *)_worker; - struct mailbox_transaction_context *trans, *ext_trans; - - if (dsync_guid_equals(&worker->selected_box_guid, mailbox)) { - /* already selected or previous select failed */ - i_assert(worker->selected_box != NULL || _worker->failed); - return; - } - if (worker->selected_box != NULL) - local_worker_mailbox_close(worker); - worker->selected_box_guid = *mailbox; - - if (local_mailbox_open(worker, mailbox, &worker->selected_box) <= 0) { - dsync_worker_set_failure(_worker); - return; - } - if (cache_fields != NULL && array_is_created(cache_fields)) - local_worker_set_cache_fields(worker, cache_fields); - - ext_trans = mailbox_transaction_begin(worker->selected_box, - MAILBOX_TRANSACTION_FLAG_EXTERNAL | - MAILBOX_TRANSACTION_FLAG_ASSIGN_UIDS); - trans = mailbox_transaction_begin(worker->selected_box, 0); - worker->mail = mail_alloc(trans, 0, NULL); - worker->ext_mail = mail_alloc(ext_trans, 0, NULL); -} - -static void -local_worker_msg_update_metadata(struct dsync_worker *_worker, - const struct dsync_message *msg) -{ - struct local_dsync_worker *worker = - (struct local_dsync_worker *)_worker; - struct mail_keywords *keywords; - - if (msg->modseq > 1) { - (void)mailbox_enable(worker->mail->box, - MAILBOX_FEATURE_CONDSTORE); - } - - if (!mail_set_uid(worker->mail, msg->uid)) - dsync_worker_set_failure(_worker); - else { - mail_update_flags(worker->mail, MODIFY_REPLACE, msg->flags); - - keywords = mailbox_keywords_create_valid(worker->mail->box, - msg->keywords); - mail_update_keywords(worker->mail, MODIFY_REPLACE, keywords); - mailbox_keywords_unref(&keywords); - mail_update_modseq(worker->mail, msg->modseq); - } -} - -static void -local_worker_msg_update_uid(struct dsync_worker *_worker, - uint32_t old_uid, uint32_t new_uid) -{ - struct local_dsync_worker *worker = - (struct local_dsync_worker *)_worker; - struct mail_save_context *save_ctx; - - if (!mail_set_uid(worker->ext_mail, old_uid)) { - dsync_worker_set_failure(_worker); - return; - } - - save_ctx = mailbox_save_alloc(worker->ext_mail->transaction); - mailbox_save_copy_flags(save_ctx, worker->ext_mail); - mailbox_save_set_uid(save_ctx, new_uid); - if (mailbox_copy(&save_ctx, worker->ext_mail) == 0) - mail_expunge(worker->ext_mail); -} - -static void local_worker_msg_expunge(struct dsync_worker *_worker, uint32_t uid) -{ - struct local_dsync_worker *worker = - (struct local_dsync_worker *)_worker; - - if (mail_set_uid(worker->mail, uid)) - mail_expunge(worker->mail); -} - -static void -local_worker_msg_save_set_metadata(struct local_dsync_worker *worker, - struct mailbox *box, - struct mail_save_context *save_ctx, - const struct dsync_message *msg) -{ - struct mail_keywords *keywords; - - i_assert(msg->uid != 0); - - if (msg->modseq > 1) - (void)mailbox_enable(box, MAILBOX_FEATURE_CONDSTORE); - - keywords = str_array_length(msg->keywords) == 0 ? NULL : - mailbox_keywords_create_valid(box, msg->keywords); - mailbox_save_set_flags(save_ctx, msg->flags, keywords); - if (keywords != NULL) - mailbox_keywords_unref(&keywords); - mailbox_save_set_uid(save_ctx, msg->uid); - mailbox_save_set_save_date(save_ctx, msg->save_date); - mailbox_save_set_min_modseq(save_ctx, msg->modseq); - - array_append(&worker->saved_uids, &msg->uid, 1); -} - -static void -local_worker_msg_copy(struct dsync_worker *_worker, - const mailbox_guid_t *src_mailbox, uint32_t src_uid, - const struct dsync_message *dest_msg, - dsync_worker_copy_callback_t *callback, void *context) -{ - struct local_dsync_worker *worker = - (struct local_dsync_worker *)_worker; - struct mailbox *src_box; - struct mailbox_transaction_context *src_trans; - struct mail *src_mail; - struct mail_save_context *save_ctx; - int ret; - - if (local_mailbox_open(worker, src_mailbox, &src_box) <= 0) { - callback(FALSE, context); - return; - } - - src_trans = mailbox_transaction_begin(src_box, 0); - src_mail = mail_alloc(src_trans, 0, NULL); - if (!mail_set_uid(src_mail, src_uid)) - ret = -1; - else { - save_ctx = mailbox_save_alloc(worker->ext_mail->transaction); - local_worker_msg_save_set_metadata(worker, worker->mail->box, - save_ctx, dest_msg); - ret = mailbox_copy(&save_ctx, src_mail); - } - - mail_free(&src_mail); - (void)mailbox_transaction_commit(&src_trans); - mailbox_free(&src_box); - - callback(ret == 0, context); -} - -static void dsync_worker_try_finish(struct local_dsync_worker *worker) -{ - if (worker->finish_callback == NULL) - return; - if (worker->save_io != NULL || worker->reading_mail) - return; - - i_assert(worker->finishing); - i_assert(!worker->finished); - worker->finishing = FALSE; - worker->finished = TRUE; - worker->finish_callback(!worker->worker.failed, worker->finish_context); -} - -static void -local_worker_save_msg_continue(struct local_dsync_worker *worker) -{ - struct mailbox *dest_box = worker->ext_mail->box; - dsync_worker_save_callback_t *callback; - ssize_t ret; - bool save_failed = FALSE; - - while ((ret = i_stream_read(worker->save_input)) > 0 || ret == -2) { - if (mailbox_save_continue(worker->save_ctx) < 0) { - save_failed = TRUE; - ret = -1; - break; - } - } - if (ret == 0) { - if (worker->save_io != NULL) - return; - worker->save_io = - io_add(i_stream_get_fd(worker->save_input), IO_READ, - local_worker_save_msg_continue, worker); - return; - } - i_assert(ret == -1); - - /* drop save_io before destroying save_input, so that save_input's - destroy callback can add io back to its fd. */ - if (worker->save_io != NULL) - io_remove(&worker->save_io); - if (worker->save_input->stream_errno != 0) { - errno = worker->save_input->stream_errno; - i_error("read(msg input) failed: %m"); - mailbox_save_cancel(&worker->save_ctx); - dsync_worker_set_failure(&worker->worker); - } else if (save_failed) { - mailbox_save_cancel(&worker->save_ctx); - dsync_worker_set_failure(&worker->worker); - } else { - i_assert(worker->save_input->eof); - if (mailbox_save_finish(&worker->save_ctx) < 0) { - i_error("Can't save message to mailbox %s: %s", - mailbox_get_vname(dest_box), - mailbox_get_last_error(dest_box, NULL)); - dsync_worker_set_failure(&worker->worker); - } - } - callback = worker->save_callback; - worker->save_callback = NULL; - i_stream_unref(&worker->save_input); - - dsync_worker_try_finish(worker); - /* call the callback last, since it could call worker code again and - cause problems (e.g. if _try_finish() is called again, it could - cause a duplicate finish_callback()) */ - callback(worker->save_context); -} - -static void -local_worker_msg_save(struct dsync_worker *_worker, - const struct dsync_message *msg, - const struct dsync_msg_static_data *data, - dsync_worker_save_callback_t *callback, - void *context) -{ - struct local_dsync_worker *worker = - (struct local_dsync_worker *)_worker; - struct mailbox *dest_box = worker->ext_mail->box; - struct mail_save_context *save_ctx; - - i_assert(worker->save_input == NULL); - - save_ctx = mailbox_save_alloc(worker->ext_mail->transaction); - if (*msg->guid != '\0') - mailbox_save_set_guid(save_ctx, msg->guid); - local_worker_msg_save_set_metadata(worker, worker->mail->box, - save_ctx, msg); - if (*data->pop3_uidl != '\0') - mailbox_save_set_pop3_uidl(save_ctx, data->pop3_uidl); - if (data->pop3_order > 0) - mailbox_save_set_pop3_order(save_ctx, data->pop3_order); - - mailbox_save_set_received_date(save_ctx, data->received_date, 0); - - if (mailbox_save_begin(&save_ctx, data->input) < 0) { - i_error("Can't save message to mailbox %s: %s", - mailbox_get_vname(dest_box), - mailbox_get_last_error(dest_box, NULL)); - dsync_worker_set_failure(_worker); - callback(context); - return; - } - - worker->save_callback = callback; - worker->save_context = context; - worker->save_input = data->input; - worker->save_ctx = save_ctx; - i_stream_ref(worker->save_input); - local_worker_save_msg_continue(worker); -} - -static void local_worker_msg_save_cancel(struct dsync_worker *_worker) -{ - struct local_dsync_worker *worker = - (struct local_dsync_worker *)_worker; - - if (worker->save_input == NULL) - return; - - if (worker->save_io != NULL) - io_remove(&worker->save_io); - mailbox_save_cancel(&worker->save_ctx); - i_stream_unref(&worker->save_input); -} - -static void local_worker_msg_get_done(struct local_dsync_worker *worker) -{ - const struct local_dsync_worker_msg_get *gets; - struct local_dsync_worker_msg_get get; - unsigned int count; - - worker->reading_mail = FALSE; - - gets = array_get(&worker->msg_get_queue, &count); - if (count == 0) - dsync_worker_try_finish(worker); - else { - get = gets[0]; - array_delete(&worker->msg_get_queue, 0, 1); - local_worker_msg_get_next(worker, &get); - } -} - -static void local_worker_msg_box_close(struct local_dsync_worker *worker) -{ - struct mailbox_transaction_context *trans; - struct mailbox *box; - - if (worker->get_mail == NULL) - return; - - box = worker->get_mail->box; - trans = worker->get_mail->transaction; - - mail_free(&worker->get_mail); - (void)mailbox_transaction_commit(&trans); - mailbox_free(&box); - memset(&worker->get_mailbox, 0, sizeof(worker->get_mailbox)); -} - -static void -local_worker_msg_get_next(struct local_dsync_worker *worker, - const struct local_dsync_worker_msg_get *get) -{ - struct dsync_msg_static_data data; - struct mailbox_transaction_context *trans; - struct mailbox *box; - const char *value; - - i_assert(!worker->reading_mail); - - if (!dsync_guid_equals(&worker->get_mailbox, &get->mailbox)) { - local_worker_msg_box_close(worker); - if (local_mailbox_open(worker, &get->mailbox, &box) <= 0) { - get->callback(DSYNC_MSG_GET_RESULT_FAILED, - NULL, get->context); - return; - } - worker->get_mailbox = get->mailbox; - - trans = mailbox_transaction_begin(box, 0); - worker->get_mail = mail_alloc(trans, MAIL_FETCH_UIDL_BACKEND | - MAIL_FETCH_RECEIVED_DATE | - MAIL_FETCH_STREAM_HEADER | - MAIL_FETCH_STREAM_BODY, NULL); - } - - if (!mail_set_uid(worker->get_mail, get->uid)) { - get->callback(DSYNC_MSG_GET_RESULT_EXPUNGED, - NULL, get->context); - return; - } - - memset(&data, 0, sizeof(data)); - if (mail_get_special(worker->get_mail, MAIL_FETCH_UIDL_BACKEND, - &data.pop3_uidl) < 0) - data.pop3_uidl = ""; - else - data.pop3_uidl = t_strdup(data.pop3_uidl); - if (mail_get_special(worker->get_mail, MAIL_FETCH_POP3_ORDER, &value) < 0 || - str_to_uint(value, &data.pop3_order) < 0) - data.pop3_order = 0; - if (mail_get_received_date(worker->get_mail, &data.received_date) < 0 || - mail_get_stream(worker->get_mail, NULL, NULL, &data.input) < 0) { - get->callback(worker->get_mail->expunged ? - DSYNC_MSG_GET_RESULT_EXPUNGED : - DSYNC_MSG_GET_RESULT_FAILED, NULL, get->context); - } else { - worker->reading_mail = TRUE; - data.input = i_stream_create_limit(data.input, (uoff_t)-1); - i_stream_set_destroy_callback(data.input, - local_worker_msg_get_done, - worker); - get->callback(DSYNC_MSG_GET_RESULT_SUCCESS, - &data, get->context); - } -} - -static void -local_worker_msg_get(struct dsync_worker *_worker, - const mailbox_guid_t *mailbox, uint32_t uid, - dsync_worker_msg_callback_t *callback, void *context) -{ - struct local_dsync_worker *worker = - (struct local_dsync_worker *)_worker; - struct local_dsync_worker_msg_get get; - - memset(&get, 0, sizeof(get)); - get.mailbox = *mailbox; - get.uid = uid; - get.callback = callback; - get.context = context; - - if (!worker->reading_mail) - local_worker_msg_get_next(worker, &get); - else - array_append(&worker->msg_get_queue, &get, 1); -} - -static void -local_worker_finish(struct dsync_worker *_worker, - dsync_worker_finish_callback_t *callback, void *context) -{ - struct local_dsync_worker *worker = - (struct local_dsync_worker *)_worker; - - i_assert(!worker->finishing); - - worker->finishing = TRUE; - worker->finished = FALSE; - worker->finish_callback = callback; - worker->finish_context = context; - - dsync_worker_try_finish(worker); -} - -struct dsync_worker_vfuncs local_dsync_worker = { - local_worker_deinit, - - local_worker_is_output_full, - local_worker_output_flush, - - local_worker_mailbox_iter_init, - local_worker_mailbox_iter_next, - local_worker_mailbox_iter_deinit, - - local_worker_subs_iter_init, - local_worker_subs_iter_next, - local_worker_subs_iter_next_un, - local_worker_subs_iter_deinit, - local_worker_set_subscribed, - - local_worker_msg_iter_init, - local_worker_msg_iter_next, - local_worker_msg_iter_deinit, - - local_worker_create_mailbox, - local_worker_delete_mailbox, - local_worker_delete_dir, - local_worker_rename_mailbox, - local_worker_update_mailbox, - - local_worker_select_mailbox, - local_worker_msg_update_metadata, - local_worker_msg_update_uid, - local_worker_msg_expunge, - local_worker_msg_copy, - local_worker_msg_save, - local_worker_msg_save_cancel, - local_worker_msg_get, - local_worker_finish -};
--- a/src/doveadm/dsync/dsync-worker-private.h Fri May 04 05:35:36 2012 +0300 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,105 +0,0 @@ -#ifndef DSYNC_WORKER_PRIVATE_H -#define DSYNC_WORKER_PRIVATE_H - -#include "dsync-worker.h" - -struct mail_user; - -struct dsync_worker_vfuncs { - void (*deinit)(struct dsync_worker *); - - bool (*is_output_full)(struct dsync_worker *worker); - int (*output_flush)(struct dsync_worker *worker); - - struct dsync_worker_mailbox_iter * - (*mailbox_iter_init)(struct dsync_worker *worker); - int (*mailbox_iter_next)(struct dsync_worker_mailbox_iter *iter, - struct dsync_mailbox *dsync_box_r); - int (*mailbox_iter_deinit)(struct dsync_worker_mailbox_iter *iter); - - struct dsync_worker_subs_iter * - (*subs_iter_init)(struct dsync_worker *worker); - int (*subs_iter_next)(struct dsync_worker_subs_iter *iter, - struct dsync_worker_subscription *rec_r); - int (*subs_iter_next_un)(struct dsync_worker_subs_iter *iter, - struct dsync_worker_unsubscription *rec_r); - int (*subs_iter_deinit)(struct dsync_worker_subs_iter *iter); - void (*set_subscribed)(struct dsync_worker *worker, - const char *name, time_t last_change, bool set); - - struct dsync_worker_msg_iter * - (*msg_iter_init)(struct dsync_worker *worker, - const mailbox_guid_t mailboxes[], - unsigned int mailbox_count); - int (*msg_iter_next)(struct dsync_worker_msg_iter *iter, - unsigned int *mailbox_idx_r, - struct dsync_message *msg_r); - int (*msg_iter_deinit)(struct dsync_worker_msg_iter *iter); - - void (*create_mailbox)(struct dsync_worker *worker, - const struct dsync_mailbox *dsync_box); - void (*delete_mailbox)(struct dsync_worker *worker, - const struct dsync_mailbox *dsync_box); - void (*delete_dir)(struct dsync_worker *worker, - const struct dsync_mailbox *dsync_box); - void (*rename_mailbox)(struct dsync_worker *worker, - const mailbox_guid_t *mailbox, - const struct dsync_mailbox *dsync_box); - void (*update_mailbox)(struct dsync_worker *worker, - const struct dsync_mailbox *dsync_box); - - void (*select_mailbox)(struct dsync_worker *worker, - const mailbox_guid_t *mailbox, - const ARRAY_TYPE(mailbox_cache_field) *cache_fields); - void (*msg_update_metadata)(struct dsync_worker *worker, - const struct dsync_message *msg); - void (*msg_update_uid)(struct dsync_worker *worker, - uint32_t old_uid, uint32_t new_uid); - void (*msg_expunge)(struct dsync_worker *worker, uint32_t uid); - void (*msg_copy)(struct dsync_worker *worker, - const mailbox_guid_t *src_mailbox, uint32_t src_uid, - const struct dsync_message *dest_msg, - dsync_worker_copy_callback_t *callback, void *context); - void (*msg_save)(struct dsync_worker *worker, - const struct dsync_message *msg, - const struct dsync_msg_static_data *data, - dsync_worker_save_callback_t *callback, - void *context); - void (*msg_save_cancel)(struct dsync_worker *worker); - void (*msg_get)(struct dsync_worker *worker, - const mailbox_guid_t *mailbox, uint32_t uid, - dsync_worker_msg_callback_t *callback, void *context); - void (*finish)(struct dsync_worker *worker, - dsync_worker_finish_callback_t *callback, void *context); -}; - -struct dsync_worker { - struct dsync_worker_vfuncs v; - - io_callback_t *input_callback, *output_callback; - void *input_context, *output_context; - - unsigned int readonly:1; - unsigned int failed:1; - unsigned int verbose:1; - unsigned int unexpected_changes:1; -}; - -struct dsync_worker_mailbox_iter { - struct dsync_worker *worker; - bool failed; -}; - -struct dsync_worker_subs_iter { - struct dsync_worker *worker; - bool failed; -}; - -struct dsync_worker_msg_iter { - struct dsync_worker *worker; - bool failed; -}; - -void dsync_worker_set_failure(struct dsync_worker *worker); - -#endif
--- a/src/doveadm/dsync/dsync-worker.c Fri May 04 05:35:36 2012 +0300 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,285 +0,0 @@ -/* Copyright (c) 2009-2012 Dovecot authors, see the included COPYING file */ - -#include "lib.h" -#include "istream.h" -#include "dsync-worker-private.h" - -void dsync_worker_deinit(struct dsync_worker **_worker) -{ - struct dsync_worker *worker = *_worker; - - *_worker = NULL; - worker->v.deinit(worker); -} - -void dsync_worker_set_readonly(struct dsync_worker *worker) -{ - worker->readonly = TRUE; -} - -void dsync_worker_set_verbose(struct dsync_worker *worker) -{ - worker->verbose = TRUE; -} - -void dsync_worker_set_input_callback(struct dsync_worker *worker, - io_callback_t *callback, void *context) -{ - worker->input_callback = callback; - worker->input_context = context; -} - -bool dsync_worker_is_output_full(struct dsync_worker *worker) -{ - return worker->v.is_output_full(worker); -} - -void dsync_worker_set_output_callback(struct dsync_worker *worker, - io_callback_t *callback, void *context) -{ - worker->output_callback = callback; - worker->output_context = context; -} - -int dsync_worker_output_flush(struct dsync_worker *worker) -{ - return worker->v.output_flush(worker); -} - -struct dsync_worker_mailbox_iter * -dsync_worker_mailbox_iter_init(struct dsync_worker *worker) -{ - return worker->v.mailbox_iter_init(worker); -} - -int dsync_worker_mailbox_iter_next(struct dsync_worker_mailbox_iter *iter, - struct dsync_mailbox *dsync_box_r) -{ - int ret; - - T_BEGIN { - ret = iter->worker->v.mailbox_iter_next(iter, dsync_box_r); - } T_END; - return ret; -} - -int dsync_worker_mailbox_iter_deinit(struct dsync_worker_mailbox_iter **_iter) -{ - struct dsync_worker_mailbox_iter *iter = *_iter; - - *_iter = NULL; - return iter->worker->v.mailbox_iter_deinit(iter); -} - -struct dsync_worker_subs_iter * -dsync_worker_subs_iter_init(struct dsync_worker *worker) -{ - return worker->v.subs_iter_init(worker); -} - -int dsync_worker_subs_iter_next(struct dsync_worker_subs_iter *iter, - struct dsync_worker_subscription *rec_r) -{ - return iter->worker->v.subs_iter_next(iter, rec_r); -} - -int dsync_worker_subs_iter_next_un(struct dsync_worker_subs_iter *iter, - struct dsync_worker_unsubscription *rec_r) -{ - return iter->worker->v.subs_iter_next_un(iter, rec_r); -} - -int dsync_worker_subs_iter_deinit(struct dsync_worker_subs_iter **_iter) -{ - struct dsync_worker_subs_iter *iter = *_iter; - - *_iter = NULL; - return iter->worker->v.subs_iter_deinit(iter); -} - -void dsync_worker_set_subscribed(struct dsync_worker *worker, - const char *name, time_t last_change, bool set) -{ - worker->v.set_subscribed(worker, name, last_change, set); -} - -struct dsync_worker_msg_iter * -dsync_worker_msg_iter_init(struct dsync_worker *worker, - const mailbox_guid_t mailboxes[], - unsigned int mailbox_count) -{ - i_assert(mailbox_count > 0); - return worker->v.msg_iter_init(worker, mailboxes, mailbox_count); -} - -int dsync_worker_msg_iter_next(struct dsync_worker_msg_iter *iter, - unsigned int *mailbox_idx_r, - struct dsync_message *msg_r) -{ - int ret; - - T_BEGIN { - ret = iter->worker->v.msg_iter_next(iter, mailbox_idx_r, msg_r); - } T_END; - return ret; -} - -int dsync_worker_msg_iter_deinit(struct dsync_worker_msg_iter **_iter) -{ - struct dsync_worker_msg_iter *iter = *_iter; - - *_iter = NULL; - return iter->worker->v.msg_iter_deinit(iter); -} - -void dsync_worker_create_mailbox(struct dsync_worker *worker, - const struct dsync_mailbox *dsync_box) -{ - i_assert(dsync_box->uid_validity != 0 || - dsync_mailbox_is_noselect(dsync_box)); - - if (!worker->readonly) T_BEGIN { - worker->v.create_mailbox(worker, dsync_box); - } T_END; -} - -void dsync_worker_delete_mailbox(struct dsync_worker *worker, - const struct dsync_mailbox *dsync_box) -{ - if (!worker->readonly) T_BEGIN { - worker->v.delete_mailbox(worker, dsync_box); - } T_END; -} - -void dsync_worker_delete_dir(struct dsync_worker *worker, - const struct dsync_mailbox *dsync_box) -{ - if (!worker->readonly) T_BEGIN { - worker->v.delete_dir(worker, dsync_box); - } T_END; -} - -void dsync_worker_rename_mailbox(struct dsync_worker *worker, - const mailbox_guid_t *mailbox, - const struct dsync_mailbox *dsync_box) -{ - if (!worker->readonly) T_BEGIN { - worker->v.rename_mailbox(worker, mailbox, dsync_box); - } T_END; -} - -void dsync_worker_update_mailbox(struct dsync_worker *worker, - const struct dsync_mailbox *dsync_box) -{ - if (!worker->readonly) T_BEGIN { - worker->v.update_mailbox(worker, dsync_box); - } T_END; -} - -void dsync_worker_select_mailbox(struct dsync_worker *worker, - const struct dsync_mailbox *box) -{ - T_BEGIN { - worker->v.select_mailbox(worker, &box->mailbox_guid, - &box->cache_fields); - } T_END; -} - -void dsync_worker_msg_update_metadata(struct dsync_worker *worker, - const struct dsync_message *msg) -{ - if (!worker->failed && !worker->readonly) - worker->v.msg_update_metadata(worker, msg); -} - -void dsync_worker_msg_update_uid(struct dsync_worker *worker, - uint32_t old_uid, uint32_t new_uid) -{ - if (!worker->failed && !worker->readonly) - worker->v.msg_update_uid(worker, old_uid, new_uid); -} - -void dsync_worker_msg_expunge(struct dsync_worker *worker, uint32_t uid) -{ - if (!worker->failed && !worker->readonly) - worker->v.msg_expunge(worker, uid); -} - -void dsync_worker_msg_copy(struct dsync_worker *worker, - const mailbox_guid_t *src_mailbox, uint32_t src_uid, - const struct dsync_message *dest_msg, - dsync_worker_copy_callback_t *callback, - void *context) -{ - if (!worker->failed && !worker->readonly) { - T_BEGIN { - worker->v.msg_copy(worker, src_mailbox, src_uid, - dest_msg, callback, context); - } T_END; - } else { - callback(FALSE, context); - } -} - -void dsync_worker_msg_save(struct dsync_worker *worker, - const struct dsync_message *msg, - const struct dsync_msg_static_data *data, - dsync_worker_save_callback_t *callback, - void *context) -{ - if (!worker->readonly) { - if (worker->failed) - callback(context); - else T_BEGIN { - worker->v.msg_save(worker, msg, data, - callback, context); - } T_END; - } else { - const unsigned char *d; - size_t size; - - while ((i_stream_read_data(data->input, &d, &size, 0)) > 0) - i_stream_skip(data->input, size); - callback(context); - } -} - -void dsync_worker_msg_save_cancel(struct dsync_worker *worker) -{ - worker->v.msg_save_cancel(worker); -} - -void dsync_worker_msg_get(struct dsync_worker *worker, - const mailbox_guid_t *mailbox, uint32_t uid, - dsync_worker_msg_callback_t *callback, void *context) -{ - i_assert(uid != 0); - - if (worker->failed) - callback(DSYNC_MSG_GET_RESULT_FAILED, NULL, context); - else T_BEGIN { - worker->v.msg_get(worker, mailbox, uid, callback, context); - } T_END; -} - -void dsync_worker_finish(struct dsync_worker *worker, - dsync_worker_finish_callback_t *callback, - void *context) -{ - worker->v.finish(worker, callback, context); -} - -void dsync_worker_set_failure(struct dsync_worker *worker) -{ - worker->failed = TRUE; -} - -bool dsync_worker_has_failed(struct dsync_worker *worker) -{ - return worker->failed; -} - -bool dsync_worker_has_unexpected_changes(struct dsync_worker *worker) -{ - return worker->unexpected_changes; -}
--- a/src/doveadm/dsync/dsync-worker.h Fri May 04 05:35:36 2012 +0300 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,165 +0,0 @@ -#ifndef DSYNC_WORKER_H -#define DSYNC_WORKER_H - -#include "ioloop.h" -#include "dsync-data.h" - -enum dsync_msg_get_result { - DSYNC_MSG_GET_RESULT_SUCCESS, - DSYNC_MSG_GET_RESULT_EXPUNGED, - DSYNC_MSG_GET_RESULT_FAILED -}; - -struct dsync_worker_subscription { - const char *vname, *storage_name, *ns_prefix; - time_t last_change; -}; -struct dsync_worker_unsubscription { - /* SHA1 sum of the mailbox's storage name, i.e. without namespace - prefix */ - mailbox_guid_t name_sha1; - const char *ns_prefix; - time_t last_change; -}; - -typedef void dsync_worker_copy_callback_t(bool success, void *context); -typedef void dsync_worker_save_callback_t(void *context); -typedef void dsync_worker_msg_callback_t(enum dsync_msg_get_result result, - const struct dsync_msg_static_data *data, - void *context); -typedef void dsync_worker_finish_callback_t(bool success, void *context); - -struct dsync_worker * -dsync_worker_init_local(struct mail_user *user, const char *namespace_prefix, - char alt_char); -struct dsync_worker *dsync_worker_init_proxy_client(int fd_in, int fd_out); -void dsync_worker_deinit(struct dsync_worker **worker); - -/* Set this worker as read-only. All attempted changes are ignored. */ -void dsync_worker_set_readonly(struct dsync_worker *worker); -/* Log verbosely */ -void dsync_worker_set_verbose(struct dsync_worker *worker); - -/* If any function returns with "waiting for more data", the given callback - gets called when more data is available. */ -void dsync_worker_set_input_callback(struct dsync_worker *worker, - io_callback_t *callback, void *context); - -/* Returns TRUE if command queue is full and caller should stop sending - more commands. */ -bool dsync_worker_is_output_full(struct dsync_worker *worker); -/* The given callback gets called when more commands can be queued. */ -void dsync_worker_set_output_callback(struct dsync_worker *worker, - io_callback_t *callback, void *context); -/* Try to flush command queue. Returns 1 if all flushed, 0 if something is - still in queue, -1 if failed. */ -int dsync_worker_output_flush(struct dsync_worker *worker); - -/* Iterate though all mailboxes */ -struct dsync_worker_mailbox_iter * -dsync_worker_mailbox_iter_init(struct dsync_worker *worker); -/* Get the next available mailbox. Returns 1 if ok, 0 if waiting for more data, - -1 if there are no more mailboxes. */ -int dsync_worker_mailbox_iter_next(struct dsync_worker_mailbox_iter *iter, - struct dsync_mailbox *dsync_box_r); -/* Finish mailbox iteration. Returns 0 if ok, -1 if iteration failed. */ -int dsync_worker_mailbox_iter_deinit(struct dsync_worker_mailbox_iter **iter); - -/* Iterate though all subscriptions */ -struct dsync_worker_subs_iter * -dsync_worker_subs_iter_init(struct dsync_worker *worker); -/* Get the next subscription. Returns 1 if ok, 0 if waiting for more data, - -1 if there are no more subscriptions. */ -int dsync_worker_subs_iter_next(struct dsync_worker_subs_iter *iter, - struct dsync_worker_subscription *rec_r); -/* Like _iter_next(), but list known recent unsubscriptions. */ -int dsync_worker_subs_iter_next_un(struct dsync_worker_subs_iter *iter, - struct dsync_worker_unsubscription *rec_r); -/* Finish subscription iteration. Returns 0 if ok, -1 if iteration failed. */ -int dsync_worker_subs_iter_deinit(struct dsync_worker_subs_iter **iter); -/* Subscribe/unsubscribe mailbox */ -void dsync_worker_set_subscribed(struct dsync_worker *worker, - const char *name, time_t last_change, - bool set); - -/* Iterate through all messages in given mailboxes. The mailboxes are iterated - in the given order. */ -struct dsync_worker_msg_iter * -dsync_worker_msg_iter_init(struct dsync_worker *worker, - const mailbox_guid_t mailboxes[], - unsigned int mailbox_count); -/* Get the next available message. Also returns all expunged messages from - the end of mailbox (if next_uid-1 message exists, nothing is returned). - mailbox_idx_r contains the mailbox's index in mailboxes[] array given - to _iter_init(). Returns 1 if ok, 0 if waiting for more data, -1 if there - are no more messages. */ -int dsync_worker_msg_iter_next(struct dsync_worker_msg_iter *iter, - unsigned int *mailbox_idx_r, - struct dsync_message *msg_r); -/* Finish message iteration. Returns 0 if ok, -1 if iteration failed. */ -int dsync_worker_msg_iter_deinit(struct dsync_worker_msg_iter **iter); - -/* Create mailbox with given name, GUID and UIDVALIDITY. */ -void dsync_worker_create_mailbox(struct dsync_worker *worker, - const struct dsync_mailbox *dsync_box); -/* Delete mailbox/dir with given GUID. */ -void dsync_worker_delete_mailbox(struct dsync_worker *worker, - const struct dsync_mailbox *dsync_box); -/* Delete mailbox's directory. Fail if it would also delete mailbox. */ -void dsync_worker_delete_dir(struct dsync_worker *worker, - const struct dsync_mailbox *dsync_box); -/* Change a mailbox and its childrens' name. The name is taken from the given - dsync_box (applying name_sep if necessary). */ -void dsync_worker_rename_mailbox(struct dsync_worker *worker, - const mailbox_guid_t *mailbox, - const struct dsync_mailbox *dsync_box); -/* Find mailbox with given GUID and make sure its uid_next and - highest_modseq are up to date (but don't shrink them). */ -void dsync_worker_update_mailbox(struct dsync_worker *worker, - const struct dsync_mailbox *dsync_box); - -/* The following message syncing functions access the this selected mailbox. */ -void dsync_worker_select_mailbox(struct dsync_worker *worker, - const struct dsync_mailbox *box); -/* Update message's metadata (flags, keywords, modseq). */ -void dsync_worker_msg_update_metadata(struct dsync_worker *worker, - const struct dsync_message *msg); -/* Change message's UID. */ -void dsync_worker_msg_update_uid(struct dsync_worker *worker, - uint32_t old_uid, uint32_t new_uid); -/* Expunge given message. */ -void dsync_worker_msg_expunge(struct dsync_worker *worker, uint32_t uid); -/* Copy given message. */ -void dsync_worker_msg_copy(struct dsync_worker *worker, - const mailbox_guid_t *src_mailbox, uint32_t src_uid, - const struct dsync_message *dest_msg, - dsync_worker_copy_callback_t *callback, - void *context); -/* Save given message from the given input stream. Once saving is finished, - the given callback is called and the stream is destroyed. */ -void dsync_worker_msg_save(struct dsync_worker *worker, - const struct dsync_message *msg, - const struct dsync_msg_static_data *data, - dsync_worker_save_callback_t *callback, - void *context); -/* Cancel any pending saves */ -void dsync_worker_msg_save_cancel(struct dsync_worker *worker); -/* Get message data for saving. The callback is called once when the static - data has been received. The whole message may not have been downloaded yet, - so the caller must read the input stream until it returns EOF and then - unreference it. */ -void dsync_worker_msg_get(struct dsync_worker *worker, - const mailbox_guid_t *mailbox, uint32_t uid, - dsync_worker_msg_callback_t *callback, void *context); -/* Call the callback once all the pending commands are finished. */ -void dsync_worker_finish(struct dsync_worker *worker, - dsync_worker_finish_callback_t *callback, - void *context); - -/* Returns TRUE if some commands have failed. */ -bool dsync_worker_has_failed(struct dsync_worker *worker); -/* Returns TRUE if some UID or modseq changes didn't get assigned as - requested. */ -bool dsync_worker_has_unexpected_changes(struct dsync_worker *worker); - -#endif
--- a/src/doveadm/dsync/test-dsync-brain-msgs.c Fri May 04 05:35:36 2012 +0300 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,670 +0,0 @@ -/* Copyright (c) 2009-2012 Dovecot authors, see the included COPYING file */ - -#include "lib.h" -#include "array.h" -#include "crc32.h" -#include "dsync-brain-private.h" -#include "test-dsync-worker.h" -#include "test-dsync-common.h" - -enum test_box_add_type { - ADD_SRC, - ADD_DEST, - ADD_BOTH -}; - -struct test_dsync_mailbox { - struct dsync_brain_mailbox box; - ARRAY_DEFINE(src_msgs, struct dsync_message); - ARRAY_DEFINE(dest_msgs, struct dsync_message); -}; -ARRAY_DEFINE_TYPE(test_dsync_mailbox, struct test_dsync_mailbox); - -static ARRAY_TYPE(test_dsync_mailbox) mailboxes; -static struct test_dsync_worker *test_src_worker, *test_dest_worker; - -void dsync_brain_fail(struct dsync_brain *brain ATTR_UNUSED) {} -void dsync_brain_msg_sync_new_msgs(struct dsync_brain_mailbox_sync *sync ATTR_UNUSED) {} - -static struct test_dsync_mailbox *test_box_find(const char *name) -{ - struct test_dsync_mailbox *boxes; - unsigned int i, count; - - boxes = array_get_modifiable(&mailboxes, &count); - for (i = 0; i < count; i++) { - if (strcmp(boxes[i].box.box.name, name) == 0) - return &boxes[i]; - } - return NULL; -} - -static bool -test_box_has_guid(const char *name, const mailbox_guid_t *guid) -{ - const struct test_dsync_mailbox *box; - - box = test_box_find(name); - return box != NULL && - memcmp(box->box.box.mailbox_guid.guid, guid->guid, - sizeof(box->box.box.mailbox_guid.guid)) == 0; -} - -static struct test_dsync_mailbox * -test_box_add(enum test_box_add_type type, const char *name) -{ - struct test_dsync_mailbox *tbox; - struct dsync_mailbox *box; - - tbox = test_box_find(name); - if (tbox == NULL) { - tbox = array_append_space(&mailboxes); - i_array_init(&tbox->src_msgs, 16); - i_array_init(&tbox->dest_msgs, 16); - } - - box = i_new(struct dsync_mailbox, 1); - box->name = i_strdup(name); - - dsync_str_sha_to_guid(t_strconcat("box-", name, NULL), - &box->mailbox_guid); - dsync_str_sha_to_guid(name, &box->name_sha1); - - box->uid_validity = crc32_str(name); - box->highest_modseq = 1; - - switch (type) { - case ADD_SRC: - tbox->box.src = box; - break; - case ADD_DEST: - tbox->box.dest = box; - break; - case ADD_BOTH: - tbox->box.src = box; - tbox->box.dest = box; - break; - } - tbox->box.box.name = box->name; - tbox->box.box.mailbox_guid = box->mailbox_guid; - tbox->box.box.name_sha1 = box->name_sha1; - tbox->box.box.uid_validity = box->uid_validity; - return tbox; -} - -static void test_msg_add(struct test_dsync_mailbox *box, - enum test_box_add_type type, - const char *guid, uint32_t uid) -{ - static int msg_date = 0; - struct dsync_message msg; - - memset(&msg, 0, sizeof(msg)); - msg.guid = i_strdup(guid); - msg.uid = uid; - msg.modseq = ++box->box.box.highest_modseq; - msg.save_date = ++msg_date; - - switch (type) { - case ADD_SRC: - box->box.src->highest_modseq++; - box->box.src->uid_next = uid + 1; - array_append(&box->src_msgs, &msg, 1); - break; - case ADD_DEST: - box->box.dest->highest_modseq++; - box->box.dest->uid_next = uid + 1; - array_append(&box->dest_msgs, &msg, 1); - break; - case ADD_BOTH: - box->box.src->highest_modseq++; - box->box.dest->highest_modseq++; - box->box.src->uid_next = uid + 1; - box->box.dest->uid_next = uid + 1; - array_append(&box->src_msgs, &msg, 1); - array_append(&box->dest_msgs, &msg, 1); - break; - } - if (box->box.box.uid_next <= uid) - box->box.box.uid_next = uid + 1; -} - -static void test_msg_set_modseq(struct test_dsync_mailbox *box, - enum test_box_add_type type, - uint32_t uid, uint64_t modseq) -{ - struct dsync_message *msgs; - unsigned int i, count; - - i_assert(modseq <= box->box.box.highest_modseq); - if (type != ADD_DEST) { - msgs = array_get_modifiable(&box->src_msgs, &count); - for (i = 0; i < count; i++) { - if (msgs[i].uid == uid) { - msgs[i].modseq = modseq; - break; - } - } - i_assert(i < count); - } - if (type != ADD_SRC) { - msgs = array_get_modifiable(&box->dest_msgs, &count); - for (i = 0; i < count; i++) { - if (msgs[i].uid == uid) { - msgs[i].modseq = modseq; - break; - } - } - i_assert(i < count); - } -} - -static void test_msg_set_flags(struct test_dsync_mailbox *box, - enum test_box_add_type type, - uint32_t uid, enum mail_flags flags) -{ - unsigned char guid_128_data[GUID_128_SIZE * 2 + 1]; - struct dsync_message *msgs; - unsigned int i, count; - - box->box.box.highest_modseq++; - if (type != ADD_DEST) { - box->box.src->highest_modseq = box->box.box.highest_modseq; - msgs = array_get_modifiable(&box->src_msgs, &count); - for (i = 0; i < count; i++) { - if (msgs[i].uid == uid) { - if ((flags & DSYNC_MAIL_FLAG_EXPUNGED) != 0) { - msgs[i].guid = i_strdup(dsync_get_guid_128_str(msgs[i].guid, - guid_128_data, sizeof(guid_128_data))); - } - msgs[i].flags = flags; - msgs[i].modseq = box->box.src->highest_modseq; - break; - } - } - i_assert(i < count); - } - if (type != ADD_SRC) { - box->box.dest->highest_modseq = box->box.box.highest_modseq; - msgs = array_get_modifiable(&box->dest_msgs, &count); - for (i = 0; i < count; i++) { - if (msgs[i].uid == uid) { - if ((flags & DSYNC_MAIL_FLAG_EXPUNGED) != 0) { - msgs[i].guid = i_strdup(dsync_get_guid_128_str(msgs[i].guid, - guid_128_data, sizeof(guid_128_data))); - } - msgs[i].flags = flags; - msgs[i].modseq = box->box.dest->highest_modseq; - break; - } - } - i_assert(i < count); - } -} - -static void ATTR_SENTINEL -test_msg_set_keywords(struct test_dsync_mailbox *box, - enum test_box_add_type type, - uint32_t uid, const char *kw, ...) -{ - struct dsync_message *msgs; - unsigned int i, count; - va_list va; - ARRAY_TYPE(const_string) keywords; - - t_array_init(&keywords, 8); - array_append(&keywords, &kw, 1); - va_start(va, kw); - while ((kw = va_arg(va, const char *)) != NULL) - array_append(&keywords, &kw, 1); - va_end(va); - (void)array_append_space(&keywords); - - box->box.box.highest_modseq++; - if (type != ADD_DEST) { - box->box.src->highest_modseq = box->box.box.highest_modseq; - msgs = array_get_modifiable(&box->src_msgs, &count); - for (i = 0; i < count; i++) { - if (msgs[i].uid == uid) { - msgs[i].keywords = array_idx(&keywords, 0); - msgs[i].modseq = box->box.src->highest_modseq; - break; - } - } - i_assert(i < count); - } - if (type != ADD_SRC) { - box->box.dest->highest_modseq = box->box.box.highest_modseq; - msgs = array_get_modifiable(&box->dest_msgs, &count); - for (i = 0; i < count; i++) { - if (msgs[i].uid == uid) { - msgs[i].keywords = array_idx(&keywords, 0); - msgs[i].modseq = box->box.src->highest_modseq; - break; - } - } - i_assert(i < count); - } -} - -static void -test_dsync_sync_msgs(struct test_dsync_worker *worker, bool dest) -{ - const struct test_dsync_mailbox *boxes; - const struct dsync_message *msgs; - struct test_dsync_worker_msg test_msg; - unsigned int i, j, box_count, msg_count; - - boxes = array_get(&mailboxes, &box_count); - for (i = 0; i < box_count; i++) { - msgs = dest ? array_get(&boxes[i].dest_msgs, &msg_count) : - array_get(&boxes[i].src_msgs, &msg_count); - for (j = 0; j < msg_count; j++) { - test_msg.msg = msgs[j]; - test_msg.mailbox_idx = i; - array_append(&worker->msg_iter.msgs, &test_msg, 1); - worker->worker.input_callback(worker->worker.input_context); - } - } - - worker->msg_iter.last = TRUE; - worker->worker.input_callback(worker->worker.input_context); -} - -static struct dsync_brain *test_dsync_brain_init(void) -{ - struct dsync_brain *brain; - - brain = i_new(struct dsync_brain, 1); - brain->src_worker = dsync_worker_init_test(); - brain->dest_worker = dsync_worker_init_test(); - - test_src_worker = (struct test_dsync_worker *)brain->src_worker; - test_dest_worker = (struct test_dsync_worker *)brain->dest_worker; - return brain; -} - -static struct dsync_brain_mailbox_sync * -test_dsync_brain_sync_init(void) -{ - ARRAY_TYPE(dsync_brain_mailbox) brain_boxes; - struct dsync_brain_mailbox_sync *sync; - const struct test_dsync_mailbox *tboxes; - unsigned int i, count; - - tboxes = array_get(&mailboxes, &count); - t_array_init(&brain_boxes, count); - for (i = 0; i < count; i++) - array_append(&brain_boxes, &tboxes[i].box, 1); - - sync = dsync_brain_msg_sync_init(test_dsync_brain_init(), &brain_boxes); - dsync_brain_msg_sync_more(sync); - test_dsync_sync_msgs(test_dest_worker, TRUE); - test_dsync_sync_msgs(test_src_worker, FALSE); - return sync; -} - -static void test_dsync_brain_msg_sync_box_multi(void) -{ - struct test_dsync_mailbox *box; - struct dsync_brain_mailbox_sync *sync; - struct test_dsync_msg_event msg_event; - const struct dsync_brain_new_msg *new_msgs; - unsigned int count; - - /* test that msg syncing finds and syncs all mailboxes */ - test_begin("dsync brain msg sync box multi"); - - i_array_init(&mailboxes, 32); - box = test_box_add(ADD_BOTH, "both"); - test_msg_add(box, ADD_BOTH, "guid1", 1); - test_msg_set_flags(box, ADD_SRC, 1, MAIL_SEEN); - test_msg_set_flags(box, ADD_DEST, 1, MAIL_DRAFT); - test_msg_set_flags(box, ADD_SRC, 1, MAIL_ANSWERED); - box = test_box_add(ADD_SRC, "src"); - test_msg_add(box, ADD_SRC, "guid2", 5); - box = test_box_add(ADD_DEST, "dest"); - test_msg_add(box, ADD_DEST, "guid3", 3); - - sync = test_dsync_brain_sync_init(); - - test_assert(test_dsync_worker_next_msg_event(test_dest_worker, &msg_event)); - test_assert(msg_event.type == LAST_MSG_TYPE_UPDATE); - test_assert(test_box_has_guid("both", &msg_event.mailbox)); - test_assert(msg_event.msg.uid == 1); - test_assert(msg_event.msg.flags == MAIL_ANSWERED); - test_assert(!test_dsync_worker_next_msg_event(test_dest_worker, &msg_event)); - - new_msgs = array_get(&sync->dest_msg_iter->new_msgs, &count); - test_assert(count == 1); - test_assert(new_msgs[0].mailbox_idx == 1); - test_assert(new_msgs[0].msg->uid == 5); - test_assert(strcmp(new_msgs[0].msg->guid, "guid2") == 0); - - new_msgs = array_get(&sync->src_msg_iter->new_msgs, &count); - test_assert(count == 1); - test_assert(new_msgs[0].mailbox_idx == 2); - test_assert(new_msgs[0].msg->uid == 3); - test_assert(strcmp(new_msgs[0].msg->guid, "guid3") == 0); - - test_end(); -} - -static void test_dsync_brain_msg_sync_box(enum test_box_add_type type) -{ - struct test_dsync_mailbox *box; - struct dsync_brain_mailbox_sync *sync; - struct test_dsync_msg_event msg_event; - const struct dsync_brain_new_msg *new_msgs; - unsigned int count; - - i_array_init(&mailboxes, 32); - box = test_box_add(type, "box1"); - test_msg_add(box, type, "guid1", 1); - box = test_box_add(type, "box2"); - test_msg_add(box, type, "guid2", 2); - - sync = test_dsync_brain_sync_init(); - - test_assert(!test_dsync_worker_next_msg_event(test_dest_worker, &msg_event)); - - new_msgs = array_get(type == ADD_DEST ? &sync->src_msg_iter->new_msgs : - &sync->dest_msg_iter->new_msgs, &count); - test_assert(count == 2); - test_assert(new_msgs[0].mailbox_idx == 0); - test_assert(new_msgs[0].msg->uid == 1); - test_assert(strcmp(new_msgs[0].msg->guid, "guid1") == 0); - test_assert(new_msgs[1].mailbox_idx == 1); - test_assert(new_msgs[1].msg->uid == 2); - test_assert(strcmp(new_msgs[1].msg->guid, "guid2") == 0); -} - -static void test_dsync_brain_msg_sync_box_single(void) -{ - test_begin("dsync brain msg sync box src"); - test_dsync_brain_msg_sync_box(ADD_SRC); - test_end(); - - test_begin("dsync brain msg sync box dest"); - test_dsync_brain_msg_sync_box(ADD_DEST); - test_end(); -} - -static void test_dsync_brain_msg_sync_existing(void) -{ - struct test_dsync_mailbox *box; - struct dsync_brain_mailbox_sync *sync; - struct test_dsync_msg_event msg_event; - - test_begin("dsync brain msg sync existing"); - - i_array_init(&mailboxes, 1); - box = test_box_add(ADD_BOTH, "box"); - test_msg_add(box, ADD_BOTH, "guid1", 1); - test_msg_add(box, ADD_BOTH, "guid2", 2); - test_msg_add(box, ADD_BOTH, "guid3", 3); - test_msg_add(box, ADD_BOTH, "guid5", 5); - test_msg_add(box, ADD_BOTH, "guid6", 6); - test_msg_add(box, ADD_BOTH, "guid9", 9); - test_msg_add(box, ADD_BOTH, "guid10", 10); - test_msg_add(box, ADD_BOTH, "guid11", 11); - test_msg_add(box, ADD_BOTH, "guid12", 12); - - /* unchanged */ - test_msg_set_flags(box, ADD_BOTH, 1, MAIL_SEEN); - - /* changed, same modseq - dest has more flags so it will be used */ - test_msg_set_flags(box, ADD_SRC, 2, MAIL_ANSWERED); - test_msg_set_flags(box, ADD_DEST, 2, MAIL_ANSWERED | MAIL_SEEN); - test_msg_set_modseq(box, ADD_BOTH, 2, 2); - - /* changed, same modseq - src has more flags so it will be used */ - test_msg_set_flags(box, ADD_SRC, 3, MAIL_ANSWERED | MAIL_SEEN); - test_msg_set_flags(box, ADD_DEST, 3, MAIL_ANSWERED); - test_msg_set_modseq(box, ADD_BOTH, 3, 3); - - /* changed, dest has higher modseq */ - test_msg_set_flags(box, ADD_BOTH, 5, MAIL_DRAFT); - test_msg_set_flags(box, ADD_DEST, 5, MAIL_FLAGGED); - - /* changed, src has higher modseq */ - test_msg_set_flags(box, ADD_DEST, 6, MAIL_FLAGGED); - test_msg_set_flags(box, ADD_SRC, 6, 0); - - /* keywords changed, src has higher modseq */ - test_msg_set_keywords(box, ADD_SRC, 9, "hello", "world", NULL); - - /* flag/keyword conflict, same modseq - src has more so it - will be used */ - test_msg_set_keywords(box, ADD_SRC, 10, "foo", NULL); - test_msg_set_flags(box, ADD_SRC, 10, MAIL_SEEN); - test_msg_set_flags(box, ADD_DEST, 10, MAIL_DRAFT); - test_msg_set_modseq(box, ADD_BOTH, 10, 5); - - /* flag/keyword conflict, same modseq - dest has more so it - will be used */ - test_msg_set_keywords(box, ADD_DEST, 11, "foo", NULL); - test_msg_set_flags(box, ADD_SRC, 11, MAIL_SEEN); - test_msg_set_flags(box, ADD_DEST, 11, MAIL_DRAFT); - test_msg_set_modseq(box, ADD_BOTH, 11, 5); - - /* flag/keyword conflict, same modseq - both have same number of - flags so src will be used */ - test_msg_set_keywords(box, ADD_SRC, 12, "bar", NULL); - test_msg_set_keywords(box, ADD_DEST, 12, "foo", NULL); - test_msg_set_flags(box, ADD_SRC, 12, MAIL_SEEN); - test_msg_set_flags(box, ADD_DEST, 12, MAIL_DRAFT); - test_msg_set_modseq(box, ADD_BOTH, 12, 5); - - sync = test_dsync_brain_sync_init(); - test_assert(array_count(&sync->src_msg_iter->new_msgs) == 0); - test_assert(array_count(&sync->dest_msg_iter->new_msgs) == 0); - - test_assert(test_dsync_worker_next_msg_event(test_src_worker, &msg_event)); - test_assert(msg_event.type == LAST_MSG_TYPE_UPDATE); - test_assert(msg_event.msg.uid == 2); - test_assert(msg_event.msg.flags == (MAIL_ANSWERED | MAIL_SEEN)); - - test_assert(test_dsync_worker_next_msg_event(test_dest_worker, &msg_event)); - test_assert(msg_event.type == LAST_MSG_TYPE_UPDATE); - test_assert(msg_event.msg.uid == 3); - test_assert(msg_event.msg.flags == (MAIL_ANSWERED | MAIL_SEEN)); - - test_assert(test_dsync_worker_next_msg_event(test_src_worker, &msg_event)); - test_assert(msg_event.type == LAST_MSG_TYPE_UPDATE); - test_assert(msg_event.msg.uid == 5); - test_assert(msg_event.msg.flags == MAIL_FLAGGED); - - test_assert(test_dsync_worker_next_msg_event(test_dest_worker, &msg_event)); - test_assert(msg_event.type == LAST_MSG_TYPE_UPDATE); - test_assert(msg_event.msg.uid == 6); - test_assert(msg_event.msg.flags == 0); - - test_assert(test_dsync_worker_next_msg_event(test_dest_worker, &msg_event)); - test_assert(msg_event.type == LAST_MSG_TYPE_UPDATE); - test_assert(msg_event.msg.uid == 9); - test_assert(msg_event.msg.flags == 0); - test_assert(strcmp(msg_event.msg.keywords[0], "hello") == 0); - test_assert(strcmp(msg_event.msg.keywords[1], "world") == 0); - test_assert(msg_event.msg.keywords[2] == NULL); - - test_assert(test_dsync_worker_next_msg_event(test_dest_worker, &msg_event)); - test_assert(msg_event.type == LAST_MSG_TYPE_UPDATE); - test_assert(msg_event.msg.uid == 10); - test_assert(msg_event.msg.flags == MAIL_SEEN); - test_assert(strcmp(msg_event.msg.keywords[0], "foo") == 0); - test_assert(msg_event.msg.keywords[1] == NULL); - - test_assert(test_dsync_worker_next_msg_event(test_src_worker, &msg_event)); - test_assert(msg_event.type == LAST_MSG_TYPE_UPDATE); - test_assert(msg_event.msg.uid == 11); - test_assert(msg_event.msg.flags == MAIL_DRAFT); - test_assert(strcmp(msg_event.msg.keywords[0], "foo") == 0); - test_assert(msg_event.msg.keywords[1] == NULL); - - test_assert(test_dsync_worker_next_msg_event(test_dest_worker, &msg_event)); - test_assert(msg_event.type == LAST_MSG_TYPE_UPDATE); - test_assert(msg_event.msg.uid == 12); - test_assert(msg_event.msg.flags == MAIL_SEEN); - test_assert(strcmp(msg_event.msg.keywords[0], "bar") == 0); - test_assert(msg_event.msg.keywords[1] == NULL); - - test_assert(!test_dsync_worker_next_msg_event(test_src_worker, &msg_event)); - test_assert(!test_dsync_worker_next_msg_event(test_dest_worker, &msg_event)); - test_end(); -} - -static void test_dsync_brain_msg_sync_expunges(void) -{ - struct test_dsync_mailbox *box; - struct dsync_brain_mailbox_sync *sync; - struct test_dsync_msg_event msg_event; - - test_begin("dsync brain msg sync expunges"); - - i_array_init(&mailboxes, 1); - box = test_box_add(ADD_BOTH, "box"); - - /* expunged from dest */ - test_msg_add(box, ADD_SRC, "guid1", 1); - /* expunged from src */ - test_msg_add(box, ADD_DEST, "guid2", 2); - /* expunged from dest with expunge record */ - test_msg_add(box, ADD_BOTH, "guid3", 3); - test_msg_set_flags(box, ADD_DEST, 3, DSYNC_MAIL_FLAG_EXPUNGED); - /* expunged from src with expunge record */ - test_msg_add(box, ADD_BOTH, "guid4", 4); - test_msg_set_flags(box, ADD_SRC, 4, DSYNC_MAIL_FLAG_EXPUNGED); - /* expunged from both, with expunge record in src */ - test_msg_add(box, ADD_SRC, "guid5", 5); - test_msg_set_flags(box, ADD_SRC, 5, DSYNC_MAIL_FLAG_EXPUNGED); - /* expunged from both, with expunge record in dest */ - test_msg_add(box, ADD_DEST, "guid6", 6); - test_msg_set_flags(box, ADD_DEST, 6, DSYNC_MAIL_FLAG_EXPUNGED); - /* expunged from both, with expunge record in both */ - test_msg_add(box, ADD_BOTH, "guid7", 7); - test_msg_set_flags(box, ADD_BOTH, 7, DSYNC_MAIL_FLAG_EXPUNGED); - - sync = test_dsync_brain_sync_init(); - test_assert(array_count(&sync->src_msg_iter->new_msgs) == 0); - test_assert(array_count(&sync->dest_msg_iter->new_msgs) == 0); - - test_assert(test_dsync_worker_next_msg_event(test_src_worker, &msg_event)); - test_assert(msg_event.type == LAST_MSG_TYPE_EXPUNGE); - test_assert(msg_event.msg.uid == 1); - - test_assert(test_dsync_worker_next_msg_event(test_dest_worker, &msg_event)); - test_assert(msg_event.type == LAST_MSG_TYPE_EXPUNGE); - test_assert(msg_event.msg.uid == 2); - - test_assert(test_dsync_worker_next_msg_event(test_src_worker, &msg_event)); - test_assert(msg_event.type == LAST_MSG_TYPE_EXPUNGE); - test_assert(msg_event.msg.uid == 3); - - test_assert(test_dsync_worker_next_msg_event(test_dest_worker, &msg_event)); - test_assert(msg_event.type == LAST_MSG_TYPE_EXPUNGE); - test_assert(msg_event.msg.uid == 4); - - test_assert(!test_dsync_worker_next_msg_event(test_src_worker, &msg_event)); - test_assert(!test_dsync_worker_next_msg_event(test_dest_worker, &msg_event)); - test_end(); -} - -static void test_dsync_brain_msg_sync_uid_conflicts(void) -{ - struct test_dsync_mailbox *box; - struct dsync_brain_mailbox_sync *sync; - struct test_dsync_msg_event msg_event; - const struct dsync_brain_uid_conflict *conflicts; - const struct dsync_brain_new_msg *src_msgs, *dest_msgs; - unsigned int src_count, dest_count; - - test_begin("dsync brain msg sync uid conflicts"); - - i_array_init(&mailboxes, 16); - - /* existing guid mismatch */ - box = test_box_add(ADD_BOTH, "box1"); - test_msg_add(box, ADD_SRC, "guid1", 1); - test_msg_add(box, ADD_DEST, "guid2", 1); - - /* preserve uid */ - test_msg_add(box, ADD_BOTH, "guid3", 3); - /* extra message in src */ - test_msg_add(box, ADD_SRC, "guid4", 4); - /* extra message in dest */ - test_msg_add(box, ADD_DEST, "guid5", 5); - - /* conflict in expunged message expunged in dest */ - test_msg_add(box, ADD_SRC, "guid6", 6); - test_msg_add(box, ADD_DEST, "guid7", 6); - test_msg_set_flags(box, ADD_DEST, 6, DSYNC_MAIL_FLAG_EXPUNGED); - - /* conflict in expunged message expunged in src */ - test_msg_add(box, ADD_SRC, "guid8", 8); - test_msg_set_flags(box, ADD_SRC, 8, DSYNC_MAIL_FLAG_EXPUNGED); - test_msg_add(box, ADD_DEST, "guid9", 8); - - /* conflict in expunged message expunged in both */ - test_msg_add(box, ADD_SRC, "guid10", 10); - test_msg_set_flags(box, ADD_SRC, 10, DSYNC_MAIL_FLAG_EXPUNGED); - test_msg_add(box, ADD_DEST, "guid11", 10); - test_msg_set_flags(box, ADD_DEST, 10, DSYNC_MAIL_FLAG_EXPUNGED); - - sync = test_dsync_brain_sync_init(); - - conflicts = array_get(&sync->src_msg_iter->uid_conflicts, &src_count); - test_assert(src_count == 3); - test_assert(conflicts[0].old_uid == 1); - test_assert(conflicts[0].new_uid == 12); - test_assert(conflicts[1].old_uid == 4); - test_assert(conflicts[1].new_uid == 13); - test_assert(conflicts[2].old_uid == 6); - test_assert(conflicts[2].new_uid == 15); - - conflicts = array_get(&sync->dest_msg_iter->uid_conflicts, &dest_count); - test_assert(dest_count == 3); - test_assert(conflicts[0].old_uid == 1); - test_assert(conflicts[0].new_uid == 11); - test_assert(conflicts[1].old_uid == 5); - test_assert(conflicts[1].new_uid == 14); - test_assert(conflicts[2].old_uid == 8); - test_assert(conflicts[2].new_uid == 16); - - test_assert(!test_dsync_worker_next_msg_event(test_src_worker, &msg_event)); - test_assert(!test_dsync_worker_next_msg_event(test_dest_worker, &msg_event)); - - src_msgs = array_get(&sync->src_msg_iter->new_msgs, &src_count); - dest_msgs = array_get(&sync->dest_msg_iter->new_msgs, &dest_count); - test_assert(src_count == 3); - test_assert(dest_count == 3); - - test_assert(dest_msgs[0].msg->uid == 12); - test_assert(strcmp(dest_msgs[0].msg->guid, "guid1") == 0); - test_assert(src_msgs[0].msg->uid == 11); - test_assert(strcmp(src_msgs[0].msg->guid, "guid2") == 0); - test_assert(dest_msgs[1].msg->uid == 13); - test_assert(strcmp(dest_msgs[1].msg->guid, "guid4") == 0); - test_assert(src_msgs[1].msg->uid == 14); - test_assert(strcmp(src_msgs[1].msg->guid, "guid5") == 0); - test_assert(dest_msgs[2].msg->uid == 15); - test_assert(strcmp(dest_msgs[2].msg->guid, "guid6") == 0); - test_assert(src_msgs[2].msg->uid == 16); - test_assert(strcmp(src_msgs[2].msg->guid, "guid9") == 0); - - test_end(); -} - -int main(void) -{ - static void (*test_functions[])(void) = { - test_dsync_brain_msg_sync_box_multi, - test_dsync_brain_msg_sync_box_single, - test_dsync_brain_msg_sync_existing, - test_dsync_brain_msg_sync_expunges, - test_dsync_brain_msg_sync_uid_conflicts, - NULL - }; - - return test_run(test_functions); -}
--- a/src/doveadm/dsync/test-dsync-brain.c Fri May 04 05:35:36 2012 +0300 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,289 +0,0 @@ -/* Copyright (c) 2009-2012 Dovecot authors, see the included COPYING file */ - -#include "lib.h" -#include "array.h" -#include "dsync-brain-private.h" -#include "test-dsync-worker.h" -#include "test-dsync-common.h" - -struct master_service *master_service; -static struct test_dsync_worker *src_test_worker, *dest_test_worker; - -struct dsync_brain_mailbox_sync * -dsync_brain_msg_sync_init(struct dsync_brain *brain, - const ARRAY_TYPE(dsync_brain_mailbox) *mailboxes) -{ - struct dsync_brain_mailbox_sync *sync; - - sync = i_new(struct dsync_brain_mailbox_sync, 1); - sync->brain = brain; - i_array_init(&sync->mailboxes, array_count(mailboxes)); - array_append_array(&sync->mailboxes, mailboxes); - return sync; -} -void dsync_brain_msg_sync_more(struct dsync_brain_mailbox_sync *sync ATTR_UNUSED) {} - -void dsync_brain_msg_sync_deinit(struct dsync_brain_mailbox_sync **_sync) -{ - array_free(&(*_sync)->mailboxes); - i_free(*_sync); -} - -static void mailboxes_set_guids(struct dsync_mailbox *boxes) -{ - for (; boxes->name != NULL; boxes++) { - dsync_str_sha_to_guid(t_strconcat("box-", boxes->name, NULL), - &boxes->mailbox_guid); - dsync_str_sha_to_guid(boxes->name, &boxes->name_sha1); - } -} - -static void mailboxes_send_to_worker(struct test_dsync_worker *test_worker, - struct dsync_mailbox *boxes) -{ - unsigned int i; - - for (i = 0; boxes[i].name != NULL; i++) { - test_worker->box_iter.next_box = &boxes[i]; - test_worker->worker.input_callback(test_worker->worker.input_context); - } - test_worker->box_iter.last = TRUE; - test_worker->worker.input_callback(test_worker->worker.input_context); -} - -static void subscriptions_send_to_worker(struct test_dsync_worker *test_worker) -{ - test_worker->subs_iter.last_subs = TRUE; - test_worker->subs_iter.last_unsubs = TRUE; - test_worker->worker.input_callback(test_worker->worker.input_context); -} - -static bool -test_dsync_mailbox_create_equals(const struct dsync_mailbox *cbox, - const struct dsync_mailbox *obox) -{ - return strcmp(cbox->name, obox->name) == 0 && - memcmp(cbox->mailbox_guid.guid, obox->mailbox_guid.guid, - sizeof(cbox->mailbox_guid.guid)) == 0 && - memcmp(cbox->name_sha1.guid, obox->name_sha1.guid, - sizeof(cbox->name_sha1.guid)) == 0 && - cbox->uid_validity == obox->uid_validity && - cbox->uid_next == 1 && cbox->highest_modseq == 0; -} - -static bool -test_dsync_mailbox_delete_equals(const struct dsync_mailbox *dbox, - const struct dsync_mailbox *obox) -{ - return memcmp(dbox->mailbox_guid.guid, obox->mailbox_guid.guid, - sizeof(dbox->mailbox_guid.guid)) == 0 && - dbox->last_change == obox->last_change; -} - -static void -test_dsync_mailbox_update(const struct dsync_mailbox *bbox, - const struct dsync_mailbox *box) -{ - struct test_dsync_box_event src_event, dest_event; - - test_assert(test_dsync_worker_next_box_event(src_test_worker, &src_event)); - test_assert(test_dsync_worker_next_box_event(dest_test_worker, &dest_event)); - test_assert(src_event.type == dest_event.type && - dsync_mailboxes_equal(&src_event.box, &dest_event.box)); - - test_assert(src_event.type == LAST_BOX_TYPE_UPDATE); - test_assert(dsync_mailboxes_equal(&src_event.box, box)); - test_assert(dsync_mailboxes_equal(bbox, box)); -} - -static int -dsync_brain_mailbox_name_cmp(const struct dsync_brain_mailbox *box1, - const struct dsync_brain_mailbox *box2) -{ - return strcmp(box1->box.name, box2->box.name); -} - -static void test_dsync_brain(void) -{ - static struct dsync_mailbox src_boxes[] = { - { "box1", '/', { { 0, } }, { { 0, } }, 1234567890, 5432, 0, 1, 123123123123ULL, 3636, 0, ARRAY_INIT }, - { "box2", '/', { { 0, } }, { { 0, } }, 1234567890, 5432, 0, 1, 123123123123ULL, 3636, 0, ARRAY_INIT }, - { "box3", '/', { { 0, } }, { { 0, } }, 1234567890, 5432, 0, 1, 123123123123ULL, 3636, 0, ARRAY_INIT }, - { "box4", '/', { { 0, } }, { { 0, } }, 1234567890, 5432, 0, 1, 123123123123ULL, 3636, 0, ARRAY_INIT }, - { "box5", '/', { { 0, } }, { { 0, } }, 1234567890, 5433, 0, 1, 123123123123ULL, 3636, 0, ARRAY_INIT }, - { "box6", '/', { { 0, } }, { { 0, } }, 1234567890, 5432, 0, 1, 123123123124ULL, 3636, 0, ARRAY_INIT }, - { "boxx", '/', { { 0, } }, { { 0, } }, 1234567890, 5432, 0, 1, 123123123123ULL, 3636, 0, ARRAY_INIT }, - { "boxd1", '/', { { 0, } }, { { 0, } }, 1234567890, 5432, 0, 1, 123123123123ULL, 3636, 0, ARRAY_INIT }, - { "boxd2", '/', { { 0, } }, { { 0, } }, 1234567890, 5432, 0, 1, 123123123123ULL, 3636, DSYNC_MAILBOX_FLAG_DELETED_MAILBOX, ARRAY_INIT }, - { NULL, 0, { { 0, } }, { { 0, } }, 0, 0, 0, 0, 0, 0, 0, ARRAY_INIT } - }; - static struct dsync_mailbox dest_boxes[] = { - { "box1", '/', { { 0, } }, { { 0, } }, 1234567890, 5432, 0, 1, 123123123123ULL, 3636, 0, ARRAY_INIT }, - { "box2", '/', { { 0, } }, { { 0, } }, 1234567891, 5432, 0, 1, 123123123123ULL, 3636, 0, ARRAY_INIT }, - { "box3", '/', { { 0, } }, { { 0, } }, 1234567890, 5433, 0, 1, 123123123123ULL, 3636, 0, ARRAY_INIT }, - { "box4", '/', { { 0, } }, { { 0, } }, 1234567890, 5432, 0, 1, 123123123124ULL, 3636, 0, ARRAY_INIT }, - { "box5", '/', { { 0, } }, { { 0, } }, 1234567890, 5432, 0, 1, 123123123123ULL, 3636, 0, ARRAY_INIT }, - { "box6", '/', { { 0, } }, { { 0, } }, 1234567890, 5432, 0, 1, 123123123123ULL, 3636, 0, ARRAY_INIT }, - { "boxy", '/', { { 0, } }, { { 0, } }, 1234567890, 5432, 0, 1, 123123123123ULL, 3636, 0, ARRAY_INIT }, - { "boxd1", '/', { { 0, } }, { { 0, } }, 1234567890, 5432, 0, 1, 123123123123ULL, 3636, DSYNC_MAILBOX_FLAG_DELETED_MAILBOX, ARRAY_INIT }, - { "boxd2", '/', { { 0, } }, { { 0, } }, 1234567890, 5432, 0, 1, 123123123123ULL, 3636, 0, ARRAY_INIT }, - { NULL, 0, { { 0, } }, { { 0, } }, 0, 0, 0, 0, 0, 0, 0, ARRAY_INIT } - }; - struct dsync_brain *brain; - struct dsync_worker *src_worker, *dest_worker; - struct test_dsync_box_event box_event; - const struct dsync_brain_mailbox *brain_boxes; - unsigned int i, count; - - test_begin("dsync brain"); - - mailboxes_set_guids(src_boxes); - mailboxes_set_guids(dest_boxes); - - src_worker = dsync_worker_init_test(); - dest_worker = dsync_worker_init_test(); - src_test_worker = (struct test_dsync_worker *)src_worker; - dest_test_worker = (struct test_dsync_worker *)dest_worker; - - brain = dsync_brain_init(src_worker, dest_worker, NULL, - DSYNC_BRAIN_FLAG_LOCAL); - dsync_brain_sync(brain); - - /* have brain read the mailboxes */ - mailboxes_send_to_worker(src_test_worker, src_boxes); - mailboxes_send_to_worker(dest_test_worker, dest_boxes); - - subscriptions_send_to_worker(src_test_worker); - subscriptions_send_to_worker(dest_test_worker); - - test_assert(brain->state == DSYNC_STATE_SYNC_MSGS); - - /* check that it created/deleted missing mailboxes */ - test_assert(test_dsync_worker_next_box_event(dest_test_worker, &box_event)); - test_assert(box_event.type == LAST_BOX_TYPE_DELETE); - test_assert(test_dsync_mailbox_delete_equals(&box_event.box, &dest_boxes[8])); - - test_assert(test_dsync_worker_next_box_event(src_test_worker, &box_event)); - test_assert(box_event.type == LAST_BOX_TYPE_DELETE); - test_assert(test_dsync_mailbox_delete_equals(&box_event.box, &src_boxes[7])); - - test_assert(test_dsync_worker_next_box_event(dest_test_worker, &box_event)); - test_assert(box_event.type == LAST_BOX_TYPE_CREATE); - test_assert(test_dsync_mailbox_create_equals(&box_event.box, &src_boxes[6])); - - test_assert(test_dsync_worker_next_box_event(src_test_worker, &box_event)); - test_assert(box_event.type == LAST_BOX_TYPE_CREATE); - test_assert(test_dsync_mailbox_create_equals(&box_event.box, &dest_boxes[6])); - - test_assert(!test_dsync_worker_next_box_event(src_test_worker, &box_event)); - test_assert(!test_dsync_worker_next_box_event(dest_test_worker, &box_event)); - - array_sort(&brain->mailbox_sync->mailboxes, - dsync_brain_mailbox_name_cmp); - - /* check mailbox updates */ - brain->state++; - dsync_brain_sync(brain); - test_assert(brain->state == DSYNC_STATE_SYNC_UPDATE_MAILBOXES); - dsync_brain_sync(brain); - test_assert(brain->state == DSYNC_STATE_SYNC_END); - - brain_boxes = array_get(&brain->mailbox_sync->mailboxes, &count); - test_assert(count == 7); - for (i = 0; i < 5; i++) { - test_assert(dsync_mailboxes_equal(brain_boxes[i].src, &src_boxes[i+1])); - test_assert(dsync_mailboxes_equal(brain_boxes[i].dest, &dest_boxes[i+1])); - } - test_assert(dsync_mailboxes_equal(brain_boxes[5].src, &src_boxes[6])); - test_assert(brain_boxes[5].dest == NULL); - test_assert(brain_boxes[6].src == NULL); - test_assert(dsync_mailboxes_equal(brain_boxes[6].dest, &dest_boxes[6])); - - test_dsync_mailbox_update(&brain_boxes[0].box, &src_boxes[1]); - test_dsync_mailbox_update(&brain_boxes[1].box, &dest_boxes[2]); - test_dsync_mailbox_update(&brain_boxes[2].box, &dest_boxes[3]); - test_dsync_mailbox_update(&brain_boxes[3].box, &src_boxes[4]); - test_dsync_mailbox_update(&brain_boxes[4].box, &src_boxes[5]); - test_dsync_mailbox_update(&brain_boxes[5].box, &src_boxes[6]); - test_dsync_mailbox_update(&brain_boxes[6].box, &dest_boxes[6]); - - test_assert(!test_dsync_worker_next_box_event(src_test_worker, &box_event)); - test_assert(!test_dsync_worker_next_box_event(dest_test_worker, &box_event)); - - dsync_brain_deinit(&brain); - dsync_worker_deinit(&src_worker); - dsync_worker_deinit(&dest_worker); - - test_end(); -} - -static void test_dsync_brain_full(void) -{ - static struct dsync_mailbox boxes[] = { - { "box1", '/', { { 0, } }, { { 0, } }, 1234567890, 5432, 0, 1, 123123123123ULL, 2352, 0, ARRAY_INIT }, - { NULL, 0, { { 0, } }, { { 0, } }, 0, 0, 0, 0, 0, 0, 0, ARRAY_INIT } - }; - struct dsync_brain *brain; - struct dsync_worker *src_worker, *dest_worker; - struct test_dsync_box_event box_event; - const struct dsync_brain_mailbox *brain_boxes; - unsigned int count; - - test_begin("dsync brain full"); - - mailboxes_set_guids(boxes); - - src_worker = dsync_worker_init_test(); - dest_worker = dsync_worker_init_test(); - src_test_worker = (struct test_dsync_worker *)src_worker; - dest_test_worker = (struct test_dsync_worker *)dest_worker; - - brain = dsync_brain_init(src_worker, dest_worker, NULL, - DSYNC_BRAIN_FLAG_FULL_SYNC | - DSYNC_BRAIN_FLAG_LOCAL); - dsync_brain_sync(brain); - - /* have brain read the mailboxes */ - mailboxes_send_to_worker(src_test_worker, boxes); - mailboxes_send_to_worker(dest_test_worker, boxes); - - subscriptions_send_to_worker(src_test_worker); - subscriptions_send_to_worker(dest_test_worker); - - test_assert(brain->state == DSYNC_STATE_SYNC_MSGS); - - test_assert(!test_dsync_worker_next_box_event(src_test_worker, &box_event)); - test_assert(!test_dsync_worker_next_box_event(dest_test_worker, &box_event)); - - /* check mailbox updates */ - brain->state++; - dsync_brain_sync(brain); - test_assert(brain->state == DSYNC_STATE_SYNC_UPDATE_MAILBOXES); - dsync_brain_sync(brain); - test_assert(brain->state == DSYNC_STATE_SYNC_END); - - brain_boxes = array_get(&brain->mailbox_sync->mailboxes, &count); - test_assert(count == 1); - test_assert(dsync_mailboxes_equal(brain_boxes[0].src, &boxes[0])); - test_assert(dsync_mailboxes_equal(brain_boxes[0].dest, &boxes[0])); - test_dsync_mailbox_update(&brain_boxes[0].box, &boxes[0]); - - test_assert(!test_dsync_worker_next_box_event(src_test_worker, &box_event)); - test_assert(!test_dsync_worker_next_box_event(dest_test_worker, &box_event)); - - dsync_brain_deinit(&brain); - dsync_worker_deinit(&src_worker); - dsync_worker_deinit(&dest_worker); - - test_end(); -} - -int main(void) -{ - static void (*test_functions[])(void) = { - test_dsync_brain, - test_dsync_brain_full, - NULL - }; - return test_run(test_functions); -}
--- a/src/doveadm/dsync/test-dsync-common.c Fri May 04 05:35:36 2012 +0300 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,80 +0,0 @@ -/* Copyright (c) 2009-2012 Dovecot authors, see the included COPYING file */ - -#include "lib.h" -#include "array.h" -#include "hex-binary.h" -#include "sha1.h" -#include "dsync-data.h" -#include "test-dsync-common.h" - -const guid_128_t test_mailbox_guid1 = { - 0x12, 0x34, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef, - 0x21, 0x43, 0x54, 0x76, 0x98, 0xba, 0xdc, 0xfe -}; - -const guid_128_t test_mailbox_guid2 = { - 0xa3, 0xbd, 0x78, 0x24, 0xde, 0xfe, 0x08, 0xf7, - 0xac, 0xc7, 0xca, 0x8c, 0xe7, 0x39, 0xdb, 0xca -}; - -bool dsync_messages_equal(const struct dsync_message *m1, - const struct dsync_message *m2) -{ - unsigned int i; - - if (strcmp(m1->guid, m2->guid) != 0 || - m1->uid != m2->uid || m1->flags != m2->flags || - m1->modseq != m2->modseq || m1->save_date != m2->save_date) - return FALSE; - - if (m1->keywords == m2->keywords) - return TRUE; - if (m1->keywords == NULL) - return m2->keywords == NULL || m2->keywords[0] == NULL; - if (m2->keywords == NULL) - return m1->keywords[0] == NULL; - - for (i = 0; m1->keywords[i] != NULL && m2->keywords[i] != NULL; i++) { - if (strcasecmp(m1->keywords[i], m2->keywords[i]) != 0) - return FALSE; - } - return m1->keywords[i] == NULL && m2->keywords[i] == NULL; -} - -bool dsync_mailboxes_equal(const struct dsync_mailbox *box1, - const struct dsync_mailbox *box2) -{ - const struct mailbox_cache_field *f1 = NULL, *f2 = NULL; - unsigned int i, f1_count = 0, f2_count = 0; - - if (strcmp(box1->name, box2->name) != 0 || - box1->name_sep != box2->name_sep || - memcmp(box1->mailbox_guid.guid, box2->mailbox_guid.guid, - sizeof(box1->mailbox_guid.guid)) != 0 || - box1->uid_validity != box2->uid_validity || - box1->uid_next != box2->uid_next || - box1->highest_modseq != box2->highest_modseq) - return FALSE; - - if (array_is_created(&box1->cache_fields)) - f1 = array_get(&box1->cache_fields, &f1_count); - if (array_is_created(&box2->cache_fields)) - f2 = array_get(&box2->cache_fields, &f2_count); - if (f1_count != f2_count) - return FALSE; - for (i = 0; i < f1_count; i++) { - if (strcmp(f1[i].name, f2[i].name) != 0 || - f1[i].decision != f2[i].decision || - f1[i].last_used != f2[i].last_used) - return FALSE; - } - return TRUE; -} - -void mail_generate_guid_128_hash(const char *guid, guid_128_t guid_128_r) -{ - unsigned char sha1_sum[SHA1_RESULTLEN]; - - sha1_get_digest(guid, strlen(guid), sha1_sum); - memcpy(guid_128_r, sha1_sum, GUID_128_SIZE); -}
--- a/src/doveadm/dsync/test-dsync-common.h Fri May 04 05:35:36 2012 +0300 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,18 +0,0 @@ -#ifndef TEST_DSYNC_COMMON_H -#define TEST_DSYNC_COMMON_H - -#include "test-common.h" -#include "dsync-data.h" - -#define TEST_MAILBOX_GUID1 "1234456789abcdef2143547698badcfe" -#define TEST_MAILBOX_GUID2 "a3bd7824defe08f7acc7ca8ce739dbca" - -extern const guid_128_t test_mailbox_guid1; -extern const guid_128_t test_mailbox_guid2; - -bool dsync_messages_equal(const struct dsync_message *m1, - const struct dsync_message *m2); -bool dsync_mailboxes_equal(const struct dsync_mailbox *box1, - const struct dsync_mailbox *box2); - -#endif
--- a/src/doveadm/dsync/test-dsync-proxy-server-cmd.c Fri May 04 05:35:36 2012 +0300 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,484 +0,0 @@ -/* Copyright (c) 2009-2012 Dovecot authors, see the included COPYING file */ - -#include "lib.h" -#include "array.h" -#include "str.h" -#include "strescape.h" -#include "istream.h" -#include "ostream.h" -#include "test-common.h" -#include "dsync-proxy-server.h" -#include "test-dsync-worker.h" -#include "test-dsync-common.h" - -#define ALL_MAIL_FLAGS "\\Answered \\Flagged \\Deleted \\Seen \\Draft \\Recent" - -struct master_service *master_service; -static string_t *out; -static struct dsync_proxy_server *server; -static struct test_dsync_worker *test_worker; -static struct dsync_proxy_server_command *cur_cmd; -static const char *cur_cmd_args[20]; - -static void out_clear(void) -{ - o_stream_seek(server->output, 0); - str_truncate(out, 0); -} - -static int run_more(void) -{ - int ret; - - i_assert(cur_cmd != NULL); - - ret = cur_cmd->func(server, cur_cmd_args); - if (ret == 0) - return 0; - - cur_cmd = NULL; - return ret; -} - -static int ATTR_SENTINEL -run_cmd(const char *cmd_name, ...) -{ - va_list va; - const char *str; - unsigned int i = 0; - - i_assert(cur_cmd == NULL); - - va_start(va, cmd_name); - while ((str = va_arg(va, const char *)) != NULL) { - i_assert(i < N_ELEMENTS(cur_cmd_args)+1); - cur_cmd_args[i++] = str; - } - cur_cmd_args[i] = NULL; - va_end(va); - - cur_cmd = dsync_proxy_server_command_find(cmd_name); - i_assert(cur_cmd != NULL); - return run_more(); -} - -static void test_dsync_proxy_box_list(void) -{ - struct dsync_mailbox box; - - test_begin("proxy server box list"); - - test_assert(run_cmd("BOX-LIST", NULL) == 0); - - /* \noselect mailbox */ - memset(&box, 0, sizeof(box)); - box.name = "\t\001\r\nname\t\001\n\r"; - box.name_sep = '/'; - box.last_change = 992; - box.flags = DSYNC_MAILBOX_FLAG_NOSELECT; - test_worker->box_iter.next_box = &box; - test_assert(run_more() == 0); - test_assert(strcmp(str_c(out), t_strconcat(str_tabescape(box.name), - "\t/\t992\t1\n", NULL)) == 0); - out_clear(); - - /* selectable mailbox */ - memset(&box, 0, sizeof(box)); - box.name = "foo/bar"; - box.name_sep = '/'; - memcpy(box.mailbox_guid.guid, test_mailbox_guid1, GUID_128_SIZE); - box.uid_validity = 4275878552; - box.uid_next = 4023233417; - box.message_count = 4525; - box.highest_modseq = 18080787909545915012ULL; - box.first_recent_uid = 353; - test_worker->box_iter.next_box = &box; - - test_assert(run_more() == 0); - - test_assert(strcmp(str_c(out), "foo/bar\t/\t0\t0\t" - TEST_MAILBOX_GUID1"\t" - "4275878552\t" - "4023233417\t" - "4525\t" - "18080787909545915012\t" - "353\n") == 0); - out_clear(); - - /* last mailbox */ - test_worker->box_iter.last = TRUE; - test_assert(run_more() == 1); - test_assert(strcmp(str_c(out), "+\n") == 0); - out_clear(); - - test_end(); -} - -static void test_dsync_proxy_subs_list(void) -{ - const char *name; - struct dsync_worker_subscription subs; - struct dsync_worker_unsubscription unsubs; - - test_begin("proxy server subs list"); - - test_assert(run_cmd("SUBS-LIST", NULL) == 0); - - /* subscription */ - name = "\t\001\r\nname\t\001\n\r"; - subs.vname = name; - subs.storage_name = "\tstorage_name\n"; - subs.last_change = 1234567890; - subs.ns_prefix = "\t\001\r\nprefix\t\001\n\r"; - test_worker->subs_iter.next_subscription = &subs; - test_assert(run_more() == 0); - test_assert(strcmp(str_c(out), t_strconcat( - str_tabescape(name), "\t", - str_tabescape(subs.storage_name), "\t", - str_tabescape(subs.ns_prefix), - "\t1234567890\n", NULL)) == 0); - out_clear(); - - test_worker->subs_iter.last_subs = TRUE; - test_assert(run_more() == 0); - test_assert(strcmp(str_c(out), "+\n") == 0); - out_clear(); - - /* unsubscription */ - memcpy(unsubs.name_sha1.guid, test_mailbox_guid1, - sizeof(unsubs.name_sha1.guid)); - unsubs.ns_prefix = "\t\001\r\nprefix2\t\001\n\r"; - unsubs.last_change = 987654321; - test_worker->subs_iter.next_unsubscription = &unsubs; - test_assert(run_more() == 0); - test_assert(strcmp(str_c(out), t_strconcat(TEST_MAILBOX_GUID1, "\t", - str_tabescape(unsubs.ns_prefix), "\t987654321\n", NULL)) == 0); - out_clear(); - - test_worker->subs_iter.last_unsubs = TRUE; - test_assert(run_more() == 1); - test_assert(strcmp(str_c(out), "+\n") == 0); - out_clear(); - - test_end(); -} - -static void test_dsync_proxy_msg_list(void) -{ - static const char *test_keywords[] = { - "kw1", "kw2", NULL - }; - struct dsync_message msg; - struct test_dsync_worker_msg test_msg; - - test_begin("proxy server msg list"); - - test_assert(run_cmd("MSG-LIST", TEST_MAILBOX_GUID1, TEST_MAILBOX_GUID2, NULL) == 0); - - memset(&msg, 0, sizeof(msg)); - msg.guid = "\t\001\r\nguid\t\001\n\r"; - msg.uid = 123; - msg.modseq = 98765432101234; - msg.save_date = 1234567890; - - /* no flags */ - test_msg.msg = msg; - test_msg.mailbox_idx = 98; - array_append(&test_worker->msg_iter.msgs, &test_msg, 1); - test_assert(run_more() == 0); - test_assert(strcmp(str_c(out), t_strconcat( - "98\t", str_tabescape(msg.guid), - "\t123\t98765432101234\t\t1234567890\n", NULL)) == 0); - out_clear(); - - /* all flags, some keywords */ - msg.modseq = 1; - msg.save_date = 2; - msg.guid = "guid"; - msg.flags = MAIL_FLAGS_MASK; - msg.keywords = test_keywords; - test_msg.msg = msg; - test_msg.mailbox_idx = 76; - array_append(&test_worker->msg_iter.msgs, &test_msg, 1); - test_assert(run_more() == 0); - test_assert(strcmp(str_c(out), "76\tguid\t123\t1\t" - ALL_MAIL_FLAGS" kw1 kw2\t2\n") == 0); - out_clear(); - - /* last message */ - test_worker->msg_iter.last = TRUE; - test_assert(run_more() == 1); - test_assert(strcmp(str_c(out), "+\n") == 0); - out_clear(); - - test_end(); -} - -static void test_dsync_proxy_box_create(void) -{ - struct test_dsync_box_event event; - - test_begin("proxy server box create"); - - test_assert(run_cmd("BOX-CREATE", "noselect", "/", - "553", "1", NULL) == 1); - test_assert(test_dsync_worker_next_box_event(test_worker, &event)); - test_assert(event.type == LAST_BOX_TYPE_CREATE); - test_assert(strcmp(event.box.name, "noselect") == 0); - test_assert(event.box.name_sep == '/'); - test_assert(event.box.last_change == 553); - test_assert(event.box.flags == DSYNC_MAILBOX_FLAG_NOSELECT); - test_assert(event.box.uid_validity == 0); - - test_assert(run_cmd("BOX-CREATE", "selectable", "?", - "61", "2", TEST_MAILBOX_GUID2, "1234567890", "9876", - "4610", "28427847284728", "853", NULL) == 1); - test_assert(test_dsync_worker_next_box_event(test_worker, &event)); - test_assert(event.type == LAST_BOX_TYPE_CREATE); - test_assert(strcmp(event.box.name, "selectable") == 0); - test_assert(event.box.name_sep == '?'); - test_assert(memcmp(event.box.mailbox_guid.guid, test_mailbox_guid2, GUID_128_SIZE) == 0); - test_assert(event.box.flags == 2); - test_assert(event.box.uid_validity == 1234567890); - test_assert(event.box.uid_next == 9876); - test_assert(event.box.message_count == 4610); - test_assert(event.box.highest_modseq == 28427847284728); - test_assert(event.box.first_recent_uid == 853); - test_assert(event.box.last_change == 61); - - test_end(); -} - -static void test_dsync_proxy_box_delete(void) -{ - struct test_dsync_box_event event; - - test_begin("proxy server box delete"); - - test_assert(run_cmd("BOX-DELETE", TEST_MAILBOX_GUID1, "4351", NULL) == 1); - test_assert(test_dsync_worker_next_box_event(test_worker, &event)); - test_assert(event.type == LAST_BOX_TYPE_DELETE); - test_assert(memcmp(event.box.mailbox_guid.guid, test_mailbox_guid1, GUID_128_SIZE) == 0); - test_assert(event.box.last_change == 4351); - - test_assert(run_cmd("BOX-DELETE", TEST_MAILBOX_GUID2, "653", NULL) == 1); - test_assert(test_dsync_worker_next_box_event(test_worker, &event)); - test_assert(event.type == LAST_BOX_TYPE_DELETE); - test_assert(memcmp(event.box.mailbox_guid.guid, test_mailbox_guid2, GUID_128_SIZE) == 0); - test_assert(event.box.last_change == 653); - - test_end(); -} - -static void test_dsync_proxy_box_rename(void) -{ - struct test_dsync_box_event event; - - test_begin("proxy server box rename"); - - test_assert(run_cmd("BOX-RENAME", TEST_MAILBOX_GUID1, "name\t1", "/", NULL) == 1); - test_assert(test_dsync_worker_next_box_event(test_worker, &event)); - test_assert(event.type == LAST_BOX_TYPE_RENAME); - test_assert(memcmp(event.box.mailbox_guid.guid, test_mailbox_guid1, GUID_128_SIZE) == 0); - test_assert(strcmp(event.box.name, "name\t1") == 0); - test_assert(event.box.name_sep == '/'); - - test_assert(run_cmd("BOX-RENAME", TEST_MAILBOX_GUID2, "", "?", NULL) == 1); - test_assert(test_dsync_worker_next_box_event(test_worker, &event)); - test_assert(event.type == LAST_BOX_TYPE_RENAME); - test_assert(memcmp(event.box.mailbox_guid.guid, test_mailbox_guid2, GUID_128_SIZE) == 0); - test_assert(strcmp(event.box.name, "") == 0); - test_assert(event.box.name_sep == '?'); - - test_end(); -} - -static void test_dsync_proxy_box_update(void) -{ - struct test_dsync_box_event event; - - test_begin("proxy server box update"); - - test_assert(run_cmd("BOX-UPDATE", "updated", "/", - "53", "2", TEST_MAILBOX_GUID1, "34343", "22", - "58293", "2238427847284728", "2482", NULL) == 1); - test_assert(test_dsync_worker_next_box_event(test_worker, &event)); - test_assert(event.type == LAST_BOX_TYPE_UPDATE); - test_assert(strcmp(event.box.name, "updated") == 0); - test_assert(event.box.name_sep == '/'); - test_assert(memcmp(event.box.mailbox_guid.guid, test_mailbox_guid1, GUID_128_SIZE) == 0); - test_assert(event.box.flags == DSYNC_MAILBOX_FLAG_DELETED_MAILBOX); - test_assert(event.box.uid_validity == 34343); - test_assert(event.box.uid_next == 22); - test_assert(event.box.message_count == 58293); - test_assert(event.box.highest_modseq == 2238427847284728); - test_assert(event.box.first_recent_uid == 2482); - test_assert(event.box.last_change == 53); - - test_end(); -} - -static void test_dsync_proxy_box_select(void) -{ - test_begin("proxy server box select"); - - test_assert(run_cmd("BOX-SELECT", TEST_MAILBOX_GUID1, NULL) == 1); - test_assert(memcmp(test_worker->selected_mailbox.guid, test_mailbox_guid1, GUID_128_SIZE) == 0); - - test_assert(run_cmd("BOX-SELECT", TEST_MAILBOX_GUID2, NULL) == 1); - test_assert(memcmp(test_worker->selected_mailbox.guid, test_mailbox_guid2, GUID_128_SIZE) == 0); - - test_end(); -} - -static void test_dsync_proxy_msg_update(void) -{ - struct test_dsync_msg_event event; - - test_begin("proxy server msg update"); - - test_assert(run_cmd("MSG-UPDATE", "123", "4782782842924", - "kw1 "ALL_MAIL_FLAGS" kw2", NULL) == 1); - test_assert(test_dsync_worker_next_msg_event(test_worker, &event)); - test_assert(event.type == LAST_MSG_TYPE_UPDATE); - test_assert(event.msg.uid == 123); - test_assert(event.msg.modseq == 4782782842924); - test_assert(event.msg.flags == MAIL_FLAGS_MASK); - test_assert(strcmp(event.msg.keywords[0], "kw1") == 0); - test_assert(strcmp(event.msg.keywords[1], "kw2") == 0); - test_assert(event.msg.keywords[2] == NULL); - - test_end(); -} - -static void test_dsync_proxy_msg_uid_change(void) -{ - struct test_dsync_msg_event event; - - test_begin("proxy server msg uid change"); - - test_assert(run_cmd("MSG-UID-CHANGE", "454", "995", NULL) == 1); - test_assert(test_dsync_worker_next_msg_event(test_worker, &event)); - test_assert(event.type == LAST_MSG_TYPE_UPDATE_UID); - test_assert(event.msg.uid == 454); - test_assert(event.msg.modseq == 995); - - test_end(); -} - -static void test_dsync_proxy_msg_expunge(void) -{ - struct test_dsync_msg_event event; - - test_begin("proxy server msg expunge"); - - test_assert(run_cmd("MSG-EXPUNGE", "8585", NULL) == 1); - test_assert(test_dsync_worker_next_msg_event(test_worker, &event)); - test_assert(event.type == LAST_MSG_TYPE_EXPUNGE); - test_assert(event.msg.uid == 8585); - - test_end(); -} - -static void test_dsync_proxy_msg_copy(void) -{ - struct test_dsync_msg_event msg_event; - - test_begin("proxy server msg copy"); - - test_assert(run_cmd("MSG-COPY", TEST_MAILBOX_GUID1, "5454", - "copyguid", "5678", "74782482882924", "\\Seen foo \\Draft", - "8294284", NULL) == 1); - test_assert(test_dsync_worker_next_msg_event(test_worker, &msg_event)); - test_assert(msg_event.type == LAST_MSG_TYPE_COPY); - test_assert(memcmp(msg_event.copy_src_mailbox.guid, test_mailbox_guid1, GUID_128_SIZE) == 0); - test_assert(msg_event.copy_src_uid == 5454); - test_assert(strcmp(msg_event.msg.guid, "copyguid") == 0); - test_assert(msg_event.msg.uid == 5678); - test_assert(msg_event.msg.modseq == 74782482882924); - test_assert(msg_event.msg.flags == (MAIL_SEEN | MAIL_DRAFT)); - test_assert(strcmp(msg_event.msg.keywords[0], "foo") == 0); - test_assert(msg_event.msg.keywords[1] == NULL); - test_assert(msg_event.msg.save_date == 8294284); - - test_end(); -} - -static void test_dsync_proxy_msg_save(void) -{ - static const char *input = "..dotty\n..behavior\nfrom you\n.\nstop"; - struct test_dsync_msg_event event; - const unsigned char *data; - size_t size; - - test_begin("proxy server msg save"); - - server->input = i_stream_create_from_data(input, strlen(input)); - - test_assert(run_cmd("MSG-SAVE", "28492428", "pop3uidl", - "saveguid", "874", "33982482882924", "\\Flagged bar \\Answered", - "8294284", NULL) == 1); - test_assert(test_dsync_worker_next_msg_event(test_worker, &event)); - test_assert(event.type == LAST_MSG_TYPE_SAVE); - test_assert(event.save_data.received_date == 28492428); - test_assert(strcmp(event.save_data.pop3_uidl, "pop3uidl") == 0); - test_assert(strcmp(event.save_body, ".dotty\n.behavior\nfrom you") == 0); - - test_assert(strcmp(event.msg.guid, "saveguid") == 0); - test_assert(event.msg.uid == 874); - test_assert(event.msg.modseq == 33982482882924); - test_assert(event.msg.flags == (MAIL_FLAGGED | MAIL_ANSWERED)); - test_assert(strcmp(event.msg.keywords[0], "bar") == 0); - test_assert(event.msg.keywords[1] == NULL); - test_assert(event.msg.save_date == 8294284); - - data = i_stream_get_data(server->input, &size); - test_assert(size == 4 && memcmp(data, "stop", 4) == 0); - i_stream_destroy(&server->input); - - test_end(); -} - -static struct dsync_proxy_server * -dsync_proxy_server_init_test(buffer_t *outbuf) -{ - struct dsync_proxy_server *server; - - server = i_new(struct dsync_proxy_server, 1); - server->worker = dsync_worker_init_test(); - server->fd_in = 0; - server->fd_out = 0; - - server->cmd_pool = pool_alloconly_create("worker server cmd", 1024); - server->output = o_stream_create_buffer(outbuf); - return server; -} - -int main(void) -{ - static void (*test_functions[])(void) = { - test_dsync_proxy_box_list, - test_dsync_proxy_subs_list, - test_dsync_proxy_msg_list, - test_dsync_proxy_box_create, - test_dsync_proxy_box_delete, - test_dsync_proxy_box_rename, - test_dsync_proxy_box_update, - test_dsync_proxy_box_select, - test_dsync_proxy_msg_update, - test_dsync_proxy_msg_uid_change, - test_dsync_proxy_msg_expunge, - test_dsync_proxy_msg_copy, - test_dsync_proxy_msg_save, - NULL - }; - - test_init(); - - out = buffer_create_dynamic(default_pool, 1024); - server = dsync_proxy_server_init_test(out); - test_worker = (struct test_dsync_worker *)server->worker; - - test_run_funcs(test_functions); - return test_deinit(); -}
--- a/src/doveadm/dsync/test-dsync-proxy.c Fri May 04 05:35:36 2012 +0300 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,184 +0,0 @@ -/* Copyright (c) 2009-2012 Dovecot authors, see the included COPYING file */ - -#include "lib.h" -#include "array.h" -#include "str.h" -#include "mail-cache.h" -#include "dsync-proxy.h" -#include "test-dsync-common.h" -#include "test-common.h" - -static void test_dsync_proxy_msg(void) -{ - static const char *test_keywords[] = { - "kw1", "kw2", NULL - }; - string_t *str; - struct dsync_message msg_in, msg_out; - const char *error; - pool_t pool; - - memset(&msg_in, 0, sizeof(msg_in)); - memset(&msg_out, 0, sizeof(msg_out)); - - pool = pool_alloconly_create("msg pool", 1024); - str = t_str_new(256); - msg_in.guid = "\t\001\r\nguid\t\001\n\r"; - msg_in.uid = (uint32_t)-1; - msg_in.modseq = (uint64_t)-1; - msg_in.save_date = (1U << 31)-1; - - test_begin("dsync proxy msg"); - - /* no flags */ - dsync_proxy_msg_export(str, &msg_in); - test_assert(dsync_proxy_msg_import(pool, str_c(str), - &msg_out, &error) == 0); - test_assert(dsync_messages_equal(&msg_in, &msg_out)); - - /* expunged flag */ - msg_in.flags = DSYNC_MAIL_FLAG_EXPUNGED; - str_truncate(str, 0); - dsync_proxy_msg_export(str, &msg_in); - test_assert(dsync_proxy_msg_import(pool, str_c(str), - &msg_out, &error) == 0); - test_assert(dsync_messages_equal(&msg_in, &msg_out)); - - /* expunged flag and another flag */ - msg_in.flags = DSYNC_MAIL_FLAG_EXPUNGED | MAIL_DRAFT; - str_truncate(str, 0); - dsync_proxy_msg_export(str, &msg_in); - test_assert(dsync_proxy_msg_import(pool, str_c(str), - &msg_out, &error) == 0); - test_assert(dsync_messages_equal(&msg_in, &msg_out)); - - /* all flags, some keywords */ - msg_in.flags = MAIL_FLAGS_MASK; - msg_in.keywords = test_keywords; - str_truncate(str, 0); - dsync_proxy_msg_export(str, &msg_in); - test_assert(dsync_proxy_msg_import(pool, str_c(str), - &msg_out, &error) == 0); - test_assert(dsync_messages_equal(&msg_in, &msg_out)); - - /* errors */ - test_assert(dsync_proxy_msg_import(pool, "0", &msg_out, &error) < 0); - test_assert(dsync_proxy_msg_import(pool, "0\t0", &msg_out, &error) < 0); - test_assert(dsync_proxy_msg_import(pool, "0\t0\t0", &msg_out, &error) < 0); - test_assert(dsync_proxy_msg_import(pool, "0\t0\t0\t0", &msg_out, &error) < 0); - test_assert(dsync_proxy_msg_import(pool, "0\t0\t0\t\\\t0", &msg_out, &error) < 0); - test_assert(dsync_proxy_msg_import(pool, "0\t0\t0\t\\seen foo \\foo\t0", &msg_out, &error) < 0); - - /* flags */ - test_assert(dsync_proxy_msg_parse_flags(pool, "\\seen \\draft", &msg_out) == 0); - test_assert(msg_out.flags == (MAIL_SEEN | MAIL_DRAFT)); - test_assert(dsync_proxy_msg_parse_flags(pool, "\\answered \\flagged", &msg_out) == 0); - test_assert(msg_out.flags == (MAIL_ANSWERED | MAIL_FLAGGED)); - test_assert(dsync_proxy_msg_parse_flags(pool, "\\deleted \\recent", &msg_out) == 0); - test_assert(msg_out.flags == (MAIL_DELETED | MAIL_RECENT)); - test_assert(dsync_proxy_msg_parse_flags(pool, "\\draft draft \\seen", &msg_out) == 0); - test_assert(msg_out.flags == (MAIL_DRAFT | MAIL_SEEN)); - test_assert(strcasecmp(msg_out.keywords[0], "draft") == 0 && msg_out.keywords[1] == NULL); - - test_end(); - pool_unref(&pool); -} - -static void test_dsync_proxy_mailbox(void) -{ - static struct mailbox_cache_field cache1 = - { "cache1", MAIL_CACHE_DECISION_NO, 1234 }; - static struct mailbox_cache_field cache2 = - { "cache2", MAIL_CACHE_DECISION_TEMP | MAIL_CACHE_DECISION_FORCED, 0 }; - string_t *str; - struct dsync_mailbox box_in, box_out; - const char *error; - pool_t pool; - - memset(&box_in, 0, sizeof(box_in)); - memset(&box_out, 0, sizeof(box_out)); - - pool = pool_alloconly_create("mailbox pool", 1024); - str = t_str_new(256); - - test_begin("dsync proxy mailbox"); - - /* test \noselect mailbox */ - box_in.name = "\t\001\r\nname\t\001\n\r"; - box_in.name_sep = '/'; - box_in.flags = DSYNC_MAILBOX_FLAG_NOSELECT; - dsync_proxy_mailbox_export(str, &box_in); - test_assert(dsync_proxy_mailbox_import(pool, str_c(str), - &box_out, &error) == 0); - test_assert(dsync_mailboxes_equal(&box_in, &box_out)); - - /* real mailbox */ - i_assert(sizeof(box_in.mailbox_guid.guid) == sizeof(test_mailbox_guid1)); - memcpy(box_in.mailbox_guid.guid, test_mailbox_guid2, GUID_128_SIZE); - box_in.flags = 24242 & ~DSYNC_MAILBOX_FLAG_NOSELECT; - box_in.uid_validity = 0xf74d921b; - box_in.uid_next = 73529472; - box_in.highest_modseq = 0x123456789abcdef0ULL; - - str_truncate(str, 0); - dsync_proxy_mailbox_export(str, &box_in); - test_assert(dsync_proxy_mailbox_import(pool, str_c(str), - &box_out, &error) == 0); - test_assert(dsync_mailboxes_equal(&box_in, &box_out)); - - /* limits */ - box_in.uid_next = (uint32_t)-1; - box_in.highest_modseq = (uint64_t)-1; - - str_truncate(str, 0); - dsync_proxy_mailbox_export(str, &box_in); - test_assert(dsync_proxy_mailbox_import(pool, str_c(str), - &box_out, &error) == 0); - test_assert(dsync_mailboxes_equal(&box_in, &box_out)); - - /* mailbox with cache fields */ - t_array_init(&box_in.cache_fields, 10); - array_append(&box_in.cache_fields, &cache1, 1); - array_append(&box_in.cache_fields, &cache2, 1); - - str_truncate(str, 0); - dsync_proxy_mailbox_export(str, &box_in); - test_assert(dsync_proxy_mailbox_import(pool, str_c(str), - &box_out, &error) == 0); - test_assert(dsync_mailboxes_equal(&box_in, &box_out)); - - test_end(); - pool_unref(&pool); -} - -static void test_dsync_proxy_guid(void) -{ - mailbox_guid_t guid_in, guid_out; - string_t *str; - - test_begin("dsync proxy mailbox guid"); - - str = t_str_new(128); - memcpy(guid_in.guid, test_mailbox_guid1, sizeof(guid_in.guid)); - dsync_proxy_mailbox_guid_export(str, &guid_in); - test_assert(dsync_proxy_mailbox_guid_import(str_c(str), &guid_out) == 0); - test_assert(memcmp(guid_in.guid, guid_out.guid, sizeof(guid_in.guid)) == 0); - - test_assert(dsync_proxy_mailbox_guid_import("12345678901234567890123456789012", &guid_out) == 0); - test_assert(dsync_proxy_mailbox_guid_import("1234567890123456789012345678901", &guid_out) < 0); - test_assert(dsync_proxy_mailbox_guid_import("1234567890123456789012345678901g", &guid_out) < 0); - test_assert(dsync_proxy_mailbox_guid_import("", &guid_out) < 0); - - test_end(); -} - -int main(void) -{ - static void (*test_functions[])(void) = { - test_dsync_proxy_msg, - test_dsync_proxy_mailbox, - test_dsync_proxy_guid, - NULL - }; - return test_run(test_functions); -}
--- a/src/doveadm/dsync/test-dsync-worker.c Fri May 04 05:35:36 2012 +0300 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,481 +0,0 @@ -/* Copyright (c) 2009-2012 Dovecot authors, see the included COPYING file */ - -#include "lib.h" -#include "array.h" -#include "str.h" -#include "istream.h" -#include "test-dsync-worker.h" - -extern struct dsync_worker_vfuncs test_dsync_worker; - -struct dsync_worker *dsync_worker_init_test(void) -{ - struct test_dsync_worker *worker; - - worker = i_new(struct test_dsync_worker, 1); - worker->worker.v = test_dsync_worker; - worker->tmp_pool = pool_alloconly_create("test worker", 256); - i_array_init(&worker->box_events, 64); - i_array_init(&worker->msg_events, 64); - i_array_init(&worker->results, 64); - worker->body_stream = i_stream_create_from_data("hdr\n\nbody", 9); - return &worker->worker; -} - -static void test_worker_deinit(struct dsync_worker *_worker) -{ - struct test_dsync_worker *worker = (struct test_dsync_worker *)_worker; - - pool_unref(&worker->tmp_pool); - array_free(&worker->box_events); - array_free(&worker->msg_events); - array_free(&worker->results); - i_stream_unref(&worker->body_stream); - i_free(worker); -} - -static bool test_worker_is_output_full(struct dsync_worker *worker ATTR_UNUSED) -{ - return FALSE; -} - -static int test_worker_output_flush(struct dsync_worker *worker ATTR_UNUSED) -{ - return 1; -} - -static struct dsync_worker_mailbox_iter * -test_worker_mailbox_iter_init(struct dsync_worker *_worker) -{ - struct test_dsync_worker *worker = (struct test_dsync_worker *)_worker; - - i_assert(worker->box_iter.iter.worker == NULL); - - worker->box_iter.iter.worker = _worker; - return &worker->box_iter.iter; -} - -static int -test_worker_mailbox_iter_next(struct dsync_worker_mailbox_iter *_iter, - struct dsync_mailbox *dsync_box_r) -{ - struct test_dsync_worker_mailbox_iter *iter = - (struct test_dsync_worker_mailbox_iter *)_iter; - - if (iter->next_box == NULL) - return iter->last ? -1 : 0; - - *dsync_box_r = *iter->next_box; - iter->next_box = NULL; - return 1; -} - -static int -test_worker_mailbox_iter_deinit(struct dsync_worker_mailbox_iter *iter) -{ - struct test_dsync_worker *worker = - (struct test_dsync_worker *)iter->worker; - - memset(&worker->box_iter, 0, sizeof(worker->box_iter)); - return 0; -} - -static struct dsync_worker_subs_iter * -test_worker_subs_iter_init(struct dsync_worker *_worker) -{ - struct test_dsync_worker *worker = (struct test_dsync_worker *)_worker; - - i_assert(worker->subs_iter.iter.worker == NULL); - - worker->subs_iter.iter.worker = _worker; - return &worker->subs_iter.iter; -} - -static int -test_worker_subs_iter_next(struct dsync_worker_subs_iter *_iter, - struct dsync_worker_subscription *rec_r) -{ - struct test_dsync_worker_subs_iter *iter = - (struct test_dsync_worker_subs_iter *)_iter; - - if (iter->next_subscription == NULL) - return iter->last_subs ? -1 : 0; - - *rec_r = *iter->next_subscription; - iter->next_subscription = NULL; - return 1; -} - -static int -test_worker_subs_iter_next_un(struct dsync_worker_subs_iter *_iter, - struct dsync_worker_unsubscription *rec_r) -{ - struct test_dsync_worker_subs_iter *iter = - (struct test_dsync_worker_subs_iter *)_iter; - - if (iter->next_unsubscription == NULL) - return iter->last_unsubs ? -1 : 0; - - *rec_r = *iter->next_unsubscription; - iter->next_unsubscription = NULL; - return 1; -} - -static int -test_worker_subs_iter_deinit(struct dsync_worker_subs_iter *iter) -{ - struct test_dsync_worker *worker = - (struct test_dsync_worker *)iter->worker; - - memset(&worker->subs_iter, 0, sizeof(worker->subs_iter)); - return 0; -} - -static struct dsync_worker_msg_iter * -test_worker_msg_iter_init(struct dsync_worker *_worker, - const mailbox_guid_t mailboxes[], - unsigned int mailbox_count) -{ - struct test_dsync_worker *worker = (struct test_dsync_worker *)_worker; - - i_assert(worker->msg_iter.iter.worker == NULL); - - worker->msg_iter_mailboxes = - p_new(worker->tmp_pool, mailbox_guid_t, mailbox_count); - memcpy(worker->msg_iter_mailboxes, mailboxes, - sizeof(mailboxes[0]) * mailbox_count); - worker->msg_iter_mailbox_count = mailbox_count; - i_array_init(&worker->msg_iter.msgs, 64); - - worker->msg_iter.iter.worker = _worker; - return &worker->msg_iter.iter; -} - -static int -test_worker_msg_iter_next(struct dsync_worker_msg_iter *_iter, - unsigned int *mailbox_idx_r, - struct dsync_message *msg_r) -{ - struct test_dsync_worker_msg_iter *iter = - (struct test_dsync_worker_msg_iter *)_iter; - const struct test_dsync_worker_msg *msg; - - if (iter->idx == array_count(&iter->msgs)) - return iter->last ? -1 : 0; - - msg = array_idx(&iter->msgs, iter->idx++); - *msg_r = msg->msg; - *mailbox_idx_r = msg->mailbox_idx; - return 1; -} - -static int -test_worker_msg_iter_deinit(struct dsync_worker_msg_iter *iter) -{ - struct test_dsync_worker *worker = - (struct test_dsync_worker *)iter->worker; - - array_free(&worker->msg_iter.msgs); - memset(&worker->msg_iter, 0, sizeof(worker->msg_iter)); - return 0; -} - -static void -test_worker_set_last_box(struct dsync_worker *_worker, - const struct dsync_mailbox *dsync_box, - enum test_dsync_last_box_type type) -{ - struct test_dsync_worker *worker = (struct test_dsync_worker *)_worker; - struct test_dsync_box_event event; - - event.type = type; - - event.box = *dsync_box; - event.box.name = p_strdup(worker->tmp_pool, dsync_box->name); - array_append(&worker->box_events, &event, 1); -} - -bool test_dsync_worker_next_box_event(struct test_dsync_worker *worker, - struct test_dsync_box_event *event_r) -{ - const struct test_dsync_box_event *events; - unsigned int count; - - events = array_get(&worker->box_events, &count); - if (count == 0) - return FALSE; - - *event_r = events[0]; - array_delete(&worker->box_events, 0, 1); - return TRUE; -} - -static void -test_worker_set_subscribed(struct dsync_worker *_worker, - const char *name, time_t last_change, bool set) -{ - struct dsync_mailbox dsync_box; - - memset(&dsync_box, 0, sizeof(dsync_box)); - dsync_box.name = name; - dsync_box.last_change = last_change; - test_worker_set_last_box(_worker, &dsync_box, - set ? LAST_BOX_TYPE_SUBSCRIBE : - LAST_BOX_TYPE_UNSUBSCRIBE); -} - -static void -test_worker_create_mailbox(struct dsync_worker *_worker, - const struct dsync_mailbox *dsync_box) -{ - test_worker_set_last_box(_worker, dsync_box, LAST_BOX_TYPE_CREATE); -} - -static void -test_worker_delete_mailbox(struct dsync_worker *_worker, - const struct dsync_mailbox *dsync_box) -{ - struct test_dsync_worker *worker = (struct test_dsync_worker *)_worker; - struct test_dsync_box_event event; - - memset(&event, 0, sizeof(event)); - event.type = LAST_BOX_TYPE_DELETE; - - event.box = *dsync_box; - array_append(&worker->box_events, &event, 1); -} - -static void -test_worker_delete_dir(struct dsync_worker *_worker, - const struct dsync_mailbox *dsync_box) -{ - struct test_dsync_worker *worker = (struct test_dsync_worker *)_worker; - struct test_dsync_box_event event; - - memset(&event, 0, sizeof(event)); - event.type = LAST_BOX_TYPE_DELETE_DIR; - - event.box = *dsync_box; - array_append(&worker->box_events, &event, 1); -} - -static void -test_worker_rename_mailbox(struct dsync_worker *_worker, - const mailbox_guid_t *mailbox, - const struct dsync_mailbox *dsync_box) -{ - struct test_dsync_worker *worker = (struct test_dsync_worker *)_worker; - struct test_dsync_box_event event; - - memset(&event, 0, sizeof(event)); - event.type = LAST_BOX_TYPE_RENAME; - - event.box = *dsync_box; - event.box.mailbox_guid = *mailbox; - array_append(&worker->box_events, &event, 1); -} - -static void -test_worker_update_mailbox(struct dsync_worker *_worker, - const struct dsync_mailbox *dsync_box) -{ - test_worker_set_last_box(_worker, dsync_box, LAST_BOX_TYPE_UPDATE); -} - -static void -test_worker_select_mailbox(struct dsync_worker *_worker, - const mailbox_guid_t *mailbox, - const ARRAY_TYPE(mailbox_cache_field) *cache_fields) -{ - struct test_dsync_worker *worker = (struct test_dsync_worker *)_worker; - struct dsync_mailbox box; - - worker->selected_mailbox = *mailbox; - worker->cache_fields = cache_fields; - - memset(&box, 0, sizeof(box)); - memcpy(box.mailbox_guid.guid, mailbox, sizeof(box.mailbox_guid.guid)); -} - -static struct test_dsync_msg_event * -test_worker_set_last_msg(struct test_dsync_worker *worker, - const struct dsync_message *msg, - enum test_dsync_last_msg_type type) -{ - struct test_dsync_msg_event *event; - const char **keywords; - unsigned int i, count; - - event = array_append_space(&worker->msg_events); - event->type = type; - event->msg = *msg; - event->mailbox = worker->selected_mailbox; - event->msg.guid = p_strdup(worker->tmp_pool, msg->guid); - if (msg->keywords != NULL) { - count = str_array_length(msg->keywords); - keywords = p_new(worker->tmp_pool, const char *, count+1); - for (i = 0; i < count; i++) { - keywords[i] = p_strdup(worker->tmp_pool, - msg->keywords[i]); - } - event->msg.keywords = keywords; - } - return event; -} - -bool test_dsync_worker_next_msg_event(struct test_dsync_worker *worker, - struct test_dsync_msg_event *event_r) -{ - const struct test_dsync_msg_event *events; - unsigned int count; - - events = array_get(&worker->msg_events, &count); - if (count == 0) - return FALSE; - - *event_r = events[0]; - array_delete(&worker->msg_events, 0, 1); - return TRUE; -} - -static void -test_worker_msg_update_metadata(struct dsync_worker *_worker, - const struct dsync_message *msg) -{ - struct test_dsync_worker *worker = (struct test_dsync_worker *)_worker; - - test_worker_set_last_msg(worker, msg, LAST_MSG_TYPE_UPDATE); -} - -static void -test_worker_msg_update_uid(struct dsync_worker *_worker, - uint32_t old_uid, uint32_t new_uid) -{ - struct test_dsync_worker *worker = (struct test_dsync_worker *)_worker; - struct dsync_message msg; - - memset(&msg, 0, sizeof(msg)); - msg.uid = old_uid; - msg.modseq = new_uid; - test_worker_set_last_msg(worker, &msg, LAST_MSG_TYPE_UPDATE_UID); -} - -static void test_worker_msg_expunge(struct dsync_worker *_worker, uint32_t uid) -{ - struct test_dsync_worker *worker = (struct test_dsync_worker *)_worker; - struct dsync_message msg; - - memset(&msg, 0, sizeof(msg)); - msg.uid = uid; - test_worker_set_last_msg(worker, &msg, LAST_MSG_TYPE_EXPUNGE); -} - -static void -test_worker_msg_copy(struct dsync_worker *_worker, - const mailbox_guid_t *src_mailbox, - uint32_t src_uid, const struct dsync_message *dest_msg, - dsync_worker_copy_callback_t *callback, void *context) -{ - struct test_dsync_worker *worker = (struct test_dsync_worker *)_worker; - struct test_dsync_msg_event *event; - - event = test_worker_set_last_msg(worker, dest_msg, LAST_MSG_TYPE_COPY); - event->copy_src_mailbox = *src_mailbox; - event->copy_src_uid = src_uid; - callback(TRUE, context); -} - -static void -test_worker_msg_save(struct dsync_worker *_worker, - const struct dsync_message *msg, - const struct dsync_msg_static_data *data, - dsync_worker_save_callback_t *callback, - void *context) -{ - struct test_dsync_worker *worker = (struct test_dsync_worker *)_worker; - struct test_dsync_msg_event *event; - const unsigned char *d; - size_t size; - ssize_t ret; - string_t *body; - - event = test_worker_set_last_msg(worker, msg, LAST_MSG_TYPE_SAVE); - event->save_data.pop3_uidl = p_strdup(worker->tmp_pool, data->pop3_uidl); - event->save_data.received_date = data->received_date; - - body = t_str_new(256); - while ((ret = i_stream_read_data(data->input, &d, &size, 0)) > 0) { - str_append_n(body, d, size); - i_stream_skip(data->input, size); - } - i_assert(ret == -1); - event->save_body = p_strdup(worker->tmp_pool, str_c(body)); - - callback(context); -} - -static void -test_worker_msg_save_cancel(struct dsync_worker *_worker ATTR_UNUSED) -{ -} - -static void -test_worker_msg_get(struct dsync_worker *_worker, - const mailbox_guid_t *mailbox ATTR_UNUSED, - uint32_t uid ATTR_UNUSED, - dsync_worker_msg_callback_t *callback, void *context) -{ - struct test_dsync_worker *worker = (struct test_dsync_worker *)_worker; - struct dsync_msg_static_data data; - - memset(&data, 0, sizeof(data)); - data.pop3_uidl = "uidl"; - data.received_date = 123456; - data.input = worker->body_stream; - i_stream_seek(data.input, 0); - callback(DSYNC_MSG_GET_RESULT_SUCCESS, &data, context); -} - -static void -test_worker_finish(struct dsync_worker *_worker ATTR_UNUSED, - dsync_worker_finish_callback_t *callback, void *context) -{ - callback(TRUE, context); -} - -struct dsync_worker_vfuncs test_dsync_worker = { - test_worker_deinit, - - test_worker_is_output_full, - test_worker_output_flush, - - test_worker_mailbox_iter_init, - test_worker_mailbox_iter_next, - test_worker_mailbox_iter_deinit, - - test_worker_subs_iter_init, - test_worker_subs_iter_next, - test_worker_subs_iter_next_un, - test_worker_subs_iter_deinit, - test_worker_set_subscribed, - - test_worker_msg_iter_init, - test_worker_msg_iter_next, - test_worker_msg_iter_deinit, - - test_worker_create_mailbox, - test_worker_delete_mailbox, - test_worker_delete_dir, - test_worker_rename_mailbox, - test_worker_update_mailbox, - - test_worker_select_mailbox, - test_worker_msg_update_metadata, - test_worker_msg_update_uid, - test_worker_msg_expunge, - test_worker_msg_copy, - test_worker_msg_save, - test_worker_msg_save_cancel, - test_worker_msg_get, - test_worker_finish -};
--- a/src/doveadm/dsync/test-dsync-worker.h Fri May 04 05:35:36 2012 +0300 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,96 +0,0 @@ -#ifndef TEST_DSYNC_WORKER_H -#define TEST_DSYNC_WORKER_H - -#include "dsync-worker-private.h" - -enum test_dsync_last_box_type { - LAST_BOX_TYPE_CREATE, - LAST_BOX_TYPE_DELETE, - LAST_BOX_TYPE_DELETE_DIR, - LAST_BOX_TYPE_RENAME, - LAST_BOX_TYPE_UPDATE, - LAST_BOX_TYPE_SUBSCRIBE, - LAST_BOX_TYPE_UNSUBSCRIBE -}; - -enum test_dsync_last_msg_type { - LAST_MSG_TYPE_UPDATE, - LAST_MSG_TYPE_UPDATE_UID, - LAST_MSG_TYPE_EXPUNGE, - LAST_MSG_TYPE_COPY, - LAST_MSG_TYPE_SAVE -}; - -struct test_dsync_worker_mailbox_iter { - struct dsync_worker_mailbox_iter iter; - struct dsync_mailbox *next_box; - bool last; -}; - -struct test_dsync_worker_subs_iter { - struct dsync_worker_subs_iter iter; - struct dsync_worker_subscription *next_subscription; - struct dsync_worker_unsubscription *next_unsubscription; - bool last_subs, last_unsubs; -}; - -struct test_dsync_worker_msg { - struct dsync_message msg; - unsigned int mailbox_idx; -}; - -struct test_dsync_worker_msg_iter { - struct dsync_worker_msg_iter iter; - ARRAY_DEFINE(msgs, struct test_dsync_worker_msg); - unsigned int idx; - bool last; -}; - -struct test_dsync_worker_result { - uint32_t tag; - int result; -}; - -struct test_dsync_box_event { - enum test_dsync_last_box_type type; - struct dsync_mailbox box; -}; - -struct test_dsync_msg_event { - enum test_dsync_last_msg_type type; - struct dsync_message msg; - - mailbox_guid_t mailbox, copy_src_mailbox; - uint32_t copy_src_uid; - struct dsync_msg_static_data save_data; - const char *save_body; -}; - -struct test_dsync_worker { - struct dsync_worker worker; - struct istream *body_stream; - - struct test_dsync_worker_mailbox_iter box_iter; - struct test_dsync_worker_subs_iter subs_iter; - struct test_dsync_worker_msg_iter msg_iter; - ARRAY_DEFINE(results, struct test_dsync_worker_result); - - pool_t tmp_pool; - - ARRAY_DEFINE(box_events, struct test_dsync_box_event); - ARRAY_DEFINE(msg_events, struct test_dsync_msg_event); - - mailbox_guid_t selected_mailbox; - mailbox_guid_t *msg_iter_mailboxes; - unsigned int msg_iter_mailbox_count; - const ARRAY_TYPE(mailbox_cache_field) *cache_fields; -}; - -struct dsync_worker *dsync_worker_init_test(void); - -bool test_dsync_worker_next_box_event(struct test_dsync_worker *worker, - struct test_dsync_box_event *event_r); -bool test_dsync_worker_next_msg_event(struct test_dsync_worker *worker, - struct test_dsync_msg_event *event_r); - -#endif