Mercurial > dovecot > core-2.2

changeset 11676:cf7f6912af02 HEAD

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
dsync: Added backup command, which syncs source to destination, discarding any changes in dest. It doesn't work perfectly in all situations. Especially if destination had saved/expunged mails in INBOX, dsync can't resolve it. For non-INBOXes it deletes the mailbox and fails a bit later, so that the next dsync can do a full resync for the mailbox.
author Timo Sirainen <tss@iki.fi>
date Thu, 01 Jul 2010 17:02:20 +0100
parents f319af83ae36
children b4f029f7793a
files src/dsync/dsync-brain-msgs.c src/dsync/dsync-brain-private.h src/dsync/dsync-brain.c src/dsync/dsync-brain.h src/dsync/dsync.c
diffstat 5 files changed, 189 insertions(+), 67 deletions(-) [+]
line wrap: on
line diff
--- a/src/dsync/dsync-brain-msgs.c	Thu Jul 01 16:38:12 2010 +0100
+++ b/src/dsync/dsync-brain-msgs.c	Thu Jul 01 17:02:20 2010 +0100
@@ -77,7 +77,8 @@
 	int ret = 1;
 
 	if (iter->msg.guid == NULL) {
-		ret = dsync_worker_msg_iter_next(iter->iter, &iter->mailbox_idx,
+		ret = dsync_worker_msg_iter_next(iter->iter,
+						 &iter->mailbox_idx,
 						 &iter->msg);
 		if (ret > 0)
 			dsync_brain_guid_add(iter);
@@ -90,10 +91,32 @@
 	return ret;
 }
 
+static int
+dsync_brain_msg_iter_skip_mailbox(struct dsync_brain_mailbox_sync *sync)
+{
+	int ret;
+
+	while ((ret = dsync_brain_msg_iter_next(sync->src_msg_iter)) > 0) ;
+	if (ret == 0)
+		return 0;
+
+	while ((ret = dsync_brain_msg_iter_next(sync->dest_msg_iter)) > 0) ;
+	if (ret == 0)
+		return 0;
+
+	sync->skip_mailbox = FALSE;
+	return -1;
+}
+
 static int dsync_brain_msg_iter_next_pair(struct dsync_brain_mailbox_sync *sync)
 {
 	int ret;
 
+	if (sync->skip_mailbox) {
+		if (dsync_brain_msg_iter_skip_mailbox(sync) == 0)
+			return 0;
+	}
+
 	if ((ret = dsync_brain_msg_iter_next(sync->src_msg_iter)) <= 0)
 		return ret;
 	if ((ret = dsync_brain_msg_iter_next(sync->dest_msg_iter)) <= 0)
@@ -129,6 +152,18 @@
 
 	brain_box = array_idx_modifiable(&save_iter->sync->mailboxes,
 					 save_iter->mailbox_idx);
+
+	if (save_iter->sync->brain->backup) {
+		i_warning("Destination mailbox %s has been modified, "
+			  "need to recreate it before we can continue syncing",
+			  brain_box->box.name);
+		dsync_worker_delete_mailbox(save_iter->sync->brain->dest_worker,
+					    &brain_box->box);
+		save_iter->sync->brain->unexpected_changes = TRUE;
+		save_iter->sync->skip_mailbox = TRUE;
+		return;
+	}
+
 	new_uid = brain_box->box.uid_next++;
 
 	conflict = array_append_space(&conflict_iter->uid_conflicts);
@@ -185,7 +220,7 @@
 	int ret;
 
 	ret = dsync_message_flag_importance_cmp(src_msg, dest_msg);
-	if (ret < 0)
+	if (ret < 0 || (sync->brain->backup && ret > 0))
 		dsync_worker_msg_update_metadata(sync->dest_worker, src_msg);
 	else if (ret > 0)
 		dsync_worker_msg_update_metadata(sync->src_worker, dest_msg);
@@ -230,7 +265,7 @@
 		/* message has been expunged from dest. */
 		if (src_expunged) {
 			/* expunged from source already */
-		} else if (sync->uid_conflict) {
+		} else if (sync->uid_conflict || sync->brain->backup) {
 			/* update uid src, copy to dest */
 			dsync_brain_msg_sync_conflict(sync->src_msg_iter,
 						      sync->dest_msg_iter,
@@ -246,7 +281,7 @@
 		/* message has been expunged from src. */
 		if (dest_expunged) {
 			/* expunged from dest already */
-		} else if (sync->uid_conflict) {
+		} else if (sync->uid_conflict && !sync->brain->backup) {
 			/* update uid in dest, copy to src */
 			dsync_brain_msg_sync_conflict(sync->dest_msg_iter,
 						      sync->src_msg_iter,
@@ -281,7 +316,13 @@
 		}
 	} else if (dest_expunged) {
 		/* message expunged from destination */
-		if (!src_expunged) {
+		if (src_expunged) {
+			/* expunged from source already */
+		} else if (sync->brain->backup) {
+			dsync_brain_msg_sync_conflict(sync->src_msg_iter,
+						      sync->dest_msg_iter,
+						      src_msg);
+		} else {
 			dsync_worker_msg_expunge(sync->src_worker,
 						 src_msg->uid);
 		}
@@ -329,9 +370,11 @@
 	/* finished syncing messages in this mailbox that exist in both source
 	   and destination. if there are messages left, we can't reliably know
 	   if they should be expunged, so just copy them to the other side. */
-	if (!dsync_brain_msg_sync_mailbox_end(sync->dest_msg_iter,
-					      sync->src_msg_iter))
-		return FALSE;
+	if (!sync->brain->backup) {
+		if (!dsync_brain_msg_sync_mailbox_end(sync->dest_msg_iter,
+						      sync->src_msg_iter))
+			return FALSE;
+	}
 	if (!dsync_brain_msg_sync_mailbox_end(sync->src_msg_iter,
 					      sync->dest_msg_iter))
 		return FALSE;
--- a/src/dsync/dsync-brain-private.h	Thu Jul 01 16:38:12 2010 +0100
+++ b/src/dsync/dsync-brain-private.h	Thu Jul 01 17:02:20 2010 +0100
@@ -104,6 +104,7 @@
 	struct dsync_brain_msg_iter *dest_msg_iter;
 
 	unsigned int uid_conflict:1;
+	unsigned int skip_mailbox:1;
 };
 
 struct dsync_brain {
@@ -124,6 +125,8 @@
 
 	unsigned int failed:1;
 	unsigned int verbose:1;
+	unsigned int backup:1;
+	unsigned int unexpected_changes:1;
 };
 
 void dsync_brain_fail(struct dsync_brain *brain);
--- a/src/dsync/dsync-brain.c	Thu Jul 01 16:38:12 2010 +0100
+++ b/src/dsync/dsync-brain.c	Thu Jul 01 17:02:20 2010 +0100
@@ -25,6 +25,7 @@
 	brain->mailbox = i_strdup(mailbox);
 	brain->flags = flags;
 	brain->verbose = (flags & DSYNC_BRAIN_FLAG_VERBOSE) != 0;
+	brain->backup = (flags & DSYNC_BRAIN_FLAG_BACKUP) != 0;
 	return brain;
 }
 
@@ -235,20 +236,68 @@
 	pool_unref(&list->pool);
 }
 
+enum dsync_brain_mailbox_action {
+	DSYNC_BRAIN_MAILBOX_ACTION_NONE,
+	DSYNC_BRAIN_MAILBOX_ACTION_CREATE,
+	DSYNC_BRAIN_MAILBOX_ACTION_DELETE
+};
+
+static void
+dsync_brain_mailbox_action(struct dsync_brain *brain,
+			   enum dsync_brain_mailbox_action action,
+			   struct dsync_worker *action_worker,
+			   struct dsync_mailbox *action_box)
+{
+	struct dsync_mailbox new_box;
+
+	if (brain->backup && action_worker == brain->src_worker) {
+		/* backup mode: switch actions */
+		action_worker = brain->dest_worker;
+		switch (action) {
+		case DSYNC_BRAIN_MAILBOX_ACTION_NONE:
+			break;
+		case DSYNC_BRAIN_MAILBOX_ACTION_CREATE:
+			action = DSYNC_BRAIN_MAILBOX_ACTION_DELETE;
+			break;
+		case DSYNC_BRAIN_MAILBOX_ACTION_DELETE:
+			action = DSYNC_BRAIN_MAILBOX_ACTION_CREATE;
+			break;
+		}
+	}
+
+	switch (action) {
+	case DSYNC_BRAIN_MAILBOX_ACTION_NONE:
+		break;
+	case DSYNC_BRAIN_MAILBOX_ACTION_CREATE:
+		new_box = *action_box;
+		new_box.uid_next = 0;
+		new_box.highest_modseq = 0;
+		dsync_worker_create_mailbox(action_worker, &new_box);
+		break;
+	case DSYNC_BRAIN_MAILBOX_ACTION_DELETE:
+		if (!dsync_mailbox_is_noselect(action_box))
+			dsync_worker_delete_mailbox(action_worker, action_box);
+		else
+			dsync_worker_delete_dir(action_worker, action_box);
+		break;
+	}
+}
+
 static void dsync_brain_sync_mailboxes(struct dsync_brain *brain)
 {
-	struct dsync_mailbox *const *src_boxes, *const *dest_boxes, new_box;
+	struct dsync_mailbox *const *src_boxes, *const *dest_boxes, *action_box;
 	unsigned int src, dest, src_count, dest_count;
+	enum dsync_brain_mailbox_action action;
+	struct dsync_worker *action_worker;
 	bool src_deleted, dest_deleted;
 	int ret;
 
-	memset(&new_box, 0, sizeof(new_box));
-
 	/* create/delete missing mailboxes. the mailboxes are sorted by
 	   GUID, so we can do this quickly. */
 	src_boxes = array_get(&brain->src_mailbox_list->mailboxes, &src_count);
 	dest_boxes = array_get(&brain->dest_mailbox_list->mailboxes, &dest_count);
 	for (src = dest = 0; src < src_count && dest < dest_count; ) {
+		action = DSYNC_BRAIN_MAILBOX_ACTION_NONE;
 		src_deleted = (src_boxes[src]->flags &
 			       DSYNC_MAILBOX_FLAG_DELETED_MAILBOX) != 0;
 		dest_deleted = (dest_boxes[dest]->flags &
@@ -258,74 +307,73 @@
 		if (ret < 0) {
 			/* exists only in source */
 			if (!src_deleted) {
-				new_box = *src_boxes[src];
-				new_box.uid_next = 0;
-				new_box.highest_modseq = 0;
-				dsync_worker_create_mailbox(brain->dest_worker,
-							    &new_box);
+				action = DSYNC_BRAIN_MAILBOX_ACTION_CREATE;
+				action_worker = brain->dest_worker;
+				action_box = src_boxes[src];
 			}
 			src++;
 		} else if (ret > 0) {
 			/* exists only in dest */
 			if (!dest_deleted) {
-				new_box = *dest_boxes[dest];
-				new_box.uid_next = 0;
-				new_box.highest_modseq = 0;
-				dsync_worker_create_mailbox(brain->src_worker,
-							    &new_box);
+				action = DSYNC_BRAIN_MAILBOX_ACTION_CREATE;
+				action_worker = brain->src_worker;
+				action_box = dest_boxes[dest];
 			}
 			dest++;
 		} else if (src_deleted) {
 			/* delete from dest too */
 			if (!dest_deleted) {
-				dsync_worker_delete_mailbox(brain->dest_worker,
-							    src_boxes[src]);
+				action = DSYNC_BRAIN_MAILBOX_ACTION_DELETE;
+				action_worker = brain->dest_worker;
+				action_box = dest_boxes[dest];
 			}
 			src++; dest++;
 		} else if (dest_deleted) {
 			/* delete from src too */
-			dsync_worker_delete_mailbox(brain->src_worker,
-						    dest_boxes[dest]);
+			action = DSYNC_BRAIN_MAILBOX_ACTION_DELETE;
+			action_worker = brain->src_worker;
+			action_box = src_boxes[src];
 			src++; dest++;
 		} else {
 			src++; dest++;
 		}
+		dsync_brain_mailbox_action(brain, action,
+					   action_worker, action_box);
 	}
 	for (; src < src_count; src++) {
 		if ((src_boxes[src]->flags &
 		     DSYNC_MAILBOX_FLAG_DELETED_MAILBOX) != 0)
 			continue;
 
-		new_box = *src_boxes[src];
-		new_box.uid_next = 0;
-		new_box.highest_modseq = 0;
-		dsync_worker_create_mailbox(brain->dest_worker, &new_box);
+		dsync_brain_mailbox_action(brain,
+			DSYNC_BRAIN_MAILBOX_ACTION_CREATE,
+			brain->dest_worker, src_boxes[src]);
 	}
 	for (; dest < dest_count; dest++) {
 		if ((dest_boxes[dest]->flags &
 		     DSYNC_MAILBOX_FLAG_DELETED_MAILBOX) != 0)
 			continue;
 
-		new_box = *dest_boxes[dest];
-		new_box.uid_next = 0;
-		new_box.highest_modseq = 0;
-		dsync_worker_create_mailbox(brain->src_worker, &new_box);
+		dsync_brain_mailbox_action(brain,
+			DSYNC_BRAIN_MAILBOX_ACTION_CREATE,
+			brain->src_worker, dest_boxes[dest]);
 	}
 }
 
 static void dsync_brain_sync_dirs(struct dsync_brain *brain)
 {
-	struct dsync_mailbox *const *src_boxes, *const *dest_boxes, new_box;
+	struct dsync_mailbox *const *src_boxes, *const *dest_boxes, *action_box;
 	unsigned int src, dest, src_count, dest_count;
+	enum dsync_brain_mailbox_action action;
+	struct dsync_worker *action_worker;
 	bool src_deleted, dest_deleted;
 	int ret;
 
-	memset(&new_box, 0, sizeof(new_box));
-
 	/* create/delete missing directories. */
 	src_boxes = array_get(&brain->src_mailbox_list->dirs, &src_count);
 	dest_boxes = array_get(&brain->dest_mailbox_list->dirs, &dest_count);
 	for (src = dest = 0; src < src_count && dest < dest_count; ) {
+		action = DSYNC_BRAIN_MAILBOX_ACTION_NONE;
 		src_deleted = (src_boxes[src]->flags &
 			       DSYNC_MAILBOX_FLAG_DELETED_DIR) != 0;
 		dest_deleted = (dest_boxes[dest]->flags &
@@ -336,50 +384,57 @@
 		if (ret < 0) {
 			/* exists only in source */
 			if (!src_deleted) {
-				new_box = *src_boxes[src];
-				dsync_worker_create_mailbox(brain->dest_worker,
-							    &new_box);
+				action = DSYNC_BRAIN_MAILBOX_ACTION_CREATE;
+				action_worker = brain->dest_worker;
+				action_box = src_boxes[src];
 			}
 			src++;
 		} else if (ret > 0) {
 			/* exists only in dest */
 			if (!dest_deleted) {
-				new_box = *dest_boxes[dest];
-				dsync_worker_create_mailbox(brain->src_worker,
-							    &new_box);
+				action = DSYNC_BRAIN_MAILBOX_ACTION_CREATE;
+				action_worker = brain->src_worker;
+				action_box = dest_boxes[dest];
 			}
 			dest++;
 		} else if (src_deleted) {
 			/* delete from dest too */
 			if (!dest_deleted) {
-				dsync_worker_delete_dir(brain->dest_worker,
-							dest_boxes[dest]);
+				action = DSYNC_BRAIN_MAILBOX_ACTION_DELETE;
+				action_worker = brain->dest_worker;
+				action_box = dest_boxes[dest];
 			}
 			src++; dest++;
 		} else if (dest_deleted) {
 			/* delete from src too */
-			dsync_worker_delete_dir(brain->src_worker,
-						src_boxes[src]);
+			action = DSYNC_BRAIN_MAILBOX_ACTION_DELETE;
+			action_worker = brain->src_worker;
+			action_box = src_boxes[src];
 			src++; dest++;
 		} else {
 			src++; dest++;
 		}
+		i_assert(dsync_mailbox_is_noselect(action_box));
+		dsync_brain_mailbox_action(brain, action,
+					   action_worker, action_box);
 	}
 	for (; src < src_count; src++) {
 		if ((src_boxes[src]->flags &
 		     DSYNC_MAILBOX_FLAG_DELETED_DIR) != 0)
 			continue;
 
-		new_box = *src_boxes[src];
-		dsync_worker_create_mailbox(brain->dest_worker, &new_box);
+		dsync_brain_mailbox_action(brain,
+			DSYNC_BRAIN_MAILBOX_ACTION_CREATE,
+			brain->dest_worker, src_boxes[src]);
 	}
 	for (; dest < dest_count; dest++) {
 		if ((dest_boxes[dest]->flags &
 		     DSYNC_MAILBOX_FLAG_DELETED_DIR) != 0)
 			continue;
 
-		new_box = *dest_boxes[dest];
-		dsync_worker_create_mailbox(brain->src_worker, &new_box);
+		dsync_brain_mailbox_action(brain,
+			DSYNC_BRAIN_MAILBOX_ACTION_CREATE,
+			brain->src_worker, dest_boxes[dest]);
 	}
 }
 
@@ -410,8 +465,11 @@
 static void dsync_brain_sync_subscriptions(struct dsync_brain *brain)
 {
 	const struct dsync_worker_subscription *src_subs, *dest_subs;
+	const struct dsync_worker_subscription *action_subs;
+	struct dsync_worker *action_worker;
 	unsigned int src, dest, src_count, dest_count;
 	time_t last_change;
+	bool subscribe;
 	int ret;
 
 	/* subscriptions are sorted by name. */
@@ -435,33 +493,40 @@
 
 		if (ret < 0) {
 			/* subscribed only in source */
+			action_subs = &src_subs[src];
 			if (dsync_brain_is_unsubscribed(brain->dest_subs_list,
 							&src_subs[src],
 							&last_change)) {
-				dsync_worker_set_subscribed(brain->src_worker,
-							    src_subs[src].vname,
-							    last_change, FALSE);
+				action_worker = brain->src_worker;
+				subscribe = FALSE;
 			} else {
-				dsync_worker_set_subscribed(brain->dest_worker,
-							    src_subs[src].vname,
-							    last_change, TRUE);
+				action_worker = brain->dest_worker;
+				subscribe = TRUE;
 			}
 			src++;
 		} else {
 			/* subscribed only in dest */
+			action_subs = &dest_subs[dest];
 			if (dsync_brain_is_unsubscribed(brain->src_subs_list,
 							&dest_subs[dest],
 							&last_change)) {
-				dsync_worker_set_subscribed(brain->dest_worker,
-							    dest_subs[dest].vname,
-							    last_change, FALSE);
+				action_worker = brain->dest_worker;
+				subscribe = FALSE;
 			} else {
-				dsync_worker_set_subscribed(brain->src_worker,
-							    dest_subs[dest].vname,
-							    last_change, TRUE);
+				action_worker = brain->src_worker;
+				subscribe = TRUE;
 			}
 			dest++;
 		}
+
+		if (brain->backup && action_worker == brain->src_worker) {
+			/* backup mode: switch action */
+			action_worker = brain->dest_worker;
+			subscribe = !subscribe;
+			last_change = ioloop_time;
+		}
+		dsync_worker_set_subscribed(action_worker, action_subs->vname,
+					    last_change, subscribe);
 	}
 }
 
@@ -612,7 +677,8 @@
 dsync_brain_sync_rename_mailbox(struct dsync_brain *brain,
 				const struct dsync_brain_mailbox *mailbox)
 {
-	if (mailbox->src->last_change > mailbox->dest->last_change) {
+	if (mailbox->src->last_change > mailbox->dest->last_change ||
+	    brain->backup) {
 		dsync_worker_rename_mailbox(brain->dest_worker,
 					    &mailbox->box.mailbox_guid,
 					    mailbox->src);
@@ -629,7 +695,10 @@
 	const struct dsync_brain_mailbox *mailbox;
 
 	array_foreach(&brain->mailbox_sync->mailboxes, mailbox) {
-		dsync_worker_update_mailbox(brain->src_worker, &mailbox->box);
+		if (!brain->backup) {
+			dsync_worker_update_mailbox(brain->src_worker,
+						    &mailbox->box);
+		}
 		dsync_worker_update_mailbox(brain->dest_worker, &mailbox->box);
 
 		if (mailbox->src != NULL && mailbox->dest != NULL &&
@@ -735,6 +804,7 @@
 
 bool dsync_brain_has_unexpected_changes(struct dsync_brain *brain)
 {
-	return dsync_worker_has_unexpected_changes(brain->src_worker) ||
+	return brain->unexpected_changes ||
+		dsync_worker_has_unexpected_changes(brain->src_worker) ||
 		dsync_worker_has_unexpected_changes(brain->dest_worker);
 }
--- a/src/dsync/dsync-brain.h	Thu Jul 01 16:38:12 2010 +0100
+++ b/src/dsync/dsync-brain.h	Thu Jul 01 17:02:20 2010 +0100
@@ -3,7 +3,10 @@
 
 enum dsync_brain_flags {
 	DSYNC_BRAIN_FLAG_FULL_SYNC	= 0x01,
-	DSYNC_BRAIN_FLAG_VERBOSE	= 0x02
+	DSYNC_BRAIN_FLAG_VERBOSE	= 0x02,
+	/* Run in backup mode. All changes from src are forced into dest,
+	   discarding any potential changes in dest. */
+	DSYNC_BRAIN_FLAG_BACKUP		= 0x04
 };
 
 struct dsync_worker;
--- a/src/dsync/dsync.c	Thu Jul 01 16:38:12 2010 +0100
+++ b/src/dsync/dsync.c	Thu Jul 01 17:02:20 2010 +0100
@@ -195,10 +195,13 @@
 	cmd_name = argv[optind++];
 
 	if (strcmp(cmd_name, "mirror") == 0 ||
-	    strcmp(cmd_name, "convert") == 0) {
+	    strcmp(cmd_name, "convert") == 0 ||
+	    strcmp(cmd_name, "backup") == 0) {
 		if (optind == argc)
 			usage();
 
+		if (strcmp(cmd_name, "backup") == 0)
+			brain_flags |= DSYNC_BRAIN_FLAG_BACKUP;
 		if (!mirror_get_remote_cmd(argv+optind, &remote_cmd_args)) {
 			if (optind+1 != argc)
 				usage();