changeset 3972:a506ee4ec31e HEAD

Added "mail storage conversion" plugin. It can be used with IMAP, POP3 and/or LDA to convert one complete mail storage to another format. Also included a convert-tool command line tool to do it manually. Currently doesn't support preserving UID/UIDVALIDITY.
author Timo Sirainen <tss@iki.fi>
date Thu, 02 Feb 2006 22:42:44 +0200
parents 539a58176e7b
children fec2e36ddb3b
files configure.in src/plugins/convert/.cvsignore src/plugins/convert/convert-plugin.c src/plugins/convert/convert-plugin.h src/plugins/convert/convert-storage.c src/plugins/convert/convert-storage.h src/plugins/convert/convert-tool.c
diffstat 7 files changed, 331 insertions(+), 5 deletions(-) [+]
line wrap: on
line diff
--- a/configure.in	Thu Feb 02 22:20:30 2006 +0200
+++ b/configure.in	Thu Feb 02 22:42:44 2006 +0200
@@ -1628,10 +1628,10 @@
 dnl ** storage classes
 dnl **
 
-maildir_libs="../lib-storage/index/maildir/libstorage_maildir.a"
-mbox_libs="../lib-storage/index/mbox/libstorage_mbox.a"
-dbox_libs="../lib-storage/index/dbox/libstorage_dbox.a"
-index_libs="../lib-storage/index/libstorage_index.a ../lib-index/libindex.a"
+maildir_libs='$(top_srcdir)/src/lib-storage/index/maildir/libstorage_maildir.a'
+mbox_libs='$(top_srcdir)/src/lib-storage/index/mbox/libstorage_mbox.a'
+dbox_libs='$(top_srcdir)/src/lib-storage/index/dbox/libstorage_dbox.a'
+index_libs='$(top_srcdir)/src/lib-storage/index/libstorage_index.a $(top_srcdir)/src/lib-index/libindex.a'
 
 STORAGE_LIBS=
 for storage in $mail_storages; do
@@ -1640,7 +1640,6 @@
 STORAGE_LIBS="$STORAGE_LIBS $index_libs"
 AC_SUBST(STORAGE_LIBS)
 
-
 dnl **
 dnl ** SQL drivers
 dnl **
@@ -1708,6 +1707,7 @@
 src/pop3-login/Makefile
 src/util/Makefile
 src/plugins/Makefile
+src/plugins/convert/Makefile
 src/plugins/quota/Makefile
 src/plugins/imap-quota/Makefile
 src/plugins/trash/Makefile
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/src/plugins/convert/.cvsignore	Thu Feb 02 22:42:44 2006 +0200
@@ -0,0 +1,9 @@
+*.la
+*.lo
+*.o
+.deps
+.libs
+Makefile
+Makefile.in
+so_locations
+convert-tool
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/src/plugins/convert/convert-plugin.c	Thu Feb 02 22:42:44 2006 +0200
@@ -0,0 +1,33 @@
+/* Copyright (C) 2006 Timo Sirainen */
+
+#include "lib.h"
+#include "convert-storage.h"
+#include "convert-plugin.h"
+
+#include <stdlib.h>
+
+void convert_plugin_init(void)
+{
+	const char *convert_mail, *mail, *home, *user;
+
+	convert_mail = getenv("CONVERT_MAIL");
+	if (convert_mail == NULL)
+		return;
+
+	mail = getenv("MAIL");
+	if (mail == NULL)
+		i_fatal("convert plugin: MAIL unset");
+	user = getenv("USER");
+	if (mail == NULL)
+		i_fatal("convert plugin: USER unset");
+	home = getenv("HOME");
+	if (mail == NULL)
+		i_fatal("convert plugin: HOME unset");
+
+	if (convert_storage(user, home, convert_mail, mail) < 0)
+		exit(FATAL_DEFAULT);
+}
+
+void convert_plugin_deinit(void)
+{
+}
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/src/plugins/convert/convert-plugin.h	Thu Feb 02 22:42:44 2006 +0200
@@ -0,0 +1,7 @@
+#ifndef __CONVERT_PLUGIN_H
+#define __CONVERT_PLUGIN_H
+
+void convert_plugin_init(void);
+void convert_plugin_deinit(void);
+
+#endif
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/src/plugins/convert/convert-storage.c	Thu Feb 02 22:42:44 2006 +0200
@@ -0,0 +1,226 @@
+/* Copyright (C) 2006 Timo Sirainen */
+
+#include "lib.h"
+#include "file-dotlock.h"
+#include "index-storage.h"
+#include "mail-search.h"
+#include "convert-storage.h"
+
+#include <stdio.h>
+
+#define CONVERT_LOCK_FILENAME ".dovecot-convert.lock"
+
+const struct dotlock_settings dotlock_settings = {
+	NULL,
+	NULL,
+
+	60*5,
+	0,
+	60*5,
+
+	NULL,
+	NULL,
+
+	FALSE
+};
+
+static int sync_mailbox(struct mailbox *box)
+{
+	struct mailbox_sync_context *ctx;
+        struct mailbox_sync_rec sync_rec;
+	struct mailbox_status status;
+
+	ctx = mailbox_sync_init(box, MAILBOX_SYNC_FLAG_FULL_READ);
+	while (mailbox_sync_next(ctx, &sync_rec) > 0)
+		;
+	return mailbox_sync_deinit(&ctx, &status);
+}
+
+static int mailbox_copy_mails(struct mailbox *srcbox, struct mailbox *destbox)
+{
+	struct mail_search_context *ctx;
+	struct mailbox_transaction_context *src_trans, *dest_trans;
+	struct mail *mail;
+	struct mail_search_arg search_arg;
+	int ret = 0;
+
+	if (sync_mailbox(srcbox) < 0)
+		return -1;
+
+	memset(&search_arg, 0, sizeof(search_arg));
+	search_arg.type = SEARCH_ALL;
+
+	src_trans = mailbox_transaction_begin(srcbox, 0);
+	dest_trans = mailbox_transaction_begin(destbox,
+					MAILBOX_TRANSACTION_FLAG_EXTERNAL);
+
+	ctx = mailbox_search_init(src_trans, NULL, &search_arg, NULL);
+	mail = mail_alloc(src_trans,
+			  MAIL_FETCH_FLAGS | MAIL_FETCH_RECEIVED_DATE |
+			  MAIL_FETCH_STREAM_HEADER | MAIL_FETCH_STREAM_BODY |
+			  MAIL_FETCH_FROM_ENVELOPE, NULL);
+	while (mailbox_search_next(ctx, mail) > 0) {
+		struct mail_keywords *keywords;
+		const char *const *keywords_list;
+
+		keywords_list = mail_get_keywords(mail);
+		keywords = strarray_length(keywords_list) == 0 ? NULL :
+			mailbox_keywords_create(dest_trans, keywords_list);
+
+		ret = mailbox_copy(dest_trans, mail, mail_get_flags(mail),
+				   keywords, NULL);
+		mailbox_keywords_free(dest_trans, &keywords);
+		if (ret < 0)
+			break;
+	}
+
+	mail_free(&mail);
+	if (mailbox_search_deinit(&ctx) < 0)
+		ret = -1;
+
+	if (ret < 0)
+		mailbox_transaction_rollback(&dest_trans);
+	else
+		ret = mailbox_transaction_commit(&dest_trans, 0);
+
+	/* source transaction committing isn't all that important.
+	   ignore if it fails. */
+	if (ret < 0)
+		mailbox_transaction_rollback(&src_trans);
+	else
+		(void)mailbox_transaction_commit(&src_trans, 0);
+	return ret;
+}
+
+static int mailbox_convert_list_item(struct mail_storage *source_storage,
+				     struct mail_storage *dest_storage,
+				     struct mailbox_list *list)
+{
+	struct mailbox *srcbox, *destbox;
+	int ret = 0;
+
+	if ((list->flags & (MAILBOX_NONEXISTENT|MAILBOX_PLACEHOLDER)) != 0)
+		return 0;
+
+	if ((list->flags & MAILBOX_NOSELECT) != 0) {
+		if (mail_storage_mailbox_create(dest_storage,
+						list->name, TRUE) < 0) {
+			i_error("Mailbox conversion: Couldn't create mailbox "
+				"directory %s", list->name);
+			return -1;
+		}
+		return 0;
+	}
+
+	/* It's a real mailbox. First create the destination mailbox. */
+	if (mail_storage_mailbox_create(dest_storage, list->name, FALSE) < 0) {
+		i_error("Mailbox conversion: Couldn't create mailbox %s",
+			list->name);
+		return -1;
+	}
+
+	/* Open both the mailboxes.. */
+	srcbox = mailbox_open(source_storage, list->name, NULL,
+			   MAILBOX_OPEN_READONLY | MAILBOX_OPEN_KEEP_RECENT);
+	if (srcbox == NULL) {
+		i_error("Mailbox conversion: Couldn't open source mailbox %s",
+			list->name);
+		return -1;
+	}
+
+	destbox = mailbox_open(dest_storage, list->name, NULL,
+			       MAILBOX_OPEN_KEEP_RECENT);
+	if (destbox == NULL) {
+		i_error("Mailbox conversion: Couldn't open dest mailbox %s",
+			list->name);
+		mailbox_close(&srcbox);
+		return -1;
+	}
+
+	if (mailbox_copy_mails(srcbox, destbox) < 0) {
+		i_error("Mailbox conversion: Couldn't copy mailbox %s",
+			mailbox_get_name(srcbox));
+	}
+
+	mailbox_close(&srcbox);
+	mailbox_close(&destbox);
+	return ret;
+}
+
+static int mailbox_list_copy(struct mail_storage *source_storage,
+			     struct mail_storage *dest_storage)
+{
+	struct mailbox_list_context *iter;
+	struct mailbox_list *list;
+	int ret = 0;
+
+	iter = mail_storage_mailbox_list_init(source_storage, "", "*",
+                                              MAILBOX_LIST_FAST_FLAGS);
+	while ((list = mail_storage_mailbox_list_next(iter)) != NULL) {
+		if (mailbox_convert_list_item(source_storage, dest_storage,
+					      list) < 0) {
+			ret = -1;
+			break;
+		}
+	}
+	if (mail_storage_mailbox_list_deinit(&iter) < 0)
+		ret = -1;
+	return ret;
+}
+
+int convert_storage(const char *user, const char *home_dir,
+		    const char *source_data, const char *dest_data)
+{
+	struct mail_storage *source_storage, *dest_storage;
+	struct dotlock *dotlock;
+        enum mail_storage_flags flags;
+        enum mail_storage_lock_method lock_method;
+	const char *path;
+	int ret;
+
+	mail_storage_parse_env(&flags, &lock_method);
+	source_storage = mail_storage_create_with_data(source_data, user,
+						       flags, lock_method);
+	if (source_storage == NULL) {
+		/* No need for conversion. */
+		return 0;
+	}
+
+        path = t_strconcat(home_dir, "/"CONVERT_LOCK_FILENAME, NULL);
+	ret = file_dotlock_create(&dotlock_settings, path, 0, &dotlock);
+	if (ret <= 0) {
+		if (ret == 0)
+			i_error("Mailbox conversion: Lock creation timeouted");
+		return -1;
+	}
+
+	dest_storage = mail_storage_create_with_data(dest_data, user,
+						     flags, lock_method);
+	if (dest_storage == NULL) {
+		i_error("Mailbox conversion: Failed to create destination "
+			"storage with data: %s", dest_data);
+	}
+
+	ret = mailbox_list_copy(source_storage, dest_storage);
+
+	if (ret == 0) {
+		/* all finished. rename the source directory to mark the
+		   move as finished. FIXME: kind of kludgy way to get the
+		   directory.. */
+		struct index_storage *index_storage =
+			(struct index_storage *)source_storage;
+		const char *dest;
+
+		dest = t_strconcat(index_storage->dir, "-converted", NULL);
+		if (rename(index_storage->dir, dest) < 0) {
+			i_error("Mailbox conversion: rename(%s, %s) failed: %m",
+				index_storage->dir, dest);
+			/* return success anyway */
+		}
+		ret = 1;
+	}
+
+	mail_storage_destroy(&source_storage);
+	mail_storage_destroy(&dest_storage);
+	return ret;
+}
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/src/plugins/convert/convert-storage.h	Thu Feb 02 22:42:44 2006 +0200
@@ -0,0 +1,7 @@
+#ifndef __CONVERT_STORAGE_H
+#define __CONVERT_STORAGE_H
+
+int convert_storage(const char *user, const char *home_dir,
+		    const char *source_data, const char *dest_data);
+
+#endif
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/src/plugins/convert/convert-tool.c	Thu Feb 02 22:42:44 2006 +0200
@@ -0,0 +1,44 @@
+/* Copyright (C) 2006 Timo Sirainen */
+
+#include "lib.h"
+#include "ioloop.h"
+#include "randgen.h"
+#include "lib-signals.h"
+#include "convert-storage.h"
+
+/* ugly, but automake doesn't like having it built as both static and
+   dynamic object.. */
+#include "convert-storage.c"
+
+int main(int argc, const char *argv[])
+{
+	struct ioloop *ioloop;
+	int ret = 0;
+
+	lib_init();
+	lib_signals_init();
+	random_init();
+	mail_storage_init();
+	mail_storage_register_all();
+
+	if (argc <= 4) {
+		i_fatal("Usage: <username> <home dir> "
+			"<source mail env> <dest mail env>");
+	}
+
+	ioloop = io_loop_create(system_pool);
+
+	ret = convert_storage(argv[1], argv[2], argv[3], argv[4]);
+	if (ret > 0)
+		i_info("Successfully converted");
+	else if (ret == 0)
+		i_error("Source storage not found");
+	else
+		i_error("Internal failure");
+
+	io_loop_destroy(&ioloop);
+	mail_storage_deinit();
+	lib_signals_deinit();
+	lib_deinit();
+	return ret;
+}