changeset 22292:d192449acace

lib-storage: Add mail_location=..:ITERINDEX This changes mailbox list iteration to work using INDEX directory instead of the normal mail directory. This can be helpful when the indexes are stored on a faster storage.
author Timo Sirainen <timo.sirainen@dovecot.fi>
date Mon, 26 Jun 2017 20:10:17 +0300
parents 5a81b1690625
children b8a501c91a39
files src/lib-storage/index/index-storage.c src/lib-storage/list/mailbox-list-fs-flags.c src/lib-storage/list/mailbox-list-fs-iter.c src/lib-storage/list/mailbox-list-fs.c src/lib-storage/list/mailbox-list-maildir-iter.c src/lib-storage/mailbox-list.c src/lib-storage/mailbox-list.h
diffstat 7 files changed, 101 insertions(+), 14 deletions(-) [+]
line wrap: on
line diff
--- a/src/lib-storage/index/index-storage.c	Tue Jun 27 16:55:34 2017 +0300
+++ b/src/lib-storage/index/index-storage.c	Mon Jun 26 20:10:17 2017 +0300
@@ -177,7 +177,7 @@
 {
 	struct stat st;
 	enum mail_error error;
-	const char *path, *path2;
+	const char *path, *path2, *index_path;
 	int ret;
 
 	/* see if it's selectable */
@@ -194,6 +194,22 @@
 		*existence_r = MAILBOX_EXISTENCE_NONE;
 		return 0;
 	}
+
+	ret = (subdir != NULL || !box->list->set.iter_from_index_dir) ? 0 :
+		mailbox_get_path_to(box, MAILBOX_LIST_PATH_TYPE_INDEX, &index_path);
+	if (ret > 0 && strcmp(path, index_path) != 0) {
+		/* index directory is different - prefer looking it up first
+		   since it might be on a faster storage. since the directory
+		   itself exists also for \NoSelect mailboxes, we'll need to
+		   check the dovecot.index.log existence. */
+		index_path = t_strconcat(index_path, "/", box->index_prefix,
+					 ".log", NULL);
+		if (stat(index_path, &st) == 0) {
+			*existence_r = MAILBOX_EXISTENCE_SELECT;
+			return 0;
+		}
+	}
+
 	if (subdir != NULL)
 		path = t_strconcat(path, "/", subdir, NULL);
 	if (stat(path, &st) == 0) {
@@ -580,6 +596,21 @@
 
 	if ((ret = mailbox_mkdir(box, path, type)) < 0)
 		return -1;
+	if (box->list->set.iter_from_index_dir) {
+		/* need to also create the directory to index path or
+		   iteration won't find it. */
+		int ret2;
+
+		if (mailbox_get_path_to(box, MAILBOX_LIST_PATH_TYPE_INDEX, &path) <= 0)
+			i_unreached();
+		if ((ret2 = mailbox_mkdir(box, path, type)) < 0)
+			return -1;
+		if (ret == 0 && ret2 > 0) {
+			/* finish partial creation: existed in mail directory,
+			   but not in index directory. */
+			ret = 1;
+		}
+	}
 	mailbox_refresh_permissions(box);
 	if (ret == 0) {
 		/* directory already exists */
--- a/src/lib-storage/list/mailbox-list-fs-flags.c	Tue Jun 27 16:55:34 2017 +0300
+++ b/src/lib-storage/list/mailbox-list-fs-flags.c	Mon Jun 26 20:10:17 2017 +0300
@@ -118,7 +118,7 @@
 
 	*flags_r = 0;
 
-	if (*list->set.maildir_name != '\0') {
+	if (*list->set.maildir_name != '\0' && !list->set.iter_from_index_dir) {
 		/* maildir_name is set: This is the simple case that works for
 		   all mail storage formats, because the only thing that
 		   matters for existence or child checks is whether the
@@ -130,9 +130,14 @@
 	}
 	/* maildir_name is not set: Now we (may) need to use storage-specific
 	   code to determine whether the mailbox is selectable or if it has
-	   children. */
+	   children.
 
-	if (list->v.is_internal_name != NULL &&
+	   We're here also when iterating from index directory, because even
+	   though maildir_name is set, it's not used for index directory.
+	*/
+
+	if (!list->set.iter_from_index_dir &&
+	    list->v.is_internal_name != NULL &&
 	    list->v.is_internal_name(list, fname)) {
 		/* skip internal dirs. For example Maildir's cur/new/tmp */
 		*flags_r |= MAILBOX_NOSELECT;
@@ -219,7 +224,7 @@
 		return 1;
 	}
 
-	if (list->v.is_internal_name == NULL) {
+	if (list->v.is_internal_name == NULL || list->set.iter_from_index_dir) {
 		/* This mailbox format doesn't use any special directories
 		   (e.g. Maildir's cur/new/tmp). In that case we can look at
 		   the directory's link count to determine whether there are
--- a/src/lib-storage/list/mailbox-list-fs-iter.c	Tue Jun 27 16:55:34 2017 +0300
+++ b/src/lib-storage/list/mailbox-list-fs-iter.c	Mon Jun 26 20:10:17 2017 +0300
@@ -245,10 +245,18 @@
 	}
 	if (*path != '/') {
 		/* non-absolute path. add the mailbox root dir as prefix. */
-		if (!mailbox_list_get_root_path(ctx->ctx.list,
-						MAILBOX_LIST_PATH_TYPE_MAILBOX,
-						&root))
+		enum mailbox_list_path_type type =
+			ctx->ctx.list->set.iter_from_index_dir ?
+			MAILBOX_LIST_PATH_TYPE_INDEX :
+			MAILBOX_LIST_PATH_TYPE_MAILBOX;
+		if (!mailbox_list_get_root_path(ctx->ctx.list, type, &root))
 			return FALSE;
+		if (ctx->ctx.list->set.iter_from_index_dir &&
+		    ctx->ctx.list->set.mailbox_dir_name[0] != '\0') {
+			/* append "mailboxes/" to the index root */
+			root = t_strconcat(root, "/",
+				ctx->ctx.list->set.mailbox_dir_name, NULL);
+		}
 		path = *path == '\0' ? root :
 			t_strconcat(root, "/", path, NULL);
 	}
--- a/src/lib-storage/list/mailbox-list-fs.c	Tue Jun 27 16:55:34 2017 +0300
+++ b/src/lib-storage/list/mailbox-list-fs.c	Mon Jun 26 20:10:17 2017 +0300
@@ -287,12 +287,32 @@
 {
 	const char *path, *child_name, *child_path, *p;
 	char sep;
+	int ret;
 
 	if (mailbox_list_get_path(list, name, MAILBOX_LIST_PATH_TYPE_DIR,
 				  &path) <= 0)
 		i_unreached();
-	if (fs_list_rmdir(list, name, path) == 0)
-		return 0;
+	ret = fs_list_rmdir(list, name, path);
+	if (!list->set.iter_from_index_dir) {
+		/* it should exist only in the mail directory */
+		if (ret == 0)
+			return 0;
+	} else if (ret == 0 || errno == ENOENT) {
+		/* the primary list location is the index directory, but it
+		   exists in both index and mail directories. */
+		if (mailbox_list_get_path(list, name, MAILBOX_LIST_PATH_TYPE_INDEX,
+					  &path) <= 0)
+			i_unreached();
+		if (fs_list_rmdir(list, name, path) == 0)
+			return 0;
+		if (ret == 0 && errno == ENOENT) {
+			/* partial existence: exists in _DIR, but not in
+			   _INDEX. return success anyway. */
+			return 0;
+		}
+		/* a) both directories didn't exist
+		   b) index directory couldn't be rmdir()ed for some reason */
+	}
 
 	if (errno == ENOENT || errno == ENOTDIR) {
 		mailbox_list_set_error(list, MAIL_ERROR_NOTFOUND,
--- a/src/lib-storage/list/mailbox-list-maildir-iter.c	Tue Jun 27 16:55:34 2017 +0300
+++ b/src/lib-storage/list/mailbox-list-maildir-iter.c	Mon Jun 26 20:10:17 2017 +0300
@@ -441,7 +441,10 @@
 	ctx->prefix_char = strcmp(_list->name, MAILBOX_LIST_NAME_IMAPDIR) == 0 ?
 		'\0' : list->sep;
 
-	ctx->dir = _list->set.root_dir;
+	if (_list->set.iter_from_index_dir)
+		ctx->dir = _list->set.index_dir;
+	else
+		ctx->dir = _list->set.root_dir;
 
 	if ((flags & MAILBOX_LIST_ITER_SELECT_SUBSCRIBED) != 0) {
 		/* Listing only subscribed mailboxes.
--- a/src/lib-storage/mailbox-list.c	Tue Jun 27 16:55:34 2017 +0300
+++ b/src/lib-storage/mailbox-list.c	Mon Jun 26 20:10:17 2017 +0300
@@ -175,6 +175,7 @@
 	list->set.volatile_dir = p_strdup(list->pool, set->volatile_dir);
 	list->set.index_control_use_maildir_name =
 		set->index_control_use_maildir_name;
+	list->set.iter_from_index_dir = set->iter_from_index_dir;
 
 	if (*set->mailbox_dir_name == '\0')
 		list->set.mailbox_dir_name = "";
@@ -345,6 +346,9 @@
 		else if (strcmp(key, "FULLDIRNAME") == 0) {
 			set_r->index_control_use_maildir_name = TRUE;
 			dest = &set_r->maildir_name;
+		} else if (strcmp(key, "ITERINDEX") == 0) {
+			set_r->iter_from_index_dir = TRUE;
+			continue;
 		} else {
 			*error_r = t_strdup_printf("Unknown setting: %s", key);
 			return -1;
@@ -357,6 +361,11 @@
 
 	if (set_r->index_dir != NULL && strcmp(set_r->index_dir, "MEMORY") == 0)
 		set_r->index_dir = "";
+	if (set_r->iter_from_index_dir &&
+	    (set_r->index_dir == NULL || set_r->index_dir[0] == '\0')) {
+		*error_r = "ITERINDEX requires INDEX to be explicitly set";
+		return -1;
+	}
 	return 0;
 }
 
@@ -1489,9 +1498,15 @@
 		return mailbox_list_iter_deinit(&iter);
 	}
 
-	rootdir = mailbox_list_get_root_forced(list, MAILBOX_LIST_PATH_TYPE_MAILBOX);
-	if (mailbox_list_get_path(list, name, MAILBOX_LIST_PATH_TYPE_DIR, &path) <= 0)
-		i_unreached();
+	if (!list->set.iter_from_index_dir) {
+		rootdir = mailbox_list_get_root_forced(list, MAILBOX_LIST_PATH_TYPE_MAILBOX);
+		if (mailbox_list_get_path(list, name, MAILBOX_LIST_PATH_TYPE_DIR, &path) <= 0)
+			i_unreached();
+	} else {
+		rootdir = mailbox_list_get_root_forced(list, MAILBOX_LIST_PATH_TYPE_INDEX);
+		if (mailbox_list_get_path(list, name, MAILBOX_LIST_PATH_TYPE_INDEX, &path) <= 0)
+			i_unreached();
+	}
 
 	fname = strrchr(path, '/');
 	if (fname == NULL) {
--- a/src/lib-storage/mailbox-list.h	Tue Jun 27 16:55:34 2017 +0300
+++ b/src/lib-storage/mailbox-list.h	Mon Jun 26 20:10:17 2017 +0300
@@ -139,6 +139,11 @@
 	   have been the default since the beginning, but for backwards
 	   compatibility it had to be made an option. */
 	bool index_control_use_maildir_name;
+	/* Perform mailbox iteration using the index directory instead of the
+	   mail root directory. This can be helpful if the indexes are on a
+	   faster storage. This could perhaps be made the default at some point,
+	   but for now since it's less tested it's optional. */
+	bool iter_from_index_dir;
 };
 
 struct mailbox_permissions {