changeset 21573:b2d7af4cdfd8

lib-storage: Add mail_vsize_bg_after_count setting. If folder vsize calculation requires opening more than this many mails from disk (i.e. mail sizes aren't in cache already), return failure and finish the calculation via indexer process. This should be used like: protocol !indexer-worker { mail_vsize_bg_after_count = 10 }
author Timo Sirainen <timo.sirainen@dovecot.fi>
date Tue, 07 Feb 2017 16:18:55 +0200
parents bd590ae82f96
children 9d941d959776
files src/lib-storage/index/index-mailbox-size.c src/lib-storage/mail-storage-settings.c src/lib-storage/mail-storage-settings.h src/plugins/quota/quota-count.c
diffstat 4 files changed, 74 insertions(+), 1 deletions(-) [+]
line wrap: on
line diff
--- a/src/lib-storage/index/index-mailbox-size.c	Tue Feb 07 13:53:52 2017 +0200
+++ b/src/lib-storage/index/index-mailbox-size.c	Tue Feb 07 16:18:55 2017 +0200
@@ -1,6 +1,10 @@
 /* Copyright (c) 2002-2017 Dovecot authors, see the included COPYING file */
 
 #include "lib.h"
+#include "str.h"
+#include "strescape.h"
+#include "net.h"
+#include "write-full.h"
 #include "file-create-locked.h"
 #include "mail-search-build.h"
 #include "index-storage.h"
@@ -33,6 +37,9 @@
 #define VSIZE_LOCK_SUFFIX ".vsize.lock"
 #define VSIZE_UPDATE_MAX_LOCK_SECS 10
 
+#define INDEXER_SOCKET_NAME "indexer"
+#define INDEXER_HANDSHAKE "VERSION\tindexer\t1\t0\n"
+
 struct mailbox_vsize_update {
 	struct mailbox *box;
 	struct mail_index_view *view;
@@ -43,6 +50,7 @@
 	struct file_lock *lock;
 	bool rebuild;
 	bool written;
+	bool finish_in_background;
 };
 
 static void vsize_header_refresh(struct mailbox_vsize_update *update)
@@ -192,6 +200,36 @@
 	(void)mail_index_transaction_commit(&trans);
 }
 
+static void index_mailbox_vsize_notify_indexer(struct mailbox *box)
+{
+	string_t *str = t_str_new(256);
+	const char *path;
+	int fd;
+
+	path = t_strconcat(box->storage->user->set->base_dir,
+			   "/"INDEXER_SOCKET_NAME, NULL);
+	fd = net_connect_unix(path);
+	if (fd == -1) {
+		mail_storage_set_critical(box->storage,
+			"Can't start vsize building on background: "
+			"net_connect_unix(%s) failed: %m", path);
+		return;
+	}
+	str_append(str, INDEXER_HANDSHAKE);
+	str_append(str, "APPEND\t0\t");
+	str_append_tabescaped(str, box->storage->user->username);
+	str_append_c(str, '\t');
+	str_append_tabescaped(str, box->vname);
+	str_append_c(str, '\n');
+
+	if (write_full(fd, str_data(str), str_len(str)) < 0) {
+		mail_storage_set_critical(box->storage,
+			"Can't start vsize building on background: "
+			"write(%s) failed: %m", path);
+	}
+	i_close_fd(&fd);
+}
+
 void index_mailbox_vsize_update_deinit(struct mailbox_vsize_update **_update)
 {
 	struct mailbox_vsize_update *update = *_update;
@@ -206,6 +244,9 @@
 		file_lock_free(&update->lock);
 		i_close_fd(&update->lock_fd);
 	}
+	if (update->finish_in_background)
+		index_mailbox_vsize_notify_indexer(update->box);
+
 	mail_index_view_close(&update->view);
 	i_free(update->lock_path);
 	i_free(update);
@@ -243,6 +284,7 @@
 	struct mail_search_args *search_args;
 	struct mailbox_status status;
 	struct mail *mail;
+	unsigned int mails_left;
 	uint32_t seq1, seq2;
 	uoff_t vsize;
 	int ret = 0;
@@ -272,8 +314,33 @@
 	trans = mailbox_transaction_begin(update->box, 0);
 	search_ctx = mailbox_search_init(trans, search_args, NULL,
 					 MAIL_FETCH_VIRTUAL_SIZE, NULL);
+	mails_left = update->box->storage->set->mail_vsize_bg_after_count == 0 ?
+		UINT_MAX : update->box->storage->set->mail_vsize_bg_after_count;
 	while (mailbox_search_next(search_ctx, &mail)) {
-		if (mail_get_virtual_size(mail, &vsize) < 0) {
+		if (mails_left == UINT_MAX) {
+			/* we want to build the full vsize here */
+			ret = mail_get_virtual_size(mail, &vsize);
+		} else {
+			/* if vsize building wants to open too many mails from
+			   storage, return temporary failure and finish up the
+			   calculation in background. */
+			mail->lookup_abort = MAIL_LOOKUP_ABORT_NOT_IN_CACHE;
+			ret = mail_get_virtual_size(mail, &vsize);
+			mail->lookup_abort = MAIL_LOOKUP_ABORT_NEVER;
+			if (ret < 0 &&
+			    mailbox_get_last_mail_error(update->box) == MAIL_ERROR_NOTPOSSIBLE) {
+				/* size isn't in cache. */
+				if (mails_left == 0) {
+					mail_storage_set_error(update->box->storage, MAIL_ERROR_INUSE,
+						"Finishing vsize calculation on background");
+					update->finish_in_background = TRUE;
+					break;
+				}
+				mails_left--;
+				ret = mail_get_virtual_size(mail, &vsize);
+			}
+		}
+		if (ret < 0) {
 			if (mail->expunged)
 				continue;
 			ret = -1;
--- a/src/lib-storage/mail-storage-settings.c	Tue Feb 07 13:53:52 2017 +0200
+++ b/src/lib-storage/mail-storage-settings.c	Tue Feb 07 16:18:55 2017 +0200
@@ -42,6 +42,7 @@
 	DEF(SET_UINT, mail_max_keyword_length),
 	DEF(SET_TIME, mail_max_lock_timeout),
 	DEF(SET_TIME, mail_temp_scan_interval),
+	DEF(SET_UINT, mail_vsize_bg_after_count),
 	DEF(SET_BOOL, mail_save_crlf),
 	DEF(SET_ENUM, mail_fsync),
 	DEF(SET_BOOL, mmap_disable),
@@ -82,6 +83,7 @@
 	.mail_max_keyword_length = 50,
 	.mail_max_lock_timeout = 0,
 	.mail_temp_scan_interval = 7*24*60*60,
+	.mail_vsize_bg_after_count = 0,
 	.mail_save_crlf = FALSE,
 	.mail_fsync = "optimized:never:always",
 	.mmap_disable = FALSE,
--- a/src/lib-storage/mail-storage-settings.h	Tue Feb 07 13:53:52 2017 +0200
+++ b/src/lib-storage/mail-storage-settings.h	Tue Feb 07 16:18:55 2017 +0200
@@ -28,6 +28,7 @@
 	unsigned int mail_max_keyword_length;
 	unsigned int mail_max_lock_timeout;
 	unsigned int mail_temp_scan_interval;
+	unsigned int mail_vsize_bg_after_count;
 	bool mail_save_crlf;
 	const char *mail_fsync;
 	bool mmap_disable;
--- a/src/plugins/quota/quota-count.c	Tue Feb 07 13:53:52 2017 +0200
+++ b/src/plugins/quota/quota-count.c	Tue Feb 07 16:18:55 2017 +0200
@@ -55,6 +55,9 @@
 			i_error("quota: Couldn't get size of mailbox %s: %s",
 				vname, errstr);
 			ret = -1;
+		} else if (error == MAIL_ERROR_INUSE) {
+			/* started on background. don't log an error. */
+			ret = -1;
 		} else {
 			/* non-temporary error, e.g. ACLs denied access. */
 			ret = 0;