# HG changeset patch
# User Timo Sirainen <timo.sirainen@dovecot.fi>
# Date 1487351273 -7200
# Node ID fb8ef6e2c2fe1a82cb4c4f937887b6d758138542
# Parent  6d50f63cfa671b982349735e82cf601f3be1f0fe
lib-storage: Add mail_sort_max_read_count setting.

This controls how many slow mail accesses sorting can perform before it
fails:

a NO [LIMIT] Requested sort would have taken too long

The SORT reply is still returned, but it's likely not correct.

diff -r 6d50f63cfa67 -r fb8ef6e2c2fe src/lib-storage/index/index-sort-private.h
--- a/src/lib-storage/index/index-sort-private.h	Fri Feb 17 18:56:23 2017 +0200
+++ b/src/lib-storage/index/index-sort-private.h	Fri Feb 17 19:07:53 2017 +0200
@@ -7,6 +7,7 @@
 	struct mailbox_transaction_context *t;
 	enum mail_sort_type sort_program[MAX_SORT_PROGRAM_SIZE];
 	struct mail *temp_mail;
+	unsigned int slow_mails_left;
 
 	void (*sort_list_add)(struct mail_search_sort_program *program,
 			      struct mail *mail);
@@ -20,7 +21,7 @@
 };
 
 /* Returns 1 on success, 0 if mail is already expunged, -1 on other errors. */
-int index_sort_header_get(struct mail *mail, uint32_t seq,
+int index_sort_header_get(struct mail_search_sort_program *program, uint32_t seq,
 			  enum mail_sort_type sort_type, string_t *dest);
 int index_sort_node_cmp_type(struct mail_search_sort_program *program,
 			     const enum mail_sort_type *sort_program,
diff -r 6d50f63cfa67 -r fb8ef6e2c2fe src/lib-storage/index/index-sort-string.c
--- a/src/lib-storage/index/index-sort-string.c	Fri Feb 17 18:56:23 2017 +0200
+++ b/src/lib-storage/index/index-sort-string.c	Fri Feb 17 19:07:53 2017 +0200
@@ -254,7 +254,6 @@
 
 static void index_sort_zeroes(struct sort_string_context *ctx)
 {
-	struct mail *mail = ctx->program->temp_mail;
 	enum mail_sort_type sort_type = ctx->program->sort_program[0];
 	string_t *str;
 	pool_t pool;
@@ -273,7 +272,7 @@
 		i_assert(nodes[i].seq <= ctx->last_seq);
 
 		T_BEGIN {
-			if (index_sort_header_get(mail, nodes[i].seq,
+			if (index_sort_header_get(ctx->program, nodes[i].seq,
 						  sort_type, str) < 0) {
 				nodes[i].no_update = TRUE;
 				ctx->failed = TRUE;
@@ -294,7 +293,6 @@
 index_sort_get_expunged_string(struct sort_string_context *ctx, uint32_t idx,
 			       string_t *str, const char **result_r)
 {
-	struct mail *mail = ctx->program->temp_mail;
 	enum mail_sort_type sort_type = ctx->program->sort_program[0];
 	const struct mail_sort_node *nodes;
 	const char *result = NULL;
@@ -329,7 +327,7 @@
 			result = ctx->sort_strings[nodes[i].seq];
 			break;
 		}
-		if (index_sort_header_get(mail, nodes[i].seq,
+		if (index_sort_header_get(ctx->program, nodes[i].seq,
 					  sort_type, str) > 0) {
 			result = str_len(str) == 0 ? "" :
 				p_strdup(ctx->sort_string_pool, str_c(str));
@@ -353,7 +351,6 @@
 index_sort_get_string(struct sort_string_context *ctx,
 		      uint32_t idx, uint32_t seq, const char **str_r)
 {
-	struct mail *mail = ctx->program->temp_mail;
 	int ret = 1;
 
 	if (ctx->sort_strings[seq] == NULL) T_BEGIN {
@@ -361,7 +358,7 @@
 		const char *result;
 
 		str = t_str_new(256);
-		ret = index_sort_header_get(mail, seq,
+		ret = index_sort_header_get(ctx->program, seq,
 					    ctx->program->sort_program[0], str);
 		if (ret < 0)
 			ctx->failed = TRUE;
diff -r 6d50f63cfa67 -r fb8ef6e2c2fe src/lib-storage/index/index-sort.c
--- a/src/lib-storage/index/index-sort.c	Fri Feb 17 18:56:23 2017 +0200
+++ b/src/lib-storage/index/index-sort.c	Fri Feb 17 19:07:53 2017 +0200
@@ -40,8 +40,34 @@
 index_sort_program_set_mail_failed(struct mail_search_sort_program *program,
 				   struct mail *mail)
 {
-	if (mailbox_get_last_mail_error(mail->box) != MAIL_ERROR_EXPUNGED)
+	switch (mailbox_get_last_mail_error(mail->box)) {
+	case MAIL_ERROR_EXPUNGED:
+		break;
+	case MAIL_ERROR_NOTPOSSIBLE:
+		/* just change the error message */
+		mail_storage_set_error(program->t->box->storage, MAIL_ERROR_LIMIT,
+			"Requested sort would have taken too long.");
+		/* fall through */
+	default:
 		program->failed = TRUE;
+		break;
+	}
+}
+
+static time_t
+index_sort_program_set_date_failed(struct mail_search_sort_program *program,
+				   struct mail *mail)
+{
+	index_sort_program_set_mail_failed(program, mail);
+
+	if (mailbox_get_last_mail_error(mail->box) == MAIL_ERROR_LIMIT) {
+		/* limit reached - sort the rest of the mails at the end of
+		   the list by their UIDs */
+		return LONG_MAX;
+	} else {
+		/* expunged / some other error - sort in the beginning */
+		return 0;
+	}
 }
 
 static void
@@ -53,10 +79,8 @@
 
 	node = array_append_space(nodes);
 	node->seq = mail->seq;
-	if (mail_get_received_date(mail, &node->date) < 0) {
-		index_sort_program_set_mail_failed(program, mail);
-		node->date = 0;
-	}
+	if (mail_get_received_date(mail, &node->date) < 0)
+		node->date = index_sort_program_set_date_failed(program, mail);
 }
 
 static void
@@ -69,14 +93,11 @@
 
 	node = array_append_space(nodes);
 	node->seq = mail->seq;
-	if (mail_get_date(mail, &node->date, &tz) < 0) {
-		index_sort_program_set_mail_failed(program, mail);
-		node->date = 0;
-	} else if (node->date == 0) {
-		if (mail_get_received_date(mail, &node->date) < 0) {
-			index_sort_program_set_mail_failed(program, mail);
-			node->date = 0;
-		}
+	if (mail_get_date(mail, &node->date, &tz) < 0)
+		node->date = index_sort_program_set_date_failed(program, mail);
+	else if (node->date == 0) {
+		if (mail_get_received_date(mail, &node->date) < 0)
+			node->date = index_sort_program_set_date_failed(program, mail);
 	}
 }
 
@@ -148,11 +169,23 @@
 void index_sort_list_add(struct mail_search_sort_program *program,
 			 struct mail *mail)
 {
+	enum mail_lookup_abort orig_abort = mail->lookup_abort;
+	bool prev_slow = mail->mail_stream_opened ||
+		mail->mail_metadata_accessed;
+
 	i_assert(mail->transaction == program->t);
 
+	if (program->slow_mails_left == 0)
+		mail->lookup_abort = MAIL_LOOKUP_ABORT_NOT_IN_CACHE;
 	T_BEGIN {
 		program->sort_list_add(program, mail);
 	} T_END;
+	if (!prev_slow && (mail->mail_stream_opened ||
+			   mail->mail_metadata_accessed)) {
+		i_assert(program->slow_mails_left > 0);
+		program->slow_mails_left--;
+	}
+	mail->lookup_abort = orig_abort;
 }
 
 static int sort_node_date_cmp(const struct mail_sort_node_date *n1,
@@ -276,6 +309,11 @@
 	program->t = t;
 	program->temp_mail = mail_alloc(t, 0, NULL);
 
+	program->slow_mails_left =
+		program->t->box->storage->set->mail_sort_max_read_count;
+	if (program->slow_mails_left == 0)
+		program->slow_mails_left = UINT_MAX;
+
 	for (i = 0; i < MAX_SORT_PROGRAM_SIZE; i++) {
 		program->sort_program[i] = sort_program[i];
 		if (sort_program[i] == MAIL_SORT_END)
@@ -425,14 +463,30 @@
 	return 0;
 }
 
-int index_sort_header_get(struct mail *mail, uint32_t seq,
+static void
+index_sort_set_seq(struct mail_search_sort_program *program,
+		   struct mail *mail, uint32_t seq)
+{
+	if ((mail->mail_stream_opened || mail->mail_metadata_accessed) &&
+	    program->slow_mails_left > 0)
+		program->slow_mails_left--;
+	mail_set_seq(mail, seq);
+	if (program->slow_mails_left == 0) {
+		/* too many slow lookups - just return the rest of the results
+		   in whatever order. */
+		mail->lookup_abort = MAIL_LOOKUP_ABORT_NOT_IN_CACHE;
+	}
+}
+
+int index_sort_header_get(struct mail_search_sort_program *program, uint32_t seq,
 			  enum mail_sort_type sort_type, string_t *dest)
 {
+	struct mail *mail = program->temp_mail;
 	const char *str;
 	int ret;
 	bool reply_or_fw;
 
-	mail_set_seq(mail, seq);
+	index_sort_set_seq(program, mail, seq);
 	str_truncate(dest, 0);
 
 	switch (sort_type & MAIL_SORT_MASK) {
@@ -500,64 +554,54 @@
 
 			str1 = t_str_new(256);
 			str2 = t_str_new(256);
-			if (index_sort_header_get(mail, seq1, sort_type, str1) < 0)
+			if (index_sort_header_get(program, seq1, sort_type, str1) < 0)
 				index_sort_program_set_mail_failed(program, mail);
-			if (index_sort_header_get(mail, seq2, sort_type, str2) < 0)
+			if (index_sort_header_get(program, seq2, sort_type, str2) < 0)
 				index_sort_program_set_mail_failed(program, mail);
 
 			ret = strcmp(str_c(str1), str_c(str2));
 		} T_END;
 		break;
 	case MAIL_SORT_ARRIVAL:
-		mail_set_seq(mail, seq1);
-		if (mail_get_received_date(mail, &time1) < 0) {
-			index_sort_program_set_mail_failed(program, mail);
-			time1 = 0;
-		}
+		index_sort_set_seq(program, mail, seq1);
+		if (mail_get_received_date(mail, &time1) < 0)
+			time1 = index_sort_program_set_date_failed(program, mail);
 
-		mail_set_seq(mail, seq2);
-		if (mail_get_received_date(mail, &time2) < 0) {
-			index_sort_program_set_mail_failed(program, mail);
-			time2 = 0;
-		}
+		index_sort_set_seq(program, mail, seq2);
+		if (mail_get_received_date(mail, &time2) < 0)
+			time2 = index_sort_program_set_date_failed(program, mail);
 
 		ret = time1 < time2 ? -1 :
 			(time1 > time2 ? 1 : 0);
 		break;
 	case MAIL_SORT_DATE:
-		mail_set_seq(mail, seq1);
-		if (mail_get_date(mail, &time1, &tz) < 0) {
-			index_sort_program_set_mail_failed(program, mail);
-			time1 = 0;
-		} else if (time1 == 0) {
-			if (mail_get_received_date(mail, &time1) < 0) {
-				index_sort_program_set_mail_failed(program, mail);
-				time1 = 0;
-			}
+		index_sort_set_seq(program, mail, seq1);
+		if (mail_get_date(mail, &time1, &tz) < 0)
+			time1 = index_sort_program_set_date_failed(program, mail);
+		else if (time1 == 0) {
+			if (mail_get_received_date(mail, &time1) < 0)
+				time1 = index_sort_program_set_date_failed(program, mail);
 		}
 
-		mail_set_seq(mail, seq2);
-		if (mail_get_date(mail, &time2, &tz) < 0) {
-			index_sort_program_set_mail_failed(program, mail);
-			time2 = 0;
-		} else if (time2 == 0) {
-			if (mail_get_received_date(mail, &time2) < 0) {
-				index_sort_program_set_mail_failed(program, mail);
-				time2 = 0;
-			}
+		index_sort_set_seq(program, mail, seq2);
+		if (mail_get_date(mail, &time2, &tz) < 0)
+			time2 = index_sort_program_set_date_failed(program, mail);
+		else if (time2 == 0) {
+			if (mail_get_received_date(mail, &time2) < 0)
+				time2 = index_sort_program_set_date_failed(program, mail);
 		}
 
 		ret = time1 < time2 ? -1 :
 			(time1 > time2 ? 1 : 0);
 		break;
 	case MAIL_SORT_SIZE:
-		mail_set_seq(mail, seq1);
+		index_sort_set_seq(program, mail, seq1);
 		if (mail_get_virtual_size(mail, &size1) < 0) {
 			index_sort_program_set_mail_failed(program, mail);
 			size1 = 0;
 		}
 
-		mail_set_seq(mail, seq2);
+		index_sort_set_seq(program, mail, seq2);
 		if (mail_get_virtual_size(mail, &size2) < 0) {
 			index_sort_program_set_mail_failed(program, mail);
 			size2 = 0;
@@ -567,10 +611,10 @@
 			(size1 > size2 ? 1 : 0);
 		break;
 	case MAIL_SORT_RELEVANCY:
-		mail_set_seq(mail, seq1);
+		index_sort_set_seq(program, mail, seq1);
 		if (index_sort_get_relevancy(mail, &float1) < 0)
 			index_sort_program_set_mail_failed(program, mail);
-		mail_set_seq(mail, seq2);
+		index_sort_set_seq(program, mail, seq2);
 		if (index_sort_get_relevancy(mail, &float2) < 0)
 			index_sort_program_set_mail_failed(program, mail);
 
@@ -582,10 +626,10 @@
 	case MAIL_SORT_POP3_ORDER:
 		/* 32bit numbers would be enough, but since there is already
 		   existing code for uoff_t in sizes, just use them. */
-		mail_set_seq(mail, seq1);
+		index_sort_set_seq(program, mail, seq1);
 		if (index_sort_get_pop3_order(mail, &size1) < 0)
 			index_sort_program_set_mail_failed(program, mail);
-		mail_set_seq(mail, seq2);
+		index_sort_set_seq(program, mail, seq2);
 		if (index_sort_get_pop3_order(mail, &size2) < 0)
 			index_sort_program_set_mail_failed(program, mail);
 
diff -r 6d50f63cfa67 -r fb8ef6e2c2fe src/lib-storage/mail-storage-settings.c
--- a/src/lib-storage/mail-storage-settings.c	Fri Feb 17 18:56:23 2017 +0200
+++ b/src/lib-storage/mail-storage-settings.c	Fri Feb 17 19:07:53 2017 +0200
@@ -43,6 +43,7 @@
 	DEF(SET_TIME, mail_max_lock_timeout),
 	DEF(SET_TIME, mail_temp_scan_interval),
 	DEF(SET_UINT, mail_vsize_bg_after_count),
+	DEF(SET_UINT, mail_sort_max_read_count),
 	DEF(SET_BOOL, mail_save_crlf),
 	DEF(SET_ENUM, mail_fsync),
 	DEF(SET_BOOL, mmap_disable),
@@ -84,6 +85,7 @@
 	.mail_max_lock_timeout = 0,
 	.mail_temp_scan_interval = 7*24*60*60,
 	.mail_vsize_bg_after_count = 0,
+	.mail_sort_max_read_count = 0,
 	.mail_save_crlf = FALSE,
 	.mail_fsync = "optimized:never:always",
 	.mmap_disable = FALSE,
diff -r 6d50f63cfa67 -r fb8ef6e2c2fe src/lib-storage/mail-storage-settings.h
--- a/src/lib-storage/mail-storage-settings.h	Fri Feb 17 18:56:23 2017 +0200
+++ b/src/lib-storage/mail-storage-settings.h	Fri Feb 17 19:07:53 2017 +0200
@@ -29,6 +29,7 @@
 	unsigned int mail_max_lock_timeout;
 	unsigned int mail_temp_scan_interval;
 	unsigned int mail_vsize_bg_after_count;
+	unsigned int mail_sort_max_read_count;
 	bool mail_save_crlf;
 	const char *mail_fsync;
 	bool mmap_disable;