Mercurial > dovecot > core-2.2

changeset 20560:3a71ed48cdf7

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Use mail_get_*stream_because() wherever possible.
author Timo Sirainen <timo.sirainen@dovecot.fi>
date Mon, 25 Jul 2016 14:20:05 -0400
parents ad24699d4138
children 6f381c5ad896
files src/lib-imap-storage/imap-msgpart.c src/lib-storage/index/index-mail-binary.c src/lib-storage/index/index-mail-headers.c src/lib-storage/index/index-mail.c src/lib-storage/index/index-mail.h src/lib-storage/index/index-search.c src/lib-storage/mail-copy.c src/plugins/fts/fts-build-mail.c src/plugins/pop3-migration/pop3-migration-plugin.c src/pop3/pop3-commands.c
diffstat 10 files changed, 73 insertions(+), 32 deletions(-) [+]
line wrap: on
line diff
--- a/src/lib-imap-storage/imap-msgpart.c	Fri Jul 29 00:37:07 2016 +0300
+++ b/src/lib-imap-storage/imap-msgpart.c	Mon Jul 25 14:20:05 2016 -0400
@@ -575,7 +575,7 @@
 
 	if (*msgpart->section_number != '\0') {
 		/* find the MIME part */
-		if (mail_get_stream(mail, NULL, NULL, &input) < 0)
+		if (mail_get_stream_because(mail, NULL, NULL, "MIME part", &input) < 0)
 			return -1;
 
 		i_stream_seek(input, part->physical_pos);
@@ -584,7 +584,7 @@
 	} else switch (msgpart->fetch_type) {
 	case FETCH_FULL:
 		/* fetch the whole message */
-		if (mail_get_stream(mail, NULL, NULL, &input) < 0 ||
+		if (mail_get_stream_because(mail, NULL, NULL, "full mail", &input) < 0 ||
 		    mail_get_virtual_size(mail, &body_size.virtual_size) < 0)
 			return -1;
 		result_r->size_field = MAIL_FETCH_VIRTUAL_SIZE;
@@ -619,7 +619,8 @@
 		break;
 	case FETCH_BODY:
 		/* fetch the message's body */
-		if (mail_get_stream(mail, &hdr_size, &body_size, &input) < 0)
+		if (mail_get_stream_because(mail, &hdr_size, &body_size,
+					    "mail body", &input) < 0)
 			return -1;
 		result_r->size_field = MAIL_FETCH_MESSAGE_PARTS;
 		break;
--- a/src/lib-storage/index/index-mail-binary.c	Fri Jul 29 00:37:07 2016 +0300
+++ b/src/lib-storage/index/index-mail-binary.c	Mon Jul 25 14:20:05 2016 -0400
@@ -362,8 +362,8 @@
 static int
 index_mail_read_binary_to_cache(struct mail *_mail,
 				const struct message_part *part,
-				bool include_hdr, bool *binary_r,
-				bool *converted_r)
+				bool include_hdr, const char *reason,
+				bool *binary_r, bool *converted_r)
 {
 	struct index_mail *mail = (struct index_mail *)_mail;
 	struct mail_binary_cache *cache = &_mail->box->storage->binary_cache;
@@ -374,7 +374,7 @@
 	t_array_init(&ctx.blocks, 8);
 
 	mail_storage_free_binary_cache(_mail->box->storage);
-	if (mail_get_stream(_mail, NULL, NULL, &ctx.input) < 0)
+	if (mail_get_stream_because(_mail, NULL, NULL, reason, &ctx.input) < 0)
 		return -1;
 
 	if (add_binary_part(&ctx, part, include_hdr) < 0) {
@@ -491,7 +491,7 @@
 	if (!get_cached_binary_parts(mail)) {
 		/* not found. parse the whole message */
 		if (index_mail_read_binary_to_cache(_mail, all_parts, TRUE,
-						    &binary, &converted) < 0)
+						    "binary.size", &binary, &converted) < 0)
 			return -1;
 	}
 
@@ -568,7 +568,7 @@
 		converted = TRUE;
 	} else {
 		if (index_mail_read_binary_to_cache(_mail, part, include_hdr,
-						    &binary, &converted) < 0)
+						    "binary stream", &binary, &converted) < 0)
 			return -1;
 		mail->data.cache_fetch_fields |= MAIL_FETCH_STREAM_BINARY;
 	}
--- a/src/lib-storage/index/index-mail-headers.c	Fri Jul 29 00:37:07 2016 +0300
+++ b/src/lib-storage/index/index-mail-headers.c	Mon Jul 25 14:20:05 2016 -0400
@@ -427,7 +427,8 @@
 }
 
 int index_mail_parse_headers(struct index_mail *mail,
-			     struct mailbox_header_lookup_ctx *headers)
+			     struct mailbox_header_lookup_ctx *headers,
+			     const char *reason)
 {
 	struct index_mail_data *data = &mail->data;
 	struct istream *input;
@@ -435,7 +436,7 @@
 
 	old_offset = data->stream == NULL ? 0 : data->stream->v_offset;
 
-	if (mail_get_hdr_stream(&mail->mail.mail, NULL, &input) < 0)
+	if (mail_get_hdr_stream_because(&mail->mail.mail, NULL, reason, &input) < 0)
 		return -1;
 
 	index_mail_parse_header_init(mail, headers);
@@ -638,10 +639,12 @@
 		if (mail->header_seq != mail->data.seq ||
 		    index_mail_header_is_parsed(mail, field_idx) < 0) {
 			/* parse */
+			const char *reason = index_mail_cache_reason(_mail,
+				t_strdup_printf("header %s", field));
 			headers[0] = field; headers[1] = NULL;
 			headers_ctx = mailbox_header_lookup_init(_mail->box,
 								 headers);
-			ret = index_mail_parse_headers(mail, headers_ctx);
+			ret = index_mail_parse_headers(mail, headers_ctx, reason);
 			mailbox_header_lookup_unref(&headers_ctx);
 			if (ret < 0)
 				return -1;
@@ -860,11 +863,14 @@
 	struct istream *input;
 	string_t *dest;
 
+	i_assert(headers->count > 0);
 	i_assert(headers->box == _mail->box);
 
 	if (mail->data.save_bodystructure_header) {
 		/* we have to parse the header. */
-		if (index_mail_parse_headers(mail, headers) < 0)
+		const char *reason =
+			index_mail_cache_reason(_mail, "bodystructure");
+		if (index_mail_parse_headers(mail, headers, reason) < 0)
 			return -1;
 	}
 
@@ -885,7 +891,25 @@
 	/* not in cache / error */
 	p_free(mail->mail.data_pool, dest);
 
-	if (mail_get_hdr_stream(_mail, NULL, &input) < 0)
+	unsigned int first_not_found = UINT_MAX, not_found_count = 0;
+	for (unsigned int i = 0; i < headers->count; i++) {
+		if (mail_cache_field_exists(_mail->transaction->cache_view,
+					    _mail->seq, headers->idx[i]) <= 0) {
+			if (not_found_count++ == 0)
+				first_not_found = i;
+		}
+	}
+
+	const char *reason;
+	if (not_found_count == 0)
+		reason = "BUG: all headers seem to exist in cache";
+	else {
+		i_assert(first_not_found != UINT_MAX);
+		reason = index_mail_cache_reason(_mail, t_strdup_printf(
+			"%u/%u headers not cached (first=%s)",
+			not_found_count, headers->count, headers->name[first_not_found]));
+	}
+	if (mail_get_hdr_stream_because(_mail, NULL, reason, &input) < 0)
 		return -1;
 
 	if (mail->data.filter_stream != NULL)
--- a/src/lib-storage/index/index-mail.c	Fri Jul 29 00:37:07 2016 +0300
+++ b/src/lib-storage/index/index-mail.c	Mon Jul 25 14:20:05 2016 -0400
@@ -321,7 +321,9 @@
 	}
 
 	if (data->parser_ctx == NULL) {
-		if (index_mail_parse_headers(mail, NULL) < 0)
+		const char *reason =
+			index_mail_cache_reason(_mail, "mime parts");
+		if (index_mail_parse_headers(mail, NULL, reason) < 0)
 			return -1;
 	}
 
@@ -510,7 +512,8 @@
 		return 0;
 
 	old_offset = data->stream == NULL ? 0 : data->stream->v_offset;
-	if (mail_get_stream(_mail, &hdr_size, &body_size, &input) < 0)
+	if (mail_get_stream_because(_mail, &hdr_size, &body_size,
+			index_mail_cache_reason(_mail, "virtual size"), &input) < 0)
 		return -1;
 	i_stream_seek(data->stream, old_offset);
 
@@ -945,7 +948,8 @@
 	}
 
 	old_offset = mail->data.stream == NULL ? 0 : mail->data.stream->v_offset;
-	if (mail_get_stream(&mail->mail.mail, NULL, NULL, &input) < 0)
+	const char *reason = index_mail_cache_reason(&mail->mail.mail, "snippet");
+	if (mail_get_stream_because(&mail->mail.mail, NULL, NULL, reason, &input) < 0)
 		return -1;
 	i_assert(mail->data.stream != NULL);
 
@@ -1177,7 +1181,7 @@
 		if (!data->hdr_size_set) {
 			if ((data->access_part & PARSE_HDR) != 0) {
 				(void)get_cached_parts(mail);
-				if (index_mail_parse_headers(mail, NULL) < 0)
+				if (index_mail_parse_headers(mail, NULL, "parse header") < 0)
 					return -1;
 			} else {
 				if (message_get_header_size(data->stream,
@@ -1247,10 +1251,12 @@
 		    !data->save_bodystructure_body ||
 		    field == MAIL_CACHE_BODY_SNIPPET) {
 			/* we haven't parsed the header yet */
+			const char *reason =
+				index_mail_cache_reason(&mail->mail.mail, "bodystructure");
 			data->save_bodystructure_header = TRUE;
 			data->save_bodystructure_body = TRUE;
 			(void)get_cached_parts(mail);
-			if (index_mail_parse_headers(mail, NULL) < 0) {
+			if (index_mail_parse_headers(mail, NULL, reason) < 0) {
 				data->save_bodystructure_header = TRUE;
 				return -1;
 			}
@@ -1745,7 +1751,7 @@
 		hdr = mail_index_get_header(_mail->transaction->view);
 		if (!_mail->saving && _mail->uid < hdr->next_uid) {
 			if ((data->access_part & (READ_BODY | PARSE_BODY)) != 0)
-				(void)mail_get_stream(_mail, NULL, NULL, &input);
+				(void)mail_get_stream_because(_mail, NULL, NULL, "access", &input);
 			else
 				(void)mail_get_hdr_stream(_mail, NULL, &input);
 		}
@@ -1808,7 +1814,7 @@
 	}
 
 	if (mail->data.stream == NULL) {
-		(void)mail_get_stream(_mail, NULL, NULL, &input);
+		(void)mail_get_stream_because(_mail, NULL, NULL, "prefetch", &input);
 		if (mail->data.stream == NULL)
 			return TRUE;
 	}
@@ -2107,7 +2113,7 @@
 	struct index_mail *imail = (struct index_mail *)mail;
 
 	imail->data.access_part |= PARSE_HDR;
-	if (index_mail_parse_headers(imail, NULL) == 0) {
+	if (index_mail_parse_headers(imail, NULL, "precache") == 0) {
 		if (parse_body) {
 			imail->data.access_part |= PARSE_BODY;
 			(void)index_mail_parse_body(imail, 0);
@@ -2231,3 +2237,10 @@
 			p_strdup(imail->mail.data_pool, ctx->data.from_envelope);
 	}
 }
+
+const char *index_mail_cache_reason(struct mail *mail, const char *reason)
+{
+	const char *cache_reason =
+		mail_cache_get_missing_reason(mail->transaction->cache_view, mail->seq);
+	return t_strdup_printf("%s (%s)", reason, cache_reason);
+}
--- a/src/lib-storage/index/index-mail.h	Fri Jul 29 00:37:07 2016 +0300
+++ b/src/lib-storage/index/index-mail.h	Mon Jul 25 14:20:05 2016 -0400
@@ -188,7 +188,8 @@
 			     struct message_header_line *hdr,
 			     struct index_mail *mail) ATTR_NULL(1);
 int index_mail_parse_headers(struct index_mail *mail,
-			     struct mailbox_header_lookup_ctx *headers)
+			     struct mailbox_header_lookup_ctx *headers,
+			     const char *reason)
 	ATTR_NULL(2);
 int index_mail_headers_get_envelope(struct index_mail *mail);
 
@@ -266,4 +267,6 @@
 				  unsigned int field_idx);
 void index_mail_save_finish(struct mail_save_context *ctx);
 
+const char *index_mail_cache_reason(struct mail *mail, const char *reason);
+
 #endif
--- a/src/lib-storage/index/index-search.c	Fri Jul 29 00:37:07 2016 +0300
+++ b/src/lib-storage/index/index-search.c	Mon Jul 25 14:20:05 2016 -0400
@@ -713,8 +713,8 @@
 	} else if (have_headers) {
 		/* we need to read the entire header */
 		ret = have_body ?
-			mail_get_stream(ctx->cur_mail, NULL, NULL, &input) :
-			mail_get_hdr_stream(ctx->cur_mail, NULL, &input);
+			mail_get_stream_because(ctx->cur_mail, NULL, NULL, "search", &input) :
+			mail_get_hdr_stream_because(ctx->cur_mail, NULL, "search", &input);
 		if (ret < 0)
 			failed = TRUE;
 		else {
@@ -766,7 +766,7 @@
 		/* we didn't search headers. */
 		struct message_size hdr_size;
 
-		if (mail_get_stream(ctx->cur_mail, &hdr_size, NULL, &input) < 0)
+		if (mail_get_stream_because(ctx->cur_mail, &hdr_size, NULL, "search", &input) < 0)
 			return -1;
 		i_stream_seek(input, hdr_size.physical_size);
 	}
--- a/src/lib-storage/mail-copy.c	Fri Jul 29 00:37:07 2016 +0300
+++ b/src/lib-storage/mail-copy.c	Mon Jul 25 14:20:05 2016 -0400
@@ -66,7 +66,7 @@
 	   to help anything. */
 	pmail->v.set_uid_cache_updates(mail, TRUE);
 
-	if (mail_get_stream(mail, NULL, NULL, &input) < 0) {
+	if (mail_get_stream_because(mail, NULL, NULL, "copying", &input) < 0) {
 		mail_copy_set_failed(ctx, mail, "stream");
 		return -1;
 	}
--- a/src/plugins/fts/fts-build-mail.c	Fri Jul 29 00:37:07 2016 +0300
+++ b/src/plugins/fts/fts-build-mail.c	Mon Jul 25 14:20:05 2016 -0400
@@ -468,7 +468,7 @@
 	const char *error;
 	int ret;
 
-	if (mail_get_stream(mail, NULL, NULL, &input) < 0) {
+	if (mail_get_stream_because(mail, NULL, NULL, "fts indexing", &input) < 0) {
 		if (mail->expunged)
 			return 0;
 		i_error("Failed to read mailbox %s mail UID=%u stream: %s",
--- a/src/plugins/pop3-migration/pop3-migration-plugin.c	Fri Jul 29 00:37:07 2016 +0300
+++ b/src/plugins/pop3-migration/pop3-migration-plugin.c	Mon Jul 25 14:20:05 2016 -0400
@@ -276,7 +276,7 @@
 	   So we'll try to avoid this by falling back to full FETCH BODY[]
 	   (and/or RETR) and we'll parse the header ourself from it. This
 	   should work around any similar bugs in all IMAP/POP3 servers. */
-	if (mail_get_stream(mail, &hdr_size, NULL, &input) < 0) {
+	if (mail_get_stream_because(mail, &hdr_size, NULL, "pop3-migration", &input) < 0) {
 		errstr = mailbox_get_last_error(mail->box, &error);
 		i_error("pop3_migration: Failed to get body for msg %u: %s",
 			mail->seq, errstr);
--- a/src/pop3/pop3-commands.c	Fri Jul 29 00:37:07 2016 +0300
+++ b/src/pop3/pop3-commands.c	Mon Jul 25 14:20:05 2016 -0400
@@ -438,7 +438,7 @@
 }
 
 static int fetch(struct client *client, unsigned int msgnum, uoff_t body_lines,
-		 uoff_t *byte_counter)
+		 const char *reason, uoff_t *byte_counter)
 {
         struct fetch_context *ctx;
 	int ret;
@@ -451,7 +451,7 @@
 			       MAIL_FETCH_STREAM_BODY, NULL);
 	mail_set_seq(ctx->mail, msgnum_to_seq(client, msgnum));
 
-	if (mail_get_stream(ctx->mail, NULL, NULL, &ctx->stream) < 0) {
+	if (mail_get_stream_because(ctx->mail, NULL, NULL, reason, &ctx->stream) < 0) {
 		ret = client_reply_msg_expunged(client, msgnum);
 		fetch_deinit(ctx);
 		return ret;
@@ -495,7 +495,7 @@
 		client->last_seen_pop3_msn = msgnum+1;
 
 	client->retr_count++;
-	return fetch(client, msgnum, (uoff_t)-1, &client->retr_bytes);
+	return fetch(client, msgnum, (uoff_t)-1, "RETR", &client->retr_bytes);
 }
 
 static int cmd_rset(struct client *client, const char *args ATTR_UNUSED)
@@ -556,7 +556,7 @@
 		return -1;
 
 	client->top_count++;
-	return fetch(client, msgnum, max_lines, &client->top_bytes);
+	return fetch(client, msgnum, max_lines, "TOP", &client->top_bytes);
 }
 
 struct cmd_uidl_context {