Mercurial > dovecot > core-2.2

changeset 12874:0461e23ae57c

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Search supports now prefetching data for returned mails. Dropped imapc's own prefetching. mail_prefetch_count specifies how many mails can be kept open and issue a prefetch. This works using posix_fadvise(POSIX_FADV_WILLNEED) for maildir, sdbox and cydir backends. Apparently only Linux supports this. imapc backend also implements this internally by sending wanted IMAP commands to remote server. The command pipelining helps with latency. This change also makes it actually possible for imapc backend to first check if wanted data is already cached in local index and avoid sending unnecessary IMAP commands to remote server.
author Timo Sirainen <tss@iki.fi>
date Thu, 31 Mar 2011 11:10:22 +0300
parents b47f17b173f5
children ac8a28a8b5c0
files configure.in doc/example-config/conf.d/10-mail.conf src/lib-storage/index/cydir/cydir-mail.c src/lib-storage/index/cydir/cydir-storage.c src/lib-storage/index/dbox-multi/mdbox-mail.c src/lib-storage/index/dbox-single/sdbox-mail.c src/lib-storage/index/dbox-single/sdbox-storage.c src/lib-storage/index/imapc/Makefile.am src/lib-storage/index/imapc/imapc-mail-fetch.c src/lib-storage/index/imapc/imapc-mail.c src/lib-storage/index/imapc/imapc-mail.h src/lib-storage/index/imapc/imapc-mailbox.c src/lib-storage/index/imapc/imapc-search.c src/lib-storage/index/imapc/imapc-storage.c src/lib-storage/index/imapc/imapc-storage.h src/lib-storage/index/index-mail.c src/lib-storage/index/index-mail.h src/lib-storage/index/index-search-private.h src/lib-storage/index/index-search.c src/lib-storage/index/maildir/maildir-mail.c src/lib-storage/index/maildir/maildir-storage.c src/lib-storage/index/mbox/mbox-mail.c src/lib-storage/index/raw/raw-mail.c src/lib-storage/mail-storage-private.h src/lib-storage/mail-storage-settings.c src/lib-storage/mail-storage-settings.h src/lib-storage/mail.c src/lib-storage/test-mail.c src/plugins/virtual/virtual-mail.c src/plugins/virtual/virtual-search.c
diffstat 30 files changed, 577 insertions(+), 541 deletions(-) [+]
line wrap: on
line diff
--- a/configure.in	Thu Mar 31 09:48:37 2011 +0300
+++ b/configure.in	Thu Mar 31 11:10:22 2011 +0300
@@ -379,7 +379,8 @@
 	       setrlimit setproctitle seteuid setreuid setegid setresgid \
 	       strtoull strtoll strtouq strtoq \
 	       setpriority quotactl getmntent kqueue kevent backtrace_symbols \
-	       walkcontext dirfd clearenv malloc_usable_size glob fallocate)
+	       walkcontext dirfd clearenv malloc_usable_size glob fallocate \
+	       posix_fadvise)
 
 AC_CHECK_LIB(rt, clock_gettime, [
   AC_DEFINE(HAVE_CLOCK_GETTIME,, Define if you have the clock_gettime function)
--- a/doc/example-config/conf.d/10-mail.conf	Thu Mar 31 09:48:37 2011 +0300
+++ b/doc/example-config/conf.d/10-mail.conf	Thu Mar 31 11:10:22 2011 +0300
@@ -228,6 +228,10 @@
 # the extra CRs wrong and cause problems.
 #mail_save_crlf = no
 
+# Max number of mails to keep open and prefetch to memory. This only works with
+# some mailbox formats and/or operating systems.
+#mail_prefetch_count = 0
+
 ##
 ## Maildir-specific settings
 ##
--- a/src/lib-storage/index/cydir/cydir-mail.c	Thu Mar 31 09:48:37 2011 +0300
+++ b/src/lib-storage/index/cydir/cydir-mail.c	Thu Mar 31 11:10:22 2011 +0300
@@ -130,6 +130,7 @@
 	index_mail_set_seq,
 	index_mail_set_uid,
 	index_mail_set_uid_cache_updates,
+	index_mail_prefetch,
 
 	index_mail_get_flags,
 	index_mail_get_keywords,
--- a/src/lib-storage/index/cydir/cydir-storage.c	Thu Mar 31 09:48:37 2011 +0300
+++ b/src/lib-storage/index/cydir/cydir-storage.c	Thu Mar 31 11:10:22 2011 +0300
@@ -110,7 +110,7 @@
 
 struct mail_storage cydir_storage = {
 	.name = CYDIR_STORAGE_NAME,
-	.class_flags = 0,
+	.class_flags = MAIL_STORAGE_CLASS_FLAG_FILE_PER_MSG,
 
 	.v = {
 		NULL,
--- a/src/lib-storage/index/dbox-multi/mdbox-mail.c	Thu Mar 31 09:48:37 2011 +0300
+++ b/src/lib-storage/index/dbox-multi/mdbox-mail.c	Thu Mar 31 11:10:22 2011 +0300
@@ -192,6 +192,7 @@
 	index_mail_set_seq,
 	index_mail_set_uid,
 	index_mail_set_uid_cache_updates,
+	index_mail_prefetch,
 
 	index_mail_get_flags,
 	index_mail_get_keywords,
--- a/src/lib-storage/index/dbox-single/sdbox-mail.c	Thu Mar 31 09:48:37 2011 +0300
+++ b/src/lib-storage/index/dbox-single/sdbox-mail.c	Thu Mar 31 11:10:22 2011 +0300
@@ -82,6 +82,7 @@
 	index_mail_set_seq,
 	index_mail_set_uid,
 	index_mail_set_uid_cache_updates,
+	index_mail_prefetch,
 
 	index_mail_get_flags,
 	index_mail_get_keywords,
--- a/src/lib-storage/index/dbox-single/sdbox-storage.c	Thu Mar 31 09:48:37 2011 +0300
+++ b/src/lib-storage/index/dbox-single/sdbox-storage.c	Thu Mar 31 11:10:22 2011 +0300
@@ -348,7 +348,7 @@
 
 struct mail_storage dbox_storage = {
 	.name = "dbox", /* alias */
-	.class_flags = 0,
+	.class_flags = MAIL_STORAGE_CLASS_FLAG_FILE_PER_MSG,
 
 	.v = {
                 NULL,
--- a/src/lib-storage/index/imapc/Makefile.am	Thu Mar 31 09:48:37 2011 +0300
+++ b/src/lib-storage/index/imapc/Makefile.am	Thu Mar 31 11:10:22 2011 +0300
@@ -18,9 +18,9 @@
 	imapc-connection.c \
 	imapc-list.c \
 	imapc-mail.c \
+	imapc-mail-fetch.c \
 	imapc-mailbox.c \
 	imapc-save.c \
-	imapc-search.c \
 	imapc-seqmap.c \
 	imapc-settings.c \
 	imapc-sync.c \
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/src/lib-storage/index/imapc/imapc-mail-fetch.c	Thu Mar 31 11:10:22 2011 +0300
@@ -0,0 +1,258 @@
+/* Copyright (c) 2011 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "str.h"
+#include "istream.h"
+#include "imap-arg.h"
+#include "imap-date.h"
+#include "imapc-client.h"
+#include "imapc-mail.h"
+#include "imapc-storage.h"
+
+static void
+imapc_mail_prefetch_callback(const struct imapc_command_reply *reply,
+			     void *context)
+{
+	struct imapc_mail *mail = context;
+	struct imapc_mailbox *mbox =
+		(struct imapc_mailbox *)mail->imail.mail.mail.box;
+
+	i_assert(mail->fetch_count > 0);
+
+	if (--mail->fetch_count == 0) {
+		struct imapc_mail *const *fetch_mails;
+		unsigned int i, count;
+
+		fetch_mails = array_get(&mbox->fetch_mails, &count);
+		for (i = 0; i < count; i++) {
+			if (fetch_mails[i] == mail) {
+				array_delete(&mbox->fetch_mails, i, 1);
+				break;
+			}
+		}
+		i_assert(i != count);
+		mail->fetching_fields = 0;
+	}
+
+	if (reply->state == IMAPC_COMMAND_STATE_OK)
+		;
+	else if (reply->state == IMAPC_COMMAND_STATE_NO) {
+		imapc_copy_error_from_reply(mbox->storage, MAIL_ERROR_PARAMS,
+					    reply);
+	} else {
+		mail_storage_set_critical(&mbox->storage->storage,
+			"imapc: Command failed: %s", reply->text_full);
+	}
+	imapc_client_stop(mbox->storage->client);
+}
+
+static int
+imapc_mail_send_fetch(struct mail *_mail, enum mail_fetch_field fields)
+{
+	struct imapc_mail *mail = (struct imapc_mail *)_mail;
+	struct imapc_mailbox *mbox = (struct imapc_mailbox *)_mail->box;
+	struct imapc_client_mailbox *client_box;
+	string_t *str;
+
+	if (_mail->lookup_abort != MAIL_LOOKUP_ABORT_NEVER)
+		return -1;
+
+	/* drop any fields that we may already be fetching currently */
+	fields &= ~mail->fetching_fields;
+	if (fields == 0)
+		return 0;
+
+	if ((fields & MAIL_FETCH_STREAM_BODY) != 0)
+		fields |= MAIL_FETCH_STREAM_HEADER;
+
+	if (imapc_mailbox_get_client_box(mbox, &client_box) < 0)
+		return -1;
+
+	str = t_str_new(64);
+	str_printfa(str, "UID FETCH %u (", _mail->uid);
+	if ((fields & MAIL_FETCH_RECEIVED_DATE) != 0)
+		str_append(str, "INTERNALDATE ");
+	if ((fields & MAIL_FETCH_STREAM_BODY) != 0)
+		str_append(str, "BODY.PEEK[] ");
+	else if ((fields & MAIL_FETCH_STREAM_HEADER) != 0)
+		str_append(str, "BODY.PEEK[HEADER] ");
+	str_truncate(str, str_len(str)-1);
+	str_append_c(str, ')');
+
+	pool_ref(mail->imail.mail.pool);
+	mail->fetching_fields |= fields;
+	if (mail->fetch_count++ == 0)
+		array_append(&mbox->fetch_mails, &mail, 1);
+
+	imapc_client_mailbox_cmdf(client_box, imapc_mail_prefetch_callback,
+				  mail, "%1s", str_c(str));
+	mail->imail.data.prefetch_sent = TRUE;
+	return 0;
+}
+
+bool imapc_mail_prefetch(struct mail *_mail)
+{
+	struct imapc_mail *mail = (struct imapc_mail *)_mail;
+	struct index_mail_data *data = &mail->imail.data;
+	enum mail_fetch_field fields = 0;
+
+	if ((mail->imail.wanted_fields & MAIL_FETCH_RECEIVED_DATE) != 0 &&
+	    data->received_date == (time_t)-1)
+		fields |= MAIL_FETCH_RECEIVED_DATE;
+
+	if (data->stream == NULL && data->access_part != 0) {
+		if ((data->access_part & (READ_BODY | PARSE_BODY)) != 0)
+			fields |= MAIL_FETCH_STREAM_BODY;
+		else
+			fields |= MAIL_FETCH_STREAM_HEADER;
+	}
+	if (fields != 0) T_BEGIN {
+		(void)imapc_mail_send_fetch(_mail, fields);
+	} T_END;
+	return !mail->imail.data.prefetch_sent;
+}
+
+int imapc_mail_fetch(struct mail *_mail, enum mail_fetch_field fields)
+{
+	struct imapc_mail *imail = (struct imapc_mail *)_mail;
+	struct imapc_storage *storage =
+		(struct imapc_storage *)_mail->box->storage;
+	int ret;
+
+	T_BEGIN {
+		ret = imapc_mail_send_fetch(_mail, fields);
+	} T_END;
+	if (ret < 0)
+		return -1;
+
+	while (imail->fetch_count > 0)
+		imapc_client_run(storage->client);
+	return 0;
+}
+
+static bool imapc_find_lfile_arg(const struct imapc_untagged_reply *reply,
+				 const struct imap_arg *arg, int *fd_r)
+{
+	const struct imap_arg *list;
+	unsigned int i, count;
+
+	for (i = 0; i < reply->file_args_count; i++) {
+		const struct imapc_arg_file *farg = &reply->file_args[i];
+
+		if (farg->parent_arg == arg->parent &&
+		    imap_arg_get_list_full(arg->parent, &list, &count) &&
+		    farg->list_idx < count && &list[farg->list_idx] == arg) {
+			*fd_r = farg->fd;
+			return TRUE;
+		}
+	}
+	return FALSE;
+}
+
+static void
+imapc_fetch_stream(struct imapc_mail *mail,
+		   const struct imapc_untagged_reply *reply,
+		   const struct imap_arg *arg, bool body)
+{
+	struct index_mail *imail = &mail->imail;
+	struct mail *_mail = &imail->mail.mail;
+	struct istream *input;
+	uoff_t size;
+	const char *value;
+	int fd, ret;
+
+	if (imail->data.stream != NULL) {
+		if (!body)
+			return;
+		/* maybe the existing stream has no body. replace it. */
+		i_stream_unref(&imail->data.stream);
+	}
+
+	if (arg->type == IMAP_ARG_LITERAL_SIZE) {
+		if (!imapc_find_lfile_arg(reply, arg, &fd))
+			return;
+		if ((fd = dup(fd)) == -1) {
+			i_error("dup() failed: %m");
+			return;
+		}
+		imail->data.stream = i_stream_create_fd(fd, 0, TRUE);
+	} else {
+		if (!imap_arg_get_nstring(arg, &value))
+			return;
+		if (value == NULL) {
+			mail_set_expunged(_mail);
+			return;
+		}
+		if (mail->body == NULL) {
+			mail->body = buffer_create_dynamic(default_pool,
+							   arg->str_len + 1);
+		}
+		buffer_set_used_size(mail->body, 0);
+		buffer_append(mail->body, value, arg->str_len);
+		imail->data.stream = i_stream_create_from_data(mail->body->data,
+							       mail->body->used);
+	}
+
+	i_stream_set_name(imail->data.stream,
+			  t_strdup_printf("imapc mail uid=%u", _mail->uid));
+	index_mail_set_read_buffer_size(_mail, imail->data.stream);
+
+	if (imail->mail.v.istream_opened != NULL) {
+		if (imail->mail.v.istream_opened(_mail,
+						 &imail->data.stream) < 0) {
+			i_stream_unref(&imail->data.stream);
+			return;
+		}
+	} else if (body) {
+		ret = i_stream_get_size(imail->data.stream, TRUE, &size);
+		if (ret < 0) {
+			i_stream_unref(&imail->data.stream);
+			return;
+		}
+		i_assert(ret != 0);
+		imail->data.physical_size = size;
+		/* we'll assume that the remote server is working properly and
+		   sending CRLF linefeeds */
+		imail->data.virtual_size = size;
+	}
+
+	if (index_mail_init_stream(imail, NULL, NULL, &input) < 0)
+		i_stream_unref(&imail->data.stream);
+}
+
+void imapc_mail_fetch_update(struct imapc_mail *mail,
+			     const struct imapc_untagged_reply *reply,
+			     const struct imap_arg *args)
+{
+	struct imapc_mailbox *mbox =
+		(struct imapc_mailbox *)mail->imail.mail.mail.box;
+	const char *key, *value;
+	unsigned int i;
+	time_t t;
+	int tz;
+	bool match = FALSE;
+
+	for (i = 0; args[i].type != IMAP_ARG_EOL; i += 2) {
+		if (!imap_arg_get_atom(&args[i], &key) ||
+		    args[i+1].type == IMAP_ARG_EOL)
+			break;
+
+		if (strcasecmp(key, "BODY[]") == 0) {
+			imapc_fetch_stream(mail, reply, &args[i+1], TRUE);
+			match = TRUE;
+		} else if (strcasecmp(key, "BODY[HEADER]") == 0) {
+			imapc_fetch_stream(mail, reply, &args[i+1], FALSE);
+			match = TRUE;
+		} else if (strcasecmp(key, "INTERNALDATE") == 0) {
+			if (imap_arg_get_astring(&args[i+1], &value) &&
+			    imap_parse_datetime(value, &t, &tz))
+				mail->imail.data.received_date = t;
+			match = TRUE;
+		}
+	}
+	if (!match) {
+		/* this is only a FETCH FLAGS update for the wanted mail */
+	} else {
+		imapc_client_stop(mbox->storage->client);
+	}
+}
--- a/src/lib-storage/index/imapc/imapc-mail.c	Thu Mar 31 09:48:37 2011 +0300
+++ b/src/lib-storage/index/imapc/imapc-mail.c	Thu Mar 31 11:10:22 2011 +0300
@@ -3,6 +3,7 @@
 #include "lib.h"
 #include "str.h"
 #include "istream.h"
+#include "imap-envelope.h"
 #include "imapc-mail.h"
 #include "imapc-client.h"
 #include "imapc-storage.h"
@@ -125,12 +126,66 @@
 	return index_mail_init_stream(mail, hdr_size, body_size, stream_r);
 }
 
+static bool
+imapc_mail_has_headers_in_cache(struct index_mail *mail,
+				struct mailbox_header_lookup_ctx *headers)
+{
+	struct mail *_mail = &mail->mail.mail;
+	unsigned int i;
+
+	for (i = 0; i < headers->count; i++) {
+		if (mail_cache_field_exists(_mail->transaction->cache_view,
+					    _mail->seq, headers->idx[i]) <= 0)
+			return FALSE;
+	}
+	return TRUE;
+}
+
+static void imapc_mail_set_seq(struct mail *_mail, uint32_t seq)
+{
+	struct imapc_mail *imail = (struct imapc_mail *)_mail;
+	struct index_mail *mail = &imail->imail;
+	struct mailbox_header_lookup_ctx *header_ctx;
+	time_t date;
+	uoff_t size;
+
+	index_mail_set_seq(_mail, seq);
+
+	if ((mail->wanted_fields & MAIL_FETCH_RECEIVED_DATE) != 0)
+		(void)index_mail_get_received_date(_mail, &date);
+	if ((mail->wanted_fields & MAIL_FETCH_PHYSICAL_SIZE) != 0) {
+		if (index_mail_get_physical_size(_mail, &size) < 0)
+			mail->data.access_part |= READ_HDR | READ_BODY;
+	}
+
+	if (mail->data.access_part == 0 && mail->wanted_headers != NULL) {
+		/* see if all wanted headers exist in cache */
+		if (!imapc_mail_has_headers_in_cache(mail, mail->wanted_headers))
+			mail->data.access_part |= PARSE_HDR;
+	}
+	if (mail->data.access_part == 0 &&
+	    (mail->wanted_fields & MAIL_FETCH_IMAP_ENVELOPE) != 0) {
+		/* the common code already checked this partially,
+		   but we need a guaranteed correct answer */
+		header_ctx = mailbox_header_lookup_init(_mail->box,
+							imap_envelope_headers);
+		if (!imapc_mail_has_headers_in_cache(mail, header_ctx))
+			mail->data.access_part |= PARSE_HDR;
+		mailbox_header_lookup_unref(&header_ctx);
+	}
+	/* searching code handles prefetching internally,
+	   elsewhere we want to do it immediately */
+	if (!mail->search_mail)
+		(void)imapc_mail_prefetch(_mail);
+}
+
 struct mail_vfuncs imapc_mail_vfuncs = {
 	index_mail_close,
 	imapc_mail_free,
-	index_mail_set_seq,
+	imapc_mail_set_seq,
 	index_mail_set_uid,
 	index_mail_set_uid_cache_updates,
+	imapc_mail_prefetch,
 
 	index_mail_get_flags,
 	index_mail_get_keywords,
--- a/src/lib-storage/index/imapc/imapc-mail.h	Thu Mar 31 09:48:37 2011 +0300
+++ b/src/lib-storage/index/imapc/imapc-mail.h	Thu Mar 31 11:10:22 2011 +0300
@@ -3,12 +3,16 @@
 
 #include "index-mail.h"
 
+struct imap_arg;
+struct imapc_untagged_reply;
+
 struct imapc_mail {
 	struct index_mail imail;
 
+	enum mail_fetch_field fetching_fields;
+	unsigned int fetch_count;
+
 	buffer_t *body;
-	unsigned int searching:1;
-	unsigned int fetch_one:1;
 };
 
 extern struct mail_vfuncs imapc_mail_vfuncs;
@@ -18,5 +22,10 @@
 		 enum mail_fetch_field wanted_fields,
 		 struct mailbox_header_lookup_ctx *wanted_headers);
 int imapc_mail_fetch(struct mail *mail, enum mail_fetch_field fields);
+bool imapc_mail_prefetch(struct mail *mail);
+
+void imapc_mail_fetch_update(struct imapc_mail *mail,
+			     const struct imapc_untagged_reply *reply,
+			     const struct imap_arg *args);
 
 #endif
--- a/src/lib-storage/index/imapc/imapc-mailbox.c	Thu Mar 31 09:48:37 2011 +0300
+++ b/src/lib-storage/index/imapc/imapc-mailbox.c	Thu Mar 31 11:10:22 2011 +0300
@@ -6,6 +6,7 @@
 #include "imap-util.h"
 #include "imapc-client.h"
 #include "imapc-seqmap.h"
+#include "imapc-mail.h"
 #include "imapc-sync.h"
 #include "imapc-storage.h"
 
@@ -155,6 +156,7 @@
 {
 	uint32_t lseq, rseq = reply->num;
 	struct imapc_seqmap *seqmap;
+	struct imapc_mail *const *mailp;
 	const struct imap_arg *list, *flags_list;
 	const char *atom;
 	const struct mail_index_record *rec = NULL;
@@ -200,9 +202,13 @@
 	seqmap = imapc_client_mailbox_get_seqmap(mbox->client_box);
 	lseq = imapc_seqmap_rseq_to_lseq(seqmap, rseq);
 
-	if (mbox->cur_fetch_mail != NULL && mbox->cur_fetch_mail->seq == lseq) {
-		i_assert(uid == 0 || mbox->cur_fetch_mail->uid == uid);
-		imapc_fetch_mail_update(mbox->cur_fetch_mail, reply, list);
+	array_foreach(&mbox->fetch_mails, mailp) {
+		struct imapc_mail *mail = *mailp;
+
+		if (mail->imail.mail.mail.seq == lseq) {
+			i_assert(uid == 0 || mail->imail.mail.mail.uid == uid);
+			imapc_mail_fetch_update(mail, reply, list);
+		}
 	}
 
 	imapc_mailbox_init_delayed_trans(mbox);
--- a/src/lib-storage/index/imapc/imapc-search.c	Thu Mar 31 09:48:37 2011 +0300
+++ /dev/null	Thu Jan 01 00:00:00 1970 +0000
@@ -1,427 +0,0 @@
-/* Copyright (c) 2011 Dovecot authors, see the included COPYING file */
-
-#include "lib.h"
-#include "istream.h"
-#include "write-full.h"
-#include "str.h"
-#include "imap-arg.h"
-#include "imap-date.h"
-#include "imap-util.h"
-#include "mail-user.h"
-#include "mail-search.h"
-#include "index-search-private.h"
-#include "imapc-mail.h"
-#include "imapc-client.h"
-#include "imapc-seqmap.h"
-#include "imapc-storage.h"
-
-struct imapc_search_context {
-	struct index_search_context ictx;
-
-	struct imapc_client_mailbox *client_box;
-
-	/* non-NULL during _search_next_nonblock() */
-	struct mail *cur_mail;
-
-	/* sequences of messages we're next wanting to fetch. */
-	ARRAY_TYPE(seq_range) next_seqs;
-	uint32_t next_pending_seq, saved_seq;
-
-	unsigned int fetching:1;
-	unsigned int failed:1;
-};
-
-static void imapc_search_fetch_callback(const struct imapc_command_reply *reply,
-					void *context)
-{
-	struct imapc_search_context *ctx = context;
-	struct imapc_mailbox *mbox =
-		(struct imapc_mailbox *)ctx->ictx.mail_ctx.transaction->box;
-
-	if (reply->state == IMAPC_COMMAND_STATE_OK)
-		;
-	else if (reply->state == IMAPC_COMMAND_STATE_NO) {
-		imapc_copy_error_from_reply(mbox->storage, MAIL_ERROR_PARAMS,
-					    reply);
-	} else {
-		mail_storage_set_critical(&mbox->storage->storage,
-			"imapc: Command failed: %s", reply->text_full);
-	}
-	ctx->fetching = FALSE;
-	imapc_client_mailbox_unlock(ctx->client_box);
-	imapc_client_stop(mbox->storage->client);
-}
-
-static bool
-imapc_append_wanted_fields(string_t *str, enum mail_fetch_field fields,
-			   bool want_headers)
-{
-	bool ret = FALSE;
-
-	if ((fields & (MAIL_FETCH_STREAM_BODY |
-		       MAIL_FETCH_MESSAGE_PARTS |
-		       MAIL_FETCH_NUL_STATE |
-		       MAIL_FETCH_IMAP_BODY |
-		       MAIL_FETCH_IMAP_BODYSTRUCTURE |
-		       MAIL_FETCH_PHYSICAL_SIZE |
-		       MAIL_FETCH_VIRTUAL_SIZE)) != 0) {
-		str_append(str, "BODY.PEEK[] ");
-		ret = TRUE;
-	} else if (want_headers ||
-		   (fields & (MAIL_FETCH_STREAM_HEADER |
-			      MAIL_FETCH_IMAP_ENVELOPE |
-			      MAIL_FETCH_HEADER_MD5 |
-			      MAIL_FETCH_DATE)) != 0) {
-		str_append(str, "BODY.PEEK[HEADER] ");
-		ret = TRUE;
-	}
-
-	if ((fields & MAIL_FETCH_RECEIVED_DATE) != 0) {
-		str_append(str, "INTERNALDATE ");
-		ret = TRUE;
-	}
-	return ret;
-}
-
-static int
-imapc_search_send_fetch(struct imapc_search_context *ctx,
-			const ARRAY_TYPE(seq_range) *uids)
-{
-	struct mail *mail = ctx->cur_mail;
-	struct mail_private *pmail = (struct mail_private *)mail;
-	struct imapc_mailbox *mbox = (struct imapc_mailbox *)mail->box;
-	struct imapc_client_mailbox *client_box;
-	string_t *str;
-
-	str = t_str_new(64);
-	str_append(str, "UID FETCH ");
-	imap_write_seq_range(str, uids);
-	str_append(str, " (");
-
-	if (!imapc_append_wanted_fields(str, pmail->wanted_fields,
-					pmail->wanted_headers != NULL)) {
-		/* we don't need to fetch anything */
-		return 0;
-	}
-
-	str_truncate(str, str_len(str) - 1);
-	str_append_c(str, ')');
-
-	if (imapc_mailbox_get_client_box(mbox, &client_box) < 0)
-		return -1;
-
-	ctx->client_box = client_box;
-	ctx->fetching = TRUE;
-	imapc_client_mailbox_lock(client_box);
-	imapc_client_mailbox_cmdf(client_box, imapc_search_fetch_callback,
-				  ctx, "%1s", str_c(str));
-	return 0;
-}
-
-struct mail_search_context *
-imapc_search_init(struct mailbox_transaction_context *t,
-		  struct mail_search_args *args,
-		  const enum mail_sort_type *sort_program,
-		  enum mail_fetch_field wanted_fields,
-		  struct mailbox_header_lookup_ctx *wanted_headers)
-{
-	struct imapc_search_context *ctx;
-
-	ctx = i_new(struct imapc_search_context, 1);
-	index_storage_search_init_context(&ctx->ictx, t, args, sort_program,
-					  wanted_fields, wanted_headers);
-	i_array_init(&ctx->next_seqs, 64);
-	ctx->ictx.recheck_index_args = TRUE;
-	return &ctx->ictx.mail_ctx;
-}
-
-int imapc_search_deinit(struct mail_search_context *_ctx)
-{
-	struct imapc_search_context *ctx = (struct imapc_search_context *)_ctx;
-	struct imapc_mailbox *mbox =
-		(struct imapc_mailbox *)_ctx->transaction->box;
-	int ret = ctx->failed ? -1 : 0;
-
-	while (ctx->fetching)
-		imapc_client_run(mbox->storage->client);
-
-	array_free(&ctx->next_seqs);
-	if (index_storage_search_deinit(_ctx) < 0)
-		ret = -1;
-	return ret;
-}
-
-bool imapc_search_next_nonblock(struct mail_search_context *_ctx,
-				struct mail **mail_r, bool *tryagain_r)
-{
-	struct imapc_search_context *ctx = (struct imapc_search_context *)_ctx;
-	struct mail *mail = ctx->ictx.mail;
-	struct imapc_mailbox *mbox =
-		(struct imapc_mailbox *)_ctx->transaction->box;
-	struct imapc_mail *imail = (struct imapc_mail *)mail;
-	bool ret;
-
-	imail->searching = TRUE;
-	ctx->cur_mail = mail;
-	ret = index_storage_search_next_nonblock(_ctx, mail_r, tryagain_r);
-	ctx->cur_mail = NULL;
-	imail->searching = FALSE;
-	if (!ret)
-		return FALSE;
-
-	if (!ctx->fetching) {
-		/* no need to fetch anything from remote */
-	} else if (mail_index_is_expunged(mbox->box.view, mail->seq)) {
-		/* already synced this message as expunged.
-		   remote server won't have it. */
-	} else {
-		struct imapc_seqmap *seqmap;
-		uint32_t rseq;
-
-		seqmap = imapc_client_mailbox_get_seqmap(mbox->client_box);
-		rseq = imapc_seqmap_lseq_to_rseq(seqmap, mail->seq);
-		if (rseq == 0) {
-			/* expunged from remote already. */
-		} else {
-			mbox->cur_fetch_mail = mail;
-			imapc_client_run(mbox->storage->client);
-			mbox->cur_fetch_mail = NULL;
-		}
-	}
-	return TRUE;
-}
-
-static void imapc_get_short_uid_range(struct mailbox *box,
-				      const ARRAY_TYPE(seq_range) *seqs,
-				      ARRAY_TYPE(seq_range) *uids)
-{
-	const struct seq_range *range;
-	unsigned int i, count;
-	uint32_t uid1, uid2;
-
-	range = array_get(seqs, &count);
-	for (i = 0; i < count; i++) {
-		mail_index_lookup_uid(box->view, range[i].seq1, &uid1);
-		mail_index_lookup_uid(box->view, range[i].seq2, &uid2);
-		seq_range_array_add_range(uids, uid1, uid2);
-	}
-}
-
-static int imapc_search_update_next_seqs(struct imapc_search_context *ctx)
-{
-	struct mail_search_context *_ctx = &ctx->ictx.mail_ctx;
-	uint32_t prev_seq;
-	int ret = 0;
-
-	/* add messages to the next_seqs list as long as the sequences
-	   are incrementing */
-	if (ctx->next_pending_seq == 0)
-		prev_seq = 0;
-	else {
-		prev_seq = ctx->next_pending_seq;
-		seq_range_array_add(&ctx->next_seqs, 0, prev_seq);
-	}
-	if (ctx->saved_seq != 0)
-		_ctx->seq = ctx->saved_seq;
-	while (index_storage_search_next_update_seq(_ctx)) {
-		mail_search_args_reset(_ctx->args->args, FALSE);
-		if (_ctx->seq < prev_seq) {
-			ctx->next_pending_seq = _ctx->seq;
-			break;
-		}
-		seq_range_array_add(&ctx->next_seqs, 0, _ctx->seq);
-	}
-	ctx->saved_seq = _ctx->seq;
-	if (array_count(&ctx->next_seqs) > 0) T_BEGIN {
-		ARRAY_TYPE(seq_range) uids;
-
-		t_array_init(&uids, array_count(&ctx->next_seqs)*2);
-		imapc_get_short_uid_range(_ctx->transaction->box,
-					  &ctx->next_seqs, &uids);
-		ret = imapc_search_send_fetch(ctx, &uids);
-	} T_END;
-	return ret;
-}
-
-bool imapc_search_next_update_seq(struct mail_search_context *_ctx)
-{
-	struct imapc_search_context *ctx = (struct imapc_search_context *)_ctx;
-	struct seq_range *seqs;
-	unsigned int count;
-
-	seqs = array_get_modifiable(&ctx->next_seqs, &count);
-	if (count == 0) {
-		if (imapc_search_update_next_seqs(ctx) < 0) {
-			ctx->failed = TRUE;
-			return FALSE;
-		}
-		seqs = array_get_modifiable(&ctx->next_seqs, &count);
-		if (count == 0)
-			return FALSE;
-	}
-
-	_ctx->seq = seqs[0].seq1;
-	if (seqs[0].seq1 < seqs[0].seq2)
-		seqs[0].seq1++;
-	else
-		array_delete(&ctx->next_seqs, 0, 1);
-	return TRUE;
-}
-
-static bool imapc_find_lfile_arg(const struct imapc_untagged_reply *reply,
-				 const struct imap_arg *arg, int *fd_r)
-{
-	const struct imap_arg *list;
-	unsigned int i, count;
-
-	for (i = 0; i < reply->file_args_count; i++) {
-		const struct imapc_arg_file *farg = &reply->file_args[i];
-
-		if (farg->parent_arg == arg->parent &&
-		    imap_arg_get_list_full(arg->parent, &list, &count) &&
-		    farg->list_idx < count && &list[farg->list_idx] == arg) {
-			*fd_r = farg->fd;
-			return TRUE;
-		}
-	}
-	return FALSE;
-}
-
-static void
-imapc_fetch_stream(struct imapc_mail *mail,
-		   const struct imapc_untagged_reply *reply,
-		   const struct imap_arg *arg, bool body)
-{
-	struct index_mail *imail = &mail->imail;
-	struct mail *_mail = &imail->mail.mail;
-	struct istream *input;
-	uoff_t size;
-	const char *value;
-	int fd, ret;
-
-	if (imail->data.stream != NULL)
-		return;
-
-	if (arg->type == IMAP_ARG_LITERAL_SIZE) {
-		if (!imapc_find_lfile_arg(reply, arg, &fd))
-			return;
-		if ((fd = dup(fd)) == -1) {
-			i_error("dup() failed: %m");
-			return;
-		}
-		imail->data.stream = i_stream_create_fd(fd, 0, TRUE);
-	} else {
-		if (!imap_arg_get_nstring(arg, &value))
-			return;
-		if (value == NULL) {
-			mail_set_expunged(_mail);
-			return;
-		}
-		if (mail->body == NULL) {
-			mail->body = buffer_create_dynamic(default_pool,
-							   arg->str_len + 1);
-		}
-		buffer_set_used_size(mail->body, 0);
-		buffer_append(mail->body, value, arg->str_len);
-		imail->data.stream = i_stream_create_from_data(mail->body->data,
-							       mail->body->used);
-	}
-
-	i_stream_set_name(imail->data.stream,
-			  t_strdup_printf("imapc mail uid=%u", _mail->uid));
-	index_mail_set_read_buffer_size(_mail, imail->data.stream);
-
-	if (imail->mail.v.istream_opened != NULL) {
-		if (imail->mail.v.istream_opened(_mail,
-						 &imail->data.stream) < 0) {
-			i_stream_unref(&imail->data.stream);
-			return;
-		}
-	} else if (body) {
-		ret = i_stream_get_size(imail->data.stream, TRUE, &size);
-		if (ret < 0) {
-			i_stream_unref(&imail->data.stream);
-			return;
-		}
-		i_assert(ret != 0);
-		imail->data.physical_size = size;
-		/* we'll assume that the remote server is working properly and
-		   sending CRLF linefeeds */
-		imail->data.virtual_size = size;
-	}
-
-	if (index_mail_init_stream(imail, NULL, NULL, &input) < 0)
-		i_stream_unref(&imail->data.stream);
-}
-
-void imapc_fetch_mail_update(struct mail *mail,
-			     const struct imapc_untagged_reply *reply,
-			     const struct imap_arg *args)
-{
-	struct imapc_mail *imapmail = (struct imapc_mail *)mail;
-	struct imapc_mailbox *mbox = (struct imapc_mailbox *)mail->box;
-	struct imapc_mail *imail = (struct imapc_mail *)mail;
-	const char *key, *value;
-	unsigned int i;
-	time_t t;
-	int tz;
-	bool match = FALSE;
-
-	for (i = 0; args[i].type != IMAP_ARG_EOL; i += 2) {
-		if (!imap_arg_get_atom(&args[i], &key) ||
-		    args[i+1].type == IMAP_ARG_EOL)
-			break;
-
-		if (strcasecmp(key, "BODY[]") == 0) {
-			imapc_fetch_stream(imail, reply, &args[i+1], TRUE);
-			match = TRUE;
-		} else if (strcasecmp(key, "BODY[HEADER]") == 0) {
-			imapc_fetch_stream(imail, reply, &args[i+1], FALSE);
-			match = TRUE;
-		} else if (strcasecmp(key, "INTERNALDATE") == 0) {
-			if (imap_arg_get_astring(&args[i+1], &value) &&
-			    imap_parse_datetime(value, &t, &tz))
-				imail->imail.data.received_date = t;
-			match = TRUE;
-		}
-	}
-	if (!match) {
-		/* this is only a FETCH FLAGS update for the wanted mail */
-		return;
-	}
-	if (!imapmail->fetch_one)
-		imapc_client_stop_now(mbox->storage->client);
-	else
-		imapc_client_stop(mbox->storage->client);
-}
-
-int imapc_mail_fetch(struct mail *mail, enum mail_fetch_field fields)
-{
-	struct imapc_mail *imail = (struct imapc_mail *)mail;
-	struct imapc_mailbox *mbox = (struct imapc_mailbox *)mail->box;
-	struct imapc_simple_context sctx;
-	struct imapc_client_mailbox *client_box;
-	string_t *str;
-
-	str = t_str_new(64);
-	str_printfa(str, "UID FETCH %u (", mail->uid);
-
-	if (!imapc_append_wanted_fields(str, fields, FALSE))
-		return 0;
-	if (imapc_mailbox_get_client_box(mbox, &client_box) < 0)
-		return -1;
-
-	str_truncate(str, str_len(str) - 1);
-	str_append_c(str, ')');
-
-	imapc_simple_context_init(&sctx, mbox->storage);
-	imapc_client_mailbox_cmdf(client_box, imapc_simple_callback,
-				  &sctx, "%1s", str_c(str));
-
-	imail->fetch_one = TRUE;
-	mbox->cur_fetch_mail = mail;
-	imapc_simple_run(&sctx);
-	mbox->cur_fetch_mail = NULL;
-	imail->fetch_one = FALSE;
-	return sctx.ret;
-}
--- a/src/lib-storage/index/imapc/imapc-storage.c	Thu Mar 31 09:48:37 2011 +0300
+++ b/src/lib-storage/index/imapc/imapc-storage.c	Thu Mar 31 11:10:22 2011 +0300
@@ -282,6 +282,7 @@
 	p_array_init(&mbox->untagged_callbacks, pool, 16);
 	p_array_init(&mbox->resp_text_callbacks, pool, 16);
 	p_array_init(&mbox->alt_client_boxes, pool, 4);
+	p_array_init(&mbox->fetch_mails, pool, 16);
 	imapc_mailbox_register_callbacks(mbox);
 	return &mbox->box;
 }
@@ -624,10 +625,10 @@
 		index_transaction_rollback,
 		NULL,
 		imapc_mail_alloc,
-		imapc_search_init,
-		imapc_search_deinit,
-		imapc_search_next_nonblock,
-		imapc_search_next_update_seq,
+		index_storage_search_init,
+		index_storage_search_deinit,
+		index_storage_search_next_nonblock,
+		index_storage_search_next_update_seq,
 		imapc_save_alloc,
 		imapc_save_begin,
 		imapc_save_continue,
--- a/src/lib-storage/index/imapc/imapc-storage.h	Thu Mar 31 09:48:37 2011 +0300
+++ b/src/lib-storage/index/imapc/imapc-storage.h	Thu Mar 31 11:10:22 2011 +0300
@@ -52,7 +52,7 @@
 	struct mail_index_view *sync_view, *delayed_sync_view;
 	struct timeout *to_idle_check, *to_idle_delay;
 
-	struct mail *cur_fetch_mail;
+	ARRAY_DEFINE(fetch_mails, struct imapc_mail *);
 
 	ARRAY_DEFINE(untagged_callbacks, struct imapc_mailbox_event_callback);
 	ARRAY_DEFINE(resp_text_callbacks, struct imapc_mailbox_event_callback);
@@ -79,20 +79,6 @@
 					struct mail_index_transaction_commit_result *result);
 void imapc_transaction_save_rollback(struct mail_save_context *ctx);
 
-struct mail_search_context *
-imapc_search_init(struct mailbox_transaction_context *t,
-		  struct mail_search_args *args,
-		  const enum mail_sort_type *sort_program,
-		  enum mail_fetch_field wanted_fields,
-		  struct mailbox_header_lookup_ctx *wanted_headers);
-int imapc_search_deinit(struct mail_search_context *_ctx);
-bool imapc_search_next_nonblock(struct mail_search_context *_ctx,
-				struct mail **mail_r, bool *tryagain_r);
-bool imapc_search_next_update_seq(struct mail_search_context *_ctx);
-void imapc_fetch_mail_update(struct mail *mail,
-			     const struct imapc_untagged_reply *reply,
-			     const struct imap_arg *args);
-
 void imapc_copy_error_from_reply(struct imapc_storage *storage,
 				 enum mail_error default_error,
 				 const struct imapc_command_reply *reply);
--- a/src/lib-storage/index/index-mail.c	Thu Mar 31 09:48:37 2011 +0300
+++ b/src/lib-storage/index/index-mail.c	Thu Mar 31 11:10:22 2011 +0300
@@ -17,6 +17,8 @@
 #include "istream-mail-stats.h"
 #include "index-mail.h"
 
+#include <fcntl.h>
+
 struct mail_cache_field global_cache_fields[MAIL_INDEX_CACHE_FIELD_COUNT] = {
 	{ "flags", 0, MAIL_CACHE_FIELD_BITMASK, sizeof(uint32_t), 0 },
 	{ "date.sent", 0, MAIL_CACHE_FIELD_FIXED_SIZE,
@@ -1335,6 +1337,47 @@
 	mail->data.initialized = TRUE;
 }
 
+bool index_mail_prefetch(struct mail *_mail)
+{
+	struct index_mail *mail = (struct index_mail *)_mail;
+#ifdef HAVE_POSIX_FADVISE
+	struct mail_storage *storage = _mail->box->storage;
+	struct istream *input;
+	off_t len;
+	int fd;
+
+	if ((storage->class_flags & MAIL_STORAGE_CLASS_FLAG_FILE_PER_MSG) == 0) {
+		/* we're handling only file-per-msg storages for now. */
+		return TRUE;
+	}
+	if (mail->data.access_part == 0) {
+		/* everything we need is cached */
+		return TRUE;
+	}
+
+	if (mail->data.stream == NULL) {
+		(void)mail_get_stream(_mail, NULL, NULL, &input);
+		if (mail->data.stream == NULL)
+			return TRUE;
+	}
+
+	/* tell OS to start reading the file into memory */
+	fd = i_stream_get_fd(mail->data.stream);
+	if (fd != -1) {
+		if ((mail->data.access_part & (READ_BODY | PARSE_BODY)) != 0)
+			len = 0;
+		else
+			len = MAIL_READ_HDR_BLOCK_SIZE;
+		if (posix_fadvise(fd, 0, len, POSIX_FADV_WILLNEED) < 0) {
+			i_error("posix_fadvise(%s) failed: %m",
+				i_stream_get_name(mail->data.stream));
+		}
+		mail->data.prefetch_sent = TRUE;
+	}
+#endif
+	return !mail->data.prefetch_sent;
+}
+
 bool index_mail_set_uid(struct mail *_mail, uint32_t uid)
 {
 	struct index_mail *mail = (struct index_mail *)_mail;
--- a/src/lib-storage/index/index-mail.h	Thu Mar 31 09:48:37 2011 +0300
+++ b/src/lib-storage/index/index-mail.h	Thu Mar 31 11:10:22 2011 +0300
@@ -113,6 +113,7 @@
 	unsigned int destroying_stream:1;
 	unsigned int initialized_wrapper_stream:1;
 	unsigned int destroy_callback_set:1;
+	unsigned int prefetch_sent:1;
 };
 
 struct index_mail {
@@ -156,6 +157,7 @@
 void index_mail_set_seq(struct mail *mail, uint32_t seq);
 bool index_mail_set_uid(struct mail *mail, uint32_t uid);
 void index_mail_set_uid_cache_updates(struct mail *mail, bool set);
+bool index_mail_prefetch(struct mail *mail);
 void index_mail_close(struct mail *mail);
 void index_mail_free(struct mail *mail);
 
--- a/src/lib-storage/index/index-search-private.h	Thu Mar 31 09:48:37 2011 +0300
+++ b/src/lib-storage/index/index-search-private.h	Thu Mar 31 11:10:22 2011 +0300
@@ -12,10 +12,14 @@
 	struct mailbox_header_lookup_ctx *extra_wanted_headers;
 
 	uint32_t seq1, seq2;
-	struct mail *mail;
-	struct index_mail *imail;
+	struct mail *cur_mail;
+	struct index_mail *cur_imail;
 	struct mail_thread_context *thread_ctx;
 
+	ARRAY_DEFINE(mails, struct mail *);
+	unsigned int unused_mail_idx;
+	unsigned int max_mails;
+
 	struct timeval search_start_time, last_notify;
 	struct timeval last_nonblock_timeval;
 	unsigned long long cost, next_time_check_cost;
@@ -25,14 +29,8 @@
 	unsigned int have_seqsets:1;
 	unsigned int have_index_args:1;
 	unsigned int have_mailbox_args:1;
-	unsigned int recheck_index_args:1;
 };
 
-void index_storage_search_init_context(struct index_search_context *ctx,
-				       struct mailbox_transaction_context *t,
-				       struct mail_search_args *args,
-				       const enum mail_sort_type *sort_program,
-				       enum mail_fetch_field wanted_fields,
-				       struct mailbox_header_lookup_ctx *wanted_headers);
+struct mail *index_search_get_mail(struct index_search_context *ctx);
 
 #endif
--- a/src/lib-storage/index/index-search.c	Thu Mar 31 09:48:37 2011 +0300
+++ b/src/lib-storage/index/index-search.c	Thu Mar 31 11:10:22 2011 +0300
@@ -215,7 +215,7 @@
 
 	switch (arg->type) {
 	case SEARCH_MAILBOX:
-		if (mail_get_special(ctx->mail, MAIL_FETCH_MAILBOX_NAME,
+		if (mail_get_special(ctx->cur_mail, MAIL_FETCH_MAILBOX_NAME,
 				     &str) < 0)
 			return -1;
 
@@ -223,7 +223,7 @@
 			return strcasecmp(arg->value.str, "INBOX") == 0;
 		return strcmp(str, arg->value.str) == 0;
 	case SEARCH_MAILBOX_GLOB:
-		if (mail_get_special(ctx->mail, MAIL_FETCH_MAILBOX_NAME,
+		if (mail_get_special(ctx->cur_mail, MAIL_FETCH_MAILBOX_NAME,
 				     &str) < 0)
 			return -1;
 		return imap_match(arg->value.mailbox_glob, str) == IMAP_MATCH_YES;
@@ -267,16 +267,16 @@
 		have_tz_offset = FALSE; tz_offset = 0; date = (time_t)-1;
 		switch (arg->value.date_type) {
 		case MAIL_SEARCH_DATE_TYPE_SENT:
-			if (mail_get_date(ctx->mail, &date, &tz_offset) < 0)
+			if (mail_get_date(ctx->cur_mail, &date, &tz_offset) < 0)
 				return -1;
 			have_tz_offset = TRUE;
 			break;
 		case MAIL_SEARCH_DATE_TYPE_RECEIVED:
-			if (mail_get_received_date(ctx->mail, &date) < 0)
+			if (mail_get_received_date(ctx->cur_mail, &date) < 0)
 				return -1;
 			break;
 		case MAIL_SEARCH_DATE_TYPE_SAVED:
-			if (mail_get_save_date(ctx->mail, &date) < 0)
+			if (mail_get_save_date(ctx->cur_mail, &date) < 0)
 				return -1;
 			break;
 		}
@@ -306,7 +306,7 @@
 	/* sizes */
 	case SEARCH_SMALLER:
 	case SEARCH_LARGER:
-		if (mail_get_virtual_size(ctx->mail, &virtual_size) < 0)
+		if (mail_get_virtual_size(ctx->cur_mail, &virtual_size) < 0)
 			return -1;
 
 		if (arg->type == SEARCH_SMALLER)
@@ -315,7 +315,7 @@
 			return virtual_size > arg->value.size;
 
 	case SEARCH_GUID:
-		if (mail_get_special(ctx->mail, MAIL_FETCH_GUID, &str) < 0)
+		if (mail_get_special(ctx->cur_mail, MAIL_FETCH_GUID, &str) < 0)
 			return -1;
 		return strcmp(str, arg->value.str) == 0;
 	default:
@@ -578,7 +578,7 @@
 	ret = message_search_msg(msg_search_ctx, ctx->input, ctx->part);
 	if (ret < 0 && ctx->input->stream_errno == 0) {
 		/* try again without cached parts */
-		mail_set_cache_corrupted(ctx->index_ctx->mail,
+		mail_set_cache_corrupted(ctx->index_ctx->cur_mail,
 					 MAIL_FETCH_MESSAGE_PARTS);
 
 		i_stream_seek(ctx->input, 0);
@@ -607,7 +607,7 @@
 		struct search_header_context hdr_ctx;
 
 		if (have_body &&
-		    ctx->mail->lookup_abort == MAIL_LOOKUP_ABORT_NEVER) {
+		    ctx->cur_mail->lookup_abort == MAIL_LOOKUP_ABORT_NEVER) {
 			/* just open the mail bypassing any caching, since
 			   we're going to read through the body anyway */
 			headers = NULL;
@@ -615,14 +615,14 @@
 
 		if (headers == NULL) {
 			headers_ctx = NULL;
-			if (mail_get_stream(ctx->mail, NULL, NULL, &input) < 0)
+			if (mail_get_stream(ctx->cur_mail, NULL, NULL, &input) < 0)
 				return -1;
 		} else {
 			/* FIXME: do this once in init */
 			i_assert(*headers != NULL);
 			headers_ctx =
 				mailbox_header_lookup_init(ctx->box, headers);
-			if (mail_get_header_stream(ctx->mail, headers_ctx,
+			if (mail_get_header_stream(ctx->cur_mail, headers_ctx,
 						   &input) < 0) {
 				mailbox_header_lookup_unref(&headers_ctx);
 				return -1;
@@ -630,14 +630,14 @@
 		}
 
 		memset(&hdr_ctx, 0, sizeof(hdr_ctx));
-		hdr_ctx.imail = ctx->imail;
+		hdr_ctx.imail = (struct index_mail *)mail_get_real_mail(ctx->cur_mail);
 		hdr_ctx.custom_header = TRUE;
 		hdr_ctx.args = args;
 		hdr_ctx.parse_headers = headers == NULL &&
-			index_mail_want_parse_headers(ctx->imail);
+			index_mail_want_parse_headers(hdr_ctx.imail);
 
 		if (hdr_ctx.parse_headers)
-			index_mail_parse_header_init(ctx->imail, headers_ctx);
+			index_mail_parse_header_init(hdr_ctx.imail, headers_ctx);
 		message_parse_header(input, NULL, hdr_parser_flags,
 				     search_header, &hdr_ctx);
 		if (headers_ctx != NULL)
@@ -645,7 +645,7 @@
 	} else {
 		struct message_size hdr_size;
 
-		if (mail_get_stream(ctx->mail, &hdr_size, NULL, &input) < 0)
+		if (mail_get_stream(ctx->cur_mail, &hdr_size, NULL, &input) < 0)
 			return -1;
 
 		i_stream_seek(input, hdr_size.physical_size);
@@ -654,13 +654,13 @@
 	if (have_body) {
 		struct search_body_context body_ctx;
 
-		if (ctx->mail->lookup_abort != MAIL_LOOKUP_ABORT_NEVER)
+		if (ctx->cur_mail->lookup_abort != MAIL_LOOKUP_ABORT_NEVER)
 			return -1;
 
 		memset(&body_ctx, 0, sizeof(body_ctx));
 		body_ctx.index_ctx = ctx;
 		body_ctx.input = input;
-		(void)mail_get_parts(ctx->mail, &body_ctx.part);
+		(void)mail_get_parts(ctx->cur_mail, &body_ctx.part);
 
 		ret = mail_search_args_foreach(args, search_body, &body_ctx);
 	} else {
@@ -1060,21 +1060,31 @@
 		*headers_ctx_r = mailbox_header_lookup_init(box, headers);
 }
 
-void index_storage_search_init_context(struct index_search_context *ctx,
-				       struct mailbox_transaction_context *t,
-				       struct mail_search_args *args,
-				       const enum mail_sort_type *sort_program,
-				       enum mail_fetch_field wanted_fields,
-				       struct mailbox_header_lookup_ctx *wanted_headers)
+struct mail_search_context *
+index_storage_search_init(struct mailbox_transaction_context *t,
+			  struct mail_search_args *args,
+			  const enum mail_sort_type *sort_program,
+			  enum mail_fetch_field wanted_fields,
+			  struct mailbox_header_lookup_ctx *wanted_headers)
 {
+	struct index_search_context *ctx;
 	struct mailbox_status status;
-	struct index_mail *imail;
 
+	ctx = i_new(struct index_search_context, 1);
 	ctx->mail_ctx.transaction = t;
 	ctx->box = t->box;
 	ctx->view = t->view;
 	ctx->mail_ctx.args = args;
 	ctx->mail_ctx.sort_program = index_sort_program_init(t, sort_program);
+	ctx->mail_ctx.wanted_fields = wanted_fields;
+	if (wanted_headers != NULL) {
+		ctx->mail_ctx.wanted_headers = wanted_headers;
+		mailbox_header_lookup_ref(wanted_headers);
+	}
+
+	ctx->max_mails = t->box->storage->set->mail_prefetch_count + 1;
+	if (ctx->max_mails == 0)
+		ctx->max_mails = -1U;
 	ctx->next_time_check_cost = SEARCH_INITIAL_MAX_COST;
 	if (gettimeofday(&ctx->last_nonblock_timeval, NULL) < 0)
 		i_fatal("gettimeofday() failed: %m");
@@ -1085,6 +1095,7 @@
 	i_array_init(&ctx->mail_ctx.results, 5);
 	array_create(&ctx->mail_ctx.module_contexts, default_pool,
 		     sizeof(void *), 5);
+	i_array_init(&ctx->mails, ctx->max_mails);
 
 	mail_search_args_reset(ctx->mail_ctx.args->args, TRUE);
 	if (args->have_inthreads) {
@@ -1100,32 +1111,11 @@
 				       &ctx->extra_wanted_headers);
 	}
 
-	ctx->mail = mail_alloc(t, wanted_fields, wanted_headers);
-	imail = (struct index_mail *)ctx->mail;
-	imail->search_mail = TRUE;
-	imail->mail.stats_track = TRUE;
-	imail->mail.extra_wanted_fields = ctx->extra_wanted_fields;
-	imail->mail.extra_wanted_headers = ctx->extra_wanted_headers;
-
 	search_get_seqset(ctx, status.messages, args->args);
 	(void)mail_search_args_foreach(args->args, search_init_arg, ctx);
 
 	/* Need to reset results for match_always cases */
 	mail_search_args_reset(ctx->mail_ctx.args->args, FALSE);
-}
-
-struct mail_search_context *
-index_storage_search_init(struct mailbox_transaction_context *t,
-			  struct mail_search_args *args,
-			  const enum mail_sort_type *sort_program,
-			  enum mail_fetch_field wanted_fields,
-			  struct mailbox_header_lookup_ctx *wanted_headers)
-{
-	struct index_search_context *ctx;
-
-	ctx = i_new(struct index_search_context, 1);
-	index_storage_search_init_context(ctx, t, args, sort_program,
-					  wanted_fields, wanted_headers);
 	return &ctx->mail_ctx;
 }
 
@@ -1143,6 +1133,7 @@
 int index_storage_search_deinit(struct mail_search_context *_ctx)
 {
         struct index_search_context *ctx = (struct index_search_context *)_ctx;
+	struct mail **mailp;
 	int ret;
 
 	ret = ctx->failed ? -1 : 0;
@@ -1151,6 +1142,8 @@
 	(void)mail_search_args_foreach(ctx->mail_ctx.args->args,
 				       search_arg_deinit, NULL);
 
+	if (ctx->mail_ctx.wanted_headers != NULL)
+		mailbox_header_lookup_unref(&ctx->mail_ctx.wanted_headers);
 	if (ctx->extra_wanted_headers != NULL)
 		mailbox_header_lookup_unref(&ctx->extra_wanted_headers);
 	if (ctx->mail_ctx.sort_program != NULL)
@@ -1160,8 +1153,13 @@
 	array_free(&ctx->mail_ctx.results);
 	array_free(&ctx->mail_ctx.module_contexts);
 
-	((struct index_mail *)ctx->mail)->search_mail = FALSE;
-	mail_free(&ctx->mail);
+	array_foreach_modifiable(&ctx->mails, mailp) {
+		struct index_mail *imail = (struct index_mail *)*mailp;
+
+		imail->search_mail = FALSE;
+		mail_free(mailp);
+	}
+	array_free(&ctx->mails);
 	i_free(ctx);
 	return ret;
 }
@@ -1176,27 +1174,18 @@
 	unsigned int i;
 	int ret = -1;
 
-	if (ctx->recheck_index_args) {
-		/* these were already checked in search_next_update_seq(),
-		   but someone reset the args and we have to recheck them */
-		ret = mail_search_args_foreach(ctx->mail_ctx.args->args,
-					       search_seqset_arg, ctx);
-		if (ctx->have_index_args) {
-			ret = mail_search_args_foreach(ctx->mail_ctx.args->args,
-						       search_index_arg, ctx);
-		}
-	}
-
 	if (ctx->have_mailbox_args) {
+		/* check that the mailbox name matches.
+		   this makes sense only with virtual mailboxes. */
 		ret = mail_search_args_foreach(ctx->mail_ctx.args->args,
 					       search_mailbox_arg, ctx);
 		if (ret >= 0)
 			return ret > 0;
 	}
 
-	/* try to avoid doing extra work for as long as possible */
+	/* avoid doing extra work for as long as possible */
 	for (i = 0; i < N_ELEMENTS(cache_lookups) && ret < 0; i++) {
-		ctx->mail->lookup_abort = cache_lookups[i];
+		ctx->cur_mail->lookup_abort = cache_lookups[i];
 		ret = mail_search_args_foreach(ctx->mail_ctx.args->args,
 					       search_cached_arg, ctx);
 		if (ret >= 0)
@@ -1206,7 +1195,7 @@
 		if (ret >= 0)
 			break;
 	}
-	ctx->mail->lookup_abort = MAIL_LOOKUP_ABORT_NEVER;
+	ctx->cur_mail->lookup_abort = MAIL_LOOKUP_ABORT_NEVER;
 	return ret > 0;
 }
 
@@ -1359,14 +1348,14 @@
 	return ret;
 }
 
-static int search_more(struct index_search_context *ctx,
-		       struct mail *mail)
+static int search_more_with_mail(struct index_search_context *ctx,
+				 struct mail *mail)
 {
 	struct mail_search_context *_ctx = &ctx->mail_ctx;
 	struct mailbox *box = _ctx->transaction->box;
 	struct mail_private *mail_private = (struct mail_private *)mail;
 	unsigned long long cost1, cost2;
-	bool match = FALSE;
+	bool match;
 
 	if (search_would_block(ctx)) {
 		/* this lookup is useful when a large number of
@@ -1381,12 +1370,12 @@
 	cost1 = search_mail_get_cost(mail_private);
 	while (box->v.search_next_update_seq(_ctx)) {
 		mail_set_seq(mail, _ctx->seq);
-		ctx->imail = (struct index_mail *)mail_get_real_mail(mail);
 
+		ctx->cur_mail = mail;
 		T_BEGIN {
 			match = search_match_next(ctx);
 		} T_END;
-		ctx->imail = NULL;
+		ctx->cur_mail = NULL;
 
 		if (mail->expunged)
 			_ctx->seen_lost_data = TRUE;
@@ -1411,18 +1400,89 @@
 	return -1;
 }
 
+struct mail *index_search_get_mail(struct index_search_context *ctx)
+{
+	struct index_mail *imail;
+	struct mail *const *mails, *mail;
+	unsigned int count;
+
+	if (ctx->unused_mail_idx == ctx->max_mails)
+		return NULL;
+
+	mails = array_get(&ctx->mails, &count);
+	if (ctx->unused_mail_idx < count)
+		return mails[ctx->unused_mail_idx];
+
+	mail = mail_alloc(ctx->mail_ctx.transaction,
+			  ctx->mail_ctx.wanted_fields,
+			  ctx->mail_ctx.wanted_headers);
+	imail = (struct index_mail *)mail;
+	imail->search_mail = TRUE;
+	imail->mail.stats_track = TRUE;
+	imail->mail.extra_wanted_fields = ctx->extra_wanted_fields;
+	imail->mail.extra_wanted_headers = ctx->extra_wanted_headers;
+
+	array_append(&ctx->mails, &mail, 1);
+	return mail;
+}
+
+static int search_more(struct index_search_context *ctx,
+		       struct mail **mail_r)
+{
+	struct mail *mail, *const *mails;
+	unsigned int count;
+	int ret = 0;
+
+	while ((mail = index_search_get_mail(ctx)) != NULL) {
+		ret = search_more_with_mail(ctx, mail);
+		if (ret <= 0)
+			break;
+		if (mail_prefetch(mail) && ctx->unused_mail_idx == 0) {
+			/* no prefetching done, return it immediately */
+			*mail_r = mail;
+			return 1;
+		}
+		ctx->unused_mail_idx++;
+	}
+
+	if (mail != NULL) {
+		if (ret == 0) {
+			/* wait */
+			return 0;
+		}
+		i_assert(ret < 0);
+		if (ctx->unused_mail_idx == 0) {
+			/* finished */
+			return -1;
+		}
+	} else {
+		/* prefetch buffer is full. */
+	}
+
+	/* return the next message */
+	i_assert(ctx->unused_mail_idx > 0);
+
+	mails = array_get(&ctx->mails, &count);
+	*mail_r = mails[0];
+	if (--ctx->unused_mail_idx > 0) {
+		array_delete(&ctx->mails, 0, 1);
+		array_append(&ctx->mails, mail_r, 1);
+	}
+	return 1;
+}
+
 bool index_storage_search_next_nonblock(struct mail_search_context *_ctx,
 					struct mail **mail_r, bool *tryagain_r)
 {
         struct index_search_context *ctx = (struct index_search_context *)_ctx;
-	struct mail *mail = ctx->mail;
+	struct mail *mail, *const *mailp;
 	uint32_t seq;
 	int ret;
 
 	*tryagain_r = FALSE;
 
 	if (_ctx->sort_program == NULL) {
-		ret = search_more(ctx, mail);
+		ret = search_more(ctx, &mail);
 		if (ret == 0) {
 			*tryagain_r = TRUE;
 			return FALSE;
@@ -1434,7 +1494,7 @@
 	}
 
 	if (!ctx->sorted) {
-		while ((ret = search_more(ctx, mail)) > 0)
+		while ((ret = search_more(ctx, &mail)) > 0)
 			index_sort_list_add(_ctx->sort_program, mail);
 
 		if (ret == 0) {
@@ -1454,8 +1514,9 @@
 	if (!index_sort_list_next(_ctx->sort_program, &seq))
 		return FALSE;
 
-	mail_set_seq(mail, seq);
-	*mail_r = mail;
+	mailp = array_idx(&ctx->mails, 0);
+	mail_set_seq(*mailp, seq);
+	*mail_r = *mailp;
 	return TRUE;
 }
 
@@ -1474,7 +1535,7 @@
 
 	if (!ctx->have_seqsets && !ctx->have_index_args &&
 	    _ctx->update_result == NULL) {
-		ctx->mail_ctx.progress_cur = _ctx->seq;
+		_ctx->progress_cur = _ctx->seq;
 		return _ctx->seq <= ctx->seq2;
 	}
 
--- a/src/lib-storage/index/maildir/maildir-mail.c	Thu Mar 31 09:48:37 2011 +0300
+++ b/src/lib-storage/index/maildir/maildir-mail.c	Thu Mar 31 11:10:22 2011 +0300
@@ -615,6 +615,7 @@
 	index_mail_set_seq,
 	index_mail_set_uid,
 	index_mail_set_uid_cache_updates,
+	index_mail_prefetch,
 
 	index_mail_get_flags,
 	index_mail_get_keywords,
--- a/src/lib-storage/index/maildir/maildir-storage.c	Thu Mar 31 09:48:37 2011 +0300
+++ b/src/lib-storage/index/maildir/maildir-storage.c	Thu Mar 31 11:10:22 2011 +0300
@@ -592,7 +592,7 @@
 
 struct mail_storage maildir_storage = {
 	.name = MAILDIR_STORAGE_NAME,
-	.class_flags = 0,
+	.class_flags = MAIL_STORAGE_CLASS_FLAG_FILE_PER_MSG,
 
 	.v = {
                 maildir_get_setting_parser_info,
--- a/src/lib-storage/index/mbox/mbox-mail.c	Thu Mar 31 09:48:37 2011 +0300
+++ b/src/lib-storage/index/mbox/mbox-mail.c	Thu Mar 31 11:10:22 2011 +0300
@@ -387,6 +387,7 @@
 	mbox_mail_set_seq,
 	mbox_mail_set_uid,
 	index_mail_set_uid_cache_updates,
+	index_mail_prefetch,
 
 	index_mail_get_flags,
 	index_mail_get_keywords,
--- a/src/lib-storage/index/raw/raw-mail.c	Thu Mar 31 09:48:37 2011 +0300
+++ b/src/lib-storage/index/raw/raw-mail.c	Thu Mar 31 11:10:22 2011 +0300
@@ -118,6 +118,7 @@
 	index_mail_set_seq,
 	index_mail_set_uid,
 	index_mail_set_uid_cache_updates,
+	index_mail_prefetch,
 
 	index_mail_get_flags,
 	index_mail_get_keywords,
--- a/src/lib-storage/mail-storage-private.h	Thu Mar 31 09:48:37 2011 +0300
+++ b/src/lib-storage/mail-storage-private.h	Thu Mar 31 11:10:22 2011 +0300
@@ -58,7 +58,9 @@
 	/* never use quota for this storage (e.g. virtual mailboxes) */
 	MAIL_STORAGE_CLASS_FLAG_NOQUOTA		= 0x08,
 	/* Storage doesn't need a mail root directory */
-	MAIL_STORAGE_CLASS_FLAG_NO_ROOT		= 0x10
+	MAIL_STORAGE_CLASS_FLAG_NO_ROOT		= 0x10,
+	/* Storage uses one file per message */
+	MAIL_STORAGE_CLASS_FLAG_FILE_PER_MSG	= 0x20
 };
 
 struct mail_storage {
@@ -276,6 +278,7 @@
 	void (*set_seq)(struct mail *mail, uint32_t seq);
 	bool (*set_uid)(struct mail *mail, uint32_t uid);
 	void (*set_uid_cache_updates)(struct mail *mail, bool set);
+	bool (*prefetch)(struct mail *mail);
 
 	enum mail_flags (*get_flags)(struct mail *mail);
 	const char *const *(*get_keywords)(struct mail *mail);
@@ -396,6 +399,8 @@
 
 	struct mail_search_args *args;
 	struct mail_search_sort_program *sort_program;
+	enum mail_fetch_field wanted_fields;
+	struct mailbox_header_lookup_ctx *wanted_headers;
 
 	/* if non-NULL, specifies that a search resulting is being updated.
 	   this can be used as a search optimization: if searched message
@@ -481,6 +486,8 @@
 void mail_storage_copy_list_error(struct mail_storage *storage,
 				  struct mailbox_list *list);
 
+/* Returns TRUE if everything should already be in memory after this call. */
+bool mail_prefetch(struct mail *mail);
 int mail_set_aborted(struct mail *mail);
 void mail_set_expunged(struct mail *mail);
 void mailbox_set_deleted(struct mailbox *box);
--- a/src/lib-storage/mail-storage-settings.c	Thu Mar 31 09:48:37 2011 +0300
+++ b/src/lib-storage/mail-storage-settings.c	Thu Mar 31 11:10:22 2011 +0300
@@ -28,6 +28,7 @@
 	DEF(SET_STR_VARS, mail_attachment_dir),
 	DEF(SET_STR, mail_attachment_hash),
 	DEF(SET_SIZE, mail_attachment_min_size),
+	DEF(SET_UINT, mail_prefetch_count),
 	DEF(SET_STR, mail_cache_fields),
 	DEF(SET_STR, mail_never_cache_fields),
 	DEF(SET_UINT, mail_cache_min_mail_count),
@@ -56,6 +57,7 @@
 	.mail_attachment_dir = "",
 	.mail_attachment_hash = "%{sha1}",
 	.mail_attachment_min_size = 1024*128,
+	.mail_prefetch_count = 0,
 	.mail_cache_fields = "flags",
 	.mail_never_cache_fields = "imap.envelope",
 	.mail_cache_min_mail_count = 0,
--- a/src/lib-storage/mail-storage-settings.h	Thu Mar 31 09:48:37 2011 +0300
+++ b/src/lib-storage/mail-storage-settings.h	Thu Mar 31 11:10:22 2011 +0300
@@ -15,6 +15,7 @@
 	const char *mail_attachment_dir;
 	const char *mail_attachment_hash;
 	uoff_t mail_attachment_min_size;
+	unsigned int mail_prefetch_count;
 	const char *mail_cache_fields;
 	const char *mail_never_cache_fields;
 	unsigned int mail_cache_min_mail_count;
--- a/src/lib-storage/mail.c	Thu Mar 31 09:48:37 2011 +0300
+++ b/src/lib-storage/mail.c	Thu Mar 31 11:10:22 2011 +0300
@@ -47,6 +47,13 @@
 	return p->v.set_uid(mail, uid);
 }
 
+bool mail_prefetch(struct mail *mail)
+{
+	struct mail_private *p = (struct mail_private *)mail;
+
+	return p->v.prefetch(mail);
+}
+
 enum mail_flags mail_get_flags(struct mail *mail)
 {
 	struct mail_private *p = (struct mail_private *)mail;
--- a/src/lib-storage/test-mail.c	Thu Mar 31 09:48:37 2011 +0300
+++ b/src/lib-storage/test-mail.c	Thu Mar 31 11:10:22 2011 +0300
@@ -53,6 +53,11 @@
 {
 }
 
+static bool test_mail_prefetch(struct mail *mail ATTR_UNUSED)
+{
+	return TRUE;
+}
+
 static enum mail_flags test_mail_get_flags(struct mail *mail ATTR_UNUSED)
 {
 	return 0;
@@ -207,6 +212,7 @@
 	test_mail_set_seq,
 	test_mail_set_uid,
 	test_mail_set_uid_cache_updates,
+	test_mail_prefetch,
 
 	test_mail_get_flags,
 	test_mail_get_keywords,
--- a/src/plugins/virtual/virtual-mail.c	Thu Mar 31 09:48:37 2011 +0300
+++ b/src/plugins/virtual/virtual-mail.c	Thu Mar 31 11:10:22 2011 +0300
@@ -161,6 +161,14 @@
 	p->v.set_uid_cache_updates(vmail->backend_mail, set);
 }
 
+static bool virtual_mail_prefetch(struct mail *mail)
+{
+	struct virtual_mail *vmail = (struct virtual_mail *)mail;
+	struct mail_private *p = (struct mail_private *)vmail->backend_mail;
+
+	return p->v.prefetch(vmail->backend_mail);
+}
+
 static int virtual_mail_handle_lost(struct virtual_mail *vmail)
 {
 	if (!vmail->lost)
@@ -384,6 +392,7 @@
 	virtual_mail_set_seq,
 	virtual_mail_set_uid,
 	virtual_mail_set_uid_cache_updates,
+	virtual_mail_prefetch,
 
 	index_mail_get_flags,
 	index_mail_get_keywords,
--- a/src/plugins/virtual/virtual-search.c	Thu Mar 31 09:48:37 2011 +0300
+++ b/src/plugins/virtual/virtual-search.c	Thu Mar 31 11:10:22 2011 +0300
@@ -166,8 +166,9 @@
 					      vctx->next_result_n, &seq))
 			return FALSE;
 		vctx->next_result_n++;
-		mail_set_seq(ictx->mail, seq);
-		*mail_r = ictx->mail;
+		*mail_r = index_search_get_mail(ictx);
+		i_assert(*mail_r != NULL);
+		mail_set_seq(*mail_r, seq);
 		return TRUE;
 	}
 	i_unreached();