# HG changeset patch
# User Timo Sirainen <tss@iki.fi>
# Date 1030154685 -10800
# Node ID a946ce1f09b7c2d913fd75ee44b33a7b211ae60e
# Parent  163675942b831814f3e4e5458fc88c09db6eccee
mbox fixes, not fully working yet but almost :)

diff -r 163675942b83 -r a946ce1f09b7 src/imap/main.c
--- a/src/imap/main.c	Fri Aug 23 19:21:36 2002 +0300
+++ b/src/imap/main.c	Sat Aug 24 05:04:45 2002 +0300
@@ -55,7 +55,7 @@
 			mail = t_strconcat("maildir:", mail, NULL);
 	}
 
-	storage = mail_storage_create_with_data(mail);
+	storage = mail_storage_create_with_data(mail, getenv("USER"));
 	if (storage == NULL) {
 		/* failed */
 		if (mail != NULL && *mail != '\0')
diff -r 163675942b83 -r a946ce1f09b7 src/lib-imap/imap-message-cache.c
--- a/src/lib-imap/imap-message-cache.c	Fri Aug 23 19:21:36 2002 +0300
+++ b/src/lib-imap/imap-message-cache.c	Sat Aug 24 05:04:45 2002 +0300
@@ -45,7 +45,7 @@
 
 	CachedMessage *open_msg;
 	IOBuffer *open_inbuf;
-	size_t open_size, open_virtual_size;
+	off_t open_virtual_size;
 
 	IOBuffer *(*inbuf_rewind)(IOBuffer *inbuf, void *context);
 	void *context;
@@ -292,8 +292,14 @@
 			message_get_header_size(cache->open_inbuf,
 						msg->hdr_size);
 
+			i_assert((off_t)msg->hdr_size->physical_size <
+				 cache->open_inbuf->size);
+			i_assert((off_t)msg->hdr_size->virtual_size <
+				 cache->open_virtual_size);
+
 			msg->body_size->lines = 0;
-			msg->body_size->physical_size = cache->open_size -
+			msg->body_size->physical_size =
+				cache->open_inbuf->size -
 				msg->hdr_size->physical_size;
 			msg->body_size->virtual_size =
 				cache->open_virtual_size -
@@ -320,8 +326,8 @@
 }
 
 void imap_msgcache_message(ImapMessageCache *cache, unsigned int uid,
-			   ImapCacheField fields, size_t virtual_size,
-			   size_t pv_headers_size, size_t pv_body_size,
+			   ImapCacheField fields, off_t virtual_size,
+			   off_t pv_headers_size, off_t pv_body_size,
 			   IOBuffer *inbuf,
 			   IOBuffer *(*inbuf_rewind)(IOBuffer *inbuf,
 						     void *context),
@@ -335,8 +341,6 @@
 
 		cache->open_msg = msg;
 		cache->open_inbuf = inbuf;
-		cache->open_size = cache->open_inbuf->stop_offset -
-			cache->open_inbuf->offset;
 		cache->open_virtual_size = virtual_size;
 
 		cache->inbuf_rewind = inbuf_rewind;
@@ -369,7 +373,7 @@
 	}
 
 	cache->open_msg = NULL;
-	cache->open_size = cache->open_virtual_size = 0;
+	cache->open_virtual_size = 0;
 }
 
 void imap_msgcache_set(ImapMessageCache *cache, unsigned int uid,
@@ -538,7 +542,8 @@
 	if (virtual_skip == 0) {
 		if (max_virtual_size < 0 && msg->body_size == NULL) {
 			msg->body_size = p_new(msg->pool, MessageSize, 1);
-			msg->body_size->physical_size = cache->open_size -
+			msg->body_size->physical_size =
+				cache->open_inbuf->size -
 				msg->hdr_size->physical_size;
 			msg->body_size->virtual_size =
 				cache->open_virtual_size -
diff -r 163675942b83 -r a946ce1f09b7 src/lib-imap/imap-message-cache.h
--- a/src/lib-imap/imap-message-cache.h	Fri Aug 23 19:21:36 2002 +0300
+++ b/src/lib-imap/imap-message-cache.h	Sat Aug 24 05:04:45 2002 +0300
@@ -37,8 +37,8 @@
    non-zero, they're set to saved to message's both physical and virtual
    sizes (ie. doesn't need to be calculated). */
 void imap_msgcache_message(ImapMessageCache *cache, unsigned int uid,
-			   ImapCacheField fields, size_t virtual_size,
-			   size_t pv_headers_size, size_t pv_body_size,
+			   ImapCacheField fields, off_t virtual_size,
+			   off_t pv_headers_size, off_t pv_body_size,
 			   IOBuffer *inbuf,
 			   IOBuffer *(*inbuf_rewind)(IOBuffer *inbuf,
 						     void *context),
diff -r 163675942b83 -r a946ce1f09b7 src/lib-imap/imap-parser.c
--- a/src/lib-imap/imap-parser.c	Fri Aug 23 19:21:36 2002 +0300
+++ b/src/lib-imap/imap-parser.c	Sat Aug 24 05:04:45 2002 +0300
@@ -284,7 +284,8 @@
 	/* expecting digits + "}" */
 	for (i = parser->cur_pos; i < data_size; i++) {
 		if (data[i] == '}') {
-			if (parser->literal_size > parser->inbuf->max_size) {
+			if (parser->literal_size >
+			    parser->inbuf->max_buffer_size) {
 				/* too long string, abort. */
 				parser->error = TRUE;
 				return FALSE;
diff -r 163675942b83 -r a946ce1f09b7 src/lib-index/Makefile.am
--- a/src/lib-index/Makefile.am	Fri Aug 23 19:21:36 2002 +0300
+++ b/src/lib-index/Makefile.am	Sat Aug 24 05:04:45 2002 +0300
@@ -1,4 +1,4 @@
-SUBDIRS = maildir
+SUBDIRS = maildir mbox
 
 noinst_LIBRARIES = libstorage_index.a
 
diff -r 163675942b83 -r a946ce1f09b7 src/lib-index/mail-index-update.c
--- a/src/lib-index/mail-index-update.c	Fri Aug 23 19:21:36 2002 +0300
+++ b/src/lib-index/mail-index-update.c	Sat Aug 24 05:04:45 2002 +0300
@@ -383,8 +383,7 @@
 				     update_header_func, &ctx);
 
 		update->rec->header_size = hdr_size.physical_size;
-		update->rec->body_size = (inbuf->stop_offset - inbuf->offset) -
-			hdr_size.physical_size;
+		update->rec->body_size = inbuf->size - inbuf->offset;
 
 		if (update->rec->full_virtual_size == 0) {
 			/* we need to calculate virtual size of the
diff -r 163675942b83 -r a946ce1f09b7 src/lib-index/mail-index.h
--- a/src/lib-index/mail-index.h	Fri Aug 23 19:21:36 2002 +0300
+++ b/src/lib-index/mail-index.h	Sat Aug 24 05:04:45 2002 +0300
@@ -43,9 +43,10 @@
 	FIELD_TYPE_CC			= 0x0080,
 	FIELD_TYPE_BCC			= 0x0100,
 	FIELD_TYPE_SUBJECT		= 0x0200,
+	FIELD_TYPE_MD5			= 0x0400,
 
-	FIELD_TYPE_LAST			= 0x0400,
-	FIELD_TYPE_MAX_BITS		= 10
+	FIELD_TYPE_LAST			= 0x0800,
+	FIELD_TYPE_MAX_BITS		= 11
 } MailField;
 
 #define IS_HEADER_FIELD(field) \
diff -r 163675942b83 -r a946ce1f09b7 src/lib-index/mbox/Makefile.am
--- a/src/lib-index/mbox/Makefile.am	Fri Aug 23 19:21:36 2002 +0300
+++ b/src/lib-index/mbox/Makefile.am	Sat Aug 24 05:04:45 2002 +0300
@@ -8,6 +8,7 @@
 
 libstorage_index_mbox_a_SOURCES = \
 	mbox-append.c \
+	mbox-from.c \
 	mbox-fsck.c \
 	mbox-index.c \
 	mbox-lock.c \
diff -r 163675942b83 -r a946ce1f09b7 src/lib-index/mbox/mbox-append.c
--- a/src/lib-index/mbox/mbox-append.c	Fri Aug 23 19:21:36 2002 +0300
+++ b/src/lib-index/mbox/mbox-append.c	Sat Aug 24 05:04:45 2002 +0300
@@ -1,21 +1,13 @@
 /* Copyright (C) 2002 Timo Sirainen */
 
 #include "lib.h"
-#include "mmap-util.h"
 #include "ioloop.h"
+#include "iobuffer.h"
+#include "hex-binary.h"
+#include "md5.h"
 #include "mbox-index.h"
 #include "mail-index-util.h"
 
-#include <time.h>
-#include <ctype.h>
-#include <unistd.h>
-#include <sys/mman.h>
-
-static const char *months[] = {
-	"Jan", "Feb", "Mar", "Apr", "May", "Jun",
-	"Jul", "Aug", "Sep", "Oct", "Nov", "Dec"
-};
-
 static MailIndexRecord *
 mail_index_record_append(MailIndex *index, time_t internal_date,
 			 size_t full_virtual_size)
@@ -33,123 +25,102 @@
 	return rec;
 }
 
-static time_t from_line_parse_date(const char *msg, size_t size)
+static void mbox_read_message(IOBuffer *inbuf, unsigned int *virtual_size)
 {
-	const char *msg_end;
-	struct tm tm;
-	int i;
-
-	/* From <sender> <date> <moreinfo> */
-	if (strncmp(msg, "From ", 5) != 0)
-		return 0;
-
-	msg_end = msg + size;
+	unsigned char *msg;
+	unsigned int i, size, startpos, vsize;
 
-	/* skip sender */
-	msg += 5;
-	while (*msg != ' ' && msg < msg_end) msg++;
-	while (*msg == ' ' && msg < msg_end) msg++;
-
-	/* next 24 chars are the date in asctime() format,
-	   eg. "Thu Nov 29 22:33:52 2001" */
-	if (msg+24 > msg_end)
-		return 0;
+	/* read until "[\r]\nFrom " is found */
+	startpos = 0; vsize = 0;
+	while (io_buffer_read_data(inbuf, &msg, &size, startpos) >= 0) {
+		for (i = startpos; i < size; i++) {
+			if (msg[i] == '\n') {
+				if (i == 0 || msg[i-1] != '\r') {
+					/* missing CR */
+					vsize++;
+				}
+			} else if (msg[i] == ' ' && i >= 5) {
+				/* See if it's space after "From" */
+				if (msg[i-5] == '\n' && msg[i-4] == 'F' &&
+				    msg[i-3] == 'r' && msg[i-2] == 'o' &&
+				    msg[i-1] == 'm') {
+					/* yes, see if we had \r too */
+					i -= 5;
+					if (i > 0 && msg[i-1] == '\r')
+						i--;
+					else
+						vsize--;
+					break;
+				}
+			}
+		}
 
-	memset(&tm, 0, sizeof(tm));
-
-	/* skip weekday */
-	msg += 4;
+		if (i < size) {
+			startpos = i;
+			break;
+		}
 
-	/* month */
-	for (i = 0; i < 12; i++) {
-		if (strncasecmp(months[i], msg, 3) == 0) {
-			tm.tm_mon = i;
-			break;
+		if (i > 0) {
+			startpos = i < 7 ? i : 7;
+			i -= startpos;
+
+			io_buffer_skip(inbuf, i);
+			vsize += i;
 		}
 	}
 
-	if (i == 12 || msg[3] != ' ')
-		return 0;
-	msg += 4;
-
-	/* day */
-	if (!i_isdigit(msg[0]) || !i_isdigit(msg[1]) || msg[2] != ' ')
-		return 0;
-	tm.tm_mday = (msg[0]-'0') * 10 + (msg[1]-'0');
-	msg += 3;
-
-	/* hour */
-	if (!i_isdigit(msg[0]) || !i_isdigit(msg[1]) || msg[2] != ':')
-		return 0;
-	tm.tm_hour = (msg[0]-'0') * 10 + (msg[1]-'0');
-	msg += 3;
+	io_buffer_skip(inbuf, startpos);
+	vsize += startpos;
 
-	/* minute */
-	if (!i_isdigit(msg[0]) || !i_isdigit(msg[1]) || msg[2] != ':')
-		return 0;
-	tm.tm_min = (msg[0]-'0') * 10 + (msg[1]-'0');
-	msg += 3;
-
-	/* second */
-	if (!i_isdigit(msg[0]) || !i_isdigit(msg[1]) || msg[2] != ' ')
-		return 0;
-	tm.tm_sec = (msg[0]-'0') * 10 + (msg[1]-'0');
-	msg += 3;
-
-	/* year */
-	if (!i_isdigit(msg[0]) || !i_isdigit(msg[1]) ||
-	    !i_isdigit(msg[2]) || !i_isdigit(msg[3]))
-		return 0;
-	tm.tm_year = (msg[0]-'0') * 1000 + (msg[1]-'0') * 100 +
-		(msg[2]-'0') * 10 + (msg[3]-'0') - 1900;
-
-	tm.tm_isdst = -1;
-	return mktime(&tm);
+	*virtual_size = vsize;
 }
 
-static void header_func(MessagePart *part __attr_unused__,
-			const char *name, unsigned int name_len,
-			const char *value, unsigned int value_len,
-			void *context)
-{
-	MailIndexRecord *rec = context;
-
-	rec->msg_flags |= mbox_header_get_flags(name, name_len,
-						value, value_len);
-}
-
-static int mbox_index_append_data(MailIndex *index, const char *msg,
-				  off_t offset, size_t physical_size,
-				  size_t virtual_size)
+static int mbox_index_append_next(MailIndex *index, IOBuffer *inbuf)
 {
 	MailIndexRecord *rec;
 	MailIndexUpdate *update;
+        MboxHeaderContext ctx;
 	time_t internal_date;
-	char location[MAX_INT_STRLEN];
-	unsigned int i;
+	off_t start_offset, stop_offset, old_size;
+	unsigned char *data, md5_digest[16];
+	unsigned int size, pos, virtual_size;
+	const char *location;
+
+	/* get the From-line */
+	pos = 0;
+	while (io_buffer_read_data(inbuf, &data, &size, pos) >= 0) {
+		for (; pos < size; pos++) {
+			if (data[pos] == '\n')
+				break;
+		}
 
-	internal_date = from_line_parse_date(msg, physical_size);
+		if (pos < size)
+			break;
+	}
+
+	if (pos == size || size <= 5 || strncmp(data, "From ", 5) != 0) {
+		/* a) no \n found, or line too long
+		   b) not a From-line */
+		index_set_error(index, "Error indexing mbox file %s: "
+				"From-line not found where expected",
+				index->mbox_path);
+		index->set_flags |= MAIL_INDEX_FLAG_FSCK;
+		return FALSE;
+	}
+
+	/* parse the From-line */
+	internal_date = mbox_from_parse_date(data, size);
 	if (internal_date <= 0)
 		internal_date = ioloop_time;
 
-	/* skip the From-line */
-	for (i = 0; i < physical_size; i++) {
-		if (msg[i] == '\n') {
-			i++;
-			break;
-		}
-	}
+	io_buffer_skip(inbuf, pos+1);
+	start_offset = inbuf->offset;
 
-	if (i == physical_size)
-		return FALSE;
+	/* now, find the ending "[\r]\nFrom " */
+	mbox_read_message(inbuf, &virtual_size);
+	stop_offset = inbuf->offset;
 
-	msg += i;
-	offset += i;
-	physical_size -= i;
-	virtual_size -= i;
-	if (i > 0 && msg[i-1] != '\r')
-		virtual_size--;
-
+	/* add message to index */
 	rec = mail_index_record_append(index, internal_date, virtual_size);
 	if (rec == NULL)
 		return FALSE;
@@ -157,11 +128,32 @@
 	update = index->update_begin(index, rec);
 
 	/* location = offset to beginning of message */
-	i_snprintf(location, sizeof(location), "%lu", (unsigned long) offset);
+	location = binary_to_hex((unsigned char *) &start_offset,
+				 sizeof(start_offset));
 	index->update_field(update, FIELD_TYPE_LOCATION, location, 0);
 
-	/* parse the header and cache wanted fields */
-	mail_index_update_headers(update, msg, physical_size, header_func, rec);
+	/* parse the header and cache wanted fields. get the message flags
+	   from Status and X-Status fields. temporarily limit the buffer size
+	   so the message body is parsed properly (FIXME: does this have
+	   side effects?) */
+	mbox_header_init_context(&ctx);
+
+        old_size = inbuf->size;
+	inbuf->size = stop_offset;
+	io_buffer_seek(inbuf, start_offset);
+
+	mail_index_update_headers(update, inbuf, mbox_header_func, &ctx);
+
+	inbuf->size = old_size;
+	io_buffer_seek(inbuf, stop_offset);
+
+	/* save message flags */
+	rec->msg_flags |= ctx.flags;
+
+	/* save MD5 */
+	md5_final(&ctx.md5, md5_digest);
+	index->update_field(update, FIELD_TYPE_MD5,
+                            binary_to_hex(md5_digest, sizeof(md5_digest)), 0);
 
 	if (!index->update_end(update)) {
 		/* failed - delete the record */
@@ -172,79 +164,9 @@
 	return TRUE;
 }
 
-int mbox_index_append_mmaped(MailIndex *index, const char *data,
-			     size_t data_size, off_t start_offset)
+int mbox_index_append(MailIndex *index, IOBuffer *inbuf)
 {
-	const char *data_start, *data_end, *start, *cr;
-	size_t size, vsize;
-	off_t pos;
-	int missing_cr_count;
-
-	/* we should start with "From ". if we don't, something's messed up
-	   and we should check the whole file instead. */
-	if (strncmp(data, "From ", 5) != 0) {
-		index->set_flags |= MAIL_INDEX_FLAG_FSCK;
-		return FALSE;
-	}
-
-	/* each message ends at "\nFrom ". first get the size of the message,
-	   then parse it. calculate the missing CR count as well. */
-	start = data; cr = NULL; missing_cr_count = 0;
-
-	data_start = data;
-	data_end = data + data_size;
-	for (; data != data_end; data++) {
-		if (*data == '\r')
-			cr = data;
-		else if (*data == '\n') {
-			if (cr != data-1)
-				missing_cr_count++;
-
-			if (data+6 < data_end && data[1] == 'F' &&
-			    data[2] == 'r' && data[3] == 'o' &&
-			    data[4] == 'm' && data[5] == ' ') {
-				/* end of message */
-				pos = (off_t) (start - data_start) +
-					start_offset;
-				size = (size_t) (data - start) + 1;
-				vsize = size + missing_cr_count;
-				if (!mbox_index_append_data(index, start, pos,
-							    size, vsize))
-					return FALSE;
-
-				missing_cr_count = 0;
-				start = data+1;
-			}
-		}
-	}
-
-	/* last message */
-	pos = (off_t) (start - data_start);
-	size = (size_t) (data - start);
-	vsize = size + missing_cr_count;
-	return mbox_index_append_data(index, start, pos, size, vsize);
-}
-
-int mbox_index_append(MailIndex *index, int fd, const char *path)
-{
-	void *mmap_base;
-	size_t mmap_length;
-	off_t pos, end_pos;
-	int ret;
-
-	/* get our current position */
-	pos = lseek(fd, 0, SEEK_CUR);
-
-	/* get the size of the file */
-	end_pos = lseek(fd, 0, SEEK_END);
-
-	if (pos == -1 || end_pos == -1) {
-		index_set_error(index, "lseek() failed with mbox file %s: %m",
-				path);
-		return FALSE;
-	}
-
-	if (pos == end_pos) {
+	if (inbuf->offset == inbuf->size) {
 		/* no new data */
 		return TRUE;
 	}
@@ -252,18 +174,27 @@
 	if (!index->set_lock(index, MAIL_LOCK_EXCLUSIVE))
 		return FALSE;
 
-	/* mmap() the file */
-	mmap_length = end_pos-pos;
-	mmap_base = mmap(NULL, mmap_length, PROT_READ, MAP_SHARED, fd, pos);
-	if (mmap_base == MAP_FAILED) {
-		index_set_error(index, "mmap() failed with mbox file %s: %m",
-				path);
-		return FALSE;
+	for (;;) {
+		if (inbuf->offset != 0) {
+			/* we're at the [\r]\n before the From-line,
+			   skip it */
+			if (!mbox_skip_crlf(inbuf)) {
+				index_set_error(index,
+						"Error indexing mbox file %s: "
+						"LF not found where expected",
+						index->mbox_path);
+
+				index->set_flags |= MAIL_INDEX_FLAG_FSCK;
+				return FALSE;
+			}
+		}
+
+		if (inbuf->offset == inbuf->size)
+			break;
+
+		if (!mbox_index_append_next(index, inbuf))
+			return FALSE;
 	}
 
-	(void)madvise(mmap_base, mmap_length, MADV_SEQUENTIAL);
-
-	ret = mbox_index_append_mmaped(index, mmap_base, mmap_length, pos);
-	(void)munmap(mmap_base, mmap_length);
-	return ret;
+	return TRUE;
 }
diff -r 163675942b83 -r a946ce1f09b7 src/lib-index/mbox/mbox-fsck.c
--- a/src/lib-index/mbox/mbox-fsck.c	Fri Aug 23 19:21:36 2002 +0300
+++ b/src/lib-index/mbox/mbox-fsck.c	Sat Aug 24 05:04:45 2002 +0300
@@ -1,106 +1,122 @@
 /* Copyright (C) 2002 Timo Sirainen */
 
 #include "lib.h"
-#include "mmap-util.h"
+#include "iobuffer.h"
+#include "hex-binary.h"
 #include "message-parser.h"
 #include "mbox-index.h"
 #include "mbox-lock.h"
 #include "mail-index-util.h"
 
+#include <unistd.h>
 #include <fcntl.h>
 
-typedef struct {
-	const char *msgid;
-	MailFlags flags;
-} HeaderContext;
+static void skip_line(IOBuffer *inbuf)
+{
+	unsigned char *msg;
+	unsigned int i, size;
 
-static void header_func(MessagePart *part __attr_unused__,
-			const char *name, unsigned int name_len,
-			const char *value, unsigned int value_len,
-			void *context)
-{
-	HeaderContext *ctx = context;
+	while (io_buffer_read_data(inbuf, &msg, &size, 0) >= 0) {
+		for (i = 0; i < size; i++) {
+			if (msg[i] == '\n')
+				break;
+		}
 
-	if (name_len != 10 || strncasecmp(name, "Message-ID", 10) != 0)
-		return;
+		if (i < size) {
+			io_buffer_skip(inbuf, i+1);
+			break;
+		}
 
-	ctx->msgid = t_strndup(value, value_len);
-	ctx->flags |= mbox_header_get_flags(name, name_len, value, value_len);
+		io_buffer_skip(inbuf, i);
+	}
 }
 
 static MailIndexRecord *
 match_next_record(MailIndex *index, MailIndexRecord *rec, unsigned int *seq,
-		  const char **data, const char *data_end)
+		  IOBuffer *inbuf)
 {
-#if 0 // FIXME
 	MessageSize hdr_size;
-        HeaderData hdr_data;
-	const char *rec_msgid, *data_next;
+	MboxHeaderContext ctx;
+	off_t old_offset;
+	unsigned char *data, current_digest[16], old_digest[16];
+	unsigned int size;
+	const char *md5sum;
 
 	/* skip the From-line */
-	while (*data != data_end && **data != '\n')
-		(*data)++;
-	(*data)++;
+	skip_line(inbuf);
 
-	if (*data >= data_end) {
-		/* end of data */
-		(void)index->expunge(index, rec, *seq, TRUE);
-		return rec;
-	}
-
-	/* find the Message-ID from the header */
-	memset(&hdr_data, 0, sizeof(hdr_data));
-	message_parse_header(NULL, *data, (size_t) (data_end-*data), &hdr_size,
-			     header_func, &hdr_data);
+	/* get the MD5 sum of fixed headers and the current message flags
+	   in Status and X-Status fields */
+        mbox_header_init_context(&ctx);
+	message_parse_header(NULL, inbuf, &hdr_size, mbox_header_func, &ctx);
+	md5_final(&ctx.md5, current_digest);
 
 	do {
+		old_offset = inbuf->offset;
 		do {
-			/* message-id must match (or be non-existant) */
-			rec_msgid = index->lookup_field(index, rec,
-							FIELD_TYPE_MESSAGEID);
-			if (hdr_data.msgid == NULL && rec_msgid != NULL)
+			/* MD5 sums must match */
+			md5sum = index->lookup_field(index, rec,
+						     FIELD_TYPE_MD5);
+			if (md5sum == NULL || strlen(md5sum) != 32 ||
+			    hex_to_binary(md5sum, old_digest) <= 0)
 				break;
-			if (hdr_data.msgid != NULL &&
-			    (rec_msgid == NULL ||
-			     strcmp(hdr_data.msgid, rec_msgid) != 0))
+
+			if (memcmp(old_digest, current_digest, 16) != 0)
 				break;
 
 			/* don't bother parsing the whole body, just make
 			   sure it ends properly */
-			data_next = *data + rec->header_size + rec->body_size;
-			if (data_next == data_end) {
+			io_buffer_skip(inbuf, rec->body_size);
+
+			if (inbuf->offset == inbuf->size) {
 				/* last message */
-			} else if (data_next+5 >= data_end ||
-				   strncmp(data_next-1, "\nFrom ", 6) != 0)
-				break;
+			} else {
+				/* read forward a bit */
+				if (io_buffer_read_data(inbuf, &data,
+							&size, 6) <= 0 ||
+				    size < 7)
+					break;
+
+				if (data[0] == '\r')
+					data++;
+				if (strncmp(data, "\nFrom ", 6) != 0)
+					break;
+			}
 
 			/* valid message, update flags */
-			if ((rec->msg_flags & hdr_data.flags) != hdr_data.flags)
-				rec->msg_flags |= hdr_data.flags;
-
-			*data = data_next;
+			if ((rec->msg_flags & ctx.flags) != ctx.flags)
+				rec->msg_flags |= ctx.flags;
 			return rec;
 		} while (0);
 
+		/* get back to beginning of message body */
+		if (inbuf->offset != old_offset)
+			io_buffer_seek(inbuf, old_offset);
+
 		/* try next message */
 		(*seq)++;
 		(void)index->expunge(index, rec, *seq, TRUE);
 		rec = index->next(index, rec);
 	} while (rec != NULL);
-#endif
+
 	return NULL;
 }
 
-static int mbox_index_fsck_mmap(MailIndex *index, const char *data, size_t size)
+static int mbox_index_fsck_buf(MailIndex *index, IOBuffer *inbuf)
 {
 	MailIndexRecord *rec;
-	const char *data_end;
-	unsigned int seq;
+	unsigned char *data;
+	unsigned int seq, size;
 
 	if (!index->set_lock(index, MAIL_LOCK_EXCLUSIVE))
 		return FALSE;
 
 	/* first make sure we start with a "From " line. */
+	while (io_buffer_read_data(inbuf, &data, &size, 5) >= 0) {
+		if (size > 5)
+			break;
+	}
+
 	if (size <= 5 || strncmp(data, "From ", 5) != 0) {
 		index_set_error(index, "File isn't in mbox format: %s",
 				index->mbox_path);
@@ -118,9 +134,21 @@
 	seq = 1;
 	rec = index->lookup(index, 1);
 
-	data_end = data + size;
 	while (rec != NULL) {
-		rec = match_next_record(index, rec, &seq, &data, data_end);
+		if (inbuf->offset != 0) {
+			/* we're at the [\r]\n before the From-line,
+			   skip it */
+			if (!mbox_skip_crlf(inbuf)) {
+				/* they just went and broke it, even while
+				   we had it locked. */
+				return FALSE;
+			}
+		}
+
+		if (inbuf->offset == inbuf->size)
+			break;
+
+		rec = match_next_record(index, rec, &seq, inbuf);
 		if (rec == NULL)
 			break;
 
@@ -128,18 +156,15 @@
 		rec = index->next(index, rec);
 	}
 
-	if (data == data_end)
+	if (inbuf->offset == inbuf->size)
 		return TRUE;
-	else {
-		return mbox_index_append_mmaped(index, data,
-						(size_t) (data_end-data), 0);
-	}
+	else
+		return mbox_index_append(index, inbuf);
 }
 
 int mbox_index_fsck(MailIndex *index)
 {
-	void *mmap_base;
-	size_t mmap_length;
+	IOBuffer *inbuf;
 	int fd, failed;
 
 	/* open the mbox file. we don't really need to open it read-write,
@@ -151,31 +176,20 @@
 		return FALSE;
 	}
 
-	mmap_base = mmap_ro_file(fd, &mmap_length);
-	if (mmap_base == MAP_FAILED) {
-		index_set_error(index, "mmap() failed with mbox file %s: %m",
-				index->mbox_path);
-		return FALSE;
-	}
-	(void)madvise(mmap_base, mmap_length, MADV_SEQUENTIAL);
-
-	if (mmap_base == NULL) {
-		/* file is empty */
-		(void)close(fd);
-		return TRUE;
-	}
+	inbuf = io_buffer_create_mmap(fd, default_pool,
+				      MAIL_MMAP_BLOCK_SIZE, -1);
 
 	/* lock the mailbox so we can be sure no-one interrupts us.
 	   we are trying to repair our index after all. */
 	if (!mbox_lock(index, index->mbox_path, fd))
 		failed = TRUE;
 	else {
-		failed = !mbox_index_fsck_mmap(index, mmap_base, mmap_length);
+		failed = !mbox_index_fsck_buf(index, inbuf);
 		(void)mbox_unlock(index, index->mbox_path, fd);
 	}
 
-	(void)munmap(mmap_base, mmap_length);
 	(void)close(fd);
+	io_buffer_destroy(inbuf);
 
 	if (failed)
 		return FALSE;
diff -r 163675942b83 -r a946ce1f09b7 src/lib-index/mbox/mbox-index.c
--- a/src/lib-index/mbox/mbox-index.c	Fri Aug 23 19:21:36 2002 +0300
+++ b/src/lib-index/mbox/mbox-index.c	Sat Aug 24 05:04:45 2002 +0300
@@ -1,41 +1,140 @@
 /* Copyright (C) 2002 Timo Sirainen */
 
 #include "lib.h"
+#include "iobuffer.h"
 #include "mbox-index.h"
 #include "mail-index-util.h"
 
 static MailIndex mbox_index;
 
-MailFlags mbox_header_get_flags(const char *name, unsigned int name_len,
-				const char *value, unsigned int value_len)
+void mbox_header_init_context(MboxHeaderContext *ctx)
+{
+	memset(ctx, 0, sizeof(MboxHeaderContext));
+	md5_init(&ctx->md5);
+}
+
+static MailFlags mbox_get_status_flags(const char *value, unsigned int len)
 {
 	MailFlags flags;
 	unsigned int i;
 
 	flags = 0;
-	if ((name_len == 6 && strncasecmp(name, "Status", 6) == 0) ||
-	    (name_len == 8 && strncasecmp(name, "X-Status", 8) == 0)) {
-		    for (i = 0; i < value_len; i++) {
-			    switch (value[i]) {
-			    case 'A':
-				    flags |= MAIL_ANSWERED;
-				    break;
-			    case 'F':
-				    flags |= MAIL_FLAGGED;
-				    break;
-			    case 'R':
-				    flags |= MAIL_SEEN;
-				    break;
-			    case 'D':
-				    flags |= MAIL_DELETED;
-				    break;
-			    }
-		    }
+	for (i = 0; i < len; i++) {
+		switch (value[i]) {
+		case 'A':
+			flags |= MAIL_ANSWERED;
+			break;
+		case 'F':
+			flags |= MAIL_FLAGGED;
+			break;
+		case 'R':
+			flags |= MAIL_SEEN;
+			break;
+		case 'D':
+			flags |= MAIL_DELETED;
+			break;
+		}
 	}
 
 	return flags;
 }
 
+void mbox_header_func(MessagePart *part __attr_unused__,
+		      const char *name, unsigned int name_len,
+		      const char *value, unsigned int value_len,
+		      void *context)
+{
+	MboxHeaderContext *ctx = context;
+	int fixed = FALSE;
+
+	/* Pretty much copy&pasted from popa3d by Solar Designer */
+	switch (*name) {
+	case 'R':
+	case 'r':
+		if (!ctx->received && name_len == 8 &&
+		    strncasecmp(name, "Received", 8) == 0) {
+			ctx->received = TRUE;
+			fixed = TRUE;
+		}
+		break;
+
+	case 'D':
+	case 'd':
+		if (name_len == 12)
+			fixed = strncasecmp(name, "Delivered-To", 12) == 0;
+		else if (name_len == 4) {
+			/* Received-header contains date too,
+			   and more trusted one */
+			fixed = !ctx->received &&
+				strncasecmp(name, "Date", 4) == 0;
+		}
+		break;
+
+	case 'M':
+	case 'm':
+		if (name_len == 10) {
+			/* Received-header contains unique ID too,
+			   and more trusted one */
+			fixed = !ctx->received &&
+				strncasecmp(name, "Message-ID", 10) == 0;
+		}
+		break;
+
+	case 'S':
+	case 's':
+		if (name_len == 6 && strncasecmp(name, "Status", 6) == 0) {
+			/* update message flags */
+			ctx->flags |= mbox_get_status_flags(value, value_len);
+		}
+		break;
+
+	case 'X':
+	case 'x':
+		/* Let the local delivery agent help generate unique ID's but
+		   don't blindly trust this header alone as it could just as
+		   easily come from the remote. */
+		if (name_len == 13)
+			fixed = strncasecmp(name, "X-Delivery-ID:", 13);
+		else if (name_len == 8 &&
+			 strncasecmp(name, "X-Status", 8) == 0) {
+			/* update message flags */
+			ctx->flags |= mbox_get_status_flags(value, value_len);
+		}
+		break;
+	}
+
+	if (fixed)
+		md5_update(&ctx->md5, value, value_len);
+}
+
+int mbox_skip_crlf(IOBuffer *inbuf)
+{
+	unsigned char *data;
+	unsigned int size;
+
+	while (io_buffer_read_data(inbuf, &data, &size, 1) >= 0) {
+		if (size > 0) {
+			if (data[0] == '\n') {
+				io_buffer_skip(inbuf, 1);
+				return TRUE;
+			}
+			if (data[0] != '\r')
+				return FALSE;
+
+			if (size > 1) {
+				if (data[1] != '\n')
+					return FALSE;
+
+				io_buffer_skip(inbuf, 2);
+				return TRUE;
+			}
+		}
+	}
+
+	/* end of file */
+	return TRUE;
+}
+
 MailIndex *mbox_index_alloc(const char *dir, const char *mbox_path)
 {
 	MailIndex *index;
diff -r 163675942b83 -r a946ce1f09b7 src/lib-index/mbox/mbox-index.h
--- a/src/lib-index/mbox/mbox-index.h	Fri Aug 23 19:21:36 2002 +0300
+++ b/src/lib-index/mbox/mbox-index.h	Sat Aug 24 05:04:45 2002 +0300
@@ -1,20 +1,32 @@
 #ifndef __MBOX_INDEX_H
 #define __MBOX_INDEX_H
 
+#include "md5.h"
 #include "mail-index.h"
 
+typedef struct {
+	MailFlags flags;
+	MD5Context md5;
+	int received;
+} MboxHeaderContext;
+
 MailIndex *mbox_index_alloc(const char *dir, const char *mbox_path);
 
-MailFlags mbox_header_get_flags(const char *name, unsigned int name_len,
-				const char *value, unsigned int value_len);
+void mbox_header_init_context(MboxHeaderContext *ctx);
+void mbox_header_func(MessagePart *part __attr_unused__,
+		      const char *name, unsigned int name_len,
+		      const char *value, unsigned int value_len,
+		      void *context);
+int mbox_skip_crlf(IOBuffer *inbuf);
 
 int mbox_index_rebuild(MailIndex *index);
 int mbox_index_sync(MailIndex *index);
 int mbox_index_fsck(MailIndex *index);
 IOBuffer *mbox_open_mail(MailIndex *index, MailIndexRecord *rec);
 
-int mbox_index_append(MailIndex *index, int fd, const char *path);
-int mbox_index_append_mmaped(MailIndex *index, const char *data,
-			     size_t data_size, off_t start_offset);
+int mbox_index_append(MailIndex *index, IOBuffer *inbuf);
+
+time_t mbox_from_parse_date(const char *msg, unsigned int size);
+const char *mbox_from_create(const char *sender, time_t time);
 
 #endif
diff -r 163675942b83 -r a946ce1f09b7 src/lib-index/mbox/mbox-open.c
--- a/src/lib-index/mbox/mbox-open.c	Fri Aug 23 19:21:36 2002 +0300
+++ b/src/lib-index/mbox/mbox-open.c	Sat Aug 24 05:04:45 2002 +0300
@@ -2,6 +2,7 @@
 
 #include "lib.h"
 #include "iobuffer.h"
+#include "hex-binary.h"
 #include "mbox-index.h"
 #include "mail-index-util.h"
 
@@ -13,8 +14,8 @@
 {
 	const char *location;
 	off_t pos, offset, stop_offset;
-	char buf[5];
-	int fd, ret, ok;
+	char buf[7], *p;
+	int fd, ret, failed;
 
 	i_assert(index->lock_type != MAIL_LOCK_UNLOCK);
 
@@ -27,8 +28,16 @@
 		return NULL;
 	}
 
-	/* location = offset */
-	offset = (off_t)strtoul(location, NULL, 10);
+	/* location = offset in hex */
+	if (strlen(location) != sizeof(offset)*2 ||
+	    hex_to_binary(location, (unsigned char *) &offset) <= 0) {
+                INDEX_MARK_CORRUPTED(index);
+		index_set_error(index, "Corrupted index file %s: "
+				"Invalid location field for record %u",
+				index->filepath, rec->uid);
+		return NULL;
+	}
+
 	stop_offset = offset + rec->header_size + rec->body_size;
 
 	fd = open(index->mbox_path, O_RDONLY);
@@ -46,21 +55,30 @@
 		return NULL;
 	}
 
-	ok = FALSE;
+	failed = TRUE;
 	if (pos == offset) {
 		/* make sure message size is valid */
 		if (lseek(fd, stop_offset, SEEK_SET) == stop_offset) {
 			/* and check that we end with either EOF or to
 			   beginning of next message */
-			ret = read(fd, buf, 5);
+			ret = read(fd, buf, 7);
 			if (ret == 0)
-				ok = TRUE; /* end of file */
-			else if (ret == 5 && strncmp(buf, "From ", 5) == 0)
-				ok = TRUE;
+				failed = FALSE; /* end of file */
+			else if (ret >= 6) {
+				/* "[\r]\nFrom " expected */
+				if (buf[0] != '\r')
+					p = buf;
+				else {
+					p = buf+1;
+					ret--;
+				}
+				if (ret >= 6 && strncmp(p, "\nFrom ", 6) == 0)
+					failed = FALSE;
+			}
 		}
 	}
 
-	if (ok) {
+	if (!failed) {
 		if (lseek(fd, offset, SEEK_SET) == offset) {
 			/* everything ok */
 			return io_buffer_create_mmap(fd, default_pool,
@@ -74,6 +92,9 @@
 	} else {
 		/* file has been updated, rescan it */
 		index->set_flags |= MAIL_INDEX_FLAG_FSCK;
+
+		index_set_error(index, "mbox file %s was modified "
+				"unexpectedly, fscking", index->mbox_path);
 	}
 
 	(void)close(fd);
diff -r 163675942b83 -r a946ce1f09b7 src/lib-index/mbox/mbox-rebuild.c
--- a/src/lib-index/mbox/mbox-rebuild.c	Fri Aug 23 19:21:36 2002 +0300
+++ b/src/lib-index/mbox/mbox-rebuild.c	Sat Aug 24 05:04:45 2002 +0300
@@ -1,6 +1,7 @@
 /* Copyright (C) 2002 Timo Sirainen */
 
 #include "lib.h"
+#include "iobuffer.h"
 #include "mbox-index.h"
 #include "mbox-lock.h"
 #include "mail-index-data.h"
@@ -14,6 +15,7 @@
 
 int mbox_index_rebuild(MailIndex *index)
 {
+	IOBuffer *inbuf;
 	struct stat st;
 	int fd;
 
@@ -25,14 +27,17 @@
 	/* reset the header */
 	mail_index_init_header(index->header);
 
-	/* we require Message-ID to be cached */
-	index->header->cache_fields |= FIELD_TYPE_MESSAGEID;
+	/* we require MD5 to be cached */
+	index->header->cache_fields |= FIELD_TYPE_MD5;
 
 	/* update indexid */
 	index->indexid = index->header->indexid;
 
-	if (msync(index->mmap_base, sizeof(MailIndexHeader), MS_SYNC) == -1)
+	if (msync(index->mmap_base, sizeof(MailIndexHeader), MS_SYNC) == -1) {
+		index_set_error(index, "msync() failed for index file %s: %m",
+				index->filepath);
 		return FALSE;
+	}
 
 	/* truncate the file first, so it won't contain
 	   any invalid data even if we crash */
@@ -61,7 +66,9 @@
 		return FALSE;
 	}
 
-	if (!mbox_index_append(index, fd, index->mbox_path)) {
+	inbuf = io_buffer_create_mmap(fd, default_pool,
+				      MAIL_MMAP_BLOCK_SIZE, -1);
+	if (!mbox_index_append(index, inbuf)) {
 		(void)mbox_unlock(index, index->mbox_path, fd);
 		(void)close(fd);
 		return FALSE;
@@ -69,6 +76,7 @@
 
 	(void)mbox_unlock(index, index->mbox_path, fd);
 	(void)close(fd);
+	io_buffer_destroy(inbuf);
 
 	/* update sync stamp */
 	if (stat(index->mbox_path, &st) == -1) {
diff -r 163675942b83 -r a946ce1f09b7 src/lib-index/mbox/mbox-sync.c
--- a/src/lib-index/mbox/mbox-sync.c	Fri Aug 23 19:21:36 2002 +0300
+++ b/src/lib-index/mbox/mbox-sync.c	Sat Aug 24 05:04:45 2002 +0300
@@ -1,6 +1,7 @@
 /* Copyright (C) 2002 Timo Sirainen */
 
 #include "lib.h"
+#include "iobuffer.h"
 #include "mbox-index.h"
 #include "mail-index-util.h"
 
@@ -9,17 +10,18 @@
 #include <fcntl.h>
 #include <sys/stat.h>
 
-static size_t get_indexed_mbox_size(MailIndex *index)
+static off_t get_indexed_mbox_size(MailIndex *index)
 {
 	MailIndexRecord *rec, *prev;
 	const char *location;
-	size_t size;
+	unsigned long size;
 
 	if (index->lock_type == MAIL_LOCK_UNLOCK) {
 		if (!mail_index_set_lock(index, MAIL_LOCK_SHARED))
 			return 0;
 	}
 
+	/* get the last record */
 	rec = index->header->messages_count == 0 ? NULL :
 		index->lookup(index, index->header->messages_count);
 	if (rec == NULL) {
@@ -34,6 +36,8 @@
 
 	size = 0;
 	if (rec != NULL) {
+		/* get the offset + size of last message, which tells the
+		   last known mbox file size */
 		location = index->lookup_field(index, rec, FIELD_TYPE_LOCATION);
 		if (location == NULL) {
 			INDEX_MARK_CORRUPTED(index);
@@ -48,11 +52,12 @@
 
 	if (index->lock_type == MAIL_LOCK_SHARED)
 		(void)mail_index_set_lock(index, MAIL_LOCK_UNLOCK);
-	return size;
+	return (off_t)size;
 }
 
 static int mbox_check_new_mail(MailIndex *index)
 {
+	IOBuffer *inbuf;
 	off_t pos;
 	int fd, ret;
 
@@ -78,8 +83,11 @@
 	}
 
 	/* add the new data */
-	ret = mbox_index_append(index, fd, index->mbox_path);
+	inbuf = io_buffer_create_mmap(fd, default_pool,
+				      MAIL_MMAP_BLOCK_SIZE, -1);
+	ret = mbox_index_append(index, inbuf);
 	(void)close(fd);
+	io_buffer_destroy(inbuf);
 
 	if (index->set_flags & MAIL_INDEX_FLAG_FSCK) {
 		/* it wasn't just new mail, reread the mbox */
diff -r 163675942b83 -r a946ce1f09b7 src/lib-storage/index/index-copy.c
--- a/src/lib-storage/index/index-copy.c	Fri Aug 23 19:21:36 2002 +0300
+++ b/src/lib-storage/index/index-copy.c	Sat Aug 24 05:04:45 2002 +0300
@@ -26,7 +26,7 @@
 	/* save it in destination mailbox */
 	failed = !cd->dest->save(cd->dest, rec->msg_flags,
 				 cd->custom_flags, rec->internal_date,
-				 inbuf, inbuf->stop_offset - inbuf->offset);
+				 inbuf, inbuf->size);
 	(void)close(inbuf->fd);
 	io_buffer_destroy(inbuf);
 	return !failed;
diff -r 163675942b83 -r a946ce1f09b7 src/lib-storage/index/maildir/maildir-storage.c
--- a/src/lib-storage/index/maildir/maildir-storage.c	Fri Aug 23 19:21:36 2002 +0300
+++ b/src/lib-storage/index/maildir/maildir-storage.c	Sat Aug 24 05:04:45 2002 +0300
@@ -18,7 +18,7 @@
 
 static const char *maildirs[] = { "cur", "new", "tmp", NULL  };
 
-static MailStorage *maildir_create(const char *data)
+static MailStorage *maildir_create(const char *data, const char *user)
 {
 	MailStorage *storage;
 	const char *home, *path;
@@ -46,12 +46,14 @@
 	memcpy(storage, &maildir_storage, sizeof(MailStorage));
 
 	storage->dir = i_strdup(data);
+	storage->user = i_strdup(user);
 	return storage;
 }
 
 static void maildir_free(MailStorage *storage)
 {
 	i_free(storage->dir);
+	i_free(storage->user);
 	i_free(storage);
 }
 
@@ -357,6 +359,7 @@
 	mail_storage_get_last_error,
 
 	NULL,
+	NULL,
 	NULL
 };
 
diff -r 163675942b83 -r a946ce1f09b7 src/lib-storage/index/mbox/mbox-save.c
--- a/src/lib-storage/index/mbox/mbox-save.c	Fri Aug 23 19:21:36 2002 +0300
+++ b/src/lib-storage/index/mbox/mbox-save.c	Sat Aug 24 05:04:45 2002 +0300
@@ -10,6 +10,38 @@
 #include <stdlib.h>
 #include <unistd.h>
 #include <fcntl.h>
+#include <netdb.h>
+
+static char my_hostdomain[256] = "";
+
+static int write_from_line(MailStorage *storage, int fd, time_t internal_date)
+{
+	const char *sender, *line, *name;
+	size_t len;
+
+	if (*my_hostdomain == '\0') {
+		struct hostent *hent;
+
+		hostpid_init();
+		hent = gethostbyname(my_hostname);
+
+		name = hent != NULL ? hent->h_name : NULL;
+		if (name == NULL) {
+			/* failed, use just the hostname */
+			name = my_hostname;
+		}
+
+		strncpy(my_hostdomain, name, 255);
+		my_hostdomain[255] = '\0';
+	}
+
+	sender = t_strconcat(storage->user, "@", my_hostdomain, NULL);
+
+	line = mbox_from_create(sender, internal_date);
+	len = strlen(line);
+
+	return (size_t)write(fd, line, len) == len;
+}
 
 int mbox_storage_save(Mailbox *box, MailFlags flags, const char *custom_flags[],
 		      time_t internal_date, IOBuffer *data, size_t data_size)
@@ -47,14 +79,15 @@
 					  "lseek() failed for mbox file %s: %m",
 					  ibox->index->mbox_path);
 		failed = TRUE;
-	}
-
-	if (!failed && !index_storage_save_into_fd(box->storage, fd,
-						   ibox->index->mbox_path,
-						   data, data_size)) {
-		/* failed, truncate file back to original size */
-		(void)ftruncate(fd, pos);
-		failed = TRUE;
+	} else {
+		if (!write_from_line(box->storage, fd, internal_date) ||
+		    !index_storage_save_into_fd(box->storage, fd,
+						ibox->index->mbox_path,
+						data, data_size)) {
+			/* failed, truncate file back to original size */
+			(void)ftruncate(fd, pos);
+			failed = TRUE;
+		}
 	}
 
 	(void)mbox_unlock(ibox->index, ibox->index->mbox_path, fd);
diff -r 163675942b83 -r a946ce1f09b7 src/lib-storage/index/mbox/mbox-storage.c
--- a/src/lib-storage/index/mbox/mbox-storage.c	Fri Aug 23 19:21:36 2002 +0300
+++ b/src/lib-storage/index/mbox/mbox-storage.c	Sat Aug 24 05:04:45 2002 +0300
@@ -40,7 +40,7 @@
 	return FALSE;
 }
 
-static MailStorage *mbox_create(const char *data)
+static MailStorage *mbox_create(const char *data, const char *user)
 {
 	MailStorage *storage;
 	const char *home, *path;
@@ -73,12 +73,14 @@
 	memcpy(storage, &mbox_storage, sizeof(MailStorage));
 
 	storage->dir = i_strdup(data);
+	storage->user = i_strdup(user);
 	return storage;
 }
 
 static void mbox_free(MailStorage *storage)
 {
 	i_free(storage->dir);
+	i_free(storage->user);
 	i_free(storage);
 }
 
@@ -346,6 +348,7 @@
 	mail_storage_get_last_error,
 
 	NULL,
+	NULL,
 	NULL
 };
 
diff -r 163675942b83 -r a946ce1f09b7 src/lib-storage/mail-storage.c
--- a/src/lib-storage/mail-storage.c	Fri Aug 23 19:21:36 2002 +0300
+++ b/src/lib-storage/mail-storage.c	Sat Aug 24 05:04:45 2002 +0300
@@ -49,7 +49,8 @@
 	}
 }
 
-MailStorage *mail_storage_create(const char *name, const char *data)
+MailStorage *mail_storage_create(const char *name, const char *data,
+				 const char *user)
 {
 	MailStorageList *list;
 
@@ -57,19 +58,19 @@
 
 	for (list = storages; list != NULL; list = list->next) {
 		if (strcasecmp(list->storage->name, name) == 0)
-			return list->storage->create(data);
+			return list->storage->create(data, user);
 	}
 
 	return NULL;
 }
 
-MailStorage *mail_storage_create_default(void)
+MailStorage *mail_storage_create_default(const char *user)
 {
 	MailStorageList *list;
 	MailStorage *storage;
 
 	for (list = storages; list != NULL; list = list->next) {
-		storage = list->storage->create(NULL);
+		storage = list->storage->create(NULL, user);
 		if (storage != NULL)
 			return storage;
 	}
@@ -89,13 +90,13 @@
 	return NULL;
 }
 
-MailStorage *mail_storage_create_with_data(const char *data)
+MailStorage *mail_storage_create_with_data(const char *data, const char *user)
 {
 	MailStorage *storage;
 	const char *p, *name;
 
 	if (data == NULL || *data == '\0')
-		return mail_storage_create_default();
+		return mail_storage_create_default(user);
 
 	/* check if we're in the form of mailformat:data
 	   (eg. maildir:Maildir) */
@@ -104,11 +105,11 @@
 
 	if (*p == ':') {
 		name = t_strdup_until(data, p);
-		storage = mail_storage_create(name, p+1);
+		storage = mail_storage_create(name, p+1, user);
 	} else {
 		storage = mail_storage_autodetect(data);
 		if (storage != NULL)
-			storage = storage->create(data);
+			storage = storage->create(data, user);
 	}
 
 	return storage;
diff -r 163675942b83 -r a946ce1f09b7 src/lib-storage/mail-storage.h
--- a/src/lib-storage/mail-storage.h	Fri Aug 23 19:21:36 2002 +0300
+++ b/src/lib-storage/mail-storage.h	Sat Aug 24 05:04:45 2002 +0300
@@ -60,7 +60,7 @@
 	char hierarchy_sep;
 
 	/* Create new instance */
-	MailStorage *(*create)(const char *data);
+	MailStorage *(*create)(const char *data, const char *user);
 
 	/* Free this instance */
 	void (*free)(MailStorage *storage);
@@ -109,6 +109,7 @@
 
 /* private: */
 	char *dir; /* root directory */
+	char *user; /* name of user accessing the storage */
 	char *error;
 };
 
@@ -223,11 +224,12 @@
 /* Create a new instance of registered mail storage class with given
    storage-specific data. If data is NULL, it tries to use defaults.
    May return NULL if anything fails. */
-MailStorage *mail_storage_create(const char *name, const char *data);
+MailStorage *mail_storage_create(const char *name, const char *data,
+				 const char *user);
 void mail_storage_destroy(MailStorage *storage);
 
-MailStorage *mail_storage_create_default(void);
-MailStorage *mail_storage_create_with_data(const char *data);
+MailStorage *mail_storage_create_default(const char *user);
+MailStorage *mail_storage_create_with_data(const char *data, const char *user);
 
 /* Set error message in storage. Critical errors are logged with i_error(),
    but user sees only "internal error" message. */
diff -r 163675942b83 -r a946ce1f09b7 src/lib/iobuffer.c
--- a/src/lib/iobuffer.c	Fri Aug 23 19:21:36 2002 +0300
+++ b/src/lib/iobuffer.c	Sat Aug 24 05:04:45 2002 +0300
@@ -50,7 +50,7 @@
 	buf->fd = fd;
 	buf->pool = pool;
 	buf->priority = priority;
-	buf->max_size = max_buffer_size;
+	buf->max_buffer_size = max_buffer_size;
 	return buf;
 }
 
@@ -65,7 +65,7 @@
 }
 
 IOBuffer *io_buffer_create_mmap(int fd, Pool pool, unsigned int block_size,
-				off_t stop_offset)
+				off_t size)
 {
 	IOBuffer *buf;
 
@@ -83,24 +83,20 @@
 	}
 
 	buf = io_buffer_create_file(fd, pool, block_size);
-	buf->stop_offset = stop_offset;
 	buf->mmaped = TRUE;
 	buf->receive = TRUE;
 
 	/* set offsets */
 	buf->start_offset = lseek(fd, 0, SEEK_CUR);
-	buf->stop_offset = stop_offset > 0 ? stop_offset :
-		lseek(fd, 0, SEEK_END);
+	buf->size = size > 0 ? size :
+		lseek(fd, 0, SEEK_END) - buf->start_offset;
 
-	if (buf->start_offset == -1 || buf->stop_offset == -1) {
+	if (buf->start_offset < 0 || buf->size < 0) {
 		i_error("io_buffer_create_mmap(): lseek() failed: %m");
-		buf->start_offset = buf->stop_offset = 0;
+		buf->start_offset = buf->size = 0;
 	}
 
-	/* fix offset alignment */
-	buf->mmap_offset = buf->start_offset & ~mmap_pagemask;
-	buf->skip = buf->mmap_offset & mmap_pagemask;
-
+	buf->skip = buf->pos = buf->start_offset;
 	return buf;
 }
 
@@ -115,7 +111,7 @@
 		if (!buf->mmaped)
 			p_free(buf->pool, buf->buffer);
 		else
-			(void)munmap(buf->buffer, buf->size);
+			(void)munmap(buf->buffer, buf->buffer_size);
 	}
         p_free(buf->pool, buf);
 }
@@ -134,9 +130,9 @@
 	buf->last_cr = FALSE;
 
 	if (buf->mmaped && buf->buffer != NULL) {
-		(void)munmap(buf->buffer, buf->size);
+		(void)munmap(buf->buffer, buf->buffer_size);
 		buf->buffer = NULL;
-		buf->size = 0;
+		buf->buffer_size = 0;
 	}
 
 	buf->mmap_offset = buf->offset = 0;
@@ -155,9 +151,9 @@
 	newbuf->pool = pool;
 
 	if (!newbuf->mmaped) {
-		newbuf->buffer = p_malloc(pool, buf->size);
+		newbuf->buffer = p_malloc(pool, buf->buffer_size);
 		memcpy(newbuf->buffer, buf->buffer + buf->skip,
-		       buf->size - buf->skip);
+		       buf->buffer_size - buf->skip);
 
 		newbuf->cr_lookup_pos -= newbuf->skip;
 		newbuf->pos -= newbuf->skip;
@@ -174,7 +170,7 @@
 {
 	i_assert(!buf->mmaped);
 
-	buf->max_size = max_size;
+	buf->max_buffer_size = max_size;
 }
 
 void io_buffer_set_send_blocking(IOBuffer *buf, unsigned int max_size,
@@ -188,7 +184,7 @@
 	buf->timeout_func = timeout_func;
 	buf->timeout_context = context;
 	buf->blocking = max_size > 0;
-	buf->max_size = max_size;
+	buf->max_buffer_size = max_size;
 }
 
 static int my_write(int fd, const void *buf, unsigned int size)
@@ -373,17 +369,18 @@
 {
 	i_assert(!buf->mmaped);
 
-	buf->size = buf->pos+size;
-	buf->size = buf->size <= IO_BUFFER_MIN_SIZE ? IO_BUFFER_MIN_SIZE :
-		nearest_power(buf->size);
+	buf->buffer_size = buf->pos+size;
+	buf->buffer_size =
+		buf->buffer_size <= IO_BUFFER_MIN_SIZE ? IO_BUFFER_MIN_SIZE :
+		nearest_power(buf->buffer_size);
 
-	if (buf->max_size > 0 && buf->size > buf->max_size)
-		buf->size = buf->max_size;
+	if (buf->max_buffer_size > 0 && buf->buffer_size > buf->max_buffer_size)
+		buf->buffer_size = buf->max_buffer_size;
 
-	buf->buffer = p_realloc(buf->pool, buf->buffer, buf->size);
+	buf->buffer = p_realloc(buf->pool, buf->buffer, buf->buffer_size);
 	if (buf->buffer == NULL) {
 		/* pool limit exceeded */
-		buf->pos = buf->size = 0;
+		buf->pos = buf->buffer_size = 0;
 	}
 }
 
@@ -456,7 +453,7 @@
 	int ret;
 
 	ret = sendfile(ctx->outbuf->fd, ctx->inbuf->fd,
-		       &ctx->inbuf->mmap_offset, ctx->size);
+		       &ctx->inbuf->offset, ctx->size);
 	if (ret < 0) {
 		if (errno != EINTR && errno != EAGAIN)
 			ctx->outbuf->closed = TRUE;
@@ -598,44 +595,43 @@
 
 int io_buffer_read_mmaped(IOBuffer *buf, unsigned int size)
 {
-	unsigned int aligned_skip;
+	off_t stop_offset, aligned_skip;
 
-	if (buf->stop_offset - buf->mmap_offset <= (off_t)buf->size) {
+	stop_offset = buf->size + buf->start_offset;
+	if (stop_offset - buf->mmap_offset <= (off_t)buf->buffer_size) {
 		/* end of file is already mapped */
 		return -1;
 	}
 
-	if (buf->pos != 0) {
-		aligned_skip = buf->skip & ~mmap_pagemask;
-		if (aligned_skip == 0 && buf->buffer != NULL) {
-			/* didn't skip enough bytes */
-			return -2;
-		}
-
-		buf->skip -= aligned_skip;
-		buf->mmap_offset += aligned_skip;
+	aligned_skip = buf->skip & ~mmap_pagemask;
+	if (aligned_skip == 0 && buf->buffer != NULL) {
+		/* didn't skip enough bytes */
+		return -2;
 	}
 
+	buf->skip -= aligned_skip;
+	buf->mmap_offset += aligned_skip;
+
 	if (buf->buffer != NULL)
-		(void)munmap(buf->buffer, buf->size);
+		(void)munmap(buf->buffer, buf->buffer_size);
 
-	buf->size = buf->stop_offset - buf->mmap_offset < (off_t)buf->max_size ?
-		buf->stop_offset - buf->mmap_offset : buf->max_size;
+	buf->buffer_size = stop_offset - buf->mmap_offset;
+	if (buf->buffer_size > buf->max_buffer_size)
+		buf->buffer_size = buf->max_buffer_size;
+	if (buf->buffer_size > size)
+		buf->buffer_size = size;
 
-	if (buf->size > size)
-		buf->size = size;
-
-	buf->buffer = mmap(NULL, buf->size, PROT_READ, MAP_PRIVATE,
+	buf->buffer = mmap(NULL, buf->buffer_size, PROT_READ, MAP_PRIVATE,
 			   buf->fd, buf->mmap_offset);
 	if (buf->buffer == MAP_FAILED) {
 		i_error("io_buffer_read_mmaped(): mmap() failed: %m");
 		return -1;
 	}
 
-	(void)madvise(buf->buffer, buf->size, MADV_SEQUENTIAL);
+	(void)madvise(buf->buffer, buf->buffer_size, MADV_SEQUENTIAL);
 
-	buf->pos = buf->size;
-	return buf->size;
+	buf->pos = buf->buffer_size;
+	return buf->buffer_size;
 }
 
 int io_buffer_read_max(IOBuffer *buf, unsigned int size)
@@ -652,29 +648,30 @@
 	if (buf->mmaped)
 		return io_buffer_read_mmaped(buf, size);
 
-	if (buf->pos == buf->size) {
+	if (buf->pos == buf->buffer_size) {
 		if (buf->skip > 0) {
 			/* remove the unused bytes from beginning of buffer */
                         io_buffer_compress(buf);
-		} else if (buf->max_size == 0 || buf->size < buf->max_size) {
+		} else if (buf->max_buffer_size == 0 ||
+			   buf->buffer_size < buf->max_buffer_size) {
 			/* buffer is full - grow it */
 			buffer_alloc_more(buf, IO_BUFFER_MIN_SIZE);
 		}
 
-		if (buf->pos == buf->size)
+		if (buf->pos == buf->buffer_size)
                         return -2; /* buffer full */
 	}
 
         /* fill the buffer */
-	if (buf->size-buf->pos < size)
-		size = buf->size - buf->pos;
+	if (buf->buffer_size-buf->pos < size)
+		size = buf->buffer_size - buf->pos;
 
 	if (!buf->file) {
 		ret = net_receive(buf->fd, buf->buffer + buf->pos,
-				  buf->size - buf->pos);
+				  buf->buffer_size - buf->pos);
 	} else {
                 ret = read(buf->fd, buf->buffer + buf->pos,
-			   buf->size - buf->pos);
+			   buf->buffer_size - buf->pos);
 		if (ret == 0)
 			ret = -1; /* EOF */
 		else if (ret < 0 && (errno == EINTR || errno == EAGAIN))
@@ -707,24 +704,17 @@
 	}
 
 	if (buf->mmaped) {
-		if (buf->stop_offset - buf->mmap_offset <= (off_t)size) {
-			/* end of file */
-			buf->mmap_offset = buf->stop_offset;
-			buf->size = 0;
-			return;
-		}
-
 		/* these point outside mmap now, next io_buffer_read_mmaped()
 		   will fix them */
 		buf->skip += size;
 		buf->pos = buf->skip;
 	} else {
-		if (buf->size == 0)
+		if (buf->buffer_size == 0)
 			buffer_alloc_more(buf, IO_BUFFER_MIN_SIZE);
 
 		size -= buf->skip;
-		while (size > buf->size) {
-			ret = io_buffer_read_max(buf, buf->size);
+		while (size > buf->buffer_size) {
+			ret = io_buffer_read_max(buf, buf->buffer_size);
 			if (ret <= 0)
 				break;
 
@@ -747,7 +737,7 @@
 
 		/* then set the wanted position, next read will
 		   pick up from there */
-		buf->mmap_offset = buf->start_offset;
+		buf->offset = buf->start_offset;
 		buf->pos = buf->skip = offset;
 	} else {
 		real_offset = buf->start_offset + offset;
@@ -844,18 +834,19 @@
 	buf->transmit = TRUE;
 
 	/* make sure we have enough space in buffer */
-	if (buf->size - buf->pos < size && buf->skip > 0) {
+	if (buf->buffer_size - buf->pos < size && buf->skip > 0) {
 		/* remove the unused bytes from beginning of buffer */
 		io_buffer_compress(buf);
 	}
 
-	if (buf->size - buf->pos < size &&
-	    (buf->max_size == 0 || size <= buf->max_size - buf->pos)) {
+	if (buf->buffer_size - buf->pos < size &&
+	    (buf->max_buffer_size == 0 ||
+	     size <= buf->max_buffer_size - buf->pos)) {
 		/* allocate more space */
                 buffer_alloc_more(buf, size);
 	}
 
-	if (buf->size - buf->pos < size)
+	if (buf->buffer_size - buf->pos < size)
                 return NULL;
 
         return buf->buffer + buf->pos;
@@ -903,9 +894,9 @@
 
 	io_buffer_reset(buf);
 
-	if (buf->size < size) {
+	if (buf->buffer_size < size) {
 		buffer_alloc_more(buf, size);
-		if (buf->size < size)
+		if (buf->buffer_size < size)
                         return -2;
 	}
 
diff -r 163675942b83 -r a946ce1f09b7 src/lib/iobuffer.h
--- a/src/lib/iobuffer.h	Fri Aug 23 19:21:36 2002 +0300
+++ b/src/lib/iobuffer.h	Sat Aug 24 05:04:45 2002 +0300
@@ -10,10 +10,8 @@
 struct _IOBuffer {
 	int fd;
 
-	unsigned int pos, skip;
-	unsigned int size, max_size;
-	off_t start_offset, stop_offset;
-	off_t offset; /* virtual offset, 0 = start_offset */
+	off_t start_offset;
+	off_t offset, size; /* virtual offset, 0 = start_offset */
 
 /* private: */
 	Pool pool;
@@ -29,7 +27,10 @@
 
 	unsigned char *buffer;
         unsigned int cr_lookup_pos; /* used only when reading a line */
+
 	off_t mmap_offset;
+	unsigned int pos, skip;
+	unsigned int buffer_size, max_buffer_size;
 
 	unsigned int file:1; /* reading/writing a file */
 	unsigned int mmaped:1; /* reading a file with mmap() */
@@ -52,7 +53,7 @@
 /* Read the file by mmap()ing it in blocks. stop_offset specifies where to
    stop reading, or 0 to end of file. */
 IOBuffer *io_buffer_create_mmap(int fd, Pool pool, unsigned int block_size,
-				off_t stop_offset);
+				off_t size);
 /* Destroy a buffer. */
 void io_buffer_destroy(IOBuffer *buf);
 /* Mark the buffer closed. Any sends/reads after this will return -1.