--- a/src/lib-imap/imap-message-cache.c	Fri Sep 06 16:29:51 2002 +0300
+++ b/src/lib-imap/imap-message-cache.c	Fri Sep 06 16:43:58 2002 +0300
@@ -48,7 +48,6 @@
 
 	CachedMessage *open_msg;
 	IOBuffer *open_inbuf;
-	uoff_t open_virtual_size;
 
 	void *context;
 };
@@ -252,6 +251,13 @@
 			msg->cached_envelope = p_strdup(msg->pool, value);
 	}
 
+	if ((fields & IMAP_CACHE_MESSAGE_BODY_SIZE) && msg->body_size == NULL) {
+		/* we don't have body size. and since we're already going
+		   to scan the whole message body, we might as well build
+		   the MessagePart. */
+                fields |= IMAP_CACHE_MESSAGE_PART;
+	}
+
 	if (fields & IMAP_CACHE_MESSAGE_PART) {
 		msg_get_part(cache);
 
@@ -275,40 +281,11 @@
 	}
 
 	if ((fields & IMAP_CACHE_MESSAGE_BODY_SIZE) && msg->body_size == NULL) {
-		msg_get_part(cache);
-
-		/* fill the body size, and while at it fill the header
-		   size as well */
-		if (msg->hdr_size == NULL)
-			msg->hdr_size = p_new(msg->pool, MessageSize, 1);
-		msg->body_size = p_new(msg->pool, MessageSize, 1);
+		i_assert(msg->part != NULL);
 
-		if (msg->part != NULL) {
-			/* easy, get it from root part */
-			*msg->hdr_size = msg->part->header_size;
-			*msg->body_size = msg->part->body_size;
-		} else {
-			/* first get the header's size, then calculate the body
-			   size from it and the total virtual size */
-			imap_msgcache_get_inbuf(cache, 0);
-			message_get_header_size(cache->open_inbuf,
-						msg->hdr_size);
-
-			/* FIXME: this may actually happen if file size
-			   is shrinked.. */
-			i_assert(msg->hdr_size->physical_size <=
-				 cache->open_inbuf->size);
-			i_assert(msg->hdr_size->virtual_size <=
-				 cache->open_virtual_size);
-
-			msg->body_size->lines = 0;
-			msg->body_size->physical_size =
-				cache->open_inbuf->size -
-				msg->hdr_size->physical_size;
-			msg->body_size->virtual_size =
-				cache->open_virtual_size -
-				msg->hdr_size->virtual_size;
-		}
+		msg->body_size = p_new(msg->pool, MessageSize, 1);
+		*msg->hdr_size = msg->part->header_size;
+		*msg->body_size = msg->part->body_size;
 	}
 
 	if ((fields & IMAP_CACHE_MESSAGE_HDR_SIZE) && msg->hdr_size == NULL) {
@@ -330,8 +307,8 @@
 }
 
 void imap_msgcache_open(ImapMessageCache *cache, unsigned int uid,
-			ImapCacheField fields, uoff_t virtual_size,
-			uoff_t pv_headers_size, uoff_t pv_body_size,
+			ImapCacheField fields,
+			uoff_t virtual_header_size, uoff_t virtual_body_size,
 			void *context)
 {
 	CachedMessage *msg;
@@ -341,22 +318,21 @@
 		imap_msgcache_close(cache);
 
 		cache->open_msg = msg;
-		cache->open_virtual_size = virtual_size;
 		cache->context = context;
 	}
 
-	if (pv_headers_size != 0 && msg->hdr_size == NULL) {
+	if (virtual_header_size != 0 && msg->hdr_size == NULL) {
 		/* physical size == virtual size */
 		msg->hdr_size = p_new(msg->pool, MessageSize, 1);
 		msg->hdr_size->physical_size = msg->hdr_size->virtual_size =
-			pv_headers_size;
+			virtual_header_size;
 	}
 
-	if (pv_body_size != 0 && msg->body_size == NULL) {
+	if (virtual_body_size != 0 && msg->body_size == NULL) {
 		/* physical size == virtual size */
 		msg->body_size = p_new(msg->pool, MessageSize, 1);
 		msg->body_size->physical_size = msg->body_size->virtual_size =
-			pv_body_size;
+			virtual_body_size;
 	}
 
 	cache_fields(cache, fields);
@@ -371,7 +347,6 @@
 	}
 
 	cache->open_msg = NULL;
-	cache->open_virtual_size = 0;
 	cache->context = NULL;
 }
 
@@ -503,14 +478,8 @@
 	size_got = FALSE;
 	if (virtual_skip == 0) {
 		if (msg->body_size == NULL) {
-			/* FIXME: may underflow */
-			msg->body_size = p_new(msg->pool, MessageSize, 1);
-			msg->body_size->physical_size =
-				cache->open_inbuf->size -
-				msg->hdr_size->physical_size;
-			msg->body_size->virtual_size =
-				cache->open_virtual_size -
-				msg->hdr_size->virtual_size;
+			cache_fields(cache, IMAP_CACHE_MESSAGE_BODY_SIZE);
+			i_assert(msg->body_size != NULL);
 		}
 
 		if (max_virtual_size >= msg->body_size->virtual_size) {

--- a/src/lib-imap/imap-message-cache.h	Fri Sep 06 16:29:51 2002 +0300
+++ b/src/lib-imap/imap-message-cache.h	Fri Sep 06 16:43:58 2002 +0300
@@ -41,12 +41,11 @@
 void imap_msgcache_clear(ImapMessageCache *cache);
 void imap_msgcache_free(ImapMessageCache *cache);
 
-/* Open the specified message. If pv_headers_size and pv_body_size is
-   non-zero, they're set to saved to message's both physical and virtual
-   sizes (ie. doesn't need to be calculated). */
+/* Open the specified message. virtual_header/body_size may be 0
+   if it's not known. */
 void imap_msgcache_open(ImapMessageCache *cache, unsigned int uid,
-			ImapCacheField fields, uoff_t virtual_size,
-			uoff_t pv_headers_size, uoff_t pv_body_size,
+			ImapCacheField fields,
+			uoff_t virtual_header_size, uoff_t virtual_body_size,
 			void *context);
 
 /* Close the IOBuffer for opened message. */

--- a/src/lib-index/mail-index-update.c	Fri Sep 06 16:29:51 2002 +0300
+++ b/src/lib-index/mail-index-update.c	Fri Sep 06 16:43:58 2002 +0300
@@ -95,7 +95,9 @@
 			index = mail_field_get_index(rec->field);
 			i_assert(index >= 0);
 
-			if (update->field_sizes[index] >= rec->full_field_size)
+			if (update->field_sizes[index] +
+			    update->field_extra_sizes[index] >
+			    rec->full_field_size)
 				return TRUE;
 		}
 		rec = mail_index_data_next(update->index->data,
@@ -234,23 +236,23 @@
 
 int mail_index_update_end(MailIndexUpdate *update)
 {
-	int failed;
+	int failed = FALSE;
 
 	i_assert(update->index->lock_type == MAIL_LOCK_EXCLUSIVE);
 
-	/* if any of the fields were newly added, or have grown larger
-	   than their old max. size, we need to move the record to end
-	   of file. */
-	if (have_new_fields(update) || have_too_large_fields(update))
-		failed = !update_by_append(update);
-	else {
-		update_by_replace(update);
-		failed = FALSE;
-	}
+	if (update->updated_fields != 0) {
+		/* if any of the fields were newly added, or have grown larger
+		   than their old max. size, we need to move the record to end
+		   of file. */
+		if (have_new_fields(update) || have_too_large_fields(update))
+			failed = !update_by_append(update);
+		else
+			update_by_replace(update);
 
-	if (!failed) {
-		/* update cached fields mask */
-		update->rec->cached_fields |= update->updated_fields;
+		if (!failed) {
+			/* update cached fields mask */
+			update->rec->cached_fields |= update->updated_fields;
+		}
 	}
 
 	pool_unref(update->pool);
@@ -387,7 +389,7 @@
 {
 	HeaderUpdateContext ctx;
 	MessagePart *part;
-	MessageSize hdr_size, body_size;
+	MessageSize hdr_size;
 	Pool pool;
 	const char *value;
 	unsigned int size;
@@ -411,14 +413,25 @@
 							update->rec,
 							FIELD_TYPE_MESSAGEPART,
 							&size);
-		if (value == NULL) {
+		if (value == NULL)
+			part = NULL;
+		else {
+			part = message_part_deserialize(pool, value, size);
+			if (part == NULL) {
+				/* corrupted, rebuild it */
+				index_set_error(update->index, "Error in index "
+						"file %s: Corrupted cached "
+						"MessagePart data",
+						update->index->filepath);
+			}
+		}
+
+		if (part == NULL) {
 			part = message_parse(pool, inbuf,
 					     update_header_func, &ctx);
 		} else {
 			/* cached, construct the bodystructure using it.
 			   also we need to parse the header.. */
-			part = message_part_deserialize(pool, value, size);
-
 			io_buffer_seek(inbuf, 0);
 			message_parse_header(NULL, inbuf, NULL,
 					     update_header_func, &ctx);
@@ -427,9 +440,6 @@
 		/* update our sizes */
 		update->rec->header_size = part->header_size.physical_size;
 		update->rec->body_size = part->body_size.physical_size;
-		update->rec->full_virtual_size =
-			part->header_size.virtual_size +
-			part->body_size.virtual_size;
 
 		if (cache_fields & FIELD_TYPE_BODY) {
 			t_push();
@@ -468,16 +478,6 @@
 
 		update->rec->header_size = hdr_size.physical_size;
 		update->rec->body_size = inbuf->size - inbuf->offset;
-
-		if (update->rec->full_virtual_size == 0) {
-			/* we need to calculate virtual size of the
-			   body as well. message_parse_header() left the
-			   inbuf point to beginning of the body. */
-			message_get_body_size(inbuf, &body_size, (uoff_t)-1);
-
-			update->rec->full_virtual_size =
-				hdr_size.virtual_size + body_size.virtual_size;
-		}
 	}
 
 	if (ctx.envelope != NULL) {

--- a/src/lib-index/mail-index-util.c	Fri Sep 06 16:29:51 2002 +0300
+++ b/src/lib-index/mail-index-util.c	Fri Sep 06 16:43:58 2002 +0300
@@ -1,7 +1,10 @@
 /* Copyright (C) 2002 Timo Sirainen */
 
 #include "lib.h"
+#include "iobuffer.h"
 #include "hostpid.h"
+#include "message-size.h"
+#include "message-part-serialize.h"
 #include "mail-index.h"
 #include "mail-index-util.h"
 
@@ -57,3 +60,57 @@
 	return fd;
 }
 
+
+int mail_index_get_virtual_size(MailIndex *index, MailIndexRecord *rec,
+				int fastscan, uoff_t *virtual_size)
+{
+	MessageSize hdr_size, body_size;
+	IOBuffer *inbuf;
+	const void *part_data;
+	unsigned int size;
+
+	if ((rec->index_flags & INDEX_MAIL_FLAG_BINARY_HEADER) &&
+	    (rec->index_flags & INDEX_MAIL_FLAG_BINARY_BODY)) {
+		/* virtual size == physical size */
+		*virtual_size += rec->header_size + rec->body_size;
+		return TRUE;
+	}
+
+	part_data = index->lookup_field_raw(index, rec,
+					    FIELD_TYPE_MESSAGEPART, &size);
+	if (part_data != NULL) {
+		/* get sizes from preparsed message structure */
+		if (!message_part_deserialize_size(part_data, size,
+						   &hdr_size, &body_size)) {
+			/* corrupted, ignore */
+			index_set_error(index, "Error in index file %s: "
+					"Corrupted cached MessagePart data",
+					index->filepath);
+		} else {
+			*virtual_size = hdr_size.virtual_size +
+				body_size.virtual_size;
+			return TRUE;
+		}
+	}
+
+	/* only way left is to actually parse the message */
+	*virtual_size = 0;
+
+	if (fastscan) {
+		/* and we don't want that */
+		return FALSE;
+	}
+
+	inbuf = index->open_mail(index, rec);
+	if (inbuf == NULL)
+		return FALSE;
+
+	/* we don't care about the difference in header/body,
+	   so parse the whole message as a "body" */
+	message_get_body_size(inbuf, &body_size, (uoff_t)-1);
+	*virtual_size = body_size.virtual_size;
+
+	(void)close(inbuf->fd);
+	io_buffer_destroy(inbuf);
+	return TRUE;
+}

--- a/src/lib-index/mail-index-util.h	Fri Sep 06 16:29:51 2002 +0300
+++ b/src/lib-index/mail-index-util.h	Fri Sep 06 16:43:58 2002 +0300
@@ -12,4 +12,11 @@
    and sets *path to the full path of the created file.  */
 int mail_index_create_temp_file(MailIndex *index, const char **path);
 
+/* Calculates virtual size for specified message. If the fastscan is FALSE
+   and the size can't be figured out from headers, the message is opened and
+   fully scanned to calculate the size. Returns TRUE if size was successfully
+   got. */
+int mail_index_get_virtual_size(MailIndex *index, MailIndexRecord *rec,
+				int fastscan, uoff_t *virtual_size);
+
 #endif

--- a/src/lib-index/mail-index.c	Fri Sep 06 16:29:51 2002 +0300
+++ b/src/lib-index/mail-index.c	Fri Sep 06 16:43:58 2002 +0300
@@ -619,10 +619,8 @@
 	   when it succeeds */
 	hdr->flags = MAIL_INDEX_FLAG_REBUILD;
 
-	/* set the fields we always want to cache - currently nothing
-	   except the location. many clients aren't interested about
-	   any of the fields. */
-	hdr->cache_fields = FIELD_TYPE_LOCATION;
+	/* set the fields we always want to cache */
+	hdr->cache_fields |= FIELD_TYPE_LOCATION | FIELD_TYPE_MESSAGEPART;
 
 	hdr->uid_validity = ioloop_time;
 	hdr->next_uid = 1;

--- a/src/lib-index/mail-index.h	Fri Sep 06 16:29:51 2002 +0300
+++ b/src/lib-index/mail-index.h	Fri Sep 06 16:43:58 2002 +0300
@@ -49,6 +49,19 @@
 	FIELD_TYPE_MAX_BITS		= 11
 } MailField;
 
+typedef enum {
+	/* If binary flags are set, it's not checked whether mail is
+	   missing CRs. So this flag may be set as an optimization for
+	   regular non-binary mails as well if it's known that it contains
+	   valid CR+LF line breaks. */
+	INDEX_MAIL_FLAG_BINARY_HEADER	= 0x0001,
+	INDEX_MAIL_FLAG_BINARY_BODY	= 0x0002,
+
+	/* Currently this means with mbox format that message flags have
+	   been changed in index, but not written into mbox file yet. */
+	INDEX_MAIL_FLAG_DIRTY		= 0x0004
+} MailIndexMailFlags;
+
 #define IS_HEADER_FIELD(field) \
 	(((field) & (FIELD_TYPE_FROM | FIELD_TYPE_TO | FIELD_TYPE_CC | \
 		     FIELD_TYPE_BCC | FIELD_TYPE_SUBJECT)) != 0)
@@ -122,17 +135,15 @@
 	time_t internal_date;
 	time_t sent_date;
 
-	uoff_t data_position;
-	unsigned int data_size;
-
-	unsigned int cached_fields;
 	uoff_t header_size;
 	uoff_t body_size;
-	uoff_t full_virtual_size;
-};
+
+	unsigned int index_flags; /* MailIndexMailFlags */
+	unsigned int cached_fields;
 
-#define MSG_HAS_VALID_CRLF_DATA(rec) \
-	((rec)->header_size + (rec)->body_size == (rec)->full_virtual_size)
+	uoff_t data_position;
+	unsigned int data_size;
+};
 
 struct _MailIndexDataRecord {
 	unsigned int field; /* MailField */
@@ -379,7 +390,8 @@
 int mail_index_compress_data(MailIndex *index);
 
 /* Max. mmap()ed size for a message */
-#define MAIL_MMAP_BLOCK_SIZE (1024*256)
+//FIXME:#define MAIL_MMAP_BLOCK_SIZE (1024*256)
+#define MAIL_MMAP_BLOCK_SIZE (1024*8) // FIXME: for debugging
 
 /* uoff_t to index file for given record */
 #define INDEX_FILE_POSITION(index, ptr) \

--- a/src/lib-index/mail-modifylog.c	Fri Sep 06 16:29:51 2002 +0300
+++ b/src/lib-index/mail-modifylog.c	Fri Sep 06 16:43:58 2002 +0300
@@ -484,8 +484,12 @@
 	path = t_strconcat(log->index->filepath,
 			   log->second_log ? ".log" : ".log.2", NULL);
 
-	if (mail_modifylog_open_and_init_file(log, path))
+	if (mail_modifylog_open_and_init_file(log, path)) {
+		/* FIXME: we want to update the _old_ file's header.
+		   and this changes the new one. and it's already closed the
+		   old one and mmap() is invalid, and we crash here.. */
 		log->header->sync_id = SYNC_ID_FULL;
+	}
 }
 
 int mail_modifylog_mark_synced(MailModifyLog *log)

--- a/src/lib-index/mbox/Makefile.am	Fri Sep 06 16:29:51 2002 +0300
+++ b/src/lib-index/mbox/Makefile.am	Fri Sep 06 16:43:58 2002 +0300
@@ -14,6 +14,7 @@
 	mbox-lock.c \
 	mbox-open.c \
 	mbox-rebuild.c \
+	mbox-rewrite.c \
 	mbox-sync.c
 
 noinst_HEADERS = \

--- a/src/lib-index/mbox/mbox-append.c	Fri Sep 06 16:29:51 2002 +0300
+++ b/src/lib-index/mbox/mbox-append.c	Fri Sep 06 16:43:58 2002 +0300
@@ -8,15 +8,13 @@
 #include "mbox-index.h"
 #include "mail-index-util.h"
 
-static MailIndexRecord *
-mail_index_record_append(MailIndex *index, time_t internal_date,
-			 size_t full_virtual_size)
+static MailIndexRecord *mail_index_record_append(MailIndex *index,
+						 time_t internal_date)
 {
 	MailIndexRecord trec, *rec;
 
 	memset(&trec, 0, sizeof(MailIndexRecord));
 	trec.internal_date = internal_date;
-	trec.full_virtual_size = full_virtual_size;
 
 	rec = &trec;
 	if (!index->append(index, &rec))
@@ -25,22 +23,17 @@
 	return rec;
 }
 
-static void mbox_read_message(IOBuffer *inbuf, unsigned int *virtual_size)
+static void mbox_read_message(IOBuffer *inbuf)
 {
 	unsigned char *msg;
-	unsigned int i, size, startpos, vsize;
+	unsigned int i, size, startpos;
 	int lastmsg;
 
 	/* read until "[\r]\nFrom " is found */
-	startpos = i = vsize = 0; lastmsg = TRUE;
+	startpos = i = 0; lastmsg = TRUE;
 	while (io_buffer_read_data(inbuf, &msg, &size, startpos) >= 0) {
 		for (i = startpos; i < size; i++) {
-			if (msg[i] == '\n') {
-				if (i == 0 || msg[i-1] != '\r') {
-					/* missing CR */
-					vsize++;
-				}
-			} else if (msg[i] == ' ' && i >= 5) {
+			if (msg[i] == ' ' && i >= 5) {
 				/* See if it's space after "From" */
 				if (msg[i-5] == '\n' && msg[i-4] == 'F' &&
 				    msg[i-3] == 'r' && msg[i-2] == 'o' &&
@@ -49,8 +42,6 @@
 					i -= 5;
 					if (i > 0 && msg[i-1] == '\r')
 						i--;
-					else
-						vsize--;
 					break;
 				}
 			}
@@ -67,7 +58,6 @@
 			i -= startpos;
 
 			io_buffer_skip(inbuf, i);
-			vsize += i;
 		}
 	}
 
@@ -79,15 +69,10 @@
 				startpos--;
 			if (startpos > 0 && msg[startpos-1] == '\r')
 				startpos--;
-			else
-				vsize--;
 		}
 	}
 
 	io_buffer_skip(inbuf, startpos);
-	vsize += startpos;
-
-	*virtual_size = vsize;
 }
 
 static int mbox_index_append_next(MailIndex *index, IOBuffer *inbuf)
@@ -98,7 +83,7 @@
 	time_t internal_date;
 	uoff_t abs_start_offset, stop_offset, old_size;
 	unsigned char *data, md5_digest[16];
-	unsigned int size, pos, virtual_size;
+	unsigned int size, pos;
 
 	/* get the From-line */
 	pos = 0;
@@ -131,18 +116,14 @@
 	abs_start_offset = inbuf->start_offset + inbuf->offset;
 
 	/* now, find the ending "[\r]\nFrom " */
-	mbox_read_message(inbuf, &virtual_size);
+	mbox_read_message(inbuf);
 	stop_offset = inbuf->offset;
 
 	/* add message to index */
-	rec = mail_index_record_append(index, internal_date, virtual_size);
+	rec = mail_index_record_append(index, internal_date);
 	if (rec == NULL)
 		return FALSE;
 
-	/* save message flags */
-	rec->msg_flags = ctx.flags;
-	mail_index_mark_flag_changes(index, rec, 0, rec->msg_flags);
-
 	update = index->update_begin(index, rec);
 
 	/* location = offset to beginning of headers in message */
@@ -166,8 +147,8 @@
 
 	/* save MD5 */
 	md5_final(&ctx.md5, md5_digest);
-	index->update_field(update, FIELD_TYPE_MD5,
-                            binary_to_hex(md5_digest, sizeof(md5_digest)), 0);
+	index->update_field_raw(update, FIELD_TYPE_MD5,
+				md5_digest, sizeof(md5_digest));
 
 	if (!index->update_end(update)) {
 		/* failed - delete the record */
@@ -175,6 +156,10 @@
 		return FALSE;
 	}
 
+	/* save message flags */
+	rec->msg_flags = ctx.flags;
+	mail_index_mark_flag_changes(index, rec, 0, rec->msg_flags);
+
 	return TRUE;
 }
 

--- a/src/lib-index/mbox/mbox-fsck.c	Fri Sep 06 16:29:51 2002 +0300
+++ b/src/lib-index/mbox/mbox-fsck.c	Fri Sep 06 16:43:58 2002 +0300
@@ -4,6 +4,7 @@
 #include "iobuffer.h"
 #include "hex-binary.h"
 #include "message-parser.h"
+#include "message-part-serialize.h"
 #include "mbox-index.h"
 #include "mbox-lock.h"
 #include "mail-index-util.h"
@@ -31,20 +32,104 @@
 	}
 }
 
-static MailIndexRecord *
-match_next_record(MailIndex *index, MailIndexRecord *rec, unsigned int *seq,
-		  IOBuffer *inbuf)
+static int verify_header_md5sum(MailIndex *index, MailIndexRecord *rec,
+				unsigned char current_digest[16])
+{
+	const unsigned char *old_digest;
+	unsigned int size;
+
+	/* MD5 sums must match */
+	old_digest = index->lookup_field_raw(index, rec, FIELD_TYPE_MD5, &size);
+	return old_digest != NULL && size == 16 &&
+                memcmp(old_digest, current_digest, 16) == 0;
+}
+
+static int verify_end_of_body(IOBuffer *inbuf, uoff_t end_offset)
 {
+	unsigned char *data;
+	unsigned int size;
+
+	/* don't bother parsing the whole body, just make
+	   sure it ends properly */
+	io_buffer_seek(inbuf, end_offset);
+
+	if (inbuf->offset == inbuf->size) {
+		/* end of file. a bit unexpected though,
+		   since \n is missing. */
+		return TRUE;
+	}
+
+	/* read forward a bit */
+	if (io_buffer_read_data(inbuf, &data, &size, 6) <= 0)
+		return FALSE;
+
+	/* either there should be the next From-line,
+	   or [\r]\n at end of file */
+	if (size > 0 && data[0] == '\r') {
+		data++; size--;
+	}
+	if (size > 0) {
+		if (data[0] != '\n')
+			return FALSE;
+
+		data++; size--;
+	}
+
+	return size == 0 || (size >= 5 && strncmp(data, "From ", 5) == 0);
+}
+
+static int mail_update_header_size(MailIndex *index, MailIndexRecord *rec,
+				   MailIndexUpdate *update,
+				   MessageSize *hdr_size)
+{
+	const void *part_data;
+	void *part_data_copy;
+	unsigned int size;
+
+	/* update index record */
+	rec->header_size = hdr_size->physical_size;
+
+	if ((rec->cached_fields & FIELD_TYPE_MESSAGEPART) == 0)
+		return TRUE;
+
+	/* update FIELD_TYPE_MESSAGEPART */
+	part_data = index->lookup_field_raw(index, rec, FIELD_TYPE_MESSAGEPART,
+					    &size);
+	if (part_data == NULL) {
+		/* well, this wasn't expected but don't bother failing */
+		return TRUE;
+	}
+
+	/* copy & update the part data */
+	part_data_copy = t_malloc(size);
+	memcpy(part_data_copy, part_data, size);
+
+	if (!message_part_serialize_update_header(part_data_copy, size,
+						  hdr_size))
+		return FALSE;
+
+	index->update_field_raw(update, FIELD_TYPE_MESSAGEPART,
+				part_data_copy, size);
+	return TRUE;
+}
+
+static int match_next_record(MailIndex *index, MailIndexRecord *rec,
+			     unsigned int seq, IOBuffer *inbuf,
+			     MailIndexRecord **next_rec)
+{
+        MailIndexUpdate *update;
 	MessageSize hdr_size;
 	MboxHeaderContext ctx;
-	uoff_t body_offset;
-	unsigned char *data, current_digest[16], old_digest[16];
-	unsigned int size;
-	const char *md5sum;
+	uoff_t header_offset, body_offset, offset;
+	unsigned char current_digest[16];
+
+	*next_rec = NULL;
 
 	/* skip the From-line */
 	skip_line(inbuf);
 
+	header_offset = inbuf->offset;
+
 	/* get the MD5 sum of fixed headers and the current message flags
 	   in Status and X-Status fields */
         mbox_header_init_context(&ctx);
@@ -53,60 +138,51 @@
 
 	body_offset = inbuf->offset;
 	do {
-		do {
-			/* MD5 sums must match */
-			md5sum = index->lookup_field(index, rec,
-						     FIELD_TYPE_MD5);
-			if (md5sum == NULL || strlen(md5sum) != 32 ||
-			    hex_to_binary(md5sum, old_digest) <= 0)
-				break;
-
-			if (memcmp(old_digest, current_digest, 16) != 0)
-				break;
-
-			/* don't bother parsing the whole body, just make
-			   sure it ends properly */
-			io_buffer_seek(inbuf, body_offset + rec->body_size);
+		if (verify_header_md5sum(index, rec, current_digest) &&
+		    verify_end_of_body(inbuf, body_offset + rec->body_size)) {
+			/* valid message */
+			update = index->update_begin(index, rec);
 
-			if (inbuf->offset == inbuf->size) {
-				/* last message */
-			} else {
-				/* read forward a bit */
-				if (io_buffer_read_data(inbuf, &data,
-							&size, 6) <= 0)
-					break;
+			/* update flags, unless we've changed them */
+			if ((rec->index_flags & INDEX_MAIL_FLAG_DIRTY) == 0) {
+				if (!index->update_flags(index, rec, seq,
+							 ctx.flags, TRUE))
+					return FALSE;
 
-				/* either there should be the next From-line,
-				   or [\r]\n at end of file */
-				if (size > 0 && data[0] == '\r') {
-					data++; size--;
-				}
-				if (size > 0) {
-					if (data[0] != '\n')
-						break;
-
-					data++; size--;
-				}
-
-				if (size > 0 &&
-				    (size < 5 ||
-				     strncmp(data, "From ", 5) != 0))
-					break;
+				/* update_flags() sets dirty flag, remove it */
+				rec->index_flags &= ~INDEX_MAIL_FLAG_DIRTY;
 			}
 
-			/* valid message, update flags */
-			if ((rec->msg_flags & ctx.flags) != ctx.flags)
-				rec->msg_flags |= ctx.flags;
-			return rec;
-		} while (0);
+			/* update location */
+			if (!mbox_mail_get_start_offset(index, rec, &offset))
+				return FALSE;
+			if (offset != header_offset) {
+				index->update_field_raw(update,
+							FIELD_TYPE_LOCATION,
+							&header_offset,
+							sizeof(uoff_t));
+			}
+
+			/* update size */
+			if (rec->header_size != hdr_size.physical_size ) {
+				if (!mail_update_header_size(index, rec,
+							     update, &hdr_size))
+					return FALSE;
+			}
+
+			if (!index->update_end(update))
+				return FALSE;
+
+			*next_rec = rec;
+			break;
+		}
 
 		/* try next message */
-		(*seq)++;
-		(void)index->expunge(index, rec, *seq, TRUE);
+		(void)index->expunge(index, rec, seq, TRUE);
 		rec = index->next(index, rec);
 	} while (rec != NULL);
 
-	return NULL;
+	return TRUE;
 }
 
 static int mbox_index_fsck_buf(MailIndex *index, IOBuffer *inbuf)
@@ -157,7 +233,9 @@
 		if (inbuf->offset == inbuf->size)
 			break;
 
-		rec = match_next_record(index, rec, &seq, inbuf);
+		if (!match_next_record(index, rec, seq, inbuf, &rec))
+			return FALSE;
+
 		if (rec == NULL) {
 			/* Get back to line before From */
 			io_buffer_seek(inbuf, from_offset);
@@ -172,7 +250,6 @@
 	while (rec != NULL) {
 		(void)index->expunge(index, rec, seq, TRUE);
 
-		seq++;
 		rec = index->next(index, rec);
 	}
 

--- a/src/lib-index/mbox/mbox-index.c	Fri Sep 06 16:29:51 2002 +0300
+++ b/src/lib-index/mbox/mbox-index.c	Fri Sep 06 16:43:58 2002 +0300
@@ -2,14 +2,17 @@
 
 #include "lib.h"
 #include "iobuffer.h"
+#include "rfc822-tokenize.h"
 #include "mbox-index.h"
 #include "mail-index-util.h"
 
 static MailIndex mbox_index;
 
-void mbox_header_init_context(MboxHeaderContext *ctx)
+void mbox_header_init_context(MboxHeaderContext *ctx,
+			      const char *custom_flags[MAIL_CUSTOM_FLAGS_COUNT])
 {
 	memset(ctx, 0, sizeof(MboxHeaderContext));
+	memcpy(ctx->custom_flags, custom_flags, sizeof(ctx->custom_flags));
 	md5_init(&ctx->md5);
 }
 
@@ -28,13 +31,13 @@
 			flags |= MAIL_FLAGGED;
 			break;
 		case 'D':
-			flags |= MAIL_DELETED;
+			flags |= MAIL_DRAFT;
 			break;
 		case 'R':
 			flags |= MAIL_SEEN;
 			break;
 		case 'T':
-			flags |= MAIL_DRAFT;
+			flags |= MAIL_DELETED;
 			break;
 		}
 	}
@@ -42,6 +45,35 @@
 	return flags;
 }
 
+static void mbox_update_custom_flags(const char *value __attr_unused__,
+				     unsigned int len __attr_unused__,
+				     int index, void *context)
+{
+	MailFlags *flags = context;
+
+	if (index >= 0)
+		*flags |= 1 << (index + MAIL_CUSTOM_FLAG_1_BIT);
+}
+
+static MailFlags
+mbox_get_keyword_flags(const char *value, unsigned int len,
+		       const char *custom_flags[MAIL_CUSTOM_FLAGS_COUNT])
+{
+	MailFlags flags;
+
+	flags = 0;
+	mbox_keywords_parse(value, len, custom_flags,
+			    mbox_update_custom_flags, &flags);
+	return flags;
+}
+
+static void
+mbox_get_custom_flags_list(const char *value, unsigned int len,
+			   const char *custom_flags[MAIL_CUSTOM_FLAGS_COUNT])
+{
+	/* FIXME */
+}
+
 void mbox_header_func(MessagePart *part __attr_unused__,
 		      const char *name, unsigned int name_len,
 		      const char *value, unsigned int value_len,
@@ -97,11 +129,21 @@
 		   don't blindly trust this header alone as it could just as
 		   easily come from the remote. */
 		if (name_len == 13)
-			fixed = strncasecmp(name, "X-Delivery-ID:", 13);
+			fixed = strncasecmp(name, "X-Delivery-ID:", 13) == 0;
 		else if (name_len == 8 &&
 			 strncasecmp(name, "X-Status", 8) == 0) {
 			/* update message flags */
 			ctx->flags |= mbox_get_status_flags(value, value_len);
+		} else if (name_len == 10 &&
+			   strncasecmp(name, "X-Keywords", 10) == 0) {
+			/* update custom message flags */
+			ctx->flags |= mbox_get_keyword_flags(value, value_len,
+							     ctx->custom_flags);
+		} else if (name_len == 10 &&
+			   strncasecmp(name, "X-IMAPbase", 10) == 0) {
+			/* update list of custom message flags */
+			mbox_get_custom_flags_list(value, value_len,
+						   ctx->custom_flags);
 		}
 		break;
 	}
@@ -110,6 +152,53 @@
 		md5_update(&ctx->md5, value, value_len);
 }
 
+void mbox_keywords_parse(const char *value, unsigned int len,
+			 const char *custom_flags[MAIL_CUSTOM_FLAGS_COUNT],
+			 void (*func)(const char *, unsigned int, int, void *),
+			 void *context)
+{
+	unsigned int custom_len[MAIL_CUSTOM_FLAGS_COUNT];
+	unsigned int item_len;
+	int i;
+
+	for (i = 0; i < MAIL_CUSTOM_FLAGS_COUNT; i++) {
+		custom_len[i] = custom_flags[i] != NULL ?
+			strlen(custom_flags[i]) : 0;
+	}
+
+	for (;;) {
+		/* skip whitespace */
+		while (len > 0 && IS_LWSP(*value)) {
+			value++;
+			len--;
+		}
+
+		if (len == 0)
+			break;
+
+		/* find the length of the item */
+		for (item_len = 0; item_len < len; item_len++) {
+			if (IS_LWSP(value[item_len]))
+				break;
+		}
+
+		/* check if it's found */
+		for (i = 0; i < MAIL_CUSTOM_FLAGS_COUNT; i++) {
+			if (custom_len[i] == item_len &&
+			    strncasecmp(custom_flags[i], value, item_len) == 0)
+				break;
+		}
+
+		if (i == MAIL_CUSTOM_FLAGS_COUNT)
+			i = -1;
+
+		func(value, item_len, i, context);
+
+		value += item_len;
+		len -= item_len;
+	}
+}
+
 int mbox_skip_crlf(IOBuffer *inbuf)
 {
 	unsigned char *data;
@@ -140,6 +229,34 @@
 	return TRUE;
 }
 
+int mbox_mail_get_start_offset(MailIndex *index, MailIndexRecord *rec,
+			       uoff_t *offset)
+{
+	const uoff_t *location;
+	unsigned int size;
+
+	location = index->lookup_field_raw(index, rec,
+					   FIELD_TYPE_LOCATION, &size);
+	if (location == NULL) {
+		INDEX_MARK_CORRUPTED(index);
+		index_set_error(index, "Corrupted index file %s: "
+				"Missing location field for record %u",
+				index->filepath, rec->uid);
+		*offset = 0;
+		return FALSE;
+	} else if (size != sizeof(uoff_t) || *location > OFF_T_MAX) {
+		INDEX_MARK_CORRUPTED(index);
+		index_set_error(index, "Corrupted index file %s: "
+				"Invalid location field for record %u",
+				index->filepath, rec->uid);
+		*offset = 0;
+		return FALSE;
+	} else {
+		*offset = *location;
+		return TRUE;
+	}
+}
+
 MailIndex *mbox_index_alloc(const char *dir, const char *mbox_path)
 {
 	MailIndex *index;
@@ -168,6 +285,17 @@
 	i_free(index);
 }
 
+static int mbox_index_update_flags(MailIndex *index, MailIndexRecord *rec,
+				   unsigned int seq, MailFlags flags,
+				   int external_change)
+{
+	if (!mail_index_update_flags(index, rec, seq, flags, external_change))
+		return FALSE;
+
+	rec->index_flags |= INDEX_MAIL_FLAG_DIRTY;
+	return TRUE;
+}
+
 static MailIndex mbox_index = {
 	mail_index_open,
 	mail_index_open_or_create,
@@ -186,7 +314,7 @@
 	mail_index_get_sequence,
 	mbox_open_mail,
 	mail_index_expunge,
-	mail_index_update_flags,
+	mbox_index_update_flags,
 	mail_index_append,
 	mail_index_update_begin,
 	mail_index_update_end,

--- a/src/lib-index/mbox/mbox-index.h	Fri Sep 06 16:29:51 2002 +0300
+++ b/src/lib-index/mbox/mbox-index.h	Fri Sep 06 16:43:58 2002 +0300
@@ -8,16 +8,24 @@
 	MailFlags flags;
 	MD5Context md5;
 	int received;
+        const char *custom_flags[MAIL_CUSTOM_FLAGS_COUNT];
 } MboxHeaderContext;
 
 MailIndex *mbox_index_alloc(const char *dir, const char *mbox_path);
 
-void mbox_header_init_context(MboxHeaderContext *ctx);
+void mbox_header_init_context(MboxHeaderContext *ctx,
+			      const char*custom_flags[MAIL_CUSTOM_FLAGS_COUNT]);
 void mbox_header_func(MessagePart *part __attr_unused__,
 		      const char *name, unsigned int name_len,
 		      const char *value, unsigned int value_len,
 		      void *context);
+void mbox_keywords_parse(const char *value, unsigned int len,
+			 const char *custom_flags[MAIL_CUSTOM_FLAGS_COUNT],
+			 void (*func)(const char *, unsigned int, int, void *),
+			 void *context);
 int mbox_skip_crlf(IOBuffer *inbuf);
+int mbox_mail_get_start_offset(MailIndex *index, MailIndexRecord *rec,
+			       uoff_t *offset);
 
 int mbox_index_rebuild(MailIndex *index);
 int mbox_index_sync(MailIndex *index);
@@ -29,4 +37,7 @@
 time_t mbox_from_parse_date(const char *msg, unsigned int size);
 const char *mbox_from_create(const char *sender, time_t time);
 
+int mbox_index_rewrite(MailIndex *index,
+		       const char *custom_flags[MAIL_CUSTOM_FLAGS_COUNT]);
+
 #endif

--- a/src/lib-index/mbox/mbox-open.c	Fri Sep 06 16:29:51 2002 +0300
+++ b/src/lib-index/mbox/mbox-open.c	Fri Sep 06 16:43:58 2002 +0300
@@ -11,35 +11,16 @@
 
 IOBuffer *mbox_open_mail(MailIndex *index, MailIndexRecord *rec)
 {
-	const uoff_t *location;
 	uoff_t offset, stop_offset;
 	off_t pos;
-	unsigned int size;
 	char buf[7], *p;
 	int fd, ret, failed;
 
 	i_assert(index->lock_type != MAIL_LOCK_UNLOCK);
 
-	location = index->lookup_field_raw(index, rec, FIELD_TYPE_LOCATION,
-					   &size);
-	if (location == NULL) {
-                INDEX_MARK_CORRUPTED(index);
-		index_set_error(index, "Corrupted index file %s: "
-				"Missing location field for record %u",
-				index->filepath, rec->uid);
+	if (!mbox_mail_get_start_offset(index, rec, &offset))
 		return NULL;
-	}
 
-	/* location = offset to beginning of headers in message */
-	if (size != sizeof(uoff_t) || *location > OFF_T_MAX) {
-		INDEX_MARK_CORRUPTED(index);
-		index_set_error(index, "Corrupted index file %s: "
-				"Invalid location field for record %u",
-				index->filepath, rec->uid);
-		return NULL;
-	}
-
-	offset = *location;
 	stop_offset = offset + rec->header_size + rec->body_size;
 
 	fd = open(index->mbox_path, O_RDONLY);

--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/src/lib-index/mbox/mbox-rewrite.c	Fri Sep 06 16:43:58 2002 +0300
@@ -0,0 +1,376 @@
+/* Copyright (C) 2002 Timo Sirainen */
+
+#include "lib.h"
+#include "iobuffer.h"
+#include "temp-string.h"
+#include "write-full.h"
+#include "mbox-index.h"
+#include "mail-index-util.h"
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <fcntl.h>
+
+typedef struct {
+	IOBuffer *outbuf;
+	unsigned int size;
+	int failed;
+
+	unsigned int seq;
+	unsigned int msg_flags;
+        const char **custom_flags;
+
+	const char *status, *x_status, *x_keywords;
+	unsigned int uid_validity;
+	unsigned int uid_last;
+} MboxRewriteContext;
+
+/* Remove dirty flag from all messages */
+static void reset_dirty_flags(MailIndex *index)
+{
+	MailIndexRecord *rec;
+
+	rec = index->lookup(index, 1);
+	while (rec != NULL) {
+		rec->index_flags &= ~INDEX_MAIL_FLAG_DIRTY;
+		rec = index->next(index, rec);
+	}
+}
+
+static int mbox_write(MailIndex *index, IOBuffer *inbuf, IOBuffer *outbuf,
+		      uoff_t end_offset)
+{
+	i_assert(inbuf->offset <= end_offset);
+
+	if (io_buffer_send_iobuffer(outbuf, inbuf,
+				    end_offset - inbuf->offset) < 0)
+		return FALSE;
+
+	if (inbuf->offset < end_offset) {
+		/* fsck should have noticed it.. */
+		index_set_error(index, "Error rewriting mbox file %s: "
+				"Unexpected end of file", index->mbox_path);
+		return FALSE;
+	}
+
+	return TRUE;
+}
+
+static const char *strip_chars(const char *value, unsigned int value_len,
+			       const char *list)
+{
+	/* leave only unknown flags, very likely none */
+	char *ret, *p;
+	unsigned int i;
+
+	ret = p = t_buffer_get(value_len+1);
+	for (i = 0; i < value_len; i++) {
+		if (strchr(list, value[i]) == NULL)
+			*p++ = value[i];
+	}
+
+	if (ret == p)
+		return NULL;
+	*p = '\0';
+        t_buffer_alloc((unsigned int) (p-ret)+1);
+	return ret;
+}
+
+static void update_stripped_custom_flags(const char *value, unsigned int len,
+					 int index, void *context)
+{
+	TempString *str = context;
+
+	if (index < 0) {
+		/* not found, keep it */
+		if (str->len != 0)
+			t_string_append_c(str, ' ');
+		t_string_append_n(str, value, len);
+	}
+}
+
+static const char *strip_custom_flags(const char *value, unsigned int len,
+				      MboxRewriteContext *ctx)
+{
+	TempString *str;
+
+	str = t_string_new(len+1);
+	mbox_keywords_parse(value, len, ctx->custom_flags,
+			    update_stripped_custom_flags, str);
+	return str->str;
+}
+
+static void header_func(MessagePart *part __attr_unused__,
+			const char *name, unsigned int name_len,
+			const char *value, unsigned int value_len,
+			void *context)
+{
+	MboxRewriteContext *ctx = context;
+	char *end;
+
+	if (ctx->failed)
+		return;
+
+	if (name_len == 6 && strncasecmp(name, "Status", 6) == 0)
+		ctx->status = strip_chars(value, value_len, "RO");
+	else if (name_len == 8 && strncasecmp(name, "X-Status", 8) == 0)
+		ctx->x_status = strip_chars(value, value_len, "ADFT");
+	else if (name_len == 10 && strncasecmp(name, "X-Keywords", 10) == 0)
+		ctx->x_keywords = strip_custom_flags(value, value_len, ctx);
+	else if (name_len == 10 && strncasecmp(name, "X-IMAPbase", 10) == 0) {
+		if (ctx->seq == 1) {
+			/* temporarily copy the value to make sure we
+			   don't overflow it */
+			t_push();
+			value = t_strndup(value, value_len);
+			ctx->uid_validity = strtoul(value, &end, 10);
+			while (*end == ' ') end++;
+			ctx->uid_last = strtoul(end, &end, 10);
+			t_pop();
+		}
+	} else {
+		/* save this header */
+		(void)io_buffer_send(ctx->outbuf, name, name_len);
+		(void)io_buffer_send(ctx->outbuf, ": ", 2);
+		(void)io_buffer_send(ctx->outbuf, value, value_len);
+		(void)io_buffer_send(ctx->outbuf, "\n", 1);
+
+		if (ctx->outbuf->closed)
+			ctx->failed = TRUE;
+		ctx->size += name_len + 2 + value_len + 1;
+	}
+}
+
+static int mbox_write_header(MailIndex *index,
+			     MailIndexRecord *rec, unsigned int seq,
+			     IOBuffer *inbuf, IOBuffer *outbuf,
+			     uoff_t end_offset,
+			     const char *custom_flags[MAIL_CUSTOM_FLAGS_COUNT])
+{
+	/* We need to update fields that define message flags. Standard fields
+	   are stored in Status and X-Status. For custom flags we use
+	   uw-imapd compatible format, by first listing them in first message's
+	   X-IMAPbase field and actually defining them in X-Keywords field.
+
+	   Format of X-IMAPbase is: <UID validity> <last used UID> <flag names>
+
+	   We don't want to sync our UIDs with the mbox file, so the UID
+	   validity is always kept different from our internal UID validity.
+	   Last used UID is also not updated, and set to 0 initially.
+	*/
+	MboxRewriteContext ctx;
+	MessageSize hdr_size;
+	const char *str, *flags;
+	unsigned int field;
+	int i;
+
+	if (inbuf->offset >= end_offset) {
+		/* fsck should have noticed it.. */
+		index_set_error(index, "Error rewriting mbox file %s: "
+				"Unexpected end of file",
+				index->mbox_path);
+		return FALSE;
+	}
+
+	/* parse the header, write the fields we don't want to change */
+	memset(&ctx, 0, sizeof(ctx));
+	ctx.outbuf = outbuf;
+	ctx.seq = seq;
+	ctx.msg_flags = rec->msg_flags;
+	ctx.custom_flags = custom_flags;
+
+	t_push();
+
+	message_parse_header(NULL, inbuf, &hdr_size, header_func, &ctx);
+
+	i_assert(hdr_size.physical_size == rec->header_size);
+
+	/* append the flag fields */
+	if (seq == 1) {
+		/* write X-IMAPbase header to first message */
+		if (ctx.uid_validity == 0)
+			ctx.uid_validity = index->header->uid_validity-1;
+
+		str = t_strdup_printf("X-IMAPbase: %u %u",
+				      ctx.uid_validity, ctx.uid_last);
+		(void)io_buffer_send(outbuf, str, strlen(str));
+
+		for (i = 0; i < MAIL_CUSTOM_FLAGS_COUNT; i++) {
+			if (custom_flags[i] != NULL) {
+				(void)io_buffer_send(outbuf, " ", 1);
+				(void)io_buffer_send(outbuf, custom_flags[i],
+						     strlen(custom_flags[i]));
+			}
+		}
+		(void)io_buffer_send(outbuf, "\n", 1);
+	}
+
+	if ((rec->msg_flags & MAIL_CUSTOM_FLAGS_MASK) ||
+	    ctx.x_keywords != NULL) {
+		/* write X-Keywords header containing custom flags */
+		(void)io_buffer_send(outbuf, "X-Keywords:", 11);
+
+		field = 1 << MAIL_CUSTOM_FLAG_1_BIT;
+		for (i = 0; i < MAIL_CUSTOM_FLAGS_COUNT; i++, field <<= 1) {
+			if ((rec->msg_flags & field) &&
+			    custom_flags[i] != NULL) {
+				(void)io_buffer_send(outbuf, " ", 1);
+				(void)io_buffer_send(outbuf, custom_flags[i],
+						     strlen(custom_flags[i]));
+			}
+		}
+
+		if (ctx.x_keywords != NULL && ctx.x_keywords[0] != '\0') {
+			/* X-Keywords that aren't custom flags */
+			(void)io_buffer_send(outbuf, " ", 1);
+			(void)io_buffer_send(outbuf, ctx.x_keywords,
+					     strlen(ctx.x_keywords));
+		}
+		(void)io_buffer_send(outbuf, "\n", 1);
+	}
+
+	/* Status field */
+	flags = (rec->msg_flags & MAIL_SEEN) ? "Status: RO" : "Status: O";
+	flags = t_strconcat(flags, ctx.status, NULL);
+	(void)io_buffer_send(outbuf, flags, strlen(flags));
+	(void)io_buffer_send(outbuf, "\n", 1);
+
+	/* X-Status field */
+	if ((rec->msg_flags & (MAIL_SYSTEM_FLAGS_MASK^MAIL_SEEN)) != 0 ||
+	    ctx.x_status != NULL) {
+		flags = t_strconcat("X-Status: ",
+				    (rec->msg_flags & MAIL_ANSWERED) ? "A" : "",
+				    (rec->msg_flags & MAIL_DRAFT) ? "D" : "",
+				    (rec->msg_flags & MAIL_FLAGGED) ? "F" : "",
+				    (rec->msg_flags & MAIL_DELETED) ? "T" : "",
+				    ctx.x_status, NULL);
+		(void)io_buffer_send(outbuf, flags, strlen(flags));
+		(void)io_buffer_send(outbuf, "\n", 1);
+	}
+	t_pop();
+
+	/* empty line ends headers */
+	(void)io_buffer_send(outbuf, "\n", 1);
+
+	return TRUE;
+}
+
+int mbox_index_rewrite(MailIndex *index,
+		       const char *custom_flags[MAIL_CUSTOM_FLAGS_COUNT])
+{
+	/* Write it to temp file and then rename() to real file.
+	   easier and much safer than moving data inside the file.
+	   This rewriting relies quite a lot on valid header/body sizes
+	   which fsck() should have ensured. */
+	MailIndexRecord *rec;
+	IOBuffer *inbuf, *outbuf;
+	uoff_t offset;
+	const uoff_t *location;
+	const char *path;
+	unsigned int size, seq;
+	int in_fd, out_fd, failed;
+
+	i_assert(index->lock_type == MAIL_LOCK_EXCLUSIVE);
+
+	if (!mbox_index_fsck(index))
+		return FALSE;
+
+	if (index->header->messages_count == 0) {
+		/* no messages in mailbox, don't bother rewriting */
+		return TRUE;
+	}
+
+	in_fd = open(index->mbox_path, O_RDWR);
+	if (in_fd == -1) {
+		index_set_error(index, "Can't open mbox file %s: %m",
+				index->mbox_path);
+		return FALSE;
+	}
+	inbuf = io_buffer_create_mmap(in_fd, default_pool,
+				      MAIL_MMAP_BLOCK_SIZE, 0);
+
+	out_fd = mail_index_create_temp_file(index, &path);
+	if (out_fd == -1) {
+		(void)close(in_fd);
+		return FALSE;
+	}
+	outbuf = io_buffer_create_file(out_fd, default_pool, 8192);
+
+	failed = FALSE; seq = 1;
+	rec = index->lookup(index, 1);
+	while (rec != NULL) {
+		/* get offset to beginning of mail headers */
+		location = index->lookup_field_raw(index, rec,
+						   FIELD_TYPE_LOCATION, &size);
+		if (size != sizeof(uoff_t) || *location <= inbuf->offset) {
+			/* fsck should have fixed it */
+			index_set_error(index, "Error rewriting mbox file %s: "
+					"Invalid location field in index",
+					index->mbox_path);
+			failed = TRUE;
+			break;
+		}
+
+		offset = *location;
+		if (offset + rec->header_size + rec->body_size > inbuf->size) {
+			index_set_error(index, "Error rewriting mbox file %s: "
+					"Invalid message size in index",
+					index->mbox_path);
+			failed = TRUE;
+			break;
+		}
+
+		/* write the From-line */
+		if (!mbox_write(index, inbuf, outbuf, offset)) {
+			failed = TRUE;
+			break;
+		}
+
+		/* write header, updating flag fields */
+		offset += rec->header_size;
+		if (!mbox_write_header(index, rec, seq, inbuf, outbuf,
+				       offset, custom_flags)) {
+			failed = TRUE;
+			break;
+		}
+
+		/* write body */
+		offset += rec->body_size;
+		if (!mbox_write(index, inbuf, outbuf, offset)) {
+			failed = TRUE;
+			break;
+		}
+
+		seq++;
+		rec = index->next(index, rec);
+	}
+
+	/* always end with a \n */
+	(void)io_buffer_send(outbuf, "\n", 1);
+	if (outbuf->closed) {
+		errno = outbuf->buf_errno;
+		index_set_error(index, "Error rewriting mbox file %s: "
+				"write() failed: %m", index->mbox_path);
+		failed = TRUE;
+	}
+
+	if (!failed) {
+		if (rename(path, index->mbox_path) == 0) {
+			/* all ok, we need to fsck the index next time */
+			index->header->flags |= MAIL_INDEX_FLAG_FSCK;
+			reset_dirty_flags(index);
+		} else {
+			index_set_error(index, "rename(%s, %s) failed: %m",
+					path, index->mbox_path);
+			failed = TRUE;
+		}
+	}
+
+	(void)close(out_fd);
+	(void)close(in_fd);
+	(void)unlink(path);
+	io_buffer_destroy(outbuf);
+	io_buffer_destroy(inbuf);
+	return failed;
+}

--- a/src/lib-index/mbox/mbox-sync.c	Fri Sep 06 16:29:51 2002 +0300
+++ b/src/lib-index/mbox/mbox-sync.c	Fri Sep 06 16:43:58 2002 +0300
@@ -13,9 +13,7 @@
 static uoff_t get_indexed_mbox_size(MailIndex *index)
 {
 	MailIndexRecord *rec, *prev;
-	const uoff_t *location;
 	uoff_t offset;
-	unsigned int size;
 
 	if (index->lock_type == MAIL_LOCK_UNLOCK) {
 		if (!mail_index_set_lock(index, MAIL_LOCK_SHARED))
@@ -39,21 +37,8 @@
 	if (rec != NULL) {
 		/* get the offset + size of last message, which tells the
 		   last known mbox file size */
-		location = index->lookup_field_raw(index, rec,
-						   FIELD_TYPE_LOCATION, &size);
-		if (location == NULL) {
-			INDEX_MARK_CORRUPTED(index);
-			index_set_error(index, "Corrupted index file %s: "
-					"Missing location field for record %u",
-					index->filepath, rec->uid);
-		} else if (size != sizeof(uoff_t) || *location > OFF_T_MAX) {
-			INDEX_MARK_CORRUPTED(index);
-			index_set_error(index, "Corrupted index file %s: "
-					"Invalid location field for record %u",
-					index->filepath, rec->uid);
-		} else {
-			offset = *location + rec->header_size + rec->body_size;
-		}
+		if (mbox_mail_get_start_offset(index, rec, &offset))
+			offset += rec->header_size + rec->body_size;
 	}
 
 	if (index->lock_type == MAIL_LOCK_SHARED)
@@ -67,49 +52,6 @@
 	return offset;
 }
 
-static int mbox_check_new_mail(MailIndex *index)
-{
-	IOBuffer *inbuf;
-	off_t pos;
-	int fd, ret;
-
-	fd = open(index->mbox_path, O_RDONLY);
-	if (fd == -1) {
-		index_set_error(index, "Can't open mbox file %s: %m",
-				index->mbox_path);
-		return FALSE;
-	}
-
-	pos = lseek(fd, (off_t)index->mbox_size, SEEK_SET);
-	if (pos == -1) {
-		index_set_error(index, "lseek() failed with mbox file %s: %m",
-				index->mbox_path);
-		(void)close(fd);
-		return FALSE;
-	}
-
-	if ((uoff_t)pos != index->mbox_size) {
-		/* someone just shrinked the file? */
-		(void)close(fd);
-		return mbox_index_fsck(index);
-	}
-
-	/* add the new data */
-	inbuf = io_buffer_create_mmap(fd, default_pool,
-				      MAIL_MMAP_BLOCK_SIZE, 0);
-	ret = mbox_index_append(index, inbuf);
-	(void)close(fd);
-	io_buffer_destroy(inbuf);
-
-	if (index->set_flags & MAIL_INDEX_FLAG_FSCK) {
-		/* it wasn't just new mail, reread the mbox */
-		index->set_flags &= ~MAIL_INDEX_FLAG_FSCK;
-		return mbox_index_fsck(index);
-	}
-
-	return ret;
-}
-
 int mbox_index_sync(MailIndex *index)
 {
 	struct stat st;
@@ -123,10 +65,11 @@
 		return FALSE;
 	}
 
-	/* |3 is simple workaround for \n at end of file, see below */
 	filesize = st.st_size;
 	if (index->file_sync_stamp == st.st_mtime &&
-	    (index->mbox_size | 3) == (filesize | 3))
+	    (index->mbox_size == filesize ||
+	     index->mbox_size == filesize-1 ||
+             index->mbox_size == filesize-2))
 		return TRUE;
 
 	/* problem .. index->mbox_size points to data after the last message.
@@ -147,12 +90,6 @@
 
 	index->file_sync_stamp = st.st_mtime;
 
-	/* file has been modified. */
-	if (index->mbox_size < filesize) {
-		/* file was grown, hopefully just new mail */
-		return mbox_check_new_mail(index);
-	} else {
-		/* something changed, scan through the whole mbox */
-		return mbox_index_fsck(index);
-	}
+	/* file has changed, scan through the whole mbox */
+	return mbox_index_fsck(index);
 }

--- a/src/lib-storage/index/index-fetch.c	Fri Sep 06 16:29:51 2002 +0300
+++ b/src/lib-storage/index/index-fetch.c	Fri Sep 06 16:43:58 2002 +0300
@@ -55,8 +55,17 @@
 
 static void index_fetch_rfc822_size(MailIndexRecord *rec, FetchContext *ctx)
 {
-	t_string_printfa(ctx->str, " RFC822.SIZE %lu",
-			 (unsigned long) rec->full_virtual_size);
+	MessageSize hdr_size, body_size;
+
+	if (!imap_msgcache_get_rfc822(ctx->cache, NULL,
+				      &hdr_size, &body_size)) {
+		i_error("Couldn't get RFC822.SIZE for UID %u (index %s)",
+			rec->uid, ctx->index->filepath);
+		return;
+	}
+
+	t_string_printfa(ctx->str, " RFC822.SIZE %"UOFF_T_FORMAT,
+			 hdr_size.virtual_size + body_size.virtual_size);
 }
 
 static void index_fetch_flags(MailIndexRecord *rec, FetchContext *ctx)
@@ -97,9 +106,8 @@
 		return;
 	}
 
-	str = t_strdup_printf(" RFC822 {%lu}\r\n",
-			      (unsigned long) (hdr_size.virtual_size +
-					       body_size.virtual_size));
+	str = t_strdup_printf(" RFC822 {%"UOFF_T_FORMAT"}\r\n",
+			      hdr_size.virtual_size + body_size.virtual_size);
 	if (ctx->first) str++; else ctx->first = FALSE;
 	(void)io_buffer_send(ctx->outbuf, str, strlen(str));
 
@@ -120,8 +128,8 @@
 		return;
 	}
 
-	str = t_strdup_printf(" RFC822.HEADER {%lu}\r\n",
-			      (unsigned long) hdr_size.virtual_size);
+	str = t_strdup_printf(" RFC822.HEADER {%"UOFF_T_FORMAT"}\r\n",
+			      hdr_size.virtual_size);
 	if (ctx->first) str++; else ctx->first = FALSE;
 	(void)io_buffer_send(ctx->outbuf, str, strlen(str));
 	(void)message_send(ctx->outbuf, inbuf, &hdr_size, 0, (uoff_t)-1);
@@ -139,8 +147,8 @@
 		return;
 	}
 
-	str = t_strdup_printf(" RFC822.TEXT {%lu}\r\n",
-			      (unsigned long) body_size.virtual_size);
+	str = t_strdup_printf(" RFC822.TEXT {%"UOFF_T_FORMAT"}\r\n",
+			      body_size.virtual_size);
 	if (ctx->first) str++; else ctx->first = FALSE;
 	(void)io_buffer_send(ctx->outbuf, str, strlen(str));
 	(void)message_send(ctx->outbuf, inbuf, &body_size, 0, (uoff_t)-1);
@@ -182,6 +190,7 @@
 static void index_msgcache_open(FetchContext *ctx, MailIndexRecord *rec)
 {
 	ImapCacheField fields;
+	uoff_t virtual_header_size, virtual_body_size;
 	void *mail_cache_context;
 
 	fields = index_get_cache(ctx->fetch_data);
@@ -190,16 +199,16 @@
 
         mail_cache_context = index_msgcache_get_context(ctx->index, rec);
 
-	if (MSG_HAS_VALID_CRLF_DATA(rec)) {
-		imap_msgcache_open(ctx->cache, rec->uid, fields,
-				   rec->full_virtual_size,
-				   rec->header_size, rec->body_size,
-				   mail_cache_context);
-	} else {
-		imap_msgcache_open(ctx->cache, rec->uid, fields,
-				   rec->full_virtual_size, 0, 0,
-				   mail_cache_context);
-	}
+	virtual_header_size =
+		(rec->index_flags & INDEX_MAIL_FLAG_BINARY_HEADER) ?
+		rec->header_size : 0;
+	virtual_body_size =
+		(rec->index_flags & INDEX_MAIL_FLAG_BINARY_BODY) ?
+		rec->body_size : 0;
+
+	imap_msgcache_open(ctx->cache, rec->uid, fields,
+			   virtual_header_size, virtual_body_size,
+			   mail_cache_context);
 }
 
 static int index_fetch_mail(MailIndex *index __attr_unused__,

--- a/src/lib-storage/index/index-msgcache.c	Fri Sep 06 16:29:51 2002 +0300
+++ b/src/lib-storage/index/index-msgcache.c	Fri Sep 06 16:43:58 2002 +0300
@@ -71,14 +71,24 @@
 static MessagePart *index_msgcache_get_cached_parts(Pool pool, void *context)
 {
 	IndexMsgcacheContext *ctx = context;
+	MessagePart *part;
 	const void *part_data;
 	unsigned int part_size;
 
 	part_data = ctx->index->lookup_field_raw(ctx->index, ctx->rec,
 						 FIELD_TYPE_MESSAGEPART,
 						 &part_size);
-	return part_data == NULL ? NULL :
-		message_part_deserialize(pool, part_data, part_size);
+	if (part_data == NULL)
+		return NULL;
+
+	part = message_part_deserialize(pool, part_data, part_size);
+	if (part == NULL) {
+		i_error("Error in index file %s: Corrupted cached "
+			"MessagePart data", ctx->index->filepath);
+		return NULL;
+	}
+
+	return part;
 }
 
 ImapMessageCacheIface index_msgcache_iface = {

--- a/src/lib-storage/index/index-save.c	Fri Sep 06 16:29:51 2002 +0300
+++ b/src/lib-storage/index/index-save.c	Fri Sep 06 16:43:58 2002 +0300
@@ -47,6 +47,8 @@
 	last_cr = FALSE;
 
 	while (data_size > 0) {
+		/* FIXME: we're using nonblocking I/O, meaning if there's no
+		   data we'll eat all CPU! */
 		ret = io_buffer_read(buf);
 		if (ret < 0) {
 			mail_storage_set_critical(storage,

--- a/src/lib-storage/index/index-search.c	Fri Sep 06 16:29:51 2002 +0300
+++ b/src/lib-storage/index/index-search.c	Fri Sep 06 16:43:58 2002 +0300
@@ -6,6 +6,7 @@
 #include "rfc822-tokenize.h"
 #include "imap-date.h"
 #include "index-storage.h"
+#include "mail-index-util.h"
 #include "mail-search.h"
 
 #include <stdlib.h>
@@ -110,6 +111,7 @@
 				  const char *value)
 {
 	time_t t;
+	uoff_t size;
 
 	switch (type) {
 	case SEARCH_ALL:
@@ -164,11 +166,15 @@
 			return FALSE;
 		return rec->sent_date >= t;
 
-	/* sizes */
+	/* sizes, only with fastscanning */
 	case SEARCH_SMALLER:
-		return rec->full_virtual_size < str_to_uoff_t(value);
+		if (!mail_index_get_virtual_size(ibox->index, rec, TRUE, &size))
+			return -1;
+		return size < str_to_uoff_t(value);
 	case SEARCH_LARGER:
-		return rec->full_virtual_size > str_to_uoff_t(value);
+		if (!mail_index_get_virtual_size(ibox->index, rec, TRUE, &size))
+			return -1;
+		return size > str_to_uoff_t(value);
 
 	default:
 		return -1;
@@ -252,6 +258,46 @@
 	}
 }
 
+/* Returns >0 = matched, 0 = not matched, -1 = unknown */
+static int search_arg_match_slow(MailIndex *index, MailIndexRecord *rec,
+				 MailSearchArgType type, const char *value)
+{
+	uoff_t size;
+
+	switch (type) {
+	/* sizes, only with fastscanning */
+	case SEARCH_SMALLER:
+		if (!mail_index_get_virtual_size(index, rec, FALSE, &size))
+			return -1;
+		return size < str_to_uoff_t(value);
+	case SEARCH_LARGER:
+		if (!mail_index_get_virtual_size(index, rec, FALSE, &size))
+			return -1;
+		return size > str_to_uoff_t(value);
+
+	default:
+		return -1;
+	}
+}
+
+static void search_slow_arg(MailSearchArg *arg, void *context)
+{
+	SearchIndexContext *ctx = context;
+
+	switch (search_arg_match_slow(ctx->ibox->index, ctx->rec,
+				      arg->type, arg->value.str)) {
+	case -1:
+		/* unknown */
+		break;
+	case 0:
+		ARG_SET_RESULT(arg, -1);
+		break;
+	default:
+		ARG_SET_RESULT(arg, 1);
+		break;
+	}
+}
+
 /* needle must be uppercased */
 static int header_value_match(const char *haystack, unsigned int haystack_len,
 			      const char *needle)
@@ -633,6 +679,7 @@
 
 		mail_search_args_foreach(args, search_index_arg, &ctx);
 		mail_search_args_foreach(args, search_cached_arg, &ctx);
+		mail_search_args_foreach(args, search_slow_arg, &ctx);
 
 		if (search_arg_match_text(ibox, rec, args) &&
 		    args->result == 1) {

--- a/src/lib-storage/index/mbox/mbox-storage.c	Fri Sep 06 16:29:51 2002 +0300
+++ b/src/lib-storage/index/mbox/mbox-storage.c	Fri Sep 06 16:43:58 2002 +0300
@@ -362,6 +362,24 @@
 	}
 }
 
+static void mbox_storage_close(Mailbox *box)
+{
+	IndexMailbox *ibox = (IndexMailbox *) box;
+
+	if (!ibox->index->set_lock(ibox->index, MAIL_LOCK_EXCLUSIVE))
+		mail_storage_set_index_error(ibox);
+	else {
+		/* update flags by rewrite mbox file */
+		mbox_index_rewrite(ibox->index,
+				   flags_file_list_get(ibox->flagsfile));
+		flags_file_list_unref(ibox->flagsfile);
+
+		(void)ibox->index->set_lock(ibox->index, MAIL_LOCK_UNLOCK);
+	}
+
+	index_storage_close(box);
+}
+
 MailStorage mbox_storage = {
 	"mbox", /* name */
 
@@ -389,7 +407,7 @@
 	NULL, /* name */
 	NULL, /* storage */
 
-	index_storage_close,
+	mbox_storage_close,
 	index_storage_get_status,
 	index_storage_sync,
 	index_storage_expunge,