--- a/src/lib-index/mail-cache.c	Fri Aug 29 17:31:06 2003 +0300
+++ b/src/lib-index/mail-cache.c	Wed Sep 03 01:33:33 2003 +0300
@@ -109,8 +109,9 @@
 	sizeof(struct mail_sent_date),
 	sizeof(time_t),
 	sizeof(uoff_t),
+	sizeof(uoff_t),
 
-	0, 0, 0, 0, 0, 0,
+	0, 0, 0, 0, 0,
 
 	/* variable sized */
 	(unsigned int)-1, (unsigned int)-1, (unsigned int)-1, (unsigned int)-1,
@@ -1761,6 +1762,26 @@
 	return TRUE;
 }
 
+int mail_cache_update_location_offset(struct mail_cache *cache,
+				      struct mail_index_record *rec,
+				      uoff_t offset)
+{
+	void *data;
+	size_t size;
+
+	i_assert(cache->locks > 0);
+
+	if (!cache_lookup_field(cache, rec, MAIL_CACHE_LOCATION_OFFSET,
+				&data, &size)) {
+		mail_cache_set_corrupted(cache,
+			"Missing location offset for record %u", rec->uid);
+		return FALSE;
+	}
+
+	memcpy(data, &offset, sizeof(offset));
+	return TRUE;
+}
+
 void *mail_cache_get_mmaped(struct mail_cache *cache, size_t *size)
 {
 	if (!mmap_update(cache, 0, 0))

--- a/src/lib-index/mail-cache.h	Fri Aug 29 17:31:06 2003 +0300
+++ b/src/lib-index/mail-cache.h	Wed Sep 03 01:33:33 2003 +0300
@@ -17,6 +17,7 @@
 	MAIL_CACHE_SENT_DATE		= 0x00000008,
 	MAIL_CACHE_RECEIVED_DATE	= 0x00000010,
 	MAIL_CACHE_VIRTUAL_FULL_SIZE	= 0x00000020,
+	MAIL_CACHE_PHYSICAL_BODY_SIZE	= 0x00000040,
 
 	/* variable sized field */
 	MAIL_CACHE_HEADERS1		= 0x40000000,
@@ -34,7 +35,8 @@
 					  MAIL_CACHE_MD5 |
 					  MAIL_CACHE_SENT_DATE |
 					  MAIL_CACHE_RECEIVED_DATE |
-					  MAIL_CACHE_VIRTUAL_FULL_SIZE,
+					  MAIL_CACHE_VIRTUAL_FULL_SIZE |
+					  MAIL_CACHE_PHYSICAL_BODY_SIZE,
 	MAIL_CACHE_HEADERS_MASK		= MAIL_CACHE_HEADERS1 |
 					  MAIL_CACHE_HEADERS2 |
 					  MAIL_CACHE_HEADERS3 |
@@ -157,6 +159,12 @@
 				  struct mail_index_record *rec,
 				  enum mail_index_record_flag flags);
 
+/* Update location offset. External locking is assumed to take care of locking
+   readers out to prevent race conditions. */
+int mail_cache_update_location_offset(struct mail_cache *cache,
+				      struct mail_index_record *rec,
+				      uoff_t offset);
+
 /* Return the whole file mmaped. */
 void *mail_cache_get_mmaped(struct mail_cache *cache, size_t *size);
 

--- a/src/lib-index/mail-index.c	Fri Aug 29 17:31:06 2003 +0300
+++ b/src/lib-index/mail-index.c	Wed Sep 03 01:33:33 2003 +0300
@@ -76,6 +76,7 @@
 
 	index->sync_id = hdr->sync_id;
 	index->sync_stamp = hdr->sync_stamp;
+	index->sync_size = hdr->sync_size;
 	index->mmap_used_length = hdr->used_file_size;
 	return TRUE;
 }
@@ -440,8 +441,10 @@
 		keep_fsck = (index->set_flags & MAIL_INDEX_HDR_FLAG_FSCK) != 0;
 		mail_index_update_header_changes(index);
 
-		if (index->sync_dirty_stamp == 0)
+		if (index->sync_dirty_stamp == 0) {
 			index->header->sync_stamp = index->sync_stamp;
+			index->header->sync_size = index->sync_size;
+		}
 
 		/* remove the FSCK flag only after successful fsync() */
 		if (mail_index_sync_file(index) && !keep_fsck) {

--- a/src/lib-index/mail-index.h	Fri Aug 29 17:31:06 2003 +0300
+++ b/src/lib-index/mail-index.h	Wed Sep 03 01:33:33 2003 +0300
@@ -134,6 +134,7 @@
 	uint32_t first_unseen_uid_lowwater;
 	uint32_t first_deleted_uid_lowwater;
 
+	uint64_t sync_size;
 	uint32_t sync_stamp;
 };
 
@@ -298,7 +299,6 @@
 	unsigned int mbox_sync_counter;
 
 	/* last mbox sync: */
-	uoff_t mbox_size;
 	dev_t mbox_dev;
 	ino_t mbox_ino;
 
@@ -320,6 +320,7 @@
 
         enum mail_lock_type lock_type;
 	time_t sync_stamp, sync_dirty_stamp;
+	uoff_t sync_size;
 	time_t next_dirty_flags_flush;
 	unsigned int first_recent_uid;
 

--- a/src/lib-index/mbox/Makefile.am	Fri Aug 29 17:31:06 2003 +0300
+++ b/src/lib-index/mbox/Makefile.am	Wed Sep 03 01:33:33 2003 +0300
@@ -7,6 +7,7 @@
 	-I$(top_srcdir)/src/lib-index
 
 libindex_mbox_a_SOURCES = \
+	istream-mbox.c \
 	mbox-append.c \
 	mbox-from.c \
 	mbox-index.c \

--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/src/lib-index/mbox/istream-mbox.c	Wed Sep 03 01:33:33 2003 +0300
@@ -0,0 +1,136 @@
+/* Copyright (C) 2003 Timo Sirainen */
+
+#include "lib.h"
+#include "buffer.h"
+#include "message-parser.h"
+#include "istream-internal.h"
+#include "mbox-index.h"
+
+struct mbox_istream {
+	struct _istream istream;
+
+	struct istream *input;
+
+	buffer_t *headers;
+	uoff_t body_offset, body_size;
+	struct message_size header_size;
+};
+
+static void _close(struct _iostream *stream __attr_unused__)
+{
+}
+
+static void _destroy(struct _iostream *stream)
+{
+	struct mbox_istream *mstream = (struct mbox_istream *) stream;
+
+	i_stream_unref(mstream->input);
+	buffer_free(mstream->headers);
+}
+
+static void _set_max_buffer_size(struct _iostream *stream, size_t max_size)
+{
+	struct mbox_istream *mstream = (struct mbox_istream *) stream;
+
+	i_stream_set_max_buffer_size(mstream->input, max_size);
+}
+
+static void _set_blocking(struct _iostream *stream, int timeout_msecs,
+			  void (*timeout_cb)(void *), void *context)
+{
+	struct mbox_istream *mstream = (struct mbox_istream *) stream;
+
+	i_stream_set_blocking(mstream->input, timeout_msecs,
+			      timeout_cb, context);
+}
+
+static ssize_t _read(struct _istream *stream)
+{
+	struct mbox_istream *mstream = (struct mbox_istream *) stream;
+	ssize_t ret;
+
+	if (stream->istream.v_offset < mstream->header_size.virtual_size) {
+		/* we don't support mixing headers and body.
+		   it shouldn't be needed. */
+		return -2;
+	}
+
+	if (mstream->input->v_offset - mstream->header_size.physical_size !=
+	    stream->istream.v_offset - mstream->header_size.virtual_size) {
+		i_stream_seek(mstream->input, stream->istream.v_offset -
+			      mstream->header_size.virtual_size +
+			      mstream->header_size.physical_size);
+	}
+
+	ret = i_stream_read(mstream->input);
+	mstream->istream.skip = 0;
+	mstream->istream.buffer =
+		i_stream_get_data(mstream->input, &mstream->istream.pos);
+	return ret;
+}
+
+static void _seek(struct _istream *stream, uoff_t v_offset)
+{
+	struct mbox_istream *mstream = (struct mbox_istream *) stream;
+
+	stream->istream.v_offset = v_offset;
+	if (v_offset < mstream->header_size.virtual_size) {
+		/* still in headers */
+		stream->skip = v_offset;
+		stream->pos = mstream->header_size.virtual_size;
+		stream->buffer = buffer_get_data(mstream->headers, NULL);
+	} else {
+		/* body - use our real input stream */
+		stream->skip = stream->pos = 0;
+		stream->buffer = NULL;
+
+		v_offset += mstream->header_size.physical_size -
+			mstream->header_size.virtual_size;
+		i_stream_seek(mstream->input, v_offset);
+	}
+}
+
+static void _skip(struct _istream *stream, uoff_t count)
+{
+	i_stream_seek(&stream->istream, stream->istream.v_offset + count);
+}
+
+struct istream *i_stream_create_mbox(pool_t pool, struct istream *input,
+				     uoff_t body_size)
+{
+	struct mbox_istream *mstream;
+
+	mstream = p_new(pool, struct mbox_istream, 1);
+	mstream->input = input;
+	mstream->body_size = body_size;
+
+	if (body_size == 0) {
+		/* possibly broken message, find the next From-line
+		   and make sure header parser won't pass it. */
+		mbox_skip_header(input);
+		i_stream_set_read_limit(input, input->v_offset);
+		i_stream_seek(input, 0);
+	}
+
+	mstream->headers = buffer_create_dynamic(default_pool,
+						 8192, (size_t)-1);
+	mbox_hide_headers(input, mstream->headers,
+			  &mstream->header_size);
+	mstream->body_offset = input->v_offset;
+	i_stream_set_read_limit(input, mstream->body_offset + body_size);
+
+	mstream->istream.buffer = buffer_get_data(mstream->headers, NULL);
+	mstream->istream.pos = mstream->header_size.virtual_size;
+
+	mstream->istream.iostream.close = _close;
+	mstream->istream.iostream.destroy = _destroy;
+	mstream->istream.iostream.set_max_buffer_size = _set_max_buffer_size;
+	mstream->istream.iostream.set_blocking = _set_blocking;
+
+	mstream->istream.read = _read;
+	mstream->istream.skip_count = _skip;
+	mstream->istream.seek = _seek;
+
+	return _i_stream_create(&mstream->istream, pool, -1, 0,
+				mstream->body_offset + body_size);
+}

--- a/src/lib-index/mbox/mbox-append.c	Fri Aug 29 17:31:06 2003 +0300
+++ b/src/lib-index/mbox/mbox-append.c	Wed Sep 03 01:33:33 2003 +0300
@@ -10,10 +10,10 @@
 #include "mail-cache.h"
 
 static int mbox_index_append_next(struct mail_index *index,
-                                  struct mail_index_record *rec,
 				  struct mail_cache_transaction_ctx *trans_ctx,
 				  struct istream *input)
 {
+	struct mail_index_record *rec;
         struct mbox_header_context ctx;
 	enum mail_index_record_flag index_flags;
 	time_t received_date;
@@ -21,7 +21,7 @@
 	const unsigned char *data;
 	unsigned char md5_digest[16];
 	size_t size, pos;
-	int dirty;
+	int dirty, save_md5 = FALSE;
 
 	/* get the From-line */
 	pos = 0;
@@ -61,15 +61,6 @@
 
 	index_flags = 0;
 
-	if (!mail_cache_add(trans_ctx, rec, MAIL_CACHE_RECEIVED_DATE,
-			    &received_date, sizeof(received_date)))
-		return -1;
-
-	/* location offset = beginning of headers in message */
-	if (!mail_cache_add(trans_ctx, rec, MAIL_CACHE_LOCATION_OFFSET,
-			    &abs_start_offset, sizeof(abs_start_offset)))
-		return -1;
-
 	/* parse the header and cache wanted fields. get the message flags
 	   from Status and X-Status fields. temporarily limit the stream length
 	   so the message body is parsed properly.
@@ -81,9 +72,9 @@
         ctx.set_read_limit = TRUE;
 
 	i_stream_seek(input, abs_start_offset - input->start_offset);
+	i_stream_set_read_limit(input, eoh_offset);
 
-	i_stream_set_read_limit(input, eoh_offset);
-	//FIXME:mail_index_update_headers(update, input, 0, mbox_header_cb, &ctx);
+	message_parse_header(NULL, input, NULL, mbox_header_cb, &ctx);
 
 	i_stream_seek(input, input->v_limit);
 	i_stream_set_read_limit(input, 0);
@@ -117,11 +108,7 @@
 		dirty = TRUE;
 	} else {
 		/* save MD5 */
-		md5_final(&ctx.md5, md5_digest);
-
-		if (!mail_cache_add(trans_ctx, rec, MAIL_CACHE_MD5,
-				    md5_digest, sizeof(md5_digest)))
-			return -1;
+                save_md5 = TRUE;
 	}
 
 	if (dirty && !index->mailbox_readonly) {
@@ -134,6 +121,11 @@
 		index_flags |= MAIL_INDEX_FLAG_DIRTY;
 	}
 
+	/* add message to index */
+	rec = index->append(index);
+	if (rec == NULL)
+		return -1;
+
 	/* save message flags */
 	rec->msg_flags = ctx.flags;
 	mail_index_mark_flag_changes(index, rec, 0, rec->msg_flags);
@@ -142,13 +134,33 @@
 			    &index_flags, sizeof(index_flags)))
 		return -1;
 
+	/* location offset = beginning of headers in message */
+	if (!mail_cache_add(trans_ctx, rec, MAIL_CACHE_LOCATION_OFFSET,
+			    &abs_start_offset, sizeof(abs_start_offset)))
+		return -1;
+
+	if (!mail_cache_add(trans_ctx, rec, MAIL_CACHE_RECEIVED_DATE,
+			    &received_date, sizeof(received_date)))
+		return -1;
+
+	if (!mail_cache_add(trans_ctx, rec, MAIL_CACHE_PHYSICAL_BODY_SIZE,
+			    &ctx.content_length, sizeof(ctx.content_length)))
+		return -1;
+
+	if (save_md5) {
+		md5_final(&ctx.md5, md5_digest);
+
+		if (!mail_cache_add(trans_ctx, rec, MAIL_CACHE_MD5,
+				    md5_digest, sizeof(md5_digest)))
+			return -1;
+	}
+
 	return 1;
 }
 
 int mbox_index_append_stream(struct mail_index *index, struct istream *input)
 {
 	struct mail_cache_transaction_ctx *trans_ctx;
-	struct mail_index_record *rec;
 	uoff_t offset;
 	int ret;
 
@@ -185,15 +197,8 @@
 			break;
 		}
 
-		/* add message to index */
-		rec = index->append(index);
-		if (rec == NULL) {
-			ret = -1;
-			break;
-		}
-
 		t_push();
-		ret = mbox_index_append_next(index, rec, trans_ctx, input);
+		ret = mbox_index_append_next(index, trans_ctx, input);
 		t_pop();
 
 		if (ret == 0) {

--- a/src/lib-index/mbox/mbox-index.c	Fri Aug 29 17:31:06 2003 +0300
+++ b/src/lib-index/mbox/mbox-index.c	Wed Sep 03 01:33:33 2003 +0300
@@ -689,9 +689,9 @@
 
 int mbox_mail_get_location(struct mail_index *index,
 			   struct mail_index_record *rec,
-			   uoff_t *offset, uoff_t *hdr_size, uoff_t *body_size)
+			   uoff_t *offset, uoff_t *body_size)
 {
-	struct message_size _hdr_size, _body_size;
+	struct message_size _body_size;
 	const void *data;
 	size_t size;
 
@@ -706,24 +706,28 @@
 		}
 	}
 
-	if (hdr_size != NULL || body_size != NULL) {
+	if (body_size != NULL) {
+		if (mail_cache_copy_fixed_field(index->cache, rec,
+						MAIL_CACHE_PHYSICAL_BODY_SIZE,
+						body_size, sizeof(uoff_t)))
+			return TRUE;
+
 		if (!mail_cache_lookup_field(index->cache, rec,
 					     MAIL_CACHE_MESSAGEPART,
 					     &data, &size)) {
 			mail_cache_set_corrupted(index->cache,
-				"Missing message_part for record %u", rec->uid);
+				"No cached body_size or message_part for "
+				"record %u", rec->uid);
 			return FALSE;
 		}
 		if (!message_part_deserialize_size(data, size,
-						   &_hdr_size, &_body_size)) {
+						   NULL, &_body_size)) {
 			mail_cache_set_corrupted(index->cache,
 				"Corrupted message_part for record %u",
 				rec->uid);
 			return FALSE;
 		}
 
-		if (hdr_size != NULL)
-			*hdr_size = _hdr_size.physical_size;
 		if (body_size != NULL)
 			*body_size = _body_size.physical_size;
 	}
@@ -731,6 +735,53 @@
 	return TRUE;
 }
 
+void mbox_hide_headers(struct istream *input, buffer_t *dest,
+		       struct message_size *hdr_size)
+{
+	struct message_header_parser_ctx *hdr_ctx;
+	struct message_header_line *hdr;
+	uoff_t virtual_size = 0;
+
+	hdr_ctx = message_parse_header_init(input, hdr_size);
+	while ((hdr = message_parse_header_next(hdr_ctx)) != NULL) {
+		if (hdr->eoh) {
+			if (dest != NULL)
+				buffer_append(dest, "\r\n", 2);
+			else
+				virtual_size += 2;
+			break;
+		}
+
+		if ((*hdr->name == 'X' &&
+		     (strcasecmp(hdr->name, "X-UID") == 0 ||
+		      strcasecmp(hdr->name, "X-IMAPbase") == 0 ||
+		      strcasecmp(hdr->name, "X-Status") == 0 ||
+		      strcasecmp(hdr->name, "X-Keywords") == 0)) ||
+		    strcasecmp(hdr->name, "Content-Length") == 0 ||
+		    strcasecmp(hdr->name, "Status") == 0) {
+			/* ignore */
+		} else if (dest != NULL) {
+			if (!hdr->continued) {
+				buffer_append(dest, hdr->name, hdr->name_len);
+				buffer_append(dest, ": ", 2);
+			}
+			buffer_append(dest, hdr->value, hdr->value_len);
+			buffer_append(dest, "\r\n", 2);
+		} else {
+			if (!hdr->continued)
+				virtual_size += hdr->name_len + 2;
+			virtual_size += hdr->value_len + 2;
+		}
+	}
+	message_parse_header_deinit(hdr_ctx);
+
+	if (dest != NULL)
+		virtual_size = buffer_get_used_size(dest);
+
+	hdr_size->virtual_size = virtual_size;
+	hdr_size->lines = 0;
+}
+
 struct mail_index *
 mbox_index_alloc(const char *mbox_path, const char *index_dir,
 		 const char *control_dir)

--- a/src/lib-index/mbox/mbox-index.h	Fri Aug 29 17:31:06 2003 +0300
+++ b/src/lib-index/mbox/mbox-index.h	Wed Sep 03 01:33:33 2003 +0300
@@ -49,7 +49,9 @@
 int mbox_verify_end_of_body(struct istream *input, uoff_t end_offset);
 int mbox_mail_get_location(struct mail_index *index,
 			   struct mail_index_record *rec,
-			   uoff_t *offset, uoff_t *hdr_size, uoff_t *full_size);
+			   uoff_t *offset, uoff_t *body_size);
+void mbox_hide_headers(struct istream *input, buffer_t *dest,
+		       struct message_size *hdr_size);
 
 struct mail_index *
 mbox_index_alloc(const char *mbox_path, const char *index_dir,
@@ -68,4 +70,7 @@
 
 int mbox_index_rewrite(struct mail_index *index);
 
+struct istream *i_stream_create_mbox(pool_t pool, struct istream *input,
+				     uoff_t body_size);
+
 #endif

--- a/src/lib-index/mbox/mbox-open.c	Fri Aug 29 17:31:06 2003 +0300
+++ b/src/lib-index/mbox/mbox-open.c	Wed Sep 03 01:33:33 2003 +0300
@@ -14,7 +14,7 @@
 			       time_t *received_date, int *deleted)
 {
 	struct istream *input;
-	uoff_t offset, hdr_size, body_size;
+	uoff_t offset, body_size;
 
 	i_assert(index->lock_type != MAIL_LOCK_UNLOCK);
 
@@ -24,7 +24,7 @@
 	if (index->inconsistent)
 		return NULL;
 
-	if (!mbox_mail_get_location(index, rec, &offset, &hdr_size, &body_size))
+	if (!mbox_mail_get_location(index, rec, &offset, &body_size))
 		return NULL;
 
 	input = mbox_get_stream(index, offset, MAIL_LOCK_SHARED);
@@ -36,6 +36,5 @@
 
 	i_assert(index->mbox_sync_counter == index->mbox_lock_counter);
 
-	i_stream_set_read_limit(input, hdr_size + body_size);
-	return input;
+	return i_stream_create_mbox(default_pool, input, body_size);
 }

--- a/src/lib-index/mbox/mbox-rewrite.c	Fri Aug 29 17:31:06 2003 +0300
+++ b/src/lib-index/mbox/mbox-rewrite.c	Wed Sep 03 01:33:33 2003 +0300
@@ -382,8 +382,8 @@
 static int mbox_write_header(struct mail_index *index,
 			     struct mail_index_record *rec, unsigned int seq,
 			     struct istream *input, struct ostream *output,
-			     uoff_t end_offset, uoff_t wanted_offset,
-			     uoff_t hdr_size, uoff_t body_size)
+			     uoff_t end_offset,
+			     uoff_t *hdr_input_size, uoff_t body_size)
 {
 	/* We need to update fields that define message flags. Standard fields
 	   are stored in Status and X-Status. For custom flags we use
@@ -399,7 +399,8 @@
 	struct mbox_rewrite_context ctx;
 	struct message_header_parser_ctx *hdr_ctx;
 	struct message_header_line *hdr;
-	struct message_size hdr_parsed_size;
+	struct message_size hdr_size;
+	uoff_t offset;
 	int force_filler;
 
 	if (input->v_offset >= end_offset) {
@@ -422,20 +423,26 @@
 	ctx.uid_last = index->header->next_uid-1;
 	ctx.custom_flags = mail_custom_flags_list_get(index->custom_flags);
 
-	i_stream_set_read_limit(input, input->v_offset + hdr_size);
+	if (body_size == 0) {
+		/* possibly broken message, find the next From-line
+		   and make sure header parser won't pass it. */
+		offset = input->v_offset;
+		mbox_skip_header(input);
+		i_stream_set_read_limit(input, input->v_offset);
+		i_stream_seek(input, offset);
+	}
 
-	hdr_ctx = message_parse_header_init(input, &hdr_parsed_size);
+	hdr_ctx = message_parse_header_init(input, &hdr_size);
 	while ((hdr = message_parse_header_next(hdr_ctx)) != NULL) {
 		t_push();
 		write_header(&ctx, hdr);
 		t_pop();
 	}
 	message_parse_header_deinit(hdr_ctx);
+	*hdr_input_size = hdr_size.physical_size;
 
 	i_stream_set_read_limit(input, 0);
 
-	i_assert(hdr_parsed_size.physical_size == hdr_size);
-
 	/* append the flag fields */
 	if (seq == 1 && !ctx.ximapbase_found) {
 		/* write X-IMAPbase header to first message */
@@ -454,8 +461,8 @@
 
 	/* write the x-keywords header last so it can fill the extra space
 	   with spaces. -1 is for ending \n. */
-	(void)mbox_write_xkeywords(&ctx, ctx.x_keywords,
-				   wanted_offset - 1, force_filler);
+	(void)mbox_write_xkeywords(&ctx, ctx.x_keywords, input->v_offset - 1,
+				   force_filler);
 	i_free(ctx.x_keywords);
 
 	t_pop();
@@ -573,7 +580,7 @@
 	struct mail_index_record *rec;
 	struct istream *input;
 	struct ostream *output;
-	uoff_t offset, hdr_size, body_size, dirty_offset, wanted_offset;
+	uoff_t offset, hdr_size, body_size, dirty_offset;
 	const char *path;
 	unsigned int seq;
 	int tmp_fd, failed, dirty, dirty_found, rewrite, no_locking;
@@ -668,7 +675,7 @@
 		if (dirty_found || dirty) {
 			/* get offset to beginning of mail headers */
 			if (!mbox_mail_get_location(index, rec, &offset,
-						    &hdr_size, &body_size)) {
+						    &body_size)) {
 				/* fsck should have fixed it */
 				failed = TRUE;
 				break;
@@ -681,7 +688,7 @@
 				break;
 			}
 
-			if (offset + hdr_size + body_size > input->v_size) {
+			if (offset + body_size > input->v_size) {
 				mail_cache_set_corrupted(index->cache,
 					"Invalid message size");
 				failed = TRUE;
@@ -703,16 +710,15 @@
 			}
 
 			/* write header, updating flag fields */
-			offset += hdr_size;
-			wanted_offset = offset - dirty_offset;
 			if (!mbox_write_header(index, rec, seq, input, output,
-					       offset, wanted_offset,
-					       hdr_size, body_size)) {
+					       offset, &hdr_size, body_size)) {
 				failed = TRUE;
 				break;
 			}
+			offset += hdr_size;
 
-			if (dirty_found && wanted_offset == output->offset) {
+			if (dirty_found &&
+			    offset - dirty_offset == output->offset) {
 				/* no need to write more, flush */
 				if (!dirty_flush(index, dirty_offset,
 						 output, tmp_fd)) {

--- a/src/lib-index/mbox/mbox-sync-full.c	Fri Aug 29 17:31:06 2003 +0300
+++ b/src/lib-index/mbox/mbox-sync-full.c	Wed Sep 03 01:33:33 2003 +0300
@@ -14,8 +14,6 @@
 #include <fcntl.h>
 #include <sys/stat.h>
 
-#if 0
-
 static void skip_line(struct istream *input)
 {
 	const unsigned char *msg;
@@ -53,59 +51,6 @@
 	return memcmp(old_digest, current_digest, 16) == 0;
 }
 
-static int mail_update_header_size(struct mail_index *index,
-				   struct mail_index_record *rec,
-				   struct mail_cache_transaction_ctx *ctx,
-				   struct message_size *hdr_size)
-{
-	const void *part_data;
-	const char *error;
-	void *part_data_copy;
-	uoff_t virtual_size;
-	size_t size;
-
-	/* update FIELD_HDR_HEADER_SIZE */
-	index->update_field_raw(update, DATA_HDR_HEADER_SIZE,
-				&hdr_size->physical_size,
-				sizeof(hdr_size->physical_size));
-
-	/* reset FIELD_HDR_VIRTUAL_SIZE - we don't know it anymore */
-        virtual_size = (uoff_t)-1;
-	index->update_field_raw(update, DATA_HDR_VIRTUAL_SIZE,
-				&virtual_size, sizeof(virtual_size));
-
-	/* update DATA_FIELD_MESSAGEPART */
-	if ((rec->data_fields & DATA_FIELD_MESSAGEPART) == 0)
-		return TRUE;
-
-	part_data = index->lookup_field_raw(index, rec, DATA_FIELD_MESSAGEPART,
-					    &size);
-	if (part_data == NULL) {
-		/* well, this wasn't expected but don't bother failing */
-		return TRUE;
-	}
-
-	t_push();
-
-	/* copy & update the part data */
-	part_data_copy = t_malloc(size);
-	memcpy(part_data_copy, part_data, size);
-
-	if (!message_part_serialize_update_header(part_data_copy, size,
-						  hdr_size, &error)) {
-		index_set_corrupted(index,
-				    "Corrupted cached message_part data (%s)",
-				    error);
-		t_pop();
-		return FALSE;
-	}
-
-	index->update_field_raw(update, DATA_FIELD_MESSAGEPART,
-				part_data_copy, size);
-	t_pop();
-	return TRUE;
-}
-
 static int mbox_check_uidvalidity(struct mail_index *index,
 				  unsigned int uid_validity)
 {
@@ -131,10 +76,10 @@
 			     unsigned int *seq, struct istream *input,
 			     struct mail_index_record **next_rec, int *dirty)
 {
-        struct mail_index_update *update;
 	struct message_size hdr_parsed_size;
 	struct mbox_header_context ctx;
 	struct mail_index_record *first_rec, *last_rec;
+        enum mail_index_record_flag index_flags;
 	uoff_t header_offset, body_offset, offset;
 	uoff_t hdr_size, body_size;
 	unsigned char current_digest[16];
@@ -151,7 +96,7 @@
 	first_seq = last_seq = 0;
 	hdr_size = 0; body_offset = 0; hdr_size_fixed = FALSE;
 	do {
-		if (!mbox_mail_get_location(index, rec, NULL, NULL, &body_size))
+		if (!mbox_mail_get_location(index, rec, &offset, &body_size))
 			return FALSE;
 
 		i_stream_seek(input, header_offset);
@@ -178,7 +123,7 @@
 				if (!mbox_check_uidvalidity(index,
 							    ctx.uid_validity)) {
 					/* uidvalidity changed, abort */
-					break;
+					return FALSE;
 				}
 
 				if (ctx.uid_last >= index->header->next_uid) {
@@ -188,7 +133,6 @@
 				}
 			}
 
-			mbox_header_free_context(&ctx);
 			i_stream_set_read_limit(input, 0);
 
 			body_offset = input->v_offset;
@@ -197,41 +141,26 @@
 		if (verify_header(index, rec, ctx.uid, current_digest) &&
 		    mbox_verify_end_of_body(input, body_offset + body_size)) {
 			/* valid message */
-			update = index->update_begin(index, rec);
 
 			/* update flags, unless we've changed them */
-			if ((rec->index_flags & INDEX_MAIL_FLAG_DIRTY) == 0) {
+			index_flags =
+				mail_cache_get_index_flags(index->cache, rec);
+			if ((index_flags & MAIL_INDEX_FLAG_DIRTY) == 0) {
 				if (!index->update_flags(index, rec, *seq,
+							 MODIFY_REPLACE,
 							 ctx.flags, TRUE))
 					return FALSE;
-
-				/* update_flags() sets dirty flag, remove it */
-				rec->index_flags &= ~INDEX_MAIL_FLAG_DIRTY;
 			} else {
 				if (rec->msg_flags != ctx.flags)
 					*dirty = TRUE;
 			}
 
 			/* update location */
-			if (!mbox_mail_get_location(index, rec, &offset,
-						    NULL, NULL))
-				return FALSE;
 			if (offset != header_offset) {
-				index->update_field_raw(update,
-							DATA_FIELD_LOCATION,
-							&header_offset,
-							sizeof(uoff_t));
-			}
-
-			/* update size */
-			if (hdr_size != hdr_parsed_size.physical_size ) {
-				if (!mail_update_header_size(index, rec, update,
-							     &hdr_parsed_size))
+				if (!mail_cache_update_location_offset(
+					index->cache, rec, header_offset))
 					return FALSE;
 			}
-
-			if (!index->update_end(update))
-				return FALSE;
 			break;
 		}
 
@@ -329,16 +258,17 @@
 			return FALSE;
 	}
 
-	if (!dirty && (index->header->flags & MAIL_INDEX_FLAG_DIRTY_MESSAGES)) {
+	if (!dirty &&
+	    (index->header->flags & MAIL_INDEX_HDR_FLAG_DIRTY_MESSAGES)) {
 		/* no flags are dirty anymore, no need to rewrite */
-		index->header->flags &= ~MAIL_INDEX_FLAG_DIRTY_MESSAGES;
+		index->header->flags &= ~MAIL_INDEX_HDR_FLAG_DIRTY_MESSAGES;
 	}
 
 	if (input->v_offset == input->v_size ||
-	    (index->set_flags & MAIL_INDEX_FLAG_REBUILD))
+	    (index->set_flags & MAIL_INDEX_HDR_FLAG_REBUILD))
 		return TRUE;
 	else
-		return mbox_index_append(index, input);
+		return mbox_index_append_stream(index, input);
 }
 
 int mbox_sync_full(struct mail_index *index)
@@ -361,7 +291,7 @@
 	} else {
 		failed = !mbox_sync_from_stream(index, input);
 		continue_offset = failed || input->v_offset == input->v_size ||
-			(index->set_flags & MAIL_INDEX_FLAG_REBUILD) ?
+			(index->set_flags & MAIL_INDEX_HDR_FLAG_REBUILD) ?
 			(uoff_t)-1 : input->v_offset;
 		i_stream_unref(input);
 	}
@@ -384,7 +314,7 @@
 		} else if (st.st_mtime == orig_st.st_mtime &&
 			   st.st_size == orig_st.st_size) {
 			i_stream_seek(input, continue_offset);
-			failed = !mbox_index_append(index, input);
+			failed = !mbox_index_append_stream(index, input);
 		} else {
 			failed = !mbox_sync_from_stream(index, input);
 		}
@@ -394,9 +324,3 @@
 
 	return !failed;
 }
-#endif
-
-int mbox_sync_full(struct mail_index *index)
-{
-	// FIXME
-}

--- a/src/lib-index/mbox/mbox-sync.c	Fri Aug 29 17:31:06 2003 +0300
+++ b/src/lib-index/mbox/mbox-sync.c	Wed Sep 03 01:33:33 2003 +0300
@@ -10,37 +10,6 @@
 #include <fcntl.h>
 #include <sys/stat.h>
 
-static uoff_t get_indexed_mbox_size(struct mail_index *index)
-{
-	struct mail_index_record *rec;
-	uoff_t offset, hdr_size, body_size;
-
-	if (index->lock_type == MAIL_LOCK_UNLOCK) {
-		if (!mail_index_set_lock(index, MAIL_LOCK_SHARED))
-			return 0;
-	}
-
-	/* get the last record */
-	rec = index->header->messages_count == 0 ? NULL :
-		index->lookup(index, index->header->messages_count);
-
-	offset = 0;
-	if (rec != NULL) {
-		/* get the offset + size of last message, which tells the
-		   last known mbox file size */
-		if (mbox_mail_get_location(index, rec, &offset,
-					   &hdr_size, &body_size))
-			offset += hdr_size + body_size;
-	}
-
-	if (offset > OFF_T_MAX) {
-		/* too large to fit in off_t */
-		return 0;
-	}
-
-	return offset + 1; /* +1 for trailing \n */
-}
-
 static int mbox_lock_and_sync_full(struct mail_index *index,
 				   enum mail_lock_type data_lock_type)
 {
@@ -110,17 +79,13 @@
 		/* mbox file was overwritten, close it if it was open */
 		index->mbox_dev = st.st_dev;
 		index->mbox_ino = st.st_ino;
-		index->mbox_size = (uoff_t)-1;
+		index->sync_size = (uoff_t)-1;
+		index->sync_stamp = (time_t)-1;
 
                 mbox_file_close_fd(index);
 	}
 
-	if (index->mbox_sync_counter == 0) {
-		/* first sync, get expected mbox size */
-		index->mbox_size = get_indexed_mbox_size(index);
-	}
-
-	if (index->sync_stamp != st.st_mtime || index->mbox_size != filesize) {
+	if (index->sync_stamp != st.st_mtime || index->sync_size != filesize) {
 		mbox_file_close_stream(index);
 
 		if (changes != NULL)
@@ -135,8 +100,8 @@
 				return FALSE;
 		}
 
-		index->mbox_size = filesize;
 		index->sync_stamp = st.st_mtime;
+		index->sync_size = filesize;
 	}
 
 	/* we need some index lock to be able to lock mbox */

--- a/src/lib-mail/message-parser.c	Fri Aug 29 17:31:06 2003 +0300
+++ b/src/lib-mail/message-parser.c	Wed Sep 03 01:33:33 2003 +0300
@@ -637,8 +637,10 @@
 	hdr_ctx = message_parse_header_init(input, hdr_size);
 	while ((hdr = message_parse_header_next(hdr_ctx)) != NULL)
 		callback(part, hdr, context);
+	message_parse_header_deinit(hdr_ctx);
+
+	/* call after the final skipping */
 	callback(part, NULL, context);
-	message_parse_header_deinit(hdr_ctx);
 }
 
 struct message_header_parser_ctx *
@@ -724,6 +726,24 @@
 				return NULL;
 			}
 
+			if (msg[0] == '\n' ||
+			    (msg[0] == '\r' && size > 1 && msg[1] == '\n')) {
+				/* end of headers - this mostly happens just
+				   with mbox where headers are read separately
+				   from body */
+				size = 0;
+				if (ctx->hdr_size != NULL)
+					ctx->hdr_size->lines++;
+				if (msg[0] == '\r')
+					ctx->skip = 2;
+				else {
+					ctx->skip = 1;
+					if (ctx->hdr_size != NULL)
+						ctx->hdr_size->virtual_size++;
+				}
+				break;
+			}
+
 			/* a) line is larger than input buffer
 			   b) header ended unexpectedly */
 			if (colon_pos == UINT_MAX) {

--- a/src/lib-mail/message-part-serialize.c	Fri Aug 29 17:31:06 2003 +0300
+++ b/src/lib-mail/message-part-serialize.c	Wed Sep 03 01:33:33 2003 +0300
@@ -264,87 +264,6 @@
 	return part;
 }
 
-static size_t get_serialized_size(unsigned int flags)
-{
-	size_t size = sizeof(unsigned int) + sizeof(uoff_t) * 5;
-
-	if ((flags & (MESSAGE_PART_FLAG_TEXT |
-		      MESSAGE_PART_FLAG_MESSAGE_RFC822)) != 0)
-		size += sizeof(unsigned int);
-	if ((flags & (MESSAGE_PART_FLAG_MULTIPART |
-		      MESSAGE_PART_FLAG_MESSAGE_RFC822)) != 0)
-		size += sizeof(unsigned int);
-	return size;
-}
-
-int message_part_serialize_update_header(void *data, size_t size,
-					 struct message_size *hdr_size,
-					 const char **error)
-{
-	unsigned char *buf = data;
-	size_t offset, part_size;
-	uoff_t uofft_size, old_size;
-	off_t pos_diff;
-	unsigned int flags;
-
-	i_assert(hdr_size->physical_size <= OFF_T_MAX);
-
-	if (size < MINIMUM_SERIALIZED_SIZE) {
-		*error = "Not enough data";
-		return FALSE;
-	}
-
-	memcpy(&flags, buf, sizeof(flags));
-	memcpy(&uofft_size, buf + sizeof(unsigned int), sizeof(uoff_t));
-
-	if (uofft_size > OFF_T_MAX) {
-		*error = "Invalid physical_size";
-		return FALSE;
-	}
-	pos_diff = (off_t)hdr_size->physical_size - (off_t)uofft_size;
-	old_size = uofft_size;
-
-	memcpy(buf + sizeof(unsigned int),
-	       &hdr_size->physical_size, sizeof(uoff_t));
-	memcpy(buf + sizeof(unsigned int) + sizeof(uoff_t),
-	       &hdr_size->virtual_size, sizeof(uoff_t));
-
-	if (pos_diff != 0) {
-		/* have to update all positions, but skip the root */
-		offset = get_serialized_size(flags) - sizeof(uoff_t);
-
-		while (offset + sizeof(unsigned int) < size) {
-			memcpy(buf + offset, &flags, sizeof(flags));
-
-			part_size = get_serialized_size(flags);
-			if (offset + part_size > size) {
-				*error = "Not enough data";
-				return FALSE;
-			}
-			memcpy(&uofft_size, buf + offset + sizeof(flags),
-			       sizeof(uoff_t));
-
-			if (uofft_size < old_size || uofft_size >= OFF_T_MAX) {
-				/* invalid offset, might cause overflow */
-				*error = "Invalid offset";
-				return FALSE;
-			}
-			uofft_size += pos_diff;
-
-			memcpy(buf + offset + sizeof(flags), &uofft_size,
-			       sizeof(uoff_t));
-			offset += part_size;
-		}
-
-		if (offset != size) {
-			*error = "Invalid size";
-			return FALSE;
-		}
-	}
-
-	return TRUE;
-}
-
 int message_part_deserialize_size(const void *data, size_t size,
 				  struct message_size *hdr_size,
 				  struct message_size *body_size)

--- a/src/lib-mail/message-part-serialize.h	Fri Aug 29 17:31:06 2003 +0300
+++ b/src/lib-mail/message-part-serialize.h	Wed Sep 03 01:33:33 2003 +0300
@@ -12,11 +12,6 @@
 struct message_part *message_part_deserialize(pool_t pool, const void *data,
 					      size_t size, const char **error);
 
-/* Update header size in serialized struct message_part. */
-int message_part_serialize_update_header(void *data, size_t size,
-					 struct message_size *hdr_size,
-					 const char **error);
-
 /* Get message size from serialized struct message_part data. */
 int message_part_deserialize_size(const void *data, size_t size,
 				  struct message_size *hdr_size,

--- a/src/lib-storage/index/mbox/mbox-expunge.c	Fri Aug 29 17:31:06 2003 +0300
+++ b/src/lib-storage/index/mbox/mbox-expunge.c	Wed Sep 03 01:33:33 2003 +0300
@@ -123,6 +123,7 @@
 static int get_from_offset(struct mail_index *index,
 			   struct mail_index_record *rec, uoff_t *offset_r)
 {
+#if 0
 	uoff_t offset, hdr_size, body_size;
 
 	if (!mbox_mail_get_location(index, rec, &offset,
@@ -131,6 +132,8 @@
 
 	*offset_r = offset + hdr_size + body_size;
 	return TRUE;
+#endif
+	abort();
 }
 
 struct mail *mbox_storage_expunge_fetch_next(struct mail_expunge_context *_ctx)