--- a/src/lib-index/Makefile.am	Mon Jun 28 19:44:38 2004 +0300
+++ b/src/lib-index/Makefile.am	Mon Jun 28 20:35:27 2004 +0300
@@ -6,6 +6,8 @@
 
 libindex_a_SOURCES = \
 	mail-cache.c \
+	mail-cache-compress.c \
+	mail-cache-decisions.c \
 	mail-cache-lookup.c \
 	mail-cache-transaction.c \
         mail-index.c \

--- a/src/lib-index/mail-cache-compress.c	Mon Jun 28 19:44:38 2004 +0300
+++ b/src/lib-index/mail-cache-compress.c	Mon Jun 28 20:35:27 2004 +0300
@@ -1,9 +1,19 @@
+/* Copyright (C) 2003-2004 Timo Sirainen */
+
+#include "lib.h"
+#include "buffer.h"
+#include "byteorder.h"
+#include "ostream.h"
+#include "mail-cache-private.h"
+
+static unsigned char null4[4] = { 0, 0, 0, 0 };
+
 static const struct mail_cache_record *
-mail_cache_compress_record(struct mail_cache *cache,
-			   struct mail_index_record *rec, int header_idx,
-			   uint32_t *size_r)
+mail_cache_compress_record(struct mail_cache_view *view, uint32_t seq,
+			   enum mail_cache_field orig_cached_fields,
+			   int header_idx, uint32_t *size_r)
 {
-	enum mail_cache_field orig_cached_fields, cached_fields, field;
+	enum mail_cache_field cached_fields, field;
 	struct mail_cache_record cache_rec;
 	buffer_t *buffer;
 	const void *data;
@@ -15,14 +25,13 @@
 	buffer = buffer_create_dynamic(pool_datastack_create(),
 				       4096, (size_t)-1);
 
-        orig_cached_fields = mail_cache_get_fields(cache, rec);
 	cached_fields = orig_cached_fields & ~MAIL_CACHE_HEADERS_MASK;
 	buffer_append(buffer, &cache_rec, sizeof(cache_rec));
 	for (i = 0, field = 1; i < 31; i++, field <<= 1) {
 		if ((cached_fields & field) == 0)
 			continue;
 
-		if (!mail_cache_lookup_field(cache, rec, field, &data, &size)) {
+		if (!mail_cache_lookup_field(view, seq, field, &data, &size)) {
 			cached_fields &= ~field;
 			continue;
 		}
@@ -44,14 +53,14 @@
 
 		for (i = 0; i <= header_idx; i++) {
 			field = mail_cache_header_fields[i];
-			if (mail_cache_lookup_field(cache, rec, field,
+			if (mail_cache_lookup_field(view, seq, field,
 						    &data, &size) && size > 1) {
 				size--; /* terminating \0 */
 				buffer_append(buffer, data, size);
 				nb_size += size;
 			}
 		}
-		buffer_append(buffer, "", 1);
+		buffer_append(buffer, null4, 1);
 		nb_size++;
 		if ((nb_size & 3) != 0)
 			buffer_append(buffer, null4, 4 - (nb_size & 3));
@@ -71,36 +80,59 @@
 	return data;
 }
 
-static int mail_cache_copy(struct mail_cache *cache, int fd)
+static int
+mail_cache_copy(struct mail_cache *cache, struct mail_index_view *view, int fd)
 {
-#if 0
-	struct mail_cache_header *hdr;
+	struct mail_cache_view *cache_view;
+	struct mail_index_transaction *t;
+	const struct mail_index_header *idx_hdr;
 	const struct mail_cache_record *cache_rec;
-	struct mail_index_record *rec;
-        enum mail_cache_field used_fields;
-	unsigned char *mmap_base;
+	struct mail_cache_header hdr;
+	struct ostream *output;
+	enum mail_cache_field keep_fields, temp_fields;
+	enum mail_cache_field cached_fields, new_fields;
 	const char *str;
-	uint32_t new_file_size, offset, size, nb_size;
-	int i, header_idx;
+	uint32_t size, nb_size, message_count, seq, first_new_seq;
+	uoff_t offset;
+	int i, header_idx, ret;
+
+	/* get sequence of first message which doesn't need it's temp fields
+	   removed. */
+	if (mail_index_get_header(view, &idx_hdr) < 0)
+		return -1;
+	if (mail_index_lookup_uid_range(view, idx_hdr->day_first_uid[7],
+					(uint32_t)-1, &first_new_seq,
+					&message_count) < 0)
+		return -1;
+	if (first_new_seq == 0)
+		first_new_seq = message_count+1;
 
-	/* pick some reasonably good file size */
-	new_file_size = cache->used_file_size -
-		nbo_to_uint32(cache->hdr->deleted_space);
-	new_file_size = (new_file_size + 1023) & ~1023;
-	if (new_file_size < MAIL_CACHE_INITIAL_SIZE)
-		new_file_size = MAIL_CACHE_INITIAL_SIZE;
+	cache_view = mail_cache_view_open(cache, view);
+	t = mail_index_transaction_begin(view, FALSE);
+	output = o_stream_create_file(fd, default_pool, 0, FALSE);
+
+	memset(&hdr, 0, sizeof(hdr));
+	hdr.indexid = cache->hdr->indexid;
+	hdr.file_seq = cache->hdr->file_seq + 1;
 
-	if (file_set_size(fd, new_file_size) < 0)
-		return mail_cache_set_syscall_error(cache, "file_set_size()");
+	memcpy(hdr.field_usage_decision_type,
+	       cache->hdr->field_usage_decision_type,
+	       sizeof(hdr.field_usage_decision_type));
+	memcpy(hdr.field_usage_last_used,
+	       cache->hdr->field_usage_last_used,
+	       sizeof(hdr.field_usage_last_used));
 
-	mmap_base = mmap(NULL, new_file_size, PROT_READ | PROT_WRITE,
-			 MAP_SHARED, fd, 0);
-	if (mmap_base == MAP_FAILED)
-		return mail_cache_set_syscall_error(cache, "mmap()");
+        keep_fields = temp_fields = 0;
+	for (i = 0; i < 32; i++) {
+		if (cache->hdr->field_usage_decision_type[i] &
+		    MAIL_CACHE_DECISION_YES)
+			keep_fields |= 1 << i;
+		else if (cache->hdr->field_usage_decision_type[i] &
+			 MAIL_CACHE_DECISION_TEMP)
+			temp_fields |= 1 << i;
+	}
 
-	/* skip file's header */
-	hdr = (struct mail_cache_header *) mmap_base;
-	offset = sizeof(*hdr);
+	offset = sizeof(hdr);
 
 	/* merge all the header pieces into one. if some message doesn't have
 	   all the required pieces, we'll just have to drop them all. */
@@ -113,136 +145,125 @@
 	if (str == NULL)
 		header_idx = -1;
 	else {
-		hdr->header_offsets[0] = uint32_to_offset(offset);
+		hdr.header_offsets[0] = mail_cache_uint32_to_offset(offset);
 		header_idx = i;
 
 		size = strlen(str) + 1;
 		nb_size = uint32_to_nbo(size);
 
-		memcpy(mmap_base + offset, &nb_size, sizeof(nb_size));
-		offset += sizeof(nb_size);
-		memcpy(mmap_base + offset, str, size);
-		offset += (size + 3) & ~3;
+		o_stream_send(output, &nb_size, sizeof(nb_size));
+		o_stream_send(output, str, size);
+		if ((size & 3) != 0)
+			o_stream_send(output, null4, 4 - (size & 3));
 	}
 
-	// FIXME: recreate index file with new cache_offsets
+	mail_index_reset_cache(t, hdr.file_seq);
+
+	ret = 0;
+	for (seq = 1; seq <= message_count; seq++) {
+		cache_rec = mail_cache_lookup(cache_view, seq, 0);
+		if (cache_rec == NULL)
+			continue;
 
-	used_fields = 0;
-	rec = cache->index->lookup(cache->index, 1);
-	while (rec != NULL) {
-		cache_rec = mail_cache_lookup(cache, rec, 0);
-		if (cache_rec == NULL)
-			rec->cache_offset = 0;
-		else if (offset_to_uint32(cache_rec->next_offset) == 0) {
-			/* just one unmodified block, copy it */
+		cached_fields = mail_cache_get_fields(cache_view, seq);
+                new_fields = cached_fields & keep_fields;
+		if ((cached_fields & temp_fields) != 0 &&
+		    seq >= first_new_seq) {
+			/* new message, keep temp fields */
+			new_fields |= cached_fields & temp_fields;
+		}
+
+		if (keep_fields == cached_fields &&
+		    mail_cache_offset_to_uint32(cache_rec->next_offset) == 0) {
+			/* just one unmodified block, save it */
 			size = nbo_to_uint32(cache_rec->size);
-			i_assert(offset + size <= new_file_size);
-
-			memcpy(mmap_base + offset, cache_rec, size);
-			rec->cache_offset = uint32_to_offset(offset);
+                        mail_index_update_cache(t, seq, output->offset);
+			o_stream_send(output, cache_rec, size);
 
-			size = (size + 3) & ~3;
-			offset += size;
+			if ((size & 3) != 0)
+				o_stream_send(output, null4, 4 - (size & 3));
 		} else {
-			/* multiple blocks, sort them into buffer */
+			/* a) dropping fields
+			   b) multiple blocks, sort them into buffer */
+                        mail_index_update_cache(t, seq, output->offset);
+
 			t_push();
-			cache_rec = mail_cache_compress_record(cache, rec,
+			cache_rec = mail_cache_compress_record(cache_view, seq,
+							       keep_fields,
 							       header_idx,
 							       &size);
-			i_assert(offset + size <= new_file_size);
-			memcpy(mmap_base + offset, cache_rec, size);
-			used_fields |= cache_rec->fields;
+			o_stream_send(output, cache_rec, size);
 			t_pop();
+		}
+	}
+	hdr.used_file_size = uint32_to_nbo(output->offset);
 
-			rec->cache_offset = uint32_to_offset(offset);
-			offset += size;
-		}
+	o_stream_unref(output);
+	mail_cache_view_close(cache_view);
 
-		rec = cache->index->next(cache->index, rec);
+	if (fdatasync(fd) < 0) {
+		mail_cache_set_syscall_error(cache, "fdatasync()");
+		(void)mail_index_transaction_rollback(t);
+		return -1;
 	}
 
-	/* update header */
-	hdr->indexid = cache->index->indexid;
-	hdr->file_seq = cache->index->hdr->cache_sync_id+1;
-	hdr->used_file_size = uint32_to_nbo(offset);
-	hdr->used_fields = used_fields;
-	hdr->field_usage_start = uint32_to_nbo(ioloop_time);
-
-	/* write everything to disk */
-	if (msync(mmap_base, offset, MS_SYNC) < 0)
-		return mail_cache_set_syscall_error(cache, "msync()");
-
-	if (munmap(mmap_base, new_file_size) < 0)
-		return mail_cache_set_syscall_error(cache, "munmap()");
-
-	if (fdatasync(fd) < 0)
-		return mail_cache_set_syscall_error(cache, "fdatasync()");
-	return TRUE;
-#endif
+	return mail_index_transaction_commit(t, &seq, &offset);
 }
 
-int mail_cache_compress(struct mail_cache *cache)
+int mail_cache_compress(struct mail_cache *cache, struct mail_index_view *view)
 {
-	int fd, ret = TRUE;
+	int fd, ret;
 
 	i_assert(cache->trans_ctx == NULL);
 
-	if (cache->anon_mmap)
-		return TRUE;
-
-	if (!cache->index->set_lock(cache->index, MAIL_LOCK_EXCLUSIVE))
-		return FALSE;
-
-	if (mail_cache_lock(cache, TRUE) <= 0)
-		return FALSE;
+	if ((ret = mail_cache_lock(cache, TRUE)) <= 0)
+		return ret;
 
 #ifdef DEBUG
 	i_warning("Compressing cache file %s", cache->filepath);
 #endif
 
-	fd = file_dotlock_open(cache->filepath, NULL, MAIL_CACHE_LOCK_TIMEOUT,
+	fd = file_dotlock_open(cache->filepath, NULL, NULL,
+			       MAIL_CACHE_LOCK_TIMEOUT,
 			       MAIL_CACHE_LOCK_CHANGE_TIMEOUT,
 			       MAIL_CACHE_LOCK_IMMEDIATE_TIMEOUT, NULL, NULL);
 	if (fd == -1) {
 		mail_cache_set_syscall_error(cache, "file_dotlock_open()");
-		return FALSE;
+		return -1;
 	}
 
-	/* now we'll begin the actual moving. keep rebuild-flag on
-	   while doing it. */
-	cache->index->hdr->flags |= MAIL_INDEX_HDR_FLAG_REBUILD;
-	if (!mail_index_fmdatasync(cache->index, cache->index->hdr_size))
-		return FALSE;
-
-	if (!mail_cache_copy(cache, fd)) {
-		(void)file_dotlock_delete(cache->filepath, fd);
-		ret = FALSE;
+	if (mail_cache_copy(cache, view, fd) < 0) {
+		(void)file_dotlock_delete(cache->filepath, NULL, fd);
+		ret = -1;
 	} else {
-		mail_cache_file_close(cache);
-		cache->fd = dup(fd);
-
-		if (file_dotlock_replace(cache->filepath, fd, FALSE) < 0) {
+		if (file_dotlock_replace(cache->filepath, NULL,
+					 -1, FALSE) < 0) {
 			mail_cache_set_syscall_error(cache,
 						     "file_dotlock_replace()");
-			ret = FALSE;
-		}
+			(void)close(fd);
+			ret = -1;
+		} else {
+			mail_cache_file_close(cache);
+			cache->fd = fd;
 
-		if (!mmap_update(cache, 0, 0))
-			ret = FALSE;
+			if (mail_cache_mmap_update(cache, 0, 0) < 0)
+				ret = -1;
+		}
 	}
 
 	/* headers could have changed, reread them */
 	memset(cache->split_offsets, 0, sizeof(cache->split_offsets));
 	memset(cache->split_headers, 0, sizeof(cache->split_headers));
 
-	if (ret) {
-		cache->index->hdr->flags &=
-			~(MAIL_INDEX_HDR_FLAG_REBUILD |
-			  MAIL_INDEX_HDR_FLAG_COMPRESS_CACHE);
-	}
+	if (mail_cache_unlock(cache) < 0)
+		return -1;
 
-	if (mail_cache_unlock(cache) < 0)
-		ret = FALSE;
-
+	if (ret == 0)
+                cache->need_compress = FALSE;
 	return ret;
 }
+
+int mail_cache_need_compress(struct mail_cache *cache)
+{
+	return cache->need_compress;
+}

--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/src/lib-index/mail-cache-decisions.c	Mon Jun 28 20:35:27 2004 +0300
@@ -0,0 +1,83 @@
+/* Copyright (C) 2004 Timo Sirainen */
+
+#include "lib.h"
+#include "write-full.h"
+#include "mail-cache-private.h"
+
+#include <stddef.h>
+
+static void
+mail_cache_set_decision_type(struct mail_cache *cache, uint32_t idx,
+			     enum mail_cache_decision_type type)
+{
+	uint8_t value = type;
+
+	/* update the header without locking, we'll just write one byte and
+	   it's very unlikely someone else tries to write different value for
+	   it at the same time. even then it's just a wrong decision which
+	   will be corrected sometimes later, not too bad.. */
+	if (pwrite_full(cache->fd, &value, 1,
+			offsetof(struct mail_cache_header,
+				 field_usage_decision_type) + idx) < 0) {
+		mail_cache_set_syscall_error(cache, "pwrite_full()");
+	}
+}
+
+void mail_cache_handle_decisions(struct mail_cache_view *view, uint32_t seq,
+				 enum mail_cache_field field)
+{
+	const struct mail_index_header *hdr;
+	unsigned int idx;
+	uint32_t uid;
+
+	idx = mail_cache_field_index(field);
+	if (view->cache->hdr->field_usage_decision_type[idx] !=
+	    MAIL_CACHE_DECISION_TEMP) {
+		/* a) forced decision
+		   b) not cached, mail_cache_mark_missing() will handle this
+		   c) permanently cached already, okay. */
+		return;
+	}
+
+	/* see if we want to change decision from TEMP to YES */
+	if (mail_index_lookup_uid(view->view, seq, &uid) < 0 ||
+	    mail_index_get_header(view->view, &hdr) < 0)
+		return;
+
+	if (uid < view->cache->field_usage_uid_highwater[idx] ||
+	    uid < hdr->day_first_uid[7]) {
+		/* a) nonordered access within this session. if client doesn't
+		      request messages in growing order, we assume it doesn't
+		      have a permanent local cache.
+		   b) accessing message older than one week. assume it's a
+		      client with no local cache. if it was just a new client
+		      generating the local cache for the first time, we'll
+		      drop back to TEMP within few months. */
+		mail_cache_set_decision_type(view->cache, idx,
+					     MAIL_CACHE_DECISION_YES);
+	} else {
+		view->cache->field_usage_uid_highwater[idx] = uid;
+	}
+}
+
+void mail_cache_mark_missing(struct mail_cache_view *view, uint32_t seq,
+			     enum mail_cache_field field)
+{
+	unsigned int idx;
+	uint32_t uid;
+
+	idx = mail_cache_field_index(field);
+	if (view->cache->hdr->field_usage_decision_type[idx] !=
+	    MAIL_CACHE_DECISION_NO) {
+		/* a) forced decision
+		   b) we're already caching it, so it just wasn't in cache */
+		return;
+	}
+
+	/* field used the first time */
+	mail_cache_set_decision_type(view->cache, idx,
+				     MAIL_CACHE_DECISION_TEMP);
+
+	if (mail_index_lookup_uid(view->view, seq, &uid) == 0)
+		view->cache->field_usage_uid_highwater[idx] = uid;
+}

--- a/src/lib-index/mail-cache-lookup.c	Mon Jun 28 19:44:38 2004 +0300
+++ b/src/lib-index/mail-cache-lookup.c	Mon Jun 28 20:35:27 2004 +0300
@@ -210,7 +210,7 @@
 static int cache_get_field(struct mail_cache *cache,
 			   struct mail_cache_record *cache_rec,
 			   enum mail_cache_field field,
-			   void **data_r, size_t *size_r)
+			   const void **data_r, size_t *size_r)
 {
 	unsigned char *buf;
 	unsigned int mask;
@@ -266,12 +266,14 @@
 	return FALSE;
 }
 
-static int cache_lookup_field(struct mail_cache_view *view, uint32_t seq,
-			      enum mail_cache_field field,
-			      void **data_r, size_t *size_r)
+int mail_cache_lookup_field(struct mail_cache_view *view, uint32_t seq,
+			    enum mail_cache_field field,
+			    const void **data_r, size_t *size_r)
 {
 	struct mail_cache_record *cache_rec;
 
+	mail_cache_handle_decisions(view, seq, field);
+
 	cache_rec = mail_cache_lookup(view, seq, field);
 	while (cache_rec != NULL) {
 		if ((cache_rec->fields & field) != 0) {
@@ -284,19 +286,6 @@
 	return FALSE;
 }
 
-int mail_cache_lookup_field(struct mail_cache_view *view, uint32_t seq,
-			    enum mail_cache_field field,
-			    const void **data_r, size_t *size_r)
-{
-	void *data;
-
-	if (!cache_lookup_field(view, seq, field, &data, size_r))
-		return FALSE;
-
-	*data_r = data;
-	return TRUE;
-}
-
 const char *
 mail_cache_lookup_string_field(struct mail_cache_view *view, uint32_t seq,
 			       enum mail_cache_field field)

--- a/src/lib-index/mail-cache-private.h	Mon Jun 28 19:44:38 2004 +0300
+++ b/src/lib-index/mail-cache-private.h	Mon Jun 28 20:35:27 2004 +0300
@@ -27,6 +27,18 @@
 #define CACHE_RECORD(cache, offset) \
 	((struct mail_cache_record *) ((char *) (cache)->mmap_base + offset))
 
+enum mail_cache_decision_type {
+	/* Not needed currently */
+	MAIL_CACHE_DECISION_NO		= 0x00,
+	/* Needed only for new mails. Drop when compressing. */
+	MAIL_CACHE_DECISION_TEMP	= 0x01,
+	/* Needed. */
+	MAIL_CACHE_DECISION_YES		= 0x02,
+
+	/* This decision has been forced manually, don't change it. */
+	MAIL_CACHE_DECISION_FORCED	= 0x80
+};
+
 struct mail_cache_header {
 	uint32_t indexid;
 	uint32_t file_seq;
@@ -36,10 +48,8 @@
 	uint32_t used_file_size;
 	uint32_t deleted_space;
 
-	uint32_t used_fields; /* enum mail_cache_field */
-
-	uint32_t field_usage_start; /* time_t */
-	uint32_t field_usage_counts[32];
+	uint32_t field_usage_last_used[32]; /* time_t */
+	uint8_t field_usage_decision_type[32];
 
 	uint32_t header_offsets[MAIL_CACHE_HEADERS_COUNT];
 };
@@ -68,10 +78,13 @@
 	enum mail_cache_field default_cache_fields;
 	enum mail_cache_field never_cache_fields;
 
+	uint32_t field_usage_uid_highwater[32];
+
         struct mail_cache_transaction_ctx *trans_ctx;
 	unsigned int locks;
 
 	unsigned int mmap_refresh:1;
+	unsigned int need_compress:1;
 	unsigned int silent:1;
 	unsigned int disabled:1;
 };
@@ -88,6 +101,7 @@
 
 uint32_t mail_cache_uint32_to_offset(uint32_t offset);
 uint32_t mail_cache_offset_to_uint32(uint32_t offset);
+unsigned int mail_cache_field_index(enum mail_cache_field field);
 
 const char *
 mail_cache_get_header_fields_str(struct mail_cache *cache, unsigned int idx);
@@ -111,6 +125,10 @@
 
 int mail_cache_mmap_update(struct mail_cache *cache,
 			   size_t offset, size_t size);
+void mail_cache_file_close(struct mail_cache *cache);
+
+void mail_cache_handle_decisions(struct mail_cache_view *view, uint32_t seq,
+				 enum mail_cache_field field);
 
 void mail_cache_set_syscall_error(struct mail_cache *cache,
 				  const char *function);

--- a/src/lib-index/mail-cache-transaction.c	Mon Jun 28 19:44:38 2004 +0300
+++ b/src/lib-index/mail-cache-transaction.c	Mon Jun 28 20:35:27 2004 +0300
@@ -145,7 +145,7 @@
 	    COMPRESS_CONTINUED_PERCENTAGE &&
 	    ctx->used_file_size >= COMPRESS_MIN_SIZE) {
 		/* too many continued rows, compress */
-		//FIXME:cache->index->set_flags |= MAIL_INDEX_HDR_FLAG_COMPRESS_CACHE;
+		cache->need_compress = TRUE;
 	}
 
 	cache->hdr->continued_record_count = uint32_to_nbo(cont);
@@ -302,8 +302,7 @@
 
 	if (ctx->next_unused_header_lowwater == MAIL_CACHE_HEADERS_COUNT) {
 		/* they're all used - compress the cache to get more */
-		/* FIXME: ctx->cache->index->set_flags |=
-			MAIL_INDEX_HDR_FLAG_COMPRESS_CACHE;*/
+		ctx->cache->need_compress = TRUE;
 	}
 
 	mail_cache_transaction_flush(ctx);
@@ -548,9 +547,8 @@
 	/* see if we've reached the max. deleted space in file */
 	max_del_space = ctx->used_file_size / 100 * COMPRESS_PERCENTAGE;
 	if (deleted_space >= max_del_space &&
-	    ctx->used_file_size >= COMPRESS_MIN_SIZE) {
-		//FIXME:cache->index->set_flags |= MAIL_INDEX_HDR_FLAG_COMPRESS_CACHE;
-	}
+	    ctx->used_file_size >= COMPRESS_MIN_SIZE)
+		cache->need_compress = TRUE;
 
 	cache->hdr->deleted_space = uint32_to_nbo(deleted_space);
 	return 0;

--- a/src/lib-index/mail-cache.c	Mon Jun 28 19:44:38 2004 +0300
+++ b/src/lib-index/mail-cache.c	Mon Jun 28 20:35:27 2004 +0300
@@ -1,7 +1,6 @@
 /* Copyright (C) 2003-2004 Timo Sirainen */
 
 #include "lib.h"
-#include "buffer.h"
 #include "byteorder.h"
 #include "file-lock.h"
 #include "file-set-size.h"
@@ -9,7 +8,6 @@
 #include "write-full.h"
 #include "mail-cache-private.h"
 
-#include <stddef.h>
 #include <unistd.h>
 #include <sys/stat.h>
 
@@ -67,6 +65,17 @@
 		(((uint32_t)buf[0] & 0x7f) << 23);
 }
 
+unsigned int mail_cache_field_index(enum mail_cache_field field)
+{
+	unsigned int i, num;
+
+	for (i = 0, num = 1; i < 32; i++, num <<= 1) {
+		if (field == num)
+			return i;
+	}
+	i_unreached();
+}
+
 void mail_cache_set_syscall_error(struct mail_cache *cache,
 				  const char *function)
 {
@@ -99,7 +108,7 @@
 	va_end(va);
 }
 
-static void mail_cache_file_close(struct mail_cache *cache)
+void mail_cache_file_close(struct mail_cache *cache)
 {
 	if (cache->mmap_base != NULL) {
 		if (munmap(cache->mmap_base, cache->mmap_length) < 0)
@@ -539,9 +548,3 @@
 {
 	i_free(view);
 }
-
-void mail_cache_mark_missing(struct mail_cache_view *view,
-			     enum mail_cache_field fields)
-{
-	// FIXME
-}

--- a/src/lib-index/mail-cache.h	Mon Jun 28 19:44:38 2004 +0300
+++ b/src/lib-index/mail-cache.h	Mon Jun 28 20:35:27 2004 +0300
@@ -28,9 +28,9 @@
 enum mail_cache_field {
 	/* fixed size fields */
 	MAIL_CACHE_INDEX_FLAGS		= 0x00000001,
-	MAIL_CACHE_SENT_DATE		= 0x00000008,
-	MAIL_CACHE_RECEIVED_DATE	= 0x00000010,
-	MAIL_CACHE_VIRTUAL_FULL_SIZE	= 0x00000020,
+	MAIL_CACHE_SENT_DATE		= 0x00000002,
+	MAIL_CACHE_RECEIVED_DATE	= 0x00000004,
+	MAIL_CACHE_VIRTUAL_FULL_SIZE	= 0x00000008,
 
 	/* variable sized field */
 	MAIL_CACHE_HEADERS1		= 0x40000000,
@@ -77,8 +77,10 @@
 			     enum mail_cache_field default_cache_fields,
 			     enum mail_cache_field never_cache_fields);
 
+/* Returns TRUE if cache should be compressed. */
+int mail_cache_need_compress(struct mail_cache *cache);
 /* Compress cache file. */
-int mail_cache_compress(struct mail_cache *cache);
+int mail_cache_compress(struct mail_cache *cache, struct mail_index_view *view);
 
 /* Reset the cache file, clearing all data. */
 int mail_cache_reset(struct mail_cache *cache);
@@ -153,9 +155,9 @@
 				enum mail_cache_field field,
 				void *buffer, size_t buffer_size);
 
-/* Mark given fields as missing, ie. they should be cached when possible. */
-void mail_cache_mark_missing(struct mail_cache_view *view,
-			     enum mail_cache_field fields);
+/* Mark given field as missing, ie. it should be cached when possible. */
+void mail_cache_mark_missing(struct mail_cache_view *view, uint32_t uid,
+			     enum mail_cache_field field);
 
 /* Return record flags. */
 enum mail_cache_record_flag

--- a/src/lib-index/mail-index-sync-update.c	Mon Jun 28 19:44:38 2004 +0300
+++ b/src/lib-index/mail-index-sync-update.c	Mon Jun 28 20:35:27 2004 +0300
@@ -1,6 +1,7 @@
 /* Copyright (C) 2004 Timo Sirainen */
 
 #include "lib.h"
+#include "ioloop.h"
 #include "buffer.h"
 #include "file-set-size.h"
 #include "mmap-util.h"
@@ -9,6 +10,8 @@
 #include "mail-transaction-log.h"
 #include "mail-transaction-util.h"
 
+#include <time.h>
+
 static void
 mail_index_header_update_counts(struct mail_index_header *hdr,
 				uint8_t old_flags, uint8_t new_flags)
@@ -306,6 +309,43 @@
 	map->write_to_disk = TRUE;
 }
 
+static void
+mail_index_update_day_headers(struct mail_index_header *hdr, uint32_t uid)
+{
+	const int max_days =
+		sizeof(hdr->day_first_uid) / sizeof(hdr->day_first_uid[0]);
+	struct tm tm;
+	time_t stamp;
+	int i, days;
+
+	/* get beginning of today */
+	tm = *localtime(&ioloop_time);
+	tm.tm_hour = 0;
+	tm.tm_min = 0;
+	tm.tm_sec = 0;
+	stamp = mktime(&tm);
+	if (stamp == (time_t)-1)
+		i_panic("mktime(today) failed");
+
+	if ((time_t)hdr->day_stamp >= stamp)
+		return;
+
+	/* get number of days since last message */
+	days = (stamp - hdr->day_stamp) / (3600*24);
+	if (days > max_days)
+		days = max_days;
+
+	/* @UNSAFE: move days forward and fill the missing days with old
+	   day_first_uid[0]. */
+	memcpy(hdr->day_first_uid + days,
+	       hdr->day_first_uid, max_days - days);
+	for (i = 1; i < days; i++)
+		hdr->day_first_uid[i] = hdr->day_first_uid[0];
+
+	hdr->day_stamp = stamp;
+	hdr->day_first_uid[0] = uid;
+}
+
 int mail_index_sync_update_index(struct mail_index_sync_ctx *sync_ctx)
 {
 	struct mail_index *index = sync_ctx->index;
@@ -314,7 +354,7 @@
 	const struct mail_transaction_header *hdr;
 	const void *data;
 	unsigned int count, old_lock_id;
-	uint32_t seq, i;
+	uint32_t seq, i, first_append_uid;
 	uoff_t offset;
 	int ret, had_dirty, skipped;
 
@@ -338,6 +378,7 @@
 	view->map = map;
 	view->map->refcount++;
 
+        first_append_uid = 0;
 	had_dirty = (map->hdr_copy.flags & MAIL_INDEX_HDR_FLAG_HAVE_DIRTY) != 0;
 	if (had_dirty)
 		map->hdr_copy.flags &= ~MAIL_INDEX_HDR_FLAG_HAVE_DIRTY;
@@ -358,6 +399,11 @@
 
 		if ((hdr->type & MAIL_TRANSACTION_APPEND) != 0) {
                         const struct mail_transaction_append_header *append_hdr;
+			const struct mail_index_record *rec;
+
+			rec = CONST_PTR_OFFSET(data, sizeof(*append_hdr));
+			if (first_append_uid == 0)
+				first_append_uid = rec->uid;
 
 			append_hdr = data;
 			if (append_hdr->record_size > map->hdr->record_size) {
@@ -367,7 +413,7 @@
 				mail_index_sync_replace_map(view, map);
 			}
 			count = (hdr->size - sizeof(*append_hdr)) /
-				 append_hdr->record_size;
+				append_hdr->record_size;
 			if (mail_index_grow(index, view->map, count) < 0) {
 				ret = -1;
 				break;
@@ -383,7 +429,6 @@
 	}
 
 	if (ret < 0) {
-		/*  */
 		mail_index_view_unlock(view);
 		return -1;
 	}
@@ -396,6 +441,9 @@
 	map->hdr_copy.log_file_seq = seq;
 	map->hdr_copy.log_file_offset = offset;
 
+	if (first_append_uid != 0)
+		mail_index_update_day_headers(&map->hdr_copy, first_append_uid);
+
 	if ((map->hdr_copy.flags & MAIL_INDEX_HDR_FLAG_HAVE_DIRTY) == 0 &&
 	    had_dirty) {
 		/* do we have dirty flags anymore? */

--- a/src/lib-index/mail-index-sync.c	Mon Jun 28 19:44:38 2004 +0300
+++ b/src/lib-index/mail-index-sync.c	Mon Jun 28 20:35:27 2004 +0300
@@ -332,8 +332,8 @@
 int mail_index_sync_end(struct mail_index_sync_ctx *ctx)
 {
 	const struct mail_index_header *hdr;
-	uint32_t seq;
-	uoff_t offset;
+	uint32_t seq, seq2;
+	uoff_t offset, offset2;
 	int ret = 0;
 
 	if (mail_transaction_log_view_is_corrupted(ctx->view->log_view))
@@ -348,8 +348,24 @@
 				hdr->log_file_seq, hdr->log_file_offset,
 				seq, offset, MAIL_TRANSACTION_TYPE_MASK) < 0)
 			ret = -1;
-		if (mail_index_sync_update_index(ctx) < 0)
+		else if (mail_index_sync_update_index(ctx) < 0)
+			ret = -1;
+	}
+
+	if (ret == 0 && mail_cache_need_compress(ctx->index->cache)) {
+		if (mail_cache_compress(ctx->index->cache, ctx->view) < 0)
 			ret = -1;
+		else {
+			/* cache_offsets have changed, sync them */
+			mail_transaction_log_get_head(ctx->index->log,
+						      &seq2, &offset2);
+			if (mail_transaction_log_view_set(ctx->view->log_view,
+					seq, offset, seq2, offset2,
+					MAIL_TRANSACTION_TYPE_MASK) < 0)
+				ret = -1;
+			else if (mail_index_sync_update_index(ctx) < 0)
+				ret = -1;
+		}
 	}
 
 	mail_index_unlock(ctx->index, ctx->lock_id);

--- a/src/lib-index/mail-index.h	Mon Jun 28 19:44:38 2004 +0300
+++ b/src/lib-index/mail-index.h	Mon Jun 28 20:35:27 2004 +0300
@@ -6,7 +6,7 @@
 #define MAIL_INDEX_MAJOR_VERSION 4
 #define MAIL_INDEX_MINOR_VERSION 0
 
-#define MAIL_INDEX_HEADER_MIN_SIZE 88
+#define MAIL_INDEX_HEADER_MIN_SIZE 120
 
 /* Number of keywords in mail_index_record. */
 #define INDEX_KEYWORDS_COUNT (3*8)
@@ -93,11 +93,15 @@
 	uint32_t log_file_seq;
 	uint32_t log_file_offset;
 
+	uint32_t sync_stamp;
 	uint64_t sync_size;
-	uint32_t sync_stamp;
 
 	uint32_t cache_file_seq;
 	uint32_t extra_records_hdr_offset;
+
+	/* daily first UIDs that have been added to index. */
+	uint32_t day_stamp;
+	uint32_t day_first_uid[8];
 };
 
 struct mail_index_record {

--- a/src/lib-storage/index/index-mail.c	Mon Jun 28 19:44:38 2004 +0300
+++ b/src/lib-storage/index/index-mail.c	Mon Jun 28 20:35:27 2004 +0300
@@ -23,7 +23,7 @@
 	size_t part_size;
 
 	if ((mail->data.cached_fields & MAIL_CACHE_MESSAGEPART) == 0) {
-		mail_cache_mark_missing(mail->trans->cache_view,
+		mail_cache_mark_missing(mail->trans->cache_view, mail->data.seq,
 					MAIL_CACHE_MESSAGEPART);
 		return NULL;
 	}
@@ -61,7 +61,8 @@
 	const char *ret;
 
 	if ((mail->data.cached_fields & field) == 0) {
-		mail_cache_mark_missing(mail->trans->cache_view, field);
+		mail_cache_mark_missing(mail->trans->cache_view,
+					mail->data.seq, field);
 		return NULL;
 	}
 
@@ -78,7 +79,8 @@
 	if (!mail_cache_copy_fixed_field(mail->trans->cache_view,
 					 mail->data.seq, field,
 					 &uoff, sizeof(uoff))) {
-		mail_cache_mark_missing(mail->trans->cache_view, field);
+		mail_cache_mark_missing(mail->trans->cache_view,
+					mail->data.seq, field);
 		uoff = (uoff_t)-1;
 	}
 
@@ -98,7 +100,7 @@
 					 mail->data.seq,
 					 MAIL_CACHE_RECEIVED_DATE,
 					 &t, sizeof(t))) {
-		mail_cache_mark_missing(mail->trans->cache_view,
+		mail_cache_mark_missing(mail->trans->cache_view, mail->data.seq,
 					MAIL_CACHE_RECEIVED_DATE);
 		t = (time_t)-1;
 	}
@@ -113,7 +115,7 @@
 					 mail->data.seq,
 					 MAIL_CACHE_SENT_DATE,
 					 sent_date, sizeof(*sent_date))) {
-		mail_cache_mark_missing(mail->trans->cache_view,
+		mail_cache_mark_missing(mail->trans->cache_view, mail->data.seq,
 					MAIL_CACHE_SENT_DATE);
 
 		sent_date->time = (time_t)-1;