--- a/src/lib-index/mail-cache.c	Tue Apr 27 00:20:15 2004 +0300
+++ /dev/null	Thu Jan 01 00:00:00 1970 +0000
@@ -1,1940 +0,0 @@
-/* Copyright (C) 2003 Timo Sirainen */
-
-#include "lib.h"
-#include "buffer.h"
-#include "byteorder.h"
-#include "file-lock.h"
-#include "file-set-size.h"
-#include "ioloop.h"
-#include "mmap-util.h"
-#include "write-full.h"
-#include "mail-index.h"
-#include "mail-index-util.h"
-#include "mail-cache.h"
-
-#include <stddef.h>
-#include <unistd.h>
-#include <sys/stat.h>
-
-/* Never compress the file if it's smaller than this */
-#define COMPRESS_MIN_SIZE (1024*50)
-
-/* Compress the file when deleted space reaches n% of total size */
-#define COMPRESS_PERCENTAGE 20
-
-/* Compress the file when n% of rows contain continued rows.
-   200% means that there's 2 continued rows per record. */
-#define COMPRESS_CONTINUED_PERCENTAGE 200
-
-/* Initial size for the file */
-#define MAIL_CACHE_INITIAL_SIZE (sizeof(struct mail_cache_header) + 10240)
-
-/* When more space is needed, grow the file n% larger than the previous size */
-#define MAIL_CACHE_GROW_PERCENTAGE 10
-
-#define MAIL_CACHE_LOCK_TIMEOUT 120
-#define MAIL_CACHE_LOCK_CHANGE_TIMEOUT 60
-#define MAIL_CACHE_LOCK_IMMEDIATE_TIMEOUT (5*60)
-
-#define CACHE_RECORD(cache, offset) \
-	((struct mail_cache_record *) ((char *) (cache)->mmap_base + offset))
-
-struct mail_cache_header {
-	uint32_t indexid;
-	uint32_t sync_id;
-
-	uint32_t continued_record_count;
-
-	uint32_t used_file_size;
-	uint32_t deleted_space;
-
-	uint32_t used_fields; /* enum mail_cache_field */
-
-	uint32_t field_usage_start; /* time_t */
-	uint32_t field_usage_counts[32];
-
-	uint32_t header_offsets[MAIL_CACHE_HEADERS_COUNT];
-};
-
-struct mail_cache_record {
-	uint32_t fields; /* enum mail_cache_field */
-	uint32_t next_offset;
-	uint32_t size; /* full record size, including this header */
-};
-
-struct mail_cache {
-	struct mail_index *index;
-
-	char *filepath;
-	int fd;
-
-	void *mmap_base;
-	size_t mmap_length;
-	uint32_t used_file_size;
-	uint32_t sync_id;
-
-	struct mail_cache_header *header;
-
-	pool_t split_header_pool;
-	uint32_t split_offsets[MAIL_CACHE_HEADERS_COUNT];
-	const char *const *split_headers[MAIL_CACHE_HEADERS_COUNT];
-
-	enum mail_cache_field default_cache_fields;
-	enum mail_cache_field never_cache_fields;
-
-        struct mail_cache_transaction_ctx *trans_ctx;
-	unsigned int locks;
-
-	unsigned int anon_mmap:1;
-	unsigned int mmap_refresh:1;
-	unsigned int silent:1;
-};
-
-struct mail_cache_transaction_ctx {
-	struct mail_cache *cache;
-
-	unsigned int next_unused_header_lowwater;
-
-	unsigned int last_idx;
-	struct mail_cache_record cache_rec;
-	buffer_t *cache_data;
-
-	unsigned int first_uid, last_uid, prev_uid;
-	enum mail_cache_field prev_fields;
-	buffer_t *index_marks, *cache_marks;
-};
-
-unsigned int mail_cache_field_sizes[32] = {
-	sizeof(enum mail_index_record_flag),
-	sizeof(uoff_t),
-	16,
-	sizeof(struct mail_sent_date),
-	sizeof(time_t),
-	sizeof(uoff_t),
-	sizeof(uoff_t),
-
-	0, 0, 0, 0, 0,
-
-	/* variable sized */
-	(unsigned int)-1, (unsigned int)-1, (unsigned int)-1, (unsigned int)-1,
-	(unsigned int)-1, (unsigned int)-1, (unsigned int)-1, (unsigned int)-1,
-	(unsigned int)-1, (unsigned int)-1, (unsigned int)-1, (unsigned int)-1,
-	(unsigned int)-1, (unsigned int)-1, (unsigned int)-1, (unsigned int)-1,
-	(unsigned int)-1, (unsigned int)-1, (unsigned int)-1, (unsigned int)-1
-};
-
-enum mail_cache_field mail_cache_header_fields[MAIL_CACHE_HEADERS_COUNT] = {
-	MAIL_CACHE_HEADERS1,
-	MAIL_CACHE_HEADERS2,
-	MAIL_CACHE_HEADERS3,
-	MAIL_CACHE_HEADERS4
-};
-
-static const unsigned char *null4[] = { 0, 0, 0, 0 };
-
-static const char *
-mail_cache_get_header_fields_str(struct mail_cache *cache, unsigned int idx);
-static int mail_cache_write(struct mail_cache_transaction_ctx *ctx);
-static struct mail_cache_record *
-mail_cache_lookup(struct mail_cache *cache,
-		  const struct mail_index_record *rec,
-		  enum mail_cache_field fields);
-
-static uint32_t uint32_to_offset(uint32_t offset)
-{
-	unsigned char buf[4];
-
-	i_assert(offset < 0x40000000);
-	i_assert((offset & 3) == 0);
-
-	offset >>= 2;
-	buf[0] = 0x80 | ((offset & 0x0fe00000) >> 21);
-	buf[1] = 0x80 | ((offset & 0x001fc000) >> 14);
-	buf[2] = 0x80 | ((offset & 0x00003f80) >> 7);
-	buf[3] = 0x80 |  (offset & 0x0000007f);
-	return *((uint32_t *) buf);
-}
-
-static uint32_t offset_to_uint32(uint32_t offset)
-{
-	const unsigned char *buf = (const unsigned char *) &offset;
-
-	if ((offset & 0x80808080) != 0x80808080)
-		return 0;
-
-	return (((uint32_t)buf[3] & 0x7f) << 2) |
-		(((uint32_t)buf[2] & 0x7f) << 9) |
-		(((uint32_t)buf[1] & 0x7f) << 16) |
-		(((uint32_t)buf[0] & 0x7f) << 23);
-}
-
-static int mail_cache_set_syscall_error(struct mail_cache *cache,
-					const char *function)
-{
-	i_assert(function != NULL);
-
-	if (ENOSPACE(errno)) {
-		cache->index->nodiskspace = TRUE;
-		return FALSE;
-	}
-
-	index_set_error(cache->index, "%s failed with index cache file %s: %m",
-			function, cache->filepath);
-	return FALSE;
-}
-
-static int mail_cache_create_memory(struct mail_cache *cache,
-				    struct mail_cache_header *hdr)
-{
-	cache->mmap_length = MAIL_CACHE_INITIAL_SIZE;
-	cache->mmap_base = mmap_anon(cache->mmap_length);
-	if (cache->mmap_base == MAP_FAILED) {
-		index_set_error(cache->index, "mmap_anon(%"PRIuSIZE_T")",
-				cache->mmap_length);
-		return FALSE;
-	}
-
-	cache->header = cache->mmap_base;
-	*cache->header = *hdr;
-
-	cache->anon_mmap = TRUE;
-	cache->filepath = i_strdup_printf("(in-memory index cache for %s)",
-					  cache->index->mailbox_path);
-	return TRUE;
-}
-
-static void mail_cache_file_close(struct mail_cache *cache)
-{
-	if (cache->anon_mmap) {
-		if (munmap_anon(cache->mmap_base, cache->mmap_length) < 0)
-			mail_cache_set_syscall_error(cache, "munmap_anon()");
-	} else if (cache->mmap_base != NULL) {
-		if (munmap(cache->mmap_base, cache->mmap_length) < 0)
-			mail_cache_set_syscall_error(cache, "munmap()");
-	}
-
-	cache->mmap_base = NULL;
-	cache->header = NULL;
-	cache->mmap_length = 0;
-
-	if (cache->fd != -1) {
-		if (close(cache->fd) < 0)
-			mail_cache_set_syscall_error(cache, "close()");
-		cache->fd = -1;
-	}
-}
-
-static int mail_cache_file_reopen(struct mail_cache *cache)
-{
-	int fd;
-
-	if (cache->anon_mmap) {
-		/* cache was set corrupted, we'll have to quit */
-		return FALSE;
-	}
-
-	fd = open(cache->filepath, O_RDWR);
-	if (fd == -1)
-		return mail_cache_set_syscall_error(cache, "open()");
-
-	mail_cache_file_close(cache);
-
-	cache->fd = fd;
-	return TRUE;
-}
-
-static int mmap_verify_header(struct mail_cache *cache)
-{
-	struct mail_cache_header *hdr;
-
-	/* check that the header is still ok */
-	if (cache->mmap_length < sizeof(struct mail_cache_header))
-		return mail_cache_set_corrupted(cache, "File too small");
-	cache->header = hdr = cache->mmap_base;
-	cache->sync_id = hdr->sync_id;
-
-	if (cache->header->indexid != cache->index->indexid) {
-		/* index id changed */
-		if (cache->header->indexid != 0)
-			mail_cache_set_corrupted(cache, "indexid changed");
-		cache->index->inconsistent = TRUE; /* easiest way to rebuild */
-		return FALSE;
-	}
-
-	if (cache->trans_ctx != NULL) {
-		/* we've updated used_file_size, do nothing */
-		return TRUE;
-	}
-
-	cache->used_file_size = nbo_to_uint32(hdr->used_file_size);
-
-	/* only check the header if we're locked */
-	if (cache->locks == 0)
-		return TRUE;
-
-	if (cache->used_file_size < sizeof(struct mail_cache_header)) {
-		mail_cache_set_corrupted(cache, "used_file_size too small");
-		return FALSE;
-	}
-	if ((cache->used_file_size % sizeof(uint32_t)) != 0) {
-		mail_cache_set_corrupted(cache, "used_file_size not aligned");
-		return FALSE;
-	}
-
-	if (cache->used_file_size > cache->mmap_length) {
-		/* maybe a crash truncated the file - just fix it */
-		hdr->used_file_size = uint32_to_nbo(cache->mmap_length & ~3);
-		if (msync(cache->mmap_base, sizeof(*hdr), MS_SYNC) < 0) 
-			return mail_cache_set_syscall_error(cache, "msync()");
-	}
-	return TRUE;
-}
-
-static int mmap_update_nocheck(struct mail_cache *cache,
-			       size_t offset, size_t size)
-{
-	struct stat st;
-
-	/* if sync id has changed, the file has to be reopened.
-	   note that if main index isn't locked, it may change again */
-	if (cache->sync_id != cache->index->cache_sync_id &&
-	    cache->mmap_base != NULL) {
-		if (!mail_cache_file_reopen(cache))
-			return -1;
-	}
-
-	if (offset < cache->mmap_length &&
-	    size <= cache->mmap_length - offset &&
-	    !cache->mmap_refresh) {
-		/* already mapped */
-		if (size != 0 || cache->anon_mmap)
-			return 1;
-
-		/* requesting the whole file - see if we need to
-		   re-mmap */
-		if (fstat(cache->fd, &st) < 0) {
-			mail_cache_set_syscall_error(cache, "fstat()");
-			return -1;
-		}
-		if ((uoff_t)st.st_size == cache->mmap_length)
-			return 1;
-	}
-	cache->mmap_refresh = FALSE;
-
-	if (cache->anon_mmap)
-		return 1;
-
-	if (cache->mmap_base != NULL) {
-		if (cache->locks != 0) {
-			/* in the middle of transaction - write the changes */
-			if (msync(cache->mmap_base, cache->mmap_length,
-				  MS_SYNC) < 0) {
-				mail_cache_set_syscall_error(cache, "msync()");
-				return -1;
-			}
-		}
-
-		if (munmap(cache->mmap_base, cache->mmap_length) < 0)
-			mail_cache_set_syscall_error(cache, "munmap()");
-	}
-
-	i_assert(cache->fd != -1);
-
-	/* map the whole file */
-	cache->header = NULL;
-	cache->mmap_length = 0;
-
-	cache->mmap_base = mmap_rw_file(cache->fd, &cache->mmap_length);
-	if (cache->mmap_base == MAP_FAILED) {
-		cache->mmap_base = NULL;
-		mail_cache_set_syscall_error(cache, "mmap()");
-		return -1;
-	}
-
-	/* re-mmaped, check header */
-	return 0;
-}
-
-static int mmap_update(struct mail_cache *cache, size_t offset, size_t size)
-{
-	int synced, ret;
-
-	for (synced = FALSE;; synced = TRUE) {
-		ret = mmap_update_nocheck(cache, offset, size);
-		if (ret > 0)
-			return TRUE;
-		if (ret < 0)
-			return FALSE;
-
-		if (!mmap_verify_header(cache))
-			return FALSE;
-
-		/* see if cache file was rebuilt - do it only once to avoid
-		   infinite looping */
-		if (cache->header->sync_id == cache->index->cache_sync_id ||
-		    synced)
-			break;
-
-		if (!mail_cache_file_reopen(cache))
-			return FALSE;
-	}
-	return TRUE;
-}
-
-static int mail_cache_open_and_verify(struct mail_cache *cache, int silent)
-{
-	struct stat st;
-
-	mail_cache_file_close(cache);
-
-	cache->fd = open(cache->filepath, O_RDWR);
-	if (cache->fd == -1) {
-		if (errno == ENOENT)
-			return 0;
-
-		mail_cache_set_syscall_error(cache, "open()");
-		return -1;
-	}
-
-	if (fstat(cache->fd, &st) < 0) {
-		mail_cache_set_syscall_error(cache, "fstat()");
-		return -1;
-	}
-
-	if (st.st_size < sizeof(struct mail_cache_header))
-		return 0;
-
-	cache->mmap_refresh = TRUE;
-	if (mmap_update_nocheck(cache, 0, sizeof(struct mail_cache_header)) < 0)
-		return -1;
-
-	/* verify that this really is the cache for wanted index */
-	cache->silent = silent;
-	if (!mmap_verify_header(cache)) {
-		cache->silent = FALSE;
-		return 0;
-	}
-
-	cache->silent = FALSE;
-	return 1;
-}
-
-static void mail_index_clear_cache_offsets(struct mail_index *index)
-{
-	struct mail_index_record *rec;
-
-	index->sync_stamp = 0;
-
-	rec = index->lookup(index, 1);
-	while (rec != NULL) {
-		rec->cache_offset = 0;
-		rec = index->next(index, rec);
-	}
-}
-
-static int mail_cache_open_or_create_file(struct mail_cache *cache,
-					  struct mail_cache_header *hdr)
-{
-	int ret, fd;
-
-	cache->filepath = i_strconcat(cache->index->filepath,
-				      MAIL_CACHE_FILE_PREFIX, NULL);
-
-	ret = mail_cache_open_and_verify(cache, FALSE);
-	if (ret != 0)
-		return ret > 0;
-
-	/* we'll have to clear cache_offsets which requires exclusive lock */
-	cache->index->inconsistent = FALSE;
-	if (!mail_index_set_lock(cache->index, MAIL_LOCK_EXCLUSIVE))
-		return FALSE;
-
-	/* maybe a rebuild.. */
-	fd = file_dotlock_open(cache->filepath, NULL, MAIL_CACHE_LOCK_TIMEOUT,
-			       MAIL_CACHE_LOCK_CHANGE_TIMEOUT,
-			       MAIL_CACHE_LOCK_IMMEDIATE_TIMEOUT, NULL, NULL);
-	if (fd == -1) {
-		mail_cache_set_syscall_error(cache, "file_dotlock_open()");
-		return FALSE;
-	}
-
-	/* see if someone else just created the cache file */
-	ret = mail_cache_open_and_verify(cache, TRUE);
-	if (ret != 0) {
-		(void)file_dotlock_delete(cache->filepath, fd);
-		return ret > 0;
-	}
-
-	/* rebuild then */
-	if (write_full(fd, hdr, sizeof(*hdr)) < 0) {
-		mail_cache_set_syscall_error(cache, "write_full()");
-		(void)file_dotlock_delete(cache->filepath, fd);
-		return FALSE;
-	}
-	if (file_set_size(fd, MAIL_CACHE_INITIAL_SIZE) < 0) {
-		mail_cache_set_syscall_error(cache, "file_set_size()");
-		(void)file_dotlock_delete(cache->filepath, fd);
-		return FALSE;
-	}
-
-	mail_index_clear_cache_offsets(cache->index);
-
-	mail_cache_file_close(cache);
-	cache->fd = dup(fd);
-
-	if (file_dotlock_replace(cache->filepath, fd, FALSE) < 0) {
-		mail_cache_set_syscall_error(cache, "file_dotlock_replace()");
-		return FALSE;
-	}
-
-	cache->mmap_refresh = TRUE;
-	if (!mmap_update(cache, 0, sizeof(struct mail_cache_header)))
-		return FALSE;
-
-	return TRUE;
-}
-
-int mail_cache_open_or_create(struct mail_index *index)
-{
-        struct mail_cache_header hdr;
-	struct mail_cache *cache;
-
-	memset(&hdr, 0, sizeof(hdr));
-	hdr.indexid = index->indexid;
-	hdr.sync_id = index->cache_sync_id;
-	hdr.used_file_size = uint32_to_nbo(sizeof(hdr));
-
-	cache = i_new(struct mail_cache, 1);
-	cache->index = index;
-	cache->fd = -1;
-        cache->split_header_pool = pool_alloconly_create("Headers", 512);
-
-	index->cache = cache;
-
-	/* we'll do anon-mmaping only if initially requested. if we fail
-	   because of out of disk space, we'll just let the main index code
-	   know it and fail. */
-	if (INDEX_IS_IN_MEMORY(index)) {
-		if (!mail_cache_create_memory(cache, &hdr)) {
-			mail_cache_free(cache);
-			return FALSE;
-		}
-	} else {
-		if (!mail_cache_open_or_create_file(cache, &hdr)) {
-			mail_cache_free(cache);
-			return FALSE;
-		}
-	}
-
-	/* unset inconsistency - we already rebuilt the cache file */
-	index->inconsistent = FALSE;
-
-	return TRUE;
-}
-
-void mail_cache_free(struct mail_cache *cache)
-{
-	i_assert(cache->trans_ctx == NULL);
-
-	cache->index->cache = NULL;
-
-	mail_cache_file_close(cache);
-
-	pool_unref(cache->split_header_pool);
-	i_free(cache->filepath);
-	i_free(cache);
-}
-
-void mail_cache_set_defaults(struct mail_cache *cache,
-			     enum mail_cache_field default_cache_fields,
-			     enum mail_cache_field never_cache_fields)
-{
-	cache->default_cache_fields = default_cache_fields;
-	cache->never_cache_fields = never_cache_fields;
-}
-
-static const struct mail_cache_record *
-mail_cache_compress_record(struct mail_cache *cache,
-			   struct mail_index_record *rec, int header_idx,
-			   uint32_t *size_r)
-{
-	enum mail_cache_field orig_cached_fields, cached_fields, field;
-	struct mail_cache_record cache_rec;
-	buffer_t *buffer;
-	const void *data;
-	size_t size, pos;
-	uint32_t nb_size;
-	int i;
-
-	memset(&cache_rec, 0, sizeof(cache_rec));
-	buffer = buffer_create_dynamic(pool_datastack_create(),
-				       4096, (size_t)-1);
-
-        orig_cached_fields = mail_cache_get_fields(cache, rec);
-	cached_fields = orig_cached_fields & ~MAIL_CACHE_HEADERS_MASK;
-	buffer_append(buffer, &cache_rec, sizeof(cache_rec));
-	for (i = 0, field = 1; i < 31; i++, field <<= 1) {
-		if ((cached_fields & field) == 0)
-			continue;
-
-		if (!mail_cache_lookup_field(cache, rec, field, &data, &size)) {
-			cached_fields &= ~field;
-			continue;
-		}
-
-		nb_size = uint32_to_nbo((uint32_t)size);
-
-		if ((field & MAIL_CACHE_FIXED_MASK) == 0)
-			buffer_append(buffer, &nb_size, sizeof(nb_size));
-		buffer_append(buffer, data, size);
-		if ((size & 3) != 0)
-			buffer_append(buffer, null4, 4 - (size & 3));
-	}
-
-	/* now merge all the headers if we have them all */
-	if ((orig_cached_fields & mail_cache_header_fields[header_idx]) != 0) {
-		nb_size = 0;
-		pos = buffer_get_used_size(buffer);
-		buffer_append(buffer, &nb_size, sizeof(nb_size));
-
-		for (i = 0; i <= header_idx; i++) {
-			field = mail_cache_header_fields[i];
-			if (mail_cache_lookup_field(cache, rec, field,
-						    &data, &size) && size > 1) {
-				size--; /* terminating \0 */
-				buffer_append(buffer, data, size);
-				nb_size += size;
-			}
-		}
-		buffer_append(buffer, "", 1);
-		nb_size++;
-		if ((nb_size & 3) != 0)
-			buffer_append(buffer, null4, 4 - (nb_size & 3));
-
-		nb_size = uint32_to_nbo(nb_size);
-		buffer_write(buffer, pos, &nb_size, sizeof(nb_size));
-
-		cached_fields |= MAIL_CACHE_HEADERS1;
-	}
-
-	cache_rec.fields = cached_fields;
-	cache_rec.size = uint32_to_nbo(buffer_get_used_size(buffer));
-	buffer_write(buffer, 0, &cache_rec, sizeof(cache_rec));
-
-	data = buffer_get_data(buffer, &size);
-	*size_r = size;
-	return data;
-}
-
-static int mail_cache_copy(struct mail_cache *cache, int fd)
-{
-	struct mail_cache_header *hdr;
-	const struct mail_cache_record *cache_rec;
-	struct mail_index_record *rec;
-        enum mail_cache_field used_fields;
-	unsigned char *mmap_base;
-	const char *str;
-	uint32_t new_file_size, offset, size, nb_size;
-	int i, header_idx;
-
-	/* pick some reasonably good file size */
-	new_file_size = cache->used_file_size -
-		nbo_to_uint32(cache->header->deleted_space);
-	new_file_size = (new_file_size + 1023) & ~1023;
-	if (new_file_size < MAIL_CACHE_INITIAL_SIZE)
-		new_file_size = MAIL_CACHE_INITIAL_SIZE;
-
-	if (file_set_size(fd, new_file_size) < 0)
-		return mail_cache_set_syscall_error(cache, "file_set_size()");
-
-	mmap_base = mmap(NULL, new_file_size, PROT_READ | PROT_WRITE,
-			 MAP_SHARED, fd, 0);
-	if (mmap_base == MAP_FAILED)
-		return mail_cache_set_syscall_error(cache, "mmap()");
-
-	/* skip file's header */
-	hdr = (struct mail_cache_header *) mmap_base;
-	offset = sizeof(*hdr);
-
-	/* merge all the header pieces into one. if some message doesn't have
-	   all the required pieces, we'll just have to drop them all. */
-	for (i = MAIL_CACHE_HEADERS_COUNT-1; i >= 0; i--) {
-		str = mail_cache_get_header_fields_str(cache, i);
-		if (str != NULL)
-			break;
-	}
-
-	if (str == NULL)
-		header_idx = -1;
-	else {
-		hdr->header_offsets[0] = uint32_to_offset(offset);
-		header_idx = i;
-
-		size = strlen(str) + 1;
-		nb_size = uint32_to_nbo(size);
-
-		memcpy(mmap_base + offset, &nb_size, sizeof(nb_size));
-		offset += sizeof(nb_size);
-		memcpy(mmap_base + offset, str, size);
-		offset += (size + 3) & ~3;
-	}
-
-	used_fields = 0;
-	rec = cache->index->lookup(cache->index, 1);
-	while (rec != NULL) {
-		cache_rec = mail_cache_lookup(cache, rec, 0);
-		if (cache_rec == NULL)
-			rec->cache_offset = 0;
-		else if (offset_to_uint32(cache_rec->next_offset) == 0) {
-			/* just one unmodified block, copy it */
-			size = nbo_to_uint32(cache_rec->size);
-			i_assert(offset + size <= new_file_size);
-
-			memcpy(mmap_base + offset, cache_rec, size);
-			rec->cache_offset = uint32_to_offset(offset);
-
-			size = (size + 3) & ~3;
-			offset += size;
-		} else {
-			/* multiple blocks, sort them into buffer */
-			t_push();
-			cache_rec = mail_cache_compress_record(cache, rec,
-							       header_idx,
-							       &size);
-			i_assert(offset + size <= new_file_size);
-			memcpy(mmap_base + offset, cache_rec, size);
-			used_fields |= cache_rec->fields;
-			t_pop();
-
-			rec->cache_offset = uint32_to_offset(offset);
-			offset += size;
-		}
-
-		rec = cache->index->next(cache->index, rec);
-	}
-
-	/* update header */
-	hdr->indexid = cache->index->indexid;
-	hdr->sync_id = cache->sync_id = cache->index->cache_sync_id =
-		++cache->index->header->cache_sync_id;
-	hdr->used_file_size = uint32_to_nbo(offset);
-	hdr->used_fields = used_fields;
-	hdr->field_usage_start = uint32_to_nbo(ioloop_time);
-
-	/* write everything to disk */
-	if (msync(mmap_base, offset, MS_SYNC) < 0)
-		return mail_cache_set_syscall_error(cache, "msync()");
-
-	if (munmap(mmap_base, new_file_size) < 0)
-		return mail_cache_set_syscall_error(cache, "munmap()");
-
-	if (fdatasync(fd) < 0)
-		return mail_cache_set_syscall_error(cache, "fdatasync()");
-	return TRUE;
-}
-
-int mail_cache_compress(struct mail_cache *cache)
-{
-	int fd, ret = TRUE;
-
-	i_assert(cache->trans_ctx == NULL);
-
-	if (cache->anon_mmap)
-		return TRUE;
-
-	if (!cache->index->set_lock(cache->index, MAIL_LOCK_EXCLUSIVE))
-		return FALSE;
-
-	if (mail_cache_lock(cache, TRUE) <= 0)
-		return FALSE;
-
-#ifdef DEBUG
-	i_warning("Compressing cache file %s", cache->filepath);
-#endif
-
-	fd = file_dotlock_open(cache->filepath, NULL, MAIL_CACHE_LOCK_TIMEOUT,
-			       MAIL_CACHE_LOCK_CHANGE_TIMEOUT,
-			       MAIL_CACHE_LOCK_IMMEDIATE_TIMEOUT, NULL, NULL);
-	if (fd == -1) {
-		mail_cache_set_syscall_error(cache, "file_dotlock_open()");
-		return FALSE;
-	}
-
-	/* now we'll begin the actual moving. keep rebuild-flag on
-	   while doing it. */
-	cache->index->header->flags |= MAIL_INDEX_HDR_FLAG_REBUILD;
-	if (!mail_index_fmdatasync(cache->index, cache->index->header_size))
-		return FALSE;
-
-	if (!mail_cache_copy(cache, fd)) {
-		(void)file_dotlock_delete(cache->filepath, fd);
-		ret = FALSE;
-	} else {
-		mail_cache_file_close(cache);
-		cache->fd = dup(fd);
-
-		if (file_dotlock_replace(cache->filepath, fd, FALSE) < 0) {
-			mail_cache_set_syscall_error(cache,
-						     "file_dotlock_replace()");
-			ret = FALSE;
-		}
-
-		if (!mmap_update(cache, 0, 0))
-			ret = FALSE;
-	}
-
-	/* headers could have changed, reread them */
-	memset(cache->split_offsets, 0, sizeof(cache->split_offsets));
-	memset(cache->split_headers, 0, sizeof(cache->split_headers));
-
-	if (ret) {
-		cache->index->header->flags &=
-			~(MAIL_INDEX_HDR_FLAG_REBUILD |
-			  MAIL_INDEX_HDR_FLAG_COMPRESS_CACHE);
-	}
-
-	if (!mail_cache_unlock(cache))
-		ret = FALSE;
-
-	return ret;
-}
-
-int mail_cache_truncate(struct mail_cache *cache)
-{
-	struct mail_cache_header hdr;
-	int ret, fd;
-
-	i_assert(cache->index->lock_type == MAIL_LOCK_EXCLUSIVE);
-
-	memset(&hdr, 0, sizeof(hdr));
-	hdr.indexid = cache->index->indexid;
-	hdr.sync_id = cache->sync_id = cache->index->cache_sync_id =
-		++cache->index->header->cache_sync_id;
-	hdr.used_file_size = uint32_to_nbo(sizeof(hdr));
-	cache->used_file_size = sizeof(hdr);
-
-	if (cache->anon_mmap) {
-		*cache->header = hdr;
-		return TRUE;
-	}
-
-	ret = mail_cache_open_and_verify(cache, TRUE);
-	if (ret != 0)
-		return ret > 0;
-
-	fd = file_dotlock_open(cache->filepath, NULL, MAIL_CACHE_LOCK_TIMEOUT,
-			       MAIL_CACHE_LOCK_CHANGE_TIMEOUT,
-			       MAIL_CACHE_LOCK_IMMEDIATE_TIMEOUT, NULL, NULL);
-	if (fd == -1) {
-		mail_cache_set_syscall_error(cache, "file_dotlock_open()");
-		return FALSE;
-	}
-
-	if (write_full(fd, &hdr, sizeof(hdr)) < 0) {
-		mail_cache_set_syscall_error(cache, "write_full()");
-		(void)file_dotlock_delete(cache->filepath, fd);
-		return FALSE;
-	}
-	if (file_set_size(fd, MAIL_CACHE_INITIAL_SIZE) < 0) {
-		mail_cache_set_syscall_error(cache, "file_set_size()");
-		(void)file_dotlock_delete(cache->filepath, fd);
-		return FALSE;
-	}
-
-	mail_cache_file_close(cache);
-	cache->fd = dup(fd);
-
-	if (file_dotlock_replace(cache->filepath, fd, FALSE) < 0) {
-		mail_cache_set_syscall_error(cache, "file_dotlock_replace()");
-		return FALSE;
-	}
-
-	cache->mmap_refresh = TRUE;
-	if (!mmap_update(cache, 0, sizeof(struct mail_cache_header)))
-		return FALSE;
-
-	return TRUE;
-}
-
-int mail_cache_mark_file_deleted(struct mail_cache *cache)
-{
-	uint32_t indexid = 0;
-
-	if (cache->anon_mmap)
-		cache->header->indexid = 0;
-	else {
-		if (pwrite(cache->fd, &indexid, sizeof(indexid), 0) < 0)
-			return mail_cache_set_syscall_error(cache, "pwrite()");
-	}
-	return TRUE;
-}
-
-int mail_cache_lock(struct mail_cache *cache, int nonblock)
-{
-	int ret;
-
-	if (cache->locks++ != 0)
-		return TRUE;
-
-	if (cache->anon_mmap)
-		return TRUE;
-
-	if (nonblock) {
-		ret = file_try_lock(cache->fd, F_WRLCK);
-		if (ret < 0)
-			mail_cache_set_syscall_error(cache, "file_try_lock()");
-	} else {
-		ret = file_wait_lock(cache->fd, F_WRLCK);
-		if (ret <= 0)
-			mail_cache_set_syscall_error(cache, "file_wait_lock()");
-	}
-
-	if (ret > 0) {
-		if (!mmap_update(cache, 0, 0)) {
-			(void)mail_cache_unlock(cache);
-			return -1;
-		}
-		if (cache->sync_id != cache->index->cache_sync_id) {
-			/* we have the cache file locked and sync_id still
-			   doesn't match. it means we crashed between updating
-			   cache file and updating sync_id in index header.
-			   just update the sync_ids so they match. */
-			i_warning("Updating broken sync_id in cache file %s",
-				  cache->filepath);
-			cache->sync_id = cache->header->sync_id =
-				cache->index->cache_sync_id;
-		}
-	}
-	return ret;
-}
-
-int mail_cache_unlock(struct mail_cache *cache)
-{
-	if (--cache->locks > 0)
-		return TRUE;
-
-	if (cache->anon_mmap)
-		return TRUE;
-
-	if (file_wait_lock(cache->fd, F_UNLCK) <= 0) {
-		mail_cache_set_syscall_error(cache, "file_wait_lock(F_UNLCK)");
-		return FALSE;
-	}
-
-	return TRUE;
-}
-
-void mail_cache_unlock_later(struct mail_cache *cache)
-{
-	cache->index->cache_later_locks++;
-}
-
-int mail_cache_is_locked(struct mail_cache *cache)
-{
-	return cache->locks > 0;
-}
-
-int mail_cache_transaction_begin(struct mail_cache *cache, int nonblock,
-				 struct mail_cache_transaction_ctx **ctx_r)
-{
-	int ret;
-
-	i_assert(cache->trans_ctx == NULL);
-
-	ret = mail_cache_lock(cache, nonblock);
-	if (ret <= 0)
-		return ret;
-
-	*ctx_r = i_new(struct mail_cache_transaction_ctx, 1);
-	(*ctx_r)->cache = cache;
-	(*ctx_r)->cache_data =
-		buffer_create_dynamic(system_pool, 8192, (size_t)-1);
-	(*ctx_r)->last_idx = (unsigned int)-1;
-
-	cache->trans_ctx = *ctx_r;
-	return 1;
-}
-
-int mail_cache_transaction_end(struct mail_cache_transaction_ctx *ctx)
-{
-	int ret = TRUE;
-
-	i_assert(ctx->cache->trans_ctx != NULL);
-
-	(void)mail_cache_transaction_rollback(ctx);
-
-	if (!mail_cache_unlock(ctx->cache))
-		ret = FALSE;
-
-	ctx->cache->trans_ctx = NULL;
-
-	if (ctx->cache_marks != NULL)
-		buffer_free(ctx->cache_marks);
-	if (ctx->index_marks != NULL)
-		buffer_free(ctx->index_marks);
-	buffer_free(ctx->cache_data);
-	i_free(ctx);
-	return ret;
-}
-
-static void mail_cache_transaction_flush(struct mail_cache_transaction_ctx *ctx)
-{
-	memset(&ctx->cache_rec, 0, sizeof(ctx->cache_rec));
-	ctx->last_idx = (unsigned int)-1;
-
-	ctx->next_unused_header_lowwater = 0;
-	ctx->first_uid = ctx->last_uid = ctx->prev_uid = 0;
-	ctx->prev_fields = 0;
-
-	if (ctx->cache_marks != NULL)
-		buffer_set_used_size(ctx->cache_marks, 0);
-	if (ctx->index_marks != NULL)
-		buffer_set_used_size(ctx->index_marks, 0);
-	buffer_set_used_size(ctx->cache_data, 0);
-}
-
-static void mark_update(buffer_t **buf, uint32_t offset, uint32_t data)
-{
-	if (*buf == NULL)
-		*buf = buffer_create_dynamic(system_pool, 1024, (size_t)-1);
-
-	/* data is in big endian, we want to update only the lowest byte */
-	buffer_append(*buf, &offset, sizeof(offset));
-	buffer_append(*buf, &data, sizeof(data));
-}
-
-static int write_mark_updates(struct mail_index *index, buffer_t *marks,
-			      const char *path, int fd)
-{
-	const uint32_t *data, *end;
-	size_t size;
-
-	data = buffer_get_data(marks, &size);
-	end = data + size/sizeof(uint32_t);
-
-	while (data < end) {
-		if (pwrite(fd, data+1, sizeof(*data), data[0]) < 0) {
-			index_file_set_syscall_error(index, path, "pwrite()");
-			return FALSE;
-		}
-		data += 2;
-	}
-	return TRUE;
-}
-
-static void write_mark_updates_in_memory(buffer_t *marks, void *mmap_base,
-					 size_t mmap_length)
-{
-	const unsigned char *data, *end;
-	uint32_t offset;
-	size_t size;
-
-	data = buffer_get_data(marks, &size);
-	end = data + size;
-
-	while (data < end) {
-		memcpy(&offset, data, sizeof(offset));
-		data += sizeof(offset);
-
-		i_assert(offset <= mmap_length - sizeof(uint32_t));
-		memcpy((char *) mmap_base + offset, data, sizeof(uint32_t));
-		data += sizeof(uint32_t);
-	}
-}
-
-static void commit_all_changes_in_memory(struct mail_cache_transaction_ctx *ctx)
-{
-	struct mail_cache *cache = ctx->cache;
-
-	if (ctx->cache_marks != NULL) {
-		write_mark_updates_in_memory(ctx->cache_marks,
-					     cache->mmap_base,
-					     cache->mmap_length);
-	}
-	if (ctx->index_marks != NULL) {
-		write_mark_updates_in_memory(ctx->index_marks,
-					     cache->index->mmap_base,
-					     cache->index->mmap_used_length);
-	}
-}
-
-static int commit_all_changes(struct mail_cache_transaction_ctx *ctx)
-{
-	struct mail_cache *cache = ctx->cache;
-	uint32_t cont;
-
-	if (ctx->cache->anon_mmap) {
-		commit_all_changes_in_memory(ctx);
-		return TRUE;
-	}
-
-	/* write everything to disk */
-	if (msync(cache->mmap_base, cache->mmap_length, MS_SYNC) < 0)
-		return mail_cache_set_syscall_error(cache, "msync()");
-
-	if (fdatasync(cache->fd) < 0)
-		return mail_cache_set_syscall_error(cache, "fdatasync()");
-
-	if (ctx->cache_marks != NULL &&
-	    buffer_get_used_size(ctx->cache_marks) != 0) {
-		/* now that we're sure it's there, set on all the used-bits */
-		if (!write_mark_updates(cache->index, ctx->cache_marks,
-					cache->filepath, cache->fd))
-			return FALSE;
-
-		/* update continued records count */
-		cont = nbo_to_uint32(cache->header->continued_record_count);
-
-		cont += buffer_get_used_size(ctx->cache_marks) /
-			(sizeof(uint32_t) * 2);
-
-		if (cont * 100 / cache->index->header->messages_count >=
-		    COMPRESS_CONTINUED_PERCENTAGE &&
-		    cache->used_file_size >= COMPRESS_MIN_SIZE) {
-			/* too many continued rows, compress */
-			cache->index->set_flags |=
-				MAIL_INDEX_HDR_FLAG_COMPRESS_CACHE;
-		}
-
-		cache->header->continued_record_count = uint32_to_nbo(cont);
-	}
-
-	/* write index last */
-	if (ctx->index_marks != NULL &&
-	    buffer_get_used_size(ctx->index_marks) != 0) {
-		if (!mail_index_fmdatasync(cache->index,
-					   cache->index->mmap_used_length))
-			return FALSE;
-
-		if (!write_mark_updates(cache->index, ctx->index_marks,
-					cache->index->filepath,
-					cache->index->fd))
-			return FALSE;
-	}
-	return TRUE;
-}
-
-int mail_cache_transaction_commit(struct mail_cache_transaction_ctx *ctx)
-{
-	int ret = TRUE;
-
-	if (ctx->last_idx != (unsigned int)-1) {
-		if (!mail_cache_write(ctx))
-			return FALSE;
-	}
-
-	ctx->cache->header->used_file_size =
-		uint32_to_nbo(ctx->cache->used_file_size);
-
-	if (!commit_all_changes(ctx))
-		ret = FALSE;
-
-	if (ctx->next_unused_header_lowwater == MAIL_CACHE_HEADERS_COUNT) {
-		/* they're all used - compress the cache to get more */
-		ctx->cache->index->set_flags |=
-			MAIL_INDEX_HDR_FLAG_COMPRESS_CACHE;
-	}
-
-	mail_cache_transaction_flush(ctx);
-	return ret;
-}
-
-int mail_cache_transaction_rollback(struct mail_cache_transaction_ctx *ctx)
-{
-	struct mail_cache *cache = ctx->cache;
-	unsigned int i;
-
-	/* no need to actually modify the file - we just didn't update
-	   used_file_size */
-	cache->used_file_size = nbo_to_uint32(cache->header->used_file_size);
-
-	/* make sure we don't cache the headers */
-	for (i = 0; i < ctx->next_unused_header_lowwater; i++) {
-		if (offset_to_uint32(cache->header->header_offsets[i]) == 0)
-			cache->split_offsets[i] = 1;
-	}
-
-	mail_cache_transaction_flush(ctx);
-	return TRUE;
-}
-
-static int mail_cache_grow(struct mail_cache *cache, uint32_t size)
-{
-	struct stat st;
-	void *base;
-	uoff_t grow_size, new_fsize;
-
-	new_fsize = cache->used_file_size + size;
-	grow_size = new_fsize / 100 * MAIL_CACHE_GROW_PERCENTAGE;
-	if (grow_size < 16384)
-		grow_size = 16384;
-
-	new_fsize += grow_size;
-	new_fsize &= ~1023;
-
-	if (cache->anon_mmap) {
-		i_assert(new_fsize < SSIZE_T_MAX);
-
-		base = mremap_anon(cache->mmap_base, cache->mmap_length,
-				   (size_t)new_fsize, MREMAP_MAYMOVE);
-		if (base == MAP_FAILED) {
-			mail_cache_set_syscall_error(cache, "mremap_anon()");
-			return FALSE;
-		}
-
-		cache->mmap_base = base;
-		cache->mmap_length = (size_t)new_fsize;
-		cache->header = cache->mmap_base;
-		return TRUE;
-	}
-
-	if (fstat(cache->fd, &st) < 0)
-		return mail_cache_set_syscall_error(cache, "fstat()");
-
-	if (cache->used_file_size + size <= (uoff_t)st.st_size) {
-		/* no need to grow, just update mmap */
-		if (!mmap_update(cache, 0, 0))
-			return FALSE;
-
-		i_assert(cache->mmap_length >= (uoff_t)st.st_size);
-		return TRUE;
-	}
-
-	if (st.st_size < (off_t)sizeof(struct mail_cache_header))
-		return mail_cache_set_corrupted(cache, "Header is missing");
-
-	if (file_set_size(cache->fd, (off_t)new_fsize) < 0)
-		return mail_cache_set_syscall_error(cache, "file_set_size()");
-
-	return mmap_update(cache, 0, 0);
-}
-
-static uint32_t mail_cache_append_space(struct mail_cache_transaction_ctx *ctx,
-					uint32_t size)
-{
-	/* NOTE: must be done within transaction or rollback would break it */
-	uint32_t offset;
-
-	i_assert((size & 3) == 0);
-
-	offset = ctx->cache->used_file_size;
-	if (offset >= 0x40000000) {
-		index_set_error(ctx->cache->index, "Cache file too large: %s",
-				ctx->cache->filepath);
-		return 0;
-	}
-
-	if (offset + size > ctx->cache->mmap_length) {
-		if (!mail_cache_grow(ctx->cache, size))
-			return 0;
-	}
-
-	ctx->cache->used_file_size += size;
-	return offset;
-}
-
-static const char *
-mail_cache_get_header_fields_str(struct mail_cache *cache, unsigned int idx)
-{
-	uint32_t offset, data_size;
-	unsigned char *buf;
-
-	offset = offset_to_uint32(cache->header->header_offsets[idx]);
-
-	if (offset == 0)
-		return NULL;
-
-	if (!mmap_update(cache, offset, 1024))
-		return NULL;
-
-	if (offset + sizeof(data_size) > cache->mmap_length) {
-		mail_cache_set_corrupted(cache, "Header %u points outside file",
-					 idx);
-		return NULL;
-	}
-
-	buf = cache->mmap_base;
-	memcpy(&data_size, buf + offset, sizeof(data_size));
-	data_size = nbo_to_uint32(data_size);
-	offset += sizeof(data_size);
-
-	if (data_size == 0) {
-		mail_cache_set_corrupted(cache,
-			"Header %u points to empty string", idx);
-		return NULL;
-	}
-
-	if (!mmap_update(cache, offset, data_size))
-		return NULL;
-
-	if (offset + data_size > cache->mmap_length) {
-		mail_cache_set_corrupted(cache, "Header %u points outside file",
-					 idx);
-		return NULL;
-	}
-
-	buf = cache->mmap_base;
-	if (buf[offset + data_size - 1] != '\0') {
-		mail_cache_set_corrupted(cache,
-			"Header %u points to invalid string", idx);
-		return NULL;
-	}
-
-	return buf + offset;
-}
-
-static const char *const *
-split_header(struct mail_cache *cache, const char *header)
-{
-	const char *const *arr, *const *tmp;
-	const char *null = NULL;
-	char *str;
-	buffer_t *buf;
-
-	if (header == NULL)
-		return NULL;
-
-	arr = t_strsplit(header, "\n");
-	buf = buffer_create_dynamic(cache->split_header_pool, 32, (size_t)-1);
-	for (tmp = arr; *tmp != NULL; tmp++) {
-		str = p_strdup(cache->split_header_pool, *tmp);
-		buffer_append(buf, &str, sizeof(str));
-	}
-	buffer_append(buf, &null, sizeof(null));
-
-	return buffer_get_data(buf, NULL);
-}
-
-const char *const *mail_cache_get_header_fields(struct mail_cache *cache,
-						unsigned int idx)
-{
-	const char *str;
-	int i;
-
-	i_assert(idx < MAIL_CACHE_HEADERS_COUNT);
-
-	/* t_strsplit() is a bit slow, so we cache it */
-	if (cache->header->header_offsets[idx] != cache->split_offsets[idx]) {
-		p_clear(cache->split_header_pool);
-
-		t_push();
-		for (i = 0; i < MAIL_CACHE_HEADERS_COUNT; i++) {
-			cache->split_offsets[i] =
-				cache->header->header_offsets[i];
-
-			str = mail_cache_get_header_fields_str(cache, i);
-			cache->split_headers[i] = split_header(cache, str);
-		}
-		t_pop();
-	}
-
-	return cache->split_headers[idx];
-}
-
-static const char *write_header_string(const char *const headers[],
-				       uint32_t *size_r)
-{
-	buffer_t *buffer;
-	size_t size;
-
-	buffer = buffer_create_dynamic(pool_datastack_create(),
-				       512, (size_t)-1);
-
-	while (*headers != NULL) {
-		if (buffer_get_used_size(buffer) != 0)
-			buffer_append(buffer, "\n", 1);
-		buffer_append(buffer, *headers, strlen(*headers));
-		headers++;
-	}
-	buffer_append(buffer, null4, 1);
-
-	size = buffer_get_used_size(buffer);
-	if ((size & 3) != 0) {
-		buffer_append(buffer, null4, 4 - (size & 3));
-		size += 4 - (size & 3);
-	}
-	*size_r = size;
-	return buffer_get_data(buffer, NULL);
-}
-
-int mail_cache_set_header_fields(struct mail_cache_transaction_ctx *ctx,
-				 unsigned int idx, const char *const headers[])
-{
-	struct mail_cache *cache = ctx->cache;
-	uint32_t offset, update_offset, size;
-	const char *header_str, *prev_str;
-
-	i_assert(*headers != NULL);
-	i_assert(idx < MAIL_CACHE_HEADERS_COUNT);
-	i_assert(idx >= ctx->next_unused_header_lowwater);
-	i_assert(offset_to_uint32(cache->header->header_offsets[idx]) == 0);
-
-	t_push();
-
-	header_str = write_header_string(headers, &size);
-	if (idx != 0) {
-		prev_str = mail_cache_get_header_fields_str(cache, idx-1);
-		if (prev_str == NULL) {
-			t_pop();
-			return FALSE;
-		}
-
-		i_assert(strcmp(header_str, prev_str) != 0);
-	}
-
-	offset = mail_cache_append_space(ctx, size + sizeof(uint32_t));
-	if (offset != 0) {
-		memcpy((char *) cache->mmap_base + offset + sizeof(uint32_t),
-		       header_str, size);
-
-		size = uint32_to_nbo(size);
-		memcpy((char *) cache->mmap_base + offset,
-		       &size, sizeof(uint32_t));
-
-		/* update cached headers */
-		cache->split_offsets[idx] = cache->header->header_offsets[idx];
-		cache->split_headers[idx] = split_header(cache, header_str);
-
-		/* mark used-bit to be updated later. not really needed for
-		   read-safety, but if transaction get rolled back we can't let
-		   this point to invalid location. */
-		update_offset = (char *) &cache->header->header_offsets[idx] -
-			(char *) cache->mmap_base;
-		mark_update(&ctx->cache_marks, update_offset,
-			    uint32_to_offset(offset));
-
-		/* make sure get_header_fields() still works for this header
-		   while the transaction isn't yet committed. */
-		ctx->next_unused_header_lowwater = idx + 1;
-	}
-
-	t_pop();
-	return offset > 0;
-}
-
-static struct mail_cache_record *
-cache_get_record(struct mail_cache *cache, uint32_t offset)
-{
-#define CACHE_PREFETCH 1024
-	struct mail_cache_record *cache_rec;
-	size_t size;
-
-	offset = offset_to_uint32(offset);
-	if (offset == 0)
-		return NULL;
-
-	if (!mmap_update(cache, offset, sizeof(*cache_rec) + CACHE_PREFETCH))
-		return NULL;
-
-	if (offset + sizeof(*cache_rec) > cache->mmap_length) {
-		mail_cache_set_corrupted(cache, "record points outside file");
-		return NULL;
-	}
-	cache_rec = CACHE_RECORD(cache, offset);
-
-	size = nbo_to_uint32(cache_rec->size);
-	if (size < sizeof(*cache_rec)) {
-		mail_cache_set_corrupted(cache, "invalid record size");
-		return NULL;
-	}
-	if (size > CACHE_PREFETCH) {
-		if (!mmap_update(cache, offset, size))
-			return NULL;
-	}
-
-	if (offset + size > cache->mmap_length) {
-		mail_cache_set_corrupted(cache, "record points outside file");
-		return NULL;
-	}
-	return cache_rec;
-}
-
-static struct mail_cache_record *
-cache_get_next_record(struct mail_cache *cache, struct mail_cache_record *rec)
-{
-	struct mail_cache_record *next;
-
-	next = cache_get_record(cache, rec->next_offset);
-	if (next != NULL && next <= rec) {
-		mail_cache_set_corrupted(cache, "next_offset points backwards");
-		return NULL;
-	}
-	return next;
-}
-
-static int mail_cache_write(struct mail_cache_transaction_ctx *ctx)
-{
-	struct mail_cache *cache = ctx->cache;
-	struct mail_cache_record *cache_rec, *next;
-	struct mail_index_record *rec;
-	uint32_t write_offset, update_offset;
-	const void *buf;
-	size_t size, buf_size;
-
-	buf = buffer_get_data(ctx->cache_data, &buf_size);
-
-	size = sizeof(*cache_rec) + buf_size;
-	ctx->cache_rec.size = uint32_to_nbo(size);
-
-	write_offset = mail_cache_append_space(ctx, size);
-	if (write_offset == 0)
-		return FALSE;
-
-	rec = INDEX_RECORD_AT(ctx->cache->index, ctx->last_idx);
-	ctx->last_idx = (unsigned int)-1;
-
-	cache_rec = cache_get_record(cache, rec->cache_offset);
-	if (cache_rec == NULL) {
-		/* first cache record - update offset in index file */
-		i_assert(cache->index->lock_type == MAIL_LOCK_EXCLUSIVE);
-
-		/* mark cache_offset to be updated later */
-		update_offset = (char *) &rec->cache_offset -
-			(char *) cache->index->mmap_base;
-		mark_update(&ctx->index_marks, update_offset,
-			    uint32_to_offset(write_offset));
-	} else {
-		/* find the last cache record */
-		while ((next = cache_get_next_record(cache, cache_rec)) != NULL)
-			cache_rec = next;
-
-		/* mark next_offset to be updated later */
-		update_offset = (char *) &cache_rec->next_offset -
-			(char *) cache->mmap_base;
-		mark_update(&ctx->cache_marks, update_offset,
-			    uint32_to_offset(write_offset));
-	}
-
-	memcpy((char *) cache->mmap_base + write_offset,
-	       &ctx->cache_rec, sizeof(ctx->cache_rec));
-	memcpy((char *) cache->mmap_base + write_offset +
-	       sizeof(ctx->cache_rec), buf, buf_size);
-
-	/* reset the write context */
-	memset(&ctx->cache_rec, 0, sizeof(ctx->cache_rec));
-	buffer_set_used_size(ctx->cache_data, 0);
-	return TRUE;
-}
-
-static struct mail_cache_record *
-mail_cache_lookup(struct mail_cache *cache, const struct mail_index_record *rec,
-		  enum mail_cache_field fields)
-{
-	struct mail_cache_record *cache_rec;
-	unsigned int idx;
-
-	if (cache->trans_ctx != NULL &&
-	    cache->trans_ctx->first_uid <= rec->uid &&
-	    cache->trans_ctx->last_uid >= rec->uid &&
-	    (cache->trans_ctx->prev_uid != rec->uid || fields == 0 ||
-	     (cache->trans_ctx->prev_fields & fields) != 0)) {
-		/* we have to auto-commit since we're not capable of looking
-		   into uncommitted records. it would be possible by checking
-		   index_marks and cache_marks, but it's just more trouble
-		   than worth. */
-		idx = INDEX_RECORD_INDEX(cache->index, rec);
-		if (cache->trans_ctx->last_idx == idx) {
-			if (!mail_cache_write(cache->trans_ctx))
-				return NULL;
-		}
-
-		if (!mail_cache_transaction_commit(cache->trans_ctx))
-			return NULL;
-	}
-
-	cache_rec = cache_get_record(cache, rec->cache_offset);
-	if (cache_rec == NULL)
-		return NULL;
-
-	return cache_rec;
-}
-
-static int get_field_num(enum mail_cache_field field)
-{
-	unsigned int mask;
-	int i;
-
-	for (i = 0, mask = 1; i < 31; i++, mask <<= 1) {
-		if ((field & mask) != 0)
-			return i;
-	}
-
-	return -1;
-}
-
-static size_t get_insert_offset(struct mail_cache_transaction_ctx *ctx,
-				enum mail_cache_field field)
-{
-	const unsigned char *buf;
-	unsigned int mask;
-	uint32_t data_size;
-	size_t offset = 0;
-	int i;
-
-	buf = buffer_get_data(ctx->cache_data, NULL);
-
-	for (i = 0, mask = 1; i < 31; i++, mask <<= 1) {
-		if ((field & mask) != 0)
-			return offset;
-
-		if ((ctx->cache_rec.fields & mask) != 0) {
-			if ((mask & MAIL_CACHE_FIXED_MASK) != 0)
-				data_size = mail_cache_field_sizes[i];
-			else {
-				memcpy(&data_size, buf + offset,
-				       sizeof(data_size));
-				data_size = nbo_to_uint32(data_size);
-				offset += sizeof(data_size);
-			}
-			offset += (data_size + 3) & ~3;
-		}
-	}
-
-	i_unreached();
-	return offset;
-}
-
-int mail_cache_add(struct mail_cache_transaction_ctx *ctx,
-		   struct mail_index_record *rec, enum mail_cache_field field,
-		   const void *data, size_t data_size)
-{
-	uint32_t nb_data_size;
-	size_t full_size, offset;
-	unsigned char *buf;
-	unsigned int idx;
-	int field_num;
-
-	i_assert(data_size > 0);
-	i_assert(data_size < (uint32_t)-1);
-
-	nb_data_size = uint32_to_nbo((uint32_t)data_size);
-
-	if ((field & MAIL_CACHE_FIXED_MASK) != 0) {
-		field_num = get_field_num(field);
-		i_assert(field_num != -1);
-		i_assert(mail_cache_field_sizes[field_num] == data_size);
-	} else if ((field & MAIL_CACHE_STRING_MASK) != 0) {
-		i_assert(((char *) data)[data_size-1] == '\0');
-	}
-
-	/* NOTE: we use index because the record pointer might not last. */
-        idx = INDEX_RECORD_INDEX(ctx->cache->index, rec);
-	if (ctx->last_idx != idx && ctx->last_idx != (unsigned int)-1) {
-		if (!mail_cache_write(ctx))
-			return FALSE;
-	}
-	ctx->last_idx = idx;
-
-	i_assert((ctx->cache_rec.fields & field) == 0);
-
-	full_size = (data_size + 3) & ~3;
-	if ((field & MAIL_CACHE_FIXED_MASK) == 0)
-		full_size += sizeof(nb_data_size);
-
-	/* fields must be ordered. find where to insert it. */
-	if (field > ctx->cache_rec.fields)
-                buf = buffer_append_space_unsafe(ctx->cache_data, full_size);
-	else {
-		offset = get_insert_offset(ctx, field);
-		buffer_copy(ctx->cache_data, offset + full_size,
-			    ctx->cache_data, offset, (size_t)-1);
-		buf = buffer_get_space_unsafe(ctx->cache_data,
-					      offset, full_size);
-	}
-	ctx->cache_rec.fields |= field;
-
-	/* @UNSAFE */
-	if ((field & MAIL_CACHE_FIXED_MASK) == 0) {
-		memcpy(buf, &nb_data_size, sizeof(nb_data_size));
-		buf += sizeof(nb_data_size);
-	}
-	memcpy(buf, data, data_size); buf += data_size;
-	if ((data_size & 3) != 0)
-		memset(buf, 0, 4 - (data_size & 3));
-
-	/* remember the transaction uid range */
-	if (rec->uid < ctx->first_uid || ctx->first_uid == 0)
-		ctx->first_uid = rec->uid;
-	if (rec->uid > ctx->last_uid)
-		ctx->last_uid = rec->uid;
-
-	if (ctx->prev_uid != rec->uid) {
-		ctx->prev_uid = rec->uid;
-		ctx->prev_fields = 0;
-	}
-	ctx->prev_fields |= field;
-
-	return TRUE;
-}
-
-int mail_cache_delete(struct mail_cache_transaction_ctx *ctx,
-		      struct mail_index_record *rec)
-{
-	struct mail_cache *cache = ctx->cache;
-	struct mail_cache_record *cache_rec;
-	uint32_t deleted_space;
-	uoff_t max_del_space;
-
-	cache_rec = mail_cache_lookup(cache, rec, 0);
-	if (cache_rec == NULL)
-		return TRUE;
-
-	/* NOTE: it would be nice to erase the cached data for the record,
-	   but some other processes might still be using them. So, we just
-	   update the deleted_space in header */
-	deleted_space = nbo_to_uint32(cache->header->deleted_space);
-
-	do {
-		deleted_space -= nbo_to_uint32(cache_rec->size);
-		cache_rec = cache_get_next_record(cache, cache_rec);
-	} while (cache_rec != NULL);
-
-	/* see if we've reached the max. deleted space in file */
-	max_del_space = cache->used_file_size / 100 * COMPRESS_PERCENTAGE;
-	if (deleted_space >= max_del_space &&
-	    cache->used_file_size >= COMPRESS_MIN_SIZE)
-		cache->index->set_flags |= MAIL_INDEX_HDR_FLAG_COMPRESS_CACHE;
-
-	cache->header->deleted_space = uint32_to_nbo(deleted_space);
-
-	return TRUE;
-}
-
-enum mail_cache_field
-mail_cache_get_fields(struct mail_cache *cache,
-		      const struct mail_index_record *rec)
-{
-	struct mail_cache_record *cache_rec;
-        enum mail_cache_field fields = 0;
-
-	cache_rec = mail_cache_lookup(cache, rec, 0);
-	while (cache_rec != NULL) {
-		fields |= cache_rec->fields;
-		cache_rec = cache_get_next_record(cache, cache_rec);
-	}
-
-	return fields;
-}
-
-static int cache_get_field(struct mail_cache *cache,
-			   struct mail_cache_record *cache_rec,
-			   enum mail_cache_field field,
-			   void **data_r, size_t *size_r)
-{
-	unsigned char *buf;
-	unsigned int mask;
-	uint32_t rec_size, data_size;
-	size_t offset, next_offset;
-	int i;
-
-	rec_size = nbo_to_uint32(cache_rec->size);
-	buf = (unsigned char *) cache_rec;
-	offset = sizeof(*cache_rec);
-
-	for (i = 0, mask = 1; i < 31; i++, mask <<= 1) {
-		if ((cache_rec->fields & mask) == 0)
-			continue;
-
-		/* all records are at least 32bit. we have to check this
-		   before getting data_size. */
-		if (offset + sizeof(uint32_t) > rec_size) {
-			mail_cache_set_corrupted(cache,
-				"Record continues outside it's allocated size");
-			return FALSE;
-		}
-
-		if ((mask & MAIL_CACHE_FIXED_MASK) != 0)
-			data_size = mail_cache_field_sizes[i];
-		else {
-			memcpy(&data_size, buf + offset, sizeof(data_size));
-			data_size = nbo_to_uint32(data_size);
-			offset += sizeof(data_size);
-		}
-
-		next_offset = offset + ((data_size + 3) & ~3);
-		if (next_offset > rec_size) {
-			mail_cache_set_corrupted(cache,
-				"Record continues outside it's allocated size");
-			return FALSE;
-		}
-
-		if (field == mask) {
-			if (data_size == 0) {
-				mail_cache_set_corrupted(cache,
-							 "Field size is 0");
-				return FALSE;
-			}
-			*data_r = buf + offset;
-			*size_r = data_size;
-			return TRUE;
-		}
-		offset = next_offset;
-	}
-
-	i_unreached();
-	return FALSE;
-}
-
-static int cache_lookup_field(struct mail_cache *cache,
-			      const struct mail_index_record *rec,
-			      enum mail_cache_field field,
-			      void **data_r, size_t *size_r)
-{
-	struct mail_cache_record *cache_rec;
-
-	cache_rec = mail_cache_lookup(cache, rec, field);
-	while (cache_rec != NULL) {
-		if ((cache_rec->fields & field) != 0) {
-			return cache_get_field(cache, cache_rec, field,
-					       data_r, size_r);
-		}
-		cache_rec = cache_get_next_record(cache, cache_rec);
-	}
-
-	return FALSE;
-}
-
-int mail_cache_lookup_field(struct mail_cache *cache,
-			    const struct mail_index_record *rec,
-			    enum mail_cache_field field,
-			    const void **data_r, size_t *size_r)
-{
-	void *data;
-
-	if (!cache_lookup_field(cache, rec, field, &data, size_r))
-		return FALSE;
-
-	*data_r = data;
-	return TRUE;
-}
-
-const char *mail_cache_lookup_string_field(struct mail_cache *cache,
-					   const struct mail_index_record *rec,
-					   enum mail_cache_field field)
-{
-	const void *data;
-	size_t size;
-
-	i_assert((field & MAIL_CACHE_STRING_MASK) != 0);
-
-	if (!mail_cache_lookup_field(cache, rec, field, &data, &size))
-		return NULL;
-
-	if (((const char *) data)[size-1] != '\0') {
-		mail_cache_set_corrupted(cache,
-			"String field %x doesn't end with NUL", field);
-		return NULL;
-	}
-	return data;
-}
-
-int mail_cache_copy_fixed_field(struct mail_cache *cache,
-				const struct mail_index_record *rec,
-				enum mail_cache_field field,
-				void *buffer, size_t buffer_size)
-{
-	const void *data;
-	size_t size;
-
-	i_assert((field & MAIL_CACHE_FIXED_MASK) != 0);
-
-	if (!mail_cache_lookup_field(cache, rec, field, &data, &size))
-		return FALSE;
-
-	if (buffer_size != size) {
-		i_panic("cache: fixed field %x wrong size "
-			"(%"PRIuSIZE_T" vs %"PRIuSIZE_T")",
-			field, size, buffer_size);
-	}
-
-	memcpy(buffer, data, buffer_size);
-	return TRUE;
-}
-
-void mail_cache_mark_missing(struct mail_cache *cache,
-			     enum mail_cache_field fields)
-{
-	// FIXME: count these
-}
-
-enum mail_index_record_flag
-mail_cache_get_index_flags(struct mail_cache *cache,
-			   const struct mail_index_record *rec)
-{
-	enum mail_index_record_flag flags;
-
-	if (!mail_cache_copy_fixed_field(cache, rec, MAIL_CACHE_INDEX_FLAGS,
-					 &flags, sizeof(flags)))
-		return 0;
-
-	return flags;
-}
-
-int mail_cache_update_index_flags(struct mail_cache *cache,
-				  struct mail_index_record *rec,
-				  enum mail_index_record_flag flags)
-{
-	void *data;
-	size_t size;
-
-	i_assert(cache->locks > 0);
-
-	if (!cache_lookup_field(cache, rec, MAIL_CACHE_INDEX_FLAGS,
-				&data, &size)) {
-		mail_cache_set_corrupted(cache,
-			"Missing index flags for record %u", rec->uid);
-		return FALSE;
-	}
-
-	memcpy(data, &flags, sizeof(flags));
-	return TRUE;
-}
-
-int mail_cache_update_location_offset(struct mail_cache *cache,
-				      struct mail_index_record *rec,
-				      uoff_t offset)
-{
-	void *data;
-	size_t size;
-
-	i_assert(cache->locks > 0);
-
-	if (!cache_lookup_field(cache, rec, MAIL_CACHE_LOCATION_OFFSET,
-				&data, &size)) {
-		mail_cache_set_corrupted(cache,
-			"Missing location offset for record %u", rec->uid);
-		return FALSE;
-	}
-
-	memcpy(data, &offset, sizeof(offset));
-	return TRUE;
-}
-
-void *mail_cache_get_mmaped(struct mail_cache *cache, size_t *size)
-{
-	if (!mmap_update(cache, 0, 0))
-		return NULL;
-
-	*size = cache->mmap_length;
-	return cache->mmap_base;
-}
-
-int mail_cache_set_corrupted(struct mail_cache *cache, const char *fmt, ...)
-{
-	va_list va;
-
-	mail_cache_mark_file_deleted(cache);
-	cache->index->inconsistent = TRUE; /* easiest way to rebuild */
-
-	if (cache->silent)
-		return FALSE;
-
-	va_start(va, fmt);
-	t_push();
-	index_set_error(cache->index, "Corrupted index cache file %s: %s",
-			cache->filepath, t_strdup_vprintf(fmt, va));
-	t_pop();
-	va_end(va);
-
-	return FALSE;
-}

--- a/src/lib-index/mail-cache.h	Tue Apr 27 00:20:15 2004 +0300
+++ /dev/null	Thu Jan 01 00:00:00 1970 +0000
@@ -1,175 +0,0 @@
-#ifndef __MAIL_CACHE_H
-#define __MAIL_CACHE_H
-
-#include "mail-index.h"
-
-#define MAIL_CACHE_FILE_PREFIX ".cache"
-
-#define MAIL_CACHE_HEADERS_COUNT 4
-
-struct mail_cache_transaction_ctx;
-
-enum mail_cache_field {
-	/* fixed size fields */
-	MAIL_CACHE_INDEX_FLAGS		= 0x00000001,
-	MAIL_CACHE_LOCATION_OFFSET	= 0x00000002,
-	MAIL_CACHE_MD5			= 0x00000004,
-	MAIL_CACHE_SENT_DATE		= 0x00000008,
-	MAIL_CACHE_RECEIVED_DATE	= 0x00000010,
-	MAIL_CACHE_VIRTUAL_FULL_SIZE	= 0x00000020,
-	MAIL_CACHE_PHYSICAL_BODY_SIZE	= 0x00000040,
-
-	/* variable sized field */
-	MAIL_CACHE_HEADERS1		= 0x40000000,
-	MAIL_CACHE_HEADERS2		= 0x20000000,
-	MAIL_CACHE_HEADERS3		= 0x10000000,
-	MAIL_CACHE_HEADERS4		= 0x08000000,
-	MAIL_CACHE_LOCATION		= 0x04000000,
-	MAIL_CACHE_BODY			= 0x02000000,
-	MAIL_CACHE_BODYSTRUCTURE	= 0x01000000,
-	MAIL_CACHE_ENVELOPE		= 0x00800000,
-	MAIL_CACHE_MESSAGEPART		= 0x00400000,
-
-	MAIL_CACHE_FIXED_MASK		= MAIL_CACHE_INDEX_FLAGS |
-					  MAIL_CACHE_LOCATION_OFFSET |
-					  MAIL_CACHE_MD5 |
-					  MAIL_CACHE_SENT_DATE |
-					  MAIL_CACHE_RECEIVED_DATE |
-					  MAIL_CACHE_VIRTUAL_FULL_SIZE |
-					  MAIL_CACHE_PHYSICAL_BODY_SIZE,
-	MAIL_CACHE_HEADERS_MASK		= MAIL_CACHE_HEADERS1 |
-					  MAIL_CACHE_HEADERS2 |
-					  MAIL_CACHE_HEADERS3 |
-					  MAIL_CACHE_HEADERS4,
-	MAIL_CACHE_STRING_MASK		= MAIL_CACHE_HEADERS_MASK |
-					  MAIL_CACHE_LOCATION |
-					  MAIL_CACHE_BODY |
-					  MAIL_CACHE_BODYSTRUCTURE |
-					  MAIL_CACHE_ENVELOPE,
-	MAIL_CACHE_BODYSTRUCTURE_MASK	= MAIL_CACHE_BODY |
-					  MAIL_CACHE_BODYSTRUCTURE |
-                                          MAIL_CACHE_MESSAGEPART
-};
-
-struct mail_sent_date {
-	time_t time;
-	int32_t timezone;
-};
-
-extern enum mail_cache_field mail_cache_header_fields[MAIL_CACHE_HEADERS_COUNT];
-
-int mail_cache_open_or_create(struct mail_index *index);
-void mail_cache_free(struct mail_cache *cache);
-
-void mail_cache_set_defaults(struct mail_cache *cache,
-			     enum mail_cache_field default_cache_fields,
-			     enum mail_cache_field never_cache_fields);
-
-/* Compress cache file. */
-int mail_cache_compress(struct mail_cache *cache);
-
-/* Truncate the cache file and update it's indexid */
-int mail_cache_truncate(struct mail_cache *cache);
-
-/* Set indexid to 0 to notify other processes using this file that they should
-   re-open it. */
-int mail_cache_mark_file_deleted(struct mail_cache *cache);
-
-/* Explicitly lock the cache file. Returns 1 if ok, 0 if nonblock is TRUE and
-   we couldn't immediately get a lock, or -1 if error. */
-int mail_cache_lock(struct mail_cache *cache, int nonblock);
-int mail_cache_unlock(struct mail_cache *cache);
-
-/* Mark the lock to be removed when unlocking index file. */
-void mail_cache_unlock_later(struct mail_cache *cache);
-
-/* Returns TRUE if cache file is locked. */
-int mail_cache_is_locked(struct mail_cache *cache);
-
-/* Begin transaction. Returns same as mail_cache_lock(). Note that if you
-   call lookup functions for messages within first and last message in
-   transaction, the transaction will be automatically committed. */
-int mail_cache_transaction_begin(struct mail_cache *cache, int nonblock,
-				 struct mail_cache_transaction_ctx **ctx_r);
-/* End transaction. Single transaction can have multiple commits/rollbacks.
-   If there's any pending changes, they will be rolled back. */
-int mail_cache_transaction_end(struct mail_cache_transaction_ctx *ctx);
-
-int mail_cache_transaction_commit(struct mail_cache_transaction_ctx *ctx);
-int mail_cache_transaction_rollback(struct mail_cache_transaction_ctx *ctx);
-
-/* Return NULL-terminated list of headers for given index, or NULL if
-   header index isn't used. */
-const char *const *mail_cache_get_header_fields(struct mail_cache *cache,
-						unsigned int idx);
-/* Set list of headers for given index. */
-int mail_cache_set_header_fields(struct mail_cache_transaction_ctx *ctx,
-				 unsigned int idx, const char *const headers[]);
-
-/* Add new field to given record. Updates are not allowed. Fixed size fields
-   must be exactly the expected size and they're converted to network byte
-   order in disk. */
-int mail_cache_add(struct mail_cache_transaction_ctx *ctx,
-		   struct mail_index_record *rec, enum mail_cache_field field,
-		   const void *data, size_t data_size);
-
-/* Mark the given record deleted. */
-int mail_cache_delete(struct mail_cache_transaction_ctx *ctx,
-		      struct mail_index_record *rec);
-
-/* Return all fields that are currently cached for record. */
-enum mail_cache_field
-mail_cache_get_fields(struct mail_cache *cache,
-		      const struct mail_index_record *rec);
-
-/* Set data_r and size_r to point to wanted field in cache file.
-   Returns TRUE if field was found. If field contains multiple fields,
-   first one found is returned. This is mostly useful for finding headers. */
-int mail_cache_lookup_field(struct mail_cache *cache,
-			    const struct mail_index_record *rec,
-			    enum mail_cache_field field,
-			    const void **data_r, size_t *size_r);
-
-/* Return string field. */
-const char *mail_cache_lookup_string_field(struct mail_cache *cache,
-					   const struct mail_index_record *rec,
-					   enum mail_cache_field field);
-
-
-/* Copy fixed size field to given buffer. buffer_size must be exactly the
-   expected size. The result will be converted to host byte order.
-   Returns TRUE if field was found. */
-int mail_cache_copy_fixed_field(struct mail_cache *cache,
-				const struct mail_index_record *rec,
-				enum mail_cache_field field,
-				void *buffer, size_t buffer_size);
-
-/* Mark given fields as missing, ie. they should be cached when possible. */
-void mail_cache_mark_missing(struct mail_cache *cache,
-			     enum mail_cache_field fields);
-
-/* Return index flags. */
-enum mail_index_record_flag
-mail_cache_get_index_flags(struct mail_cache *cache,
-			   const struct mail_index_record *rec);
-
-/* Update index flags. The cache file must be locked and the flags must be
-   already inserted to the record. */
-int mail_cache_update_index_flags(struct mail_cache *cache,
-				  struct mail_index_record *rec,
-				  enum mail_index_record_flag flags);
-
-/* Update location offset. External locking is assumed to take care of locking
-   readers out to prevent race conditions. */
-int mail_cache_update_location_offset(struct mail_cache *cache,
-				      struct mail_index_record *rec,
-				      uoff_t offset);
-
-/* Return the whole file mmaped. */
-void *mail_cache_get_mmaped(struct mail_cache *cache, size_t *size);
-
-/* "Error in index cache file %s: ...". */
-int mail_cache_set_corrupted(struct mail_cache *cache, const char *fmt, ...)
-	__attr_format__(2, 3);
-
-#endif

--- a/src/lib-index/mail-custom-flags.c	Tue Apr 27 00:20:15 2004 +0300
+++ /dev/null	Thu Jan 01 00:00:00 1970 +0000
@@ -1,592 +0,0 @@
-/* Copyright (C) 2002 Timo Sirainen */
-
-#include "lib.h"
-#include "file-lock.h"
-#include "mmap-util.h"
-#include "write-full.h"
-#include "imap-util.h"
-#include "mail-index.h"
-#include "mail-index-util.h"
-#include "mail-custom-flags.h"
-
-#include <ctype.h>
-#include <unistd.h>
-#include <fcntl.h>
-#include <sys/stat.h>
-
-/* Header is simply a counter which is increased every time the file is
-   updated. This allows other processes to easily notice if there's been
-   any changes. */
-
-#define COUNTER_SIZE 4
-#define HEADER_SIZE (COUNTER_SIZE + 1) /* 0000\n */
-
-struct mail_custom_flags {
-	struct mail_index *index;
-	char *filepath;
-	int fd;
-	int lock_type;
-
-	char sync_counter[COUNTER_SIZE];
-	char *custom_flags[MAIL_CUSTOM_FLAGS_COUNT];
-
-	void *mmap_base;
-	size_t mmap_length;
-
-	unsigned int syncing:1;
-	unsigned int noupdate:1;
-	unsigned int changed:1;
-};
-
-static int lock_file(struct mail_custom_flags *mcf, int type);
-
-static int index_cf_set_syscall_error(struct mail_custom_flags *mcf,
-				      const char *function)
-{
-	i_assert(function != NULL);
-
-	if (ENOSPACE(errno)) {
-		mcf->index->nodiskspace = TRUE;
-		return FALSE;
-	}
-
-	index_set_error(mcf->index, "%s failed with custom flags file %s: %m",
-			function, mcf->filepath);
-	return FALSE;
-}
-
-static int update_mmap(struct mail_custom_flags *mcf)
-{
-	if (mcf->mmap_base != NULL) {
-		if (munmap(mcf->mmap_base, mcf->mmap_length) < 0)
-			index_cf_set_syscall_error(mcf, "munmap()");
-	}
-
-
-	mcf->mmap_base = mcf->noupdate ?
-		mmap_ro_file(mcf->fd, &mcf->mmap_length) :
-		mmap_rw_file(mcf->fd, &mcf->mmap_length);
-	if (mcf->mmap_base == MAP_FAILED) {
-		mcf->mmap_base = NULL;
-		return index_cf_set_syscall_error(mcf, "mmap()");
-	}
-
-	(void)madvise(mcf->mmap_base, mcf->mmap_length, MADV_SEQUENTIAL);
-	return TRUE;
-}
-
-static int custom_flags_init(struct mail_custom_flags *mcf)
-{
-	static char buf[HEADER_SIZE] = "0000\n";
-	struct stat st;
-	int failed;
-
-	if (!lock_file(mcf, F_WRLCK))
-		return FALSE;
-
-	failed = FALSE;
-
-	/* make sure it's still empty after locking */
-	if (fstat(mcf->fd, &st) < 0) {
-		index_cf_set_syscall_error(mcf, "fstat()");
-		failed = TRUE;
-	} else if (st.st_size < HEADER_SIZE) {
-		/* write the header - it's a 4 byte counter as hex */
-		if (write_full(mcf->fd, buf, HEADER_SIZE) < 0) {
-			index_cf_set_syscall_error(mcf, "write_full()");
-			failed = TRUE;
-		}
-	}
-
-	if (!lock_file(mcf, F_UNLCK))
-		return FALSE;
-
-	return !failed;
-}
-
-static void custom_flags_sync(struct mail_custom_flags *mcf)
-{
-	char *data, *data_end, *line;
-	unsigned int num;
-	int i;
-
-	if (mcf->noupdate)
-		return;
-
-	memcpy(mcf->sync_counter, mcf->mmap_base, COUNTER_SIZE);
-
-	for (i = 0; i < MAIL_CUSTOM_FLAGS_COUNT; i++) {
-		if (mcf->custom_flags[i] != NULL) {
-			i_free(mcf->custom_flags[i]);
-                        mcf->custom_flags[i] = NULL;
-		}
-	}
-
-	data = mcf->mmap_base;
-	data_end = data + mcf->mmap_length;
-
-	/* this loop skips the first line, which is the header */
-	while (data != data_end) {
-		if (*data != '\n') {
-			data++;
-			continue;
-		}
-
-		/* beginning of line, get the index */
-		if (data+1 == data_end)
-			break;
-		data++;
-
-		if (!i_isdigit(*data))
-			continue;
-
-		num = 0;
-		while (data != data_end && *data >= '0' && *data <= '9') {
-			num = num*10 + (*data-'0');
-			data++;
-		}
-
-		if (num < MAIL_CUSTOM_FLAGS_COUNT) {
-			/* get the name */
-			if (data == data_end || *data != ' ')
-				continue;
-
-			line = ++data;
-			while (data != data_end && *data != '\n')
-				data++;
-
-			if (mcf->custom_flags[num] != NULL) {
-				i_warning("Error in custom flags file %s: "
-					  "Duplicated ID %u", mcf->filepath,
-					  num);
-				i_free(mcf->custom_flags[num]);
-			}
-
-			mcf->custom_flags[num] = i_strdup_until(line, data);
-		}
-	}
-}
-
-static int custom_flags_check_sync(struct mail_custom_flags *mcf)
-{
-	if (mcf->fd == -1)
-		return TRUE;
-
-	if (mcf->mmap_length != 0 &&
-	    memcmp(mcf->sync_counter, mcf->mmap_base, COUNTER_SIZE) == 0)
-		return TRUE;
-
-	/* file modified, resync */
-	if (!update_mmap(mcf))
-		return FALSE;
-
-	if (mcf->mmap_length < HEADER_SIZE && !mcf->noupdate) {
-		/* it's broken, rewrite header */
-		if (mcf->lock_type == F_RDLCK)
-			(void)lock_file(mcf, F_UNLCK);
-
-		if (lseek(mcf->fd, 0, SEEK_SET) < 0) {
-			index_cf_set_syscall_error(mcf, "lseek()");
-			return FALSE;
-		}
-
-		if (!custom_flags_init(mcf))
-			return FALSE;
-
-		if (!update_mmap(mcf))
-			return FALSE;
-	}
-
-	custom_flags_sync(mcf);
-	mcf->changed = TRUE;
-	return TRUE;
-}
-
-static int lock_file(struct mail_custom_flags *mcf, int type)
-{
-	if (mcf->lock_type == type)
-		return TRUE;
-
-	if (mcf->fd != -1) {
-		/* FIXME: possibility to use .lock file instead */
-		if (file_wait_lock(mcf->fd, type) <= 0) {
-			index_cf_set_syscall_error(mcf, "file_wait_lock()");
-			return FALSE;
-		}
-	}
-
-	mcf->lock_type = type;
-
-	if (type != F_UNLCK && !mcf->syncing) {
-		mcf->syncing = TRUE;
-		if (!custom_flags_check_sync(mcf)) {
-			mcf->syncing = FALSE;
-			return FALSE;
-		}
-
-		/* syncing may have changed locking, do it again */
-		if (!lock_file(mcf, type)) {
-			mcf->syncing = FALSE;
-			return FALSE;
-		}
-
-		mcf->syncing = FALSE;
-	}
-	return TRUE;
-}
-
-int mail_custom_flags_open_or_create(struct mail_index *index)
-{
-	struct mail_custom_flags *mcf;
-	const char *path;
-	int fd, readonly;
-
-	readonly = index->mailbox_readonly;
-
-	if (index->control_dir != NULL) {
-		path = t_strconcat(index->control_dir, "/",
-				   CUSTOM_FLAGS_FILE_NAME, NULL);
-		fd = !readonly ? open(path, O_RDWR | O_CREAT, 0660) :
-			open(path, O_RDONLY);
-		if (fd == -1 && errno == EACCES) {
-			fd = open(path, O_RDONLY);
-			readonly = TRUE;
-		}
-		if (fd == -1 && errno != EACCES && errno != ENOENT &&
-		    !ENOSPACE(errno)) {
-			index_file_set_syscall_error(index, path, "open()");
-			return FALSE;
-		}
-	} else {
-		path = NULL;
-		fd = -1;
-	}
-
-	mcf = i_new(struct mail_custom_flags, 1);
-	mcf->index = index;
-	mcf->filepath = fd != -1 ? i_strdup(path) :
-		i_strdup_printf("(in-memory custom flags for %s)",
-				index->mailbox_path);
-	mcf->fd = fd;
-	mcf->noupdate = mcf->fd == -1 || readonly;
-
-	if (fd != -1) {
-		if (!update_mmap(mcf)) {
-			(void)close(mcf->fd);
-			mcf->fd = -1;
-			mcf->noupdate = TRUE;
-		}
-
-		if (mcf->mmap_length < HEADER_SIZE && !mcf->noupdate) {
-			/* we just created it, write the header */
-			mcf->syncing = TRUE;
-			if (!custom_flags_init(mcf) || !update_mmap(mcf)) {
-				(void)close(mcf->fd);
-				mcf->fd = -1;
-				mcf->noupdate = TRUE;
-			}
-			mcf->syncing = FALSE;
-		}
-	}
-
-	mcf->index->allow_new_custom_flags = mcf->fd != -1;
-
-	custom_flags_sync(mcf);
-
-	index->custom_flags = mcf;
-	return TRUE;
-}
-
-void mail_custom_flags_free(struct mail_custom_flags *mcf)
-{
-	int i;
-
-	for (i = 0; i < MAIL_CUSTOM_FLAGS_COUNT; i++)
-		i_free(mcf->custom_flags[i]);
-
-	if (mcf->mmap_base != NULL) {
-		if (munmap(mcf->mmap_base, mcf->mmap_length) < 0)
-			index_cf_set_syscall_error(mcf, "munmap()");
-	}
-
-	if (mcf->fd != -1) {
-		if (close(mcf->fd) < 0)
-			index_cf_set_syscall_error(mcf, "close()");
-	}
-
-	i_free(mcf->filepath);
-	i_free(mcf);
-}
-
-static int custom_flags_update_counter(struct mail_custom_flags *mcf)
-{
-	int i;
-
-	if (lseek(mcf->fd, 0, SEEK_SET) < 0)
-		return index_cf_set_syscall_error(mcf, "lseek()");
-
-	for (i = COUNTER_SIZE-1; i >= 0; i--) {
-		if (mcf->sync_counter[i] == '9') {
-			mcf->sync_counter[i] = 'A';
-			break;
-		}
-
-		if (mcf->sync_counter[i] == 'F') {
-			/* digit wrapped, update next one */
-			mcf->sync_counter[i] = '0';
-		} else {
-			mcf->sync_counter[i]++;
-			break;
-		}
-	}
-
-	if (write_full(mcf->fd, mcf->sync_counter, COUNTER_SIZE) < 0)
-		return index_cf_set_syscall_error(mcf, "write_full()");
-
-	mcf->changed = TRUE;
-	return TRUE;
-}
-
-static int custom_flags_add(struct mail_custom_flags *mcf,
-			    int idx, const char *name)
-{
-	const char *buf;
-	size_t len;
-	off_t pos;
-
-	i_assert(idx < MAIL_CUSTOM_FLAGS_COUNT);
-
-	/* first update the sync counter */
-	if (!custom_flags_update_counter(mcf))
-		return FALSE;
-
-	/* add the flag */
-	pos = lseek(mcf->fd, 0, SEEK_END);
-	if (pos < 0)
-		return index_cf_set_syscall_error(mcf, "lseek()");
-
-	if (pos != (off_t)mcf->mmap_length) {
-		index_set_error(mcf->index, "Custom flags file %s was "
-				"changed by someone while we were"
-				"trying to modify it", mcf->filepath);
-		return FALSE;
-	}
-
-	buf = t_strdup_printf("\n%d %s\n", idx, name);
-	len = strlen(buf);
-
-	if (((char *) mcf->mmap_base)[mcf->mmap_length-1] == '\n') {
-		/* don't add the \n prefix */
-		buf++;
-		len--;
-	}
-
-	if (write_full(mcf->fd, buf, len) < 0)
-		return index_cf_set_syscall_error(mcf, "write_full()");
-
-	if (!update_mmap(mcf))
-		return FALSE;
-
-	return TRUE;
-}
-
-static int custom_flags_remove(struct mail_custom_flags *mcf, unsigned int idx)
-{
-	char *data, *data_end, *line;
-	unsigned int num;
-	int pos, linelen;
-
-	data = mcf->mmap_base;
-	data_end = data + mcf->mmap_length;
-
-	while (data != data_end) {
-		if (*data != '\n') {
-			data++;
-			continue;
-		}
-
-		/* beginning of line, get the index */
-		if (data+1 == data_end)
-			break;
-		line = ++data;
-
-		num = 0;
-		while (data != data_end && *data >= '0' && *data <= '9') {
-			num = num*10 + (*data-'0');
-			data++;
-		}
-
-		if (num == idx) {
-			/* remove this line */
-			while (data != data_end && data[-1] != '\n')
-				data++;
-
-			linelen = (int) (data - line);
-			pos = (int) (data - (char *) mcf->mmap_base);
-			memmove(line, data, mcf->mmap_length - pos);
-
-			mcf->mmap_length -= linelen;
-			if (ftruncate(mcf->fd, (off_t) mcf->mmap_length) < 0) {
-				index_cf_set_syscall_error(mcf, "ftruncate()");
-				return FALSE;
-			}
-
-			return TRUE;
-		}
-	}
-
-	return FALSE;
-}
-
-static int find_first_unused_flag(struct mail_custom_flags *mcf)
-{
-	int i;
-
-	for (i = 0; i < MAIL_CUSTOM_FLAGS_COUNT; i++) {
-		if (mcf->custom_flags[i] == NULL)
-			return i;
-	}
-
-	return -1;
-}
-
-static void remove_unused_custom_flags(struct mail_custom_flags *mcf,
-				       enum mail_flags used_flags)
-{
-	unsigned int i;
-
-	for (i = 0; i < MAIL_CUSTOM_FLAGS_COUNT; i++) {
-		if ((used_flags & (1 << (i + MAIL_CUSTOM_FLAG_1_BIT))) == 0) {
-			i_free(mcf->custom_flags[i]);
-			mcf->custom_flags[i] = NULL;
-
-			custom_flags_remove(mcf, i);
-		}
-	}
-}
-
-static enum mail_flags get_used_flags(struct mail_custom_flags *mcf)
-{
-	struct mail_index_record *rec;
-	enum mail_flags used_flags;
-
-	used_flags = 0;
-
-	rec = mcf->index->lookup(mcf->index, 1);
-	while (rec != NULL) {
-		used_flags |= rec->msg_flags;
-		rec = mcf->index->next(mcf->index, rec);
-	}
-
-	return used_flags;
-}
-
-static int get_flag_index(struct mail_custom_flags *mcf, const char *flag,
-			  int index_hint)
-{
-	int i, first_empty;
-
-	if (index_hint >= 0 && index_hint < MAIL_CUSTOM_FLAGS_COUNT) {
-		if (mcf->custom_flags[index_hint] != NULL &&
-		    strcasecmp(mcf->custom_flags[index_hint], flag) == 0)
-			return index_hint;
-	}
-
-	/* check existing flags */
-	for (i = 0; i < MAIL_CUSTOM_FLAGS_COUNT; i++) {
-		if (mcf->custom_flags[i] != NULL) {
-			i_assert(mcf->custom_flags[i] != '\0');
-			if (strcasecmp(mcf->custom_flags[i], flag) == 0)
-				return i;
-		}
-	}
-
-	if (mcf->noupdate)
-		return -1;
-
-	if (mcf->lock_type != F_WRLCK) {
-		/* unlock + write lock, don't directly change from
-		   read -> write lock to prevent deadlocking */
-		if (!lock_file(mcf, F_UNLCK) || !lock_file(mcf, F_WRLCK))
-			return -1;
-
-		/* list may have already changed between the lock changes,
-		   check again */
-		return get_flag_index(mcf, flag, -1);
-	}
-
-	/* new flag, add it. first find the first free flag, note that
-	   unlock+lock might have just changed it. */
-	first_empty = find_first_unused_flag(mcf);
-	if (first_empty == -1) {
-		/* all custom flags are used, see if some of them are unused */
-		remove_unused_custom_flags(mcf, get_used_flags(mcf));
-
-		first_empty = find_first_unused_flag(mcf);
-		if (first_empty == -1) {
-			/* everything is in use */
-			return -1;
-		}
-	}
-
-	if (!custom_flags_add(mcf, first_empty, flag))
-		return -1;
-
-	mcf->index->set_flags |= MAIL_INDEX_HDR_FLAG_DIRTY_CUSTOMFLAGS;
-
-	mcf->custom_flags[first_empty] = i_strdup(flag);
-	return first_empty;
-}
-
-int mail_custom_flags_fix_list(struct mail_custom_flags *mcf,
-			       enum mail_flags *flags,
-			       const char *custom_flags[], unsigned int count)
-{
-	enum mail_flags oldflags, flag;
-	int i, idx;
-
-	i_assert(count < 32);
-
-	if ((*flags & MAIL_CUSTOM_FLAGS_MASK) == 0)
-		return 1;
-
-	if (!lock_file(mcf, F_RDLCK))
-		return -1;
-
-	oldflags = *flags;
-	*flags &= MAIL_SYSTEM_FLAGS_MASK;
-
-	flag = MAIL_CUSTOM_FLAG_1;
-	for (i = 0; i < (int)count; i++, flag <<= 1) {
-		if ((oldflags & flag) && custom_flags[i] != NULL) {
-			i_assert(*custom_flags[i] != '\0');
-
-			idx = get_flag_index(mcf, custom_flags[i], i);
-			if (idx == -1) {
-				(void)lock_file(mcf, F_UNLCK);
-				return 0;
-			}
-			*flags |= 1 << (idx + MAIL_CUSTOM_FLAG_1_BIT);
-		}
-	}
-
-	if (!lock_file(mcf, F_UNLCK))
-		return -1;
-
-	return 1;
-}
-
-const char **mail_custom_flags_list_get(struct mail_custom_flags *mcf)
-{
-	return (const char **) mcf->custom_flags;
-}
-
-int mail_custom_flags_has_changes(struct mail_custom_flags *mcf)
-{
-	if (!mcf->changed)
-		return FALSE;
-	else {
-		mcf->changed = FALSE;
-		return TRUE;
-	}
-}

--- a/src/lib-index/mail-custom-flags.h	Tue Apr 27 00:20:15 2004 +0300
+++ /dev/null	Thu Jan 01 00:00:00 1970 +0000
@@ -1,31 +0,0 @@
-#ifndef __MAIL_CUSTOM_FLAGS_H
-#define __MAIL_CUSTOM_FLAGS_H
-
-/* NOTE: Contains it's own locking, unrelated to index locks. */
-
-#include "mail-index.h"
-
-#define CUSTOM_FLAGS_FILE_NAME ".customflags"
-
-int mail_custom_flags_open_or_create(struct mail_index *index);
-void mail_custom_flags_free(struct mail_custom_flags *mcf);
-
-/* Change custom flags so that they reflect the real flag numbers in
-   the file. Initially flags contains the custom flags in the order of the
-   specified list, it's modified to reflect the actual list. Returns 1 if ok,
-   0 if number of custom flags exceeded or -1 if error */
-int mail_custom_flags_fix_list(struct mail_custom_flags *mcf,
-			       enum mail_flags *flags,
-			       const char *custom_flags[], unsigned int count);
-
-/* Returns a pointer to list of flags. Note that calls to
-   mail_cutom_flags_fix_list() may modify the flags in the returned list.
-   It can modify only the flags that aren't in use anywhere, so this should
-   be safe. */
-const char **mail_custom_flags_list_get(struct mail_custom_flags *mcf);
-
-/* Returns TRUE if there's been any changes since this function was
-   called last time, or since open if this is the first call. */
-int mail_custom_flags_has_changes(struct mail_custom_flags *mcf);
-
-#endif

--- a/src/lib-index/mail-index-file.c	Tue Apr 27 00:20:15 2004 +0300
+++ /dev/null	Thu Jan 01 00:00:00 1970 +0000
@@ -1,208 +0,0 @@
-/* Copyright (C) 2003 Timo Sirainen */
-
-#include "lib.h"
-#include "file-set-size.h"
-#include "mail-index.h"
-#include "mail-index-util.h"
-
-#include <unistd.h>
-
-struct mail_index_record *mail_index_next(struct mail_index *index,
-					  struct mail_index_record *rec)
-{
-	i_assert(index->lock_type != MAIL_LOCK_UNLOCK);
-	i_assert(rec >= INDEX_RECORD_AT(index, 0));
-
-	return rec+1 == INDEX_END_RECORD(index) ? NULL : rec+1;
-}
-
-static int compress(struct mail_index *index, unsigned int remove_first_idx,
-		    unsigned int remove_last_idx)
-{
-	struct mail_index_record *rec = INDEX_RECORD_AT(index, 0);
-	unsigned int idx_limit, count;
-
-	idx_limit = MAIL_INDEX_RECORD_COUNT(index);
-	count = remove_last_idx - remove_first_idx + 1;
-
-	memmove(rec + remove_first_idx, rec + remove_last_idx + 1,
-		(idx_limit - remove_last_idx - 1) * sizeof(*rec));
-
-	index->header->used_file_size -= sizeof(*rec) * count;
-	index->mmap_used_length -= sizeof(*rec) * count;
-
-	/* not really needed since append() will initialize it as well,
-	   but may help preventing problems if change is only partially
-	   written to disk */
-	memset((char *) rec + index->mmap_used_length, 0, sizeof(*rec) * count);
-
-	return mail_index_truncate(index);
-}
-
-int mail_index_expunge_record_range(struct mail_index *index,
-				    struct mail_index_record *first_rec,
-				    struct mail_index_record *last_rec)
-{
-	struct mail_index_record *rec;
-	unsigned int first_idx, last_idx, idx_limit;
-
-        i_assert(index->lock_type == MAIL_LOCK_EXCLUSIVE);
-
-	first_idx = INDEX_RECORD_INDEX(index, first_rec);
-	last_idx = INDEX_RECORD_INDEX(index, last_rec);
-	idx_limit = MAIL_INDEX_RECORD_COUNT(index);
-
-	i_assert(first_idx <= last_idx);
-	i_assert(last_idx < idx_limit);
-
-	index->header->messages_count -= last_idx - first_idx + 1;
-	for (rec = first_rec; rec <= last_rec; rec++)
-		mail_index_mark_flag_changes(index, rec, rec->msg_flags, 0);
-
-	return compress(index, first_idx, last_idx);
-}
-
-struct mail_index_record *mail_index_lookup(struct mail_index *index,
-					    unsigned int seq)
-{
-	i_assert(index->lock_type != MAIL_LOCK_UNLOCK);
-	i_assert(seq > 0);
-
-	if (seq > index->header->messages_count)
-		return NULL;
-
-	return INDEX_RECORD_AT(index, seq-1);
-}
-
-struct mail_index_record *
-mail_index_lookup_uid_range(struct mail_index *index, unsigned int first_uid,
-			    unsigned int last_uid, unsigned int *seq_r)
-{
-	struct mail_index_record *rec_p;
-	unsigned int idx_limit, idx, left_idx, right_idx;
-
-	i_assert(index->lock_type != MAIL_LOCK_UNLOCK);
-	i_assert(first_uid > 0);
-	i_assert(first_uid <= last_uid);
-
-	rec_p = INDEX_RECORD_AT(index, 0);
-	idx_limit = MAIL_INDEX_RECORD_COUNT(index);
-
-	idx = 0;
-	left_idx = 0;
-	right_idx = idx_limit;
-
-	while (left_idx < right_idx) {
-		idx = (left_idx + right_idx) / 2;
-
-		if (rec_p[idx].uid < first_uid)
-			left_idx = idx+1;
-		else if (rec_p[idx].uid > first_uid)
-			right_idx = idx;
-		else
-			break;
-	}
-
-	if (rec_p[idx].uid < first_uid || rec_p[idx].uid > last_uid) {
-		/* could still be the next one */
-		idx++;
-		if (idx == idx_limit ||
-		    rec_p[idx].uid < first_uid || rec_p[idx].uid > last_uid) {
-			if (seq_r != NULL) *seq_r = 0;
-			return NULL;
-		}
-	}
-
-	if (seq_r != NULL)
-		*seq_r = idx + 1;
-	return rec_p + idx;
-}
-
-int mail_index_compress(struct mail_index *index)
-{
-	size_t diff;
-	off_t new_file_size;
-
-	if (index->header_size >= sizeof(struct mail_index_header))
-		return TRUE;
-
-	if (!index->set_lock(index, MAIL_LOCK_EXCLUSIVE))
-		return FALSE;
-
-	/* make sure the file is large enough */
-	diff = sizeof(struct mail_index_header) - index->header_size;
-	if (index->mmap_used_length + diff > index->mmap_full_length) {
-		/* mmap_update ftruncates the file to multiples of
-		   mail_index_record, make sure we grow it enough here. */
-		new_file_size = index->mmap_used_length + diff +
-			(sizeof(struct mail_index_record) -
-			 (diff % sizeof(struct mail_index_record)));
-		if (file_set_size(index->fd, new_file_size) < 0) {
-			index_set_syscall_error(index, "file_set_size()");
-			return FALSE;
-		}
-
-		index->header->master_sync_id++;
-		if (!mail_index_mmap_update(index))
-			return FALSE;
-	}
-
-	/* if we break, we'll have to rebuild it completely */
-	index->header->flags |= MAIL_INDEX_HDR_FLAG_REBUILD;
-	if (!mail_index_fmdatasync(index, index->header_size))
-		return FALSE;
-
-	memmove((char *) index->mmap_base + sizeof(struct mail_index_header),
-		(char *) index->mmap_base + index->header_size,
-		index->mmap_used_length - index->header_size);
-	memset((char *) index->mmap_base + index->header_size, 0, diff);
-
-	index->mmap_used_length += diff;
-	index->header_size = sizeof(struct mail_index_header);
-
-	index->header->header_size = sizeof(struct mail_index_header);
-	index->header->used_file_size += diff;
-	index->header->master_sync_id++;
- 
-	if (!mail_index_fmdatasync(index, index->mmap_used_length))
-		return FALSE;
-
-	index->header->flags &= ~MAIL_INDEX_HDR_FLAG_REBUILD;
-	return mail_index_mmap_update(index);
-}
-
-int mail_index_truncate(struct mail_index *index)
-{
-	uoff_t empty_space, truncate_threshold;
-
-	i_assert(index->lock_type == MAIL_LOCK_EXCLUSIVE);
-
-	if (index->mmap_full_length <= INDEX_FILE_MIN_SIZE(index) ||
-	    index->anon_mmap)
-		return TRUE;
-
-	/* really truncate the file only when it's almost empty */
-	empty_space = index->mmap_full_length - index->mmap_used_length;
-	truncate_threshold =
-		index->mmap_full_length / 100 * INDEX_TRUNCATE_PERCENTAGE;
-
-	if (empty_space > truncate_threshold) {
-		index->mmap_full_length = index->mmap_used_length +
-			(empty_space * INDEX_TRUNCATE_KEEP_PERCENTAGE / 100);
-
-		/* keep the size record-aligned */
-		index->mmap_full_length -= (index->mmap_full_length -
-					    index->header_size) %
-			sizeof(struct mail_index_record);
-
-		if (index->mmap_full_length < INDEX_FILE_MIN_SIZE(index))
-                        index->mmap_full_length = INDEX_FILE_MIN_SIZE(index);
-
-		if (ftruncate(index->fd, (off_t)index->mmap_full_length) < 0)
-			return index_set_syscall_error(index, "ftruncate()");
-
-		index->header->master_sync_id++;
-	}
-
-	return TRUE;
-}

--- a/src/lib-index/mail-index-fsck.c	Tue Apr 27 00:20:15 2004 +0300
+++ /dev/null	Thu Jan 01 00:00:00 1970 +0000
@@ -1,103 +0,0 @@
-/* Copyright (C) 2002 Timo Sirainen */
-
-#include "lib.h"
-#include "mail-index.h"
-#include "mail-index-util.h"
-
-#define CHECK(field) \
-	if (old_hdr->field != new_hdr->field) \
-		i_warning("fsck %s: "#field" %u != %u", \
-			  index->filepath, old_hdr->field, new_hdr->field);
-
-
-static void print_differences(struct mail_index *index,
-			      struct mail_index_header *old_hdr,
-			      struct mail_index_header *new_hdr)
-{
-	CHECK(next_uid);
-
-	CHECK(messages_count);
-	CHECK(seen_messages_count);
-	CHECK(deleted_messages_count);
-	CHECK(last_nonrecent_uid);
-
-	if (old_hdr->first_unseen_uid_lowwater >
-	    new_hdr->first_unseen_uid_lowwater) {
-		i_warning("fsck %s: first_unseen_uid_lowwater %u > %u",
-			  index->filepath,
-			  old_hdr->first_unseen_uid_lowwater,
-                          new_hdr->first_unseen_uid_lowwater);
-	}
-
-	if (old_hdr->first_deleted_uid_lowwater >
-	    new_hdr->first_deleted_uid_lowwater) {
-		i_warning("fsck %s: first_deleted_uid_lowwater %u > %u",
-			  index->filepath,
-			  old_hdr->first_deleted_uid_lowwater,
-                          new_hdr->first_deleted_uid_lowwater);
-	}
-}
-
-int mail_index_fsck(struct mail_index *index)
-{
-	struct mail_index_header old_hdr, *hdr;
-	struct mail_index_record *rec, *end_rec;
-	unsigned int max_uid;
-
-	i_assert(index->lock_type != MAIL_LOCK_SHARED);
-
-	if (!mail_index_compress(index))
-		return FALSE;
-
-	/* then we verify only the fields in the header. other problems will
-	   be noticed and fixed while reading the messages. */
-	hdr = index->header;
-	memcpy(&old_hdr, hdr, sizeof(struct mail_index_header));
-
-	hdr->messages_count = 0;
-	hdr->seen_messages_count = 0;
-	hdr->deleted_messages_count = 0;
-
-	hdr->first_unseen_uid_lowwater = 0;
-	hdr->first_deleted_uid_lowwater = 0;
-
-	rec = INDEX_RECORD_AT(index, 0);
-	end_rec = INDEX_END_RECORD(index);
-
-	max_uid = 0;
-	for (; rec < end_rec; rec++) {
-		if (rec->uid < max_uid) {
-			index_set_corrupted(index, "UIDs are not ordered "
-					    "(%u < %u)", rec->uid, max_uid);
-			return FALSE;
-		}
-		max_uid = rec->uid;
-
-		if (rec->msg_flags & MAIL_SEEN)
-			hdr->seen_messages_count++;
-		else if (hdr->first_unseen_uid_lowwater == 0)
-			hdr->first_unseen_uid_lowwater = rec->uid;
-
-		if (rec->msg_flags & MAIL_DELETED) {
-			if (hdr->first_deleted_uid_lowwater == 0)
-                                hdr->first_deleted_uid_lowwater = rec->uid;
-			hdr->deleted_messages_count++;
-		}
-		hdr->messages_count++;
-	}
-
-	if (hdr->next_uid <= max_uid)
-		hdr->next_uid = max_uid+1;
-	if (hdr->last_nonrecent_uid >= hdr->next_uid)
-		hdr->last_nonrecent_uid = hdr->next_uid-1;
-
-	if (hdr->first_unseen_uid_lowwater == 0)
-		hdr->first_unseen_uid_lowwater = hdr->next_uid;
-	if (hdr->first_deleted_uid_lowwater == 0)
-		hdr->first_deleted_uid_lowwater = hdr->next_uid;
-
-	print_differences(index, &old_hdr, hdr);
-
-	/* FSCK flag is removed automatically by set_lock() */
-	return TRUE;
-}

--- a/src/lib-index/mail-index-open.c	Tue Apr 27 00:20:15 2004 +0300
+++ /dev/null	Thu Jan 01 00:00:00 1970 +0000
@@ -1,397 +0,0 @@
-/* Copyright (C) 2002-2003 Timo Sirainen */
-
-#include "lib.h"
-#include "ioloop.h"
-#include "file-lock.h"
-#include "file-set-size.h"
-#include "hostpid.h"
-#include "mmap-util.h"
-#include "unlink-lockfiles.h"
-#include "write-full.h"
-#include "mail-index.h"
-#include "mail-index-util.h"
-#include "mail-cache.h"
-#include "mail-modifylog.h"
-#include "mail-custom-flags.h"
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <unistd.h>
-#include <fcntl.h>
-
-static int mail_index_open_init(struct mail_index *index,
-				enum mail_index_open_flags flags)
-{
-	struct mail_index_header *hdr;
-
-	hdr = index->header;
-
-	index->maildir_have_new =
-		(hdr->flags & MAIL_INDEX_FLAG_MAILDIR_NEW) != 0;
-
-	if ((hdr->flags & MAIL_INDEX_HDR_FLAG_DIRTY_MESSAGES) != 0)
-		index->next_dirty_flags_flush = ioloop_time;
-
-	/* update \Recent message counters */
-	if ((flags & MAIL_INDEX_OPEN_FLAG_UPDATE_RECENT) != 0 &&
-	    hdr->last_nonrecent_uid != hdr->next_uid-1) {
-		/* keep last_recent_uid to next_uid-1 */
-		if (index->lock_type == MAIL_LOCK_SHARED) {
-			if (!index->set_lock(index, MAIL_LOCK_UNLOCK))
-				return FALSE;
-		}
-
-		if (!index->set_lock(index, MAIL_LOCK_EXCLUSIVE))
-			return FALSE;
-
-		index->first_recent_uid = index->header->last_nonrecent_uid+1;
-		index->header->last_nonrecent_uid = index->header->next_uid-1;
-	} else {
-		index->first_recent_uid = hdr->last_nonrecent_uid+1;
-	}
-
-	if (hdr->next_uid >= MAX_ALLOWED_UID - 1000) {
-		/* UID values are getting too high, rebuild index */
-		index->set_flags |= MAIL_INDEX_HDR_FLAG_REBUILD;
-	}
-
-	if (index->lock_type == MAIL_LOCK_EXCLUSIVE) {
-		/* finally reset the modify log marks, fsck or syncing might
-		   have deleted some messages, and since we're only just
-		   opening the index, there's no need to remember them */
-		if (!mail_modifylog_mark_synced(index->modifylog))
-			return FALSE;
-	}
-
-	return TRUE;
-}
-
-static int index_open_and_fix(struct mail_index *index,
-			      enum mail_index_open_flags flags)
-{
-	int rebuilt;
-
-	if (index->header_size < sizeof(struct mail_index_header)) {
-		/* upgrading from older index file. */
-		if (!mail_index_compress(index))
-			return FALSE;
-	}
-
-	if (!mail_cache_open_or_create(index))
-		return FALSE;
-
-	/* custom flags file needs to be open before rebuilding index */
-	if (!mail_custom_flags_open_or_create(index))
-		return FALSE;
-
-	if ((index->header->flags & MAIL_INDEX_HDR_FLAG_REBUILD) != 0 ||
-	    (index->set_flags & MAIL_INDEX_HDR_FLAG_REBUILD) != 0) {
-
-		if (!index->rebuild(index))
-			return FALSE;
-
-		if ((index->header->flags & MAIL_INDEX_HDR_FLAG_REBUILD) != 0)
-			return FALSE;
-
-		/* no inconsistency problems since we're still opening
-		   the index */
-		index->inconsistent = FALSE;
-		rebuilt = TRUE;
-	} else {
-		rebuilt = FALSE;
-	}
-
-	if ((flags & _MAIL_INDEX_OPEN_FLAG_CREATING) == 0) {
-		if (!mail_modifylog_open_or_create(index))
-			return FALSE;
-	} else {
-		if (!mail_modifylog_create(index))
-			return FALSE;
-	}
-
-	if (index->header->flags & MAIL_INDEX_HDR_FLAG_FSCK) {
-		/* index needs fscking */
-		if (!index->fsck(index))
-			return FALSE;
-	}
-
-	if (!rebuilt) {
-		/* sync ourself. do it before compression which may happen
-		   as a result of this. */
-		if (!index->sync_and_lock(index, FALSE,
-					  MAIL_LOCK_SHARED, NULL) &&
-		    !index->nodiskspace)
-			return FALSE;
-	}
-
-	/* we never want to keep shared lock if syncing happens to set it.
-	   either exclusive or nothing (NOTE: drop it directly, not through
-	   index->set_lock() so mbox lock won't be affected). */
-	if (index->lock_type == MAIL_LOCK_SHARED) {
-		if (!mail_index_set_lock(index, MAIL_LOCK_UNLOCK))
-			return FALSE;
-	}
-
-	if ((flags & MAIL_INDEX_OPEN_FLAG_FAST) == 0) {
-		if (index->header->flags & MAIL_INDEX_HDR_FLAG_COMPRESS) {
-			/* remove deleted blocks from index file */
-			if (!mail_index_compress(index))
-				return FALSE;
-		}
-
-		if (index->header->flags & MAIL_INDEX_HDR_FLAG_COMPRESS_CACHE) {
-			/* remove unused space from index data file. */
-			if (!mail_cache_compress(index->cache))
-				return FALSE;
-		}
-	}
-
-	if (!mail_index_open_init(index, flags))
-		return FALSE;
-
-	return TRUE;
-}
-
-static int mail_index_read_header(struct mail_index *index,
-				  struct mail_index_header *hdr)
-{
-	ssize_t ret;
-
-	if (lseek(index->fd, 0, SEEK_SET) < 0) {
-		index_set_syscall_error(index, "seek()");
-		return -1;
-	}
-
-	ret = read(index->fd, hdr, sizeof(*hdr));
-	if (ret < 0) {
-		index_set_syscall_error(index, "read()");
-		return -1;
-	}
-
-	if (ret != sizeof(*hdr)) {
-		/* missing data */
-		return 0;
-	}
-
-	return 1;
-}
-
-static int mail_index_init_file(struct mail_index *index,
-				const struct mail_index_header *hdr)
-{
-	uoff_t file_size;
-
-	if (lseek(index->fd, 0, SEEK_SET) < 0) {
-		index_set_syscall_error(index, "lseek()");
-		return FALSE;
-	}
-
-	if (write_full(index->fd, hdr, sizeof(*hdr)) < 0) {
-		index_set_syscall_error(index, "write_full()");
-		return FALSE;
-	}
-
-	file_size = sizeof(*hdr) +
-		INDEX_MIN_RECORDS_COUNT * sizeof(struct mail_index_record);
-	if (file_set_size(index->fd, (off_t)file_size) < 0) {
-		index_set_syscall_error(index, "file_set_size()");
-		return FALSE;
-	}
-
-	return TRUE;
-}
-
-static void get_compat_data(unsigned char compat_data[4])
-{
-#ifndef WORDS_BIGENDIAN
-	compat_data[0] = MAIL_INDEX_COMPAT_LITTLE_ENDIAN;
-#else
-	compat_data[0] = 0;
-#endif
-	compat_data[1] = sizeof(uoff_t);
-	compat_data[2] = sizeof(time_t);
-	compat_data[3] = 0;
-}
-
-void mail_index_init_header(struct mail_index_header *hdr)
-{
-	i_assert(sizeof(struct mail_index_header) < 256);
-
-	memset(hdr, 0, sizeof(*hdr));
-	hdr->major_version = MAIL_INDEX_MAJOR_VERSION;
-	hdr->minor_version = MAIL_INDEX_MINOR_VERSION;
-	hdr->header_size = (uint8_t)sizeof(struct mail_index_header);
-	get_compat_data(hdr->compat_data);
-
-	hdr->indexid = ioloop_time;
-
-	/* mark the index requiring rebuild - rebuild() removes this flag
-	   when it succeeds */
-	hdr->flags = MAIL_INDEX_HDR_FLAG_REBUILD;
-
-	hdr->used_file_size = sizeof(struct mail_index_header);
-	hdr->uid_validity = ioloop_time;
-	hdr->next_uid = 1;
-}
-
-static void mail_index_cleanup_temp_files(const char *dir)
-{
-	unlink_lockfiles(dir, t_strconcat(".temp.", my_hostname, ".", NULL),
-			 ".temp.", time(NULL) - TEMP_FILE_TIMEOUT);
-}
-
-void mail_index_init(struct mail_index *index, const char *dir)
-{
-	size_t len;
-
-	index->fd = -1;
-
-	if (dir != NULL) {
-		index->dir = i_strdup(dir);
-
-		len = strlen(index->dir);
-		if (index->dir[len-1] == '/')
-			index->dir[len-1] = '\0';
-	}
-
-	index->mail_read_mmaped = getenv("MAIL_READ_MMAPED") != NULL;
-}
-
-static int mail_index_create_memory(struct mail_index *index,
-				    enum mail_index_open_flags flags)
-{
-	if ((flags & MAIL_INDEX_OPEN_FLAG_CREATE) == 0)
-		return FALSE;
-
-	flags |= _MAIL_INDEX_OPEN_FLAG_CREATING;
-
-	index->header_size = sizeof(struct mail_index_header);
-	index->mmap_full_length = INDEX_FILE_MIN_SIZE(index);
-	index->mmap_base = mmap_anon(index->mmap_full_length);
-	if (index->mmap_base == MAP_FAILED) {
-		index->mmap_base = NULL;
-		return index_set_error(index, "mmap_anon() failed: %m");
-	}
-
-	mail_index_init_header(index->mmap_base);
-	index->header = index->mmap_base;
-	index->mmap_used_length = index->header->used_file_size;
-
-	index->anon_mmap = TRUE;
-	index->lock_type = MAIL_LOCK_EXCLUSIVE;
-	index->indexid = index->header->indexid;
-	index->filepath = i_strdup_printf("(in-memory index for %s)",
-					  index->mailbox_path);
-
-	if (!index_open_and_fix(index, flags)) {
-		mail_index_close(index);
-		return FALSE;
-	}
-
-	index->opened = TRUE;
-	return TRUE;
-}
-
-static int mail_index_open_index(struct mail_index *index,
-				 enum mail_index_open_flags flags)
-{
-	struct mail_index_header hdr;
-	unsigned char compat_data[4];
-	int ret;
-
-	if ((flags & _MAIL_INDEX_OPEN_FLAG_CREATING) == 0)
-		index->lock_type = MAIL_LOCK_SHARED;
-	else
-		index->lock_type = MAIL_LOCK_EXCLUSIVE;
-
-	/* if index is being created, we'll wait here until it's finished */
-	if (!mail_index_wait_lock(index, MAIL_LOCK_TO_FLOCK(index->lock_type)))
-		return FALSE;
-#ifdef DEBUG
-	if (index->mmap_base != NULL) {
-		mprotect(index->mmap_base, index->mmap_used_length,
-			 PROT_READ|PROT_WRITE);
-	}
-#endif
-
-	if ((ret = mail_index_read_header(index, &hdr)) < 0)
-		return FALSE;
-	index->indexid = hdr.indexid;
-
-	get_compat_data(compat_data);
-	if (ret == 0 || hdr.major_version != MAIL_INDEX_MAJOR_VERSION ||
-	    (hdr.flags & MAIL_INDEX_HDR_FLAG_REBUILD) != 0 ||
-	    memcmp(compat_data, hdr.compat_data, sizeof(compat_data)) != 0 ||
-	    !mail_index_mmap_update(index)) {
-		if ((flags & MAIL_INDEX_OPEN_FLAG_CREATE) == 0)
-			return FALSE;
-
-		flags |= _MAIL_INDEX_OPEN_FLAG_CREATING;
-
-		/* so, we're creating the index */
-		if (index->lock_type != MAIL_LOCK_EXCLUSIVE) {
-			/* have to get exclusive lock first */
-			if (!mail_index_wait_lock(index, F_UNLCK))
-				return FALSE;
-			return mail_index_open_index(index, flags);
-		}
-
-		mail_index_init_header(&hdr);
-		if (!mail_index_init_file(index, &hdr))
-			return FALSE;
-
-		if (!mail_index_mmap_update(index))
-			return FALSE;
-	}
-
-	if (index->lock_type == MAIL_LOCK_SHARED) {
-		/* we don't want to keep the shared lock while opening
-		   indexes. opening should work unlocked and some
-		   things want exclusive lock */
-		if (!mail_index_wait_lock(index, F_UNLCK))
-			return FALSE;
-		index->lock_type = MAIL_LOCK_UNLOCK;
-	}
-
-	if (!index_open_and_fix(index, flags))
-		return FALSE;
-
-	if (!index->set_lock(index, MAIL_LOCK_UNLOCK))
-		return FALSE;
-
-	index->opened = TRUE;
-	return TRUE;
-}
-
-int mail_index_open(struct mail_index *index, enum mail_index_open_flags flags)
-{
-	const char *path;
-
-	i_assert(!index->opened);
-
-	if (index->dir == NULL)
-		return mail_index_create_memory(index, flags);
-
-	mail_index_cleanup_temp_files(index->dir);
-
-	/* open/create the file */
-        path = t_strconcat(index->dir, "/", INDEX_FILE_PREFIX, NULL);
-	if ((flags & MAIL_INDEX_OPEN_FLAG_CREATE) != 0)
-		index->fd = open(path, O_RDWR | O_CREAT, 0660);
-	else
-		index->fd = open(path, O_RDWR);
-	if (index->fd == -1) {
-		if (errno != ENOENT)
-			index_file_set_syscall_error(index, path, "open()");
-		return mail_index_create_memory(index, flags);
-	}
-
-	index->filepath = i_strdup(path);
-
-	if (!mail_index_open_index(index, flags)) {
-		mail_index_close(index);
-		return mail_index_create_memory(index, flags);
-	}
-
-	return TRUE;
-}

--- a/src/lib-index/mail-index-rebuild.c	Tue Apr 27 00:20:15 2004 +0300
+++ /dev/null	Thu Jan 01 00:00:00 1970 +0000
@@ -1,47 +0,0 @@
-/* Copyright (C) 2002-2003 Timo Sirainen */
-
-#include "lib.h"
-#include "mail-index.h"
-#include "mail-index-util.h"
-#include "mail-cache.h"
-
-#include <sys/mman.h>
-
-int mail_index_rebuild(struct mail_index *index)
-{
-	if (!mail_index_set_lock(index, MAIL_LOCK_EXCLUSIVE))
-		return FALSE;
-
-	index->set_flags &= ~MAIL_INDEX_HDR_FLAG_REBUILD;
-
-	/* reset the header */
-	mail_index_init_header(index->header);
-	index->mmap_used_length = index->header->used_file_size;
-
-	/* update indexid, which also means that our state has completely
-	   changed */
-	index->indexid = index->header->indexid;
-	index->inconsistent = TRUE;
-	index->rebuilding = TRUE;
-
-	if (!index->anon_mmap) {
-		if (msync(index->mmap_base, index->header_size, MS_SYNC) < 0)
-			return index_set_syscall_error(index, "msync()");
-	}
-
-	if (!mail_cache_truncate(index->cache))
-		return FALSE;
-
-	/* read the mails by syncing */
-	if (!index->sync_and_lock(index, FALSE, MAIL_LOCK_UNLOCK, NULL))
-		return FALSE;
-
-	/* rebuild is complete - remove the flag */
-	index->header->flags &= ~(MAIL_INDEX_HDR_FLAG_REBUILD |
-				  MAIL_INDEX_HDR_FLAG_FSCK);
-	index->header->flags |= index->set_flags;
-	index->set_flags = 0;
-
-	index->rebuilding = FALSE;
-	return TRUE;
-}

--- a/src/lib-index/mail-index-util.c	Tue Apr 27 00:20:15 2004 +0300
+++ /dev/null	Thu Jan 01 00:00:00 1970 +0000
@@ -1,154 +0,0 @@
-/* Copyright (C) 2002 Timo Sirainen */
-
-#include "lib.h"
-#include "istream.h"
-#include "hostpid.h"
-#include "file-lock.h"
-#include "message-size.h"
-#include "message-part-serialize.h"
-#include "mail-index.h"
-#include "mail-index-util.h"
-
-#include <unistd.h>
-#include <fcntl.h>
-
-int index_set_error(struct mail_index *index, const char *fmt, ...)
-{
-	va_list va;
-
-	i_free(index->error);
-
-	if (fmt == NULL)
-		index->error = NULL;
-	else {
-		va_start(va, fmt);
-		index->error = i_strdup_vprintf(fmt, va);
-		va_end(va);
-
-		i_error("%s", index->error);
-	}
-
-	return FALSE;
-}
-
-int index_set_corrupted(struct mail_index *index, const char *fmt, ...)
-{
-	va_list va;
-
-	INDEX_MARK_CORRUPTED(index);
-	index->inconsistent = TRUE;
-
-	va_start(va, fmt);
-	t_push();
-	index_set_error(index, "Corrupted index file %s: %s",
-			index->filepath, t_strdup_vprintf(fmt, va));
-	t_pop();
-	va_end(va);
-
-	return FALSE;
-}
-
-int index_set_syscall_error(struct mail_index *index, const char *function)
-{
-	i_assert(function != NULL);
-
-	if (ENOSPACE(errno)) {
-		index->nodiskspace = TRUE;
-		return FALSE;
-	}
-
-	index_set_error(index, "%s failed with index file %s: %m",
-			function, index->filepath);
-	return FALSE;
-}
-
-int index_file_set_syscall_error(struct mail_index *index, const char *filepath,
-				 const char *function)
-{
-	i_assert(filepath != NULL);
-	i_assert(function != NULL);
-
-	if (ENOSPACE(errno)) {
-		index->nodiskspace = TRUE;
-		return FALSE;
-	}
-
-	index_set_error(index, "%s failed with file %s: %m",
-			function, filepath);
-
-	return FALSE;
-}
-
-void index_reset_error(struct mail_index *index)
-{
-	if (index->error != NULL) {
-		i_free(index->error);
-		index->error = NULL;
-	}
-
-	index->nodiskspace = FALSE;
-}
-
-int mail_index_create_temp_file(struct mail_index *index, const char **path)
-{
-	int fd;
-
-	/* use ".temp.host.pid" as temporary file name. unlink() it first,
-	   just to be sure it's not symlinked somewhere for some reason..
-	   FIXME: this function should rather be removed entirely. With
-	   in-memory indexes index->dir is NULL, so we fallback to /tmp
-           so that mbox rewriting doesn't crash. */
-	*path = t_strconcat(index->dir != NULL ? index->dir : "/tmp",
-			    "/.temp.", my_hostname, ".", my_pid, NULL);
-	(void)unlink(*path);
-
-	/* usage of O_EXCL isn't exactly needed since the path should be
-	   trusted, but it shouldn't hurt either - if creating file fails
-	   because of it, it's because something must be wrong (race
-	   condition). also, might not won't work through NFS but that
-	   can't be helped. */
-	fd = open(*path, O_RDWR | O_CREAT | O_EXCL, 0660);
-	if (fd == -1) {
-		if (ENOSPACE(errno))
-			index->nodiskspace = TRUE;
-		else {
-			index_set_error(index, "Can't create temp index %s: %m",
-					*path);
-		}
-	}
-
-	return fd;
-}
-
-static void mail_index_lock_notify(unsigned int secs_left, void *context)
-{
-	struct mail_index *index = context;
-
-	if (index->lock_notify_cb == NULL)
-		return;
-
-	index->lock_notify_cb(MAIL_LOCK_NOTIFY_INDEX_ABORT, secs_left,
-			      index->lock_notify_context);
-}
-
-int mail_index_wait_lock(struct mail_index *index, int lock_type)
-{
-	int ret;
-
-	ret = file_wait_lock_full(index->fd, lock_type, DEFAULT_LOCK_TIMEOUT,
-				  mail_index_lock_notify, index);
-	if (ret < 0)
-		return index_set_syscall_error(index, "file_wait_lock()");
-
-	if (ret == 0) {
-		index_set_error(index, "Timeout while waiting for release of "
-				"%s fcntl() lock for index file %s",
-				lock_type == F_WRLCK ? "exclusive" : "shared",
-				index->filepath);
-		index->index_lock_timeout = TRUE;
-		return FALSE;
-	}
-
-	return TRUE;
-
-}

--- a/src/lib-index/mail-index-util.h	Tue Apr 27 00:20:15 2004 +0300
+++ /dev/null	Thu Jan 01 00:00:00 1970 +0000
@@ -1,43 +0,0 @@
-#ifndef __MAIL_INDEX_UTIL_H
-#define __MAIL_INDEX_UTIL_H
-
-/* Get index's lock state as mprotect() argument */
-#define MAIL_INDEX_PROT(index) \
-	((index)->lock_type == MAIL_LOCK_EXCLUSIVE ? (PROT_READ|PROT_WRITE) : \
-	 (index)->lock_type == MAIL_LOCK_SHARED || !(index)->opened ? \
-	 PROT_READ : PROT_NONE)
-
-/* DEBUG: Force mmap() locks with mprotect() */
-#ifdef DEBUG
-#  define debug_mprotect(mmap_base, mmap_length, index) \
-	mprotect(mmap_base, mmap_length, MAIL_INDEX_PROT(index))
-#else
-#  define debug_mprotect(mmap_base, mmap_length, index)
-#endif
-
-/* Set the current error message */
-int index_set_error(struct mail_index *index, const char *fmt, ...)
-	__attr_format__(2, 3);
-
-/* "Error in index file %s: ...". Also marks the index file as corrupted. */
-int index_set_corrupted(struct mail_index *index, const char *fmt, ...)
-	__attr_format__(2, 3);
-
-/* "%s failed with index file %s: %m" */
-int index_set_syscall_error(struct mail_index *index, const char *function);
-
-/* "%s failed with file %s: %m" */
-int index_file_set_syscall_error(struct mail_index *index, const char *filepath,
-				 const char *function);
-
-/* Reset the current error */
-void index_reset_error(struct mail_index *index);
-
-/* Create temporary file into index's directory. Returns opened file handle
-   and sets *path to the full path of the created file.  */
-int mail_index_create_temp_file(struct mail_index *index, const char **path);
-
-/* Wrapper to file_set_lock(), also calling index's lock notify callback. */
-int mail_index_wait_lock(struct mail_index *index, int lock_type);
-
-#endif

--- a/src/lib-index/mail-index.c	Tue Apr 27 00:20:15 2004 +0300
+++ /dev/null	Thu Jan 01 00:00:00 1970 +0000
@@ -1,735 +0,0 @@
-/* Copyright (C) 2002 Timo Sirainen */
-
-#include "lib.h"
-#include "ioloop.h"
-#include "file-lock.h"
-#include "file-set-size.h"
-#include "mmap-util.h"
-#include "mail-index.h"
-#include "mail-index-util.h"
-#include "mail-cache.h"
-#include "mail-modifylog.h"
-#include "mail-custom-flags.h"
-
-#include <unistd.h>
-#include <fcntl.h>
-
-static void update_header(struct mail_index *index)
-{
-	struct mail_index_header *hdr = index->header;
-
-	index->cache_sync_id = hdr->cache_sync_id;
-	index->log_sync_id = hdr->log_sync_id;
-	index->sync_stamp = hdr->sync_stamp;
-	index->sync_size = hdr->sync_size;
-}
-
-static int mmap_verify(struct mail_index *index)
-{
-	struct mail_index_header *hdr;
-	unsigned int extra;
-
-	index->mmap_used_length = 0;
-
-	if (index->mmap_full_length < sizeof(struct mail_index_header)) {
-                index_set_corrupted(index, "File too small");
-		return FALSE;
-	}
-
-	/* keep the header set even if we fail, so we can update the flags */
-	hdr = index->mmap_base;
-	index->header = hdr;
-	index->header_size = hdr->header_size;
-
-	if (index->header_size > index->mmap_full_length) {
-		index_set_corrupted(index, "Invalid header_size in header "
-				    "(%"PRIuSIZE_T")", index->header_size);
-		return FALSE;
-	}
-
-	extra = (index->mmap_full_length - index->header_size) %
-		sizeof(struct mail_index_record);
-
-	if (extra != 0) {
-		/* partial write or corrupted -
-		   truncate the file to valid length */
-		i_assert(!index->anon_mmap);
-
-		index->mmap_full_length -= extra;
-		(void)ftruncate(index->fd, (off_t)index->mmap_full_length);
-	}
-
-	if (hdr->used_file_size > index->mmap_full_length) {
-		index_set_corrupted(index,
-				    "used_file_size larger than real file size "
-				    "(%u vs %"PRIuSIZE_T")",
-				    hdr->used_file_size,
-				    index->mmap_full_length);
-		return FALSE;
-	}
-
-	if (hdr->used_file_size < index->header_size ||
-	    (hdr->used_file_size - index->header_size) %
-	    sizeof(struct mail_index_record) != 0) {
-		index_set_corrupted(index, "Invalid used_file_size in header "
-				    "(%u)", hdr->used_file_size);
-		return FALSE;
-	}
-
-	if (hdr->messages_count < hdr->seen_messages_count) {
-		index_set_corrupted(index, "Invalid seen messages count "
-				    "(%u < %u)", hdr->messages_count,
-				    hdr->seen_messages_count);
-		return FALSE;
-	}
-
-	if (hdr->messages_count < hdr->deleted_messages_count) {
-		index_set_corrupted(index, "Invalid deleted messages count "
-				    "(%u < %u)", hdr->messages_count,
-				    hdr->deleted_messages_count);
-		return FALSE;
-	}
-
-	index->master_sync_id = hdr->master_sync_id;
-	index->mmap_used_length = hdr->used_file_size;
-        update_header(index);
-	return TRUE;
-}
-
-int mail_index_mmap_update(struct mail_index *index)
-{
-	if (index->anon_mmap)
-		return mmap_verify(index);
-
-	if (index->mmap_base != NULL) {
-		index->header = (struct mail_index_header *) index->mmap_base;
-		update_header(index);
-
-		if (index->mmap_invalidate) {
-			if (msync(index->mmap_base,
-				  index->mmap_used_length,
-				  MS_SYNC | MS_INVALIDATE) < 0) {
-				index_set_syscall_error(index, "msync()");
-				return FALSE;
-			}
-		}
-
-		/* make sure file size hasn't changed */
-		if (index->header->master_sync_id == index->master_sync_id) {
-			index->mmap_used_length = index->header->used_file_size;
-			if (index->mmap_used_length > index->mmap_full_length) {
-				i_panic("Index file size was grown without "
-					"updating sync_id");
-			}
-			return TRUE;
-		}
-
-		if (!index->mmap_invalidate) {
-			if (msync(index->mmap_base,
-				  index->mmap_used_length, MS_SYNC) < 0) {
-				index_set_syscall_error(index, "msync()");
-				return FALSE;
-			}
-		}
-
-		if (munmap(index->mmap_base, index->mmap_full_length) < 0)
-			return index_set_syscall_error(index, "munmap()");
-	}
-
-	index->mmap_base = mmap_rw_file(index->fd, &index->mmap_full_length);
-	if (index->mmap_base == MAP_FAILED) {
-		index->mmap_base = NULL;
-		index->mmap_used_length = 0;
-		index_set_syscall_error(index, "mmap()");
-		return FALSE;
-	}
-
-	return mmap_verify(index);
-}
-
-void mail_index_close(struct mail_index *index)
-{
-	if (index->set_flags != 0) {
-		if (index->header != NULL) {
-#ifdef DEBUG
-			mprotect(index->mmap_base, index->mmap_used_length,
-				 PROT_READ|PROT_WRITE);
-#endif
-			index->header->flags |= index->set_flags;
-			(void)msync(index->mmap_base, index->header_size,
-				    MS_SYNC);
-		}
-		index->set_flags = 0;
-	}
-
-	index->opened = FALSE;
-	index->inconsistent = FALSE;
-
-	index->lock_type = MAIL_LOCK_UNLOCK;
-	index->header = NULL;
-
-	if (index->fd != -1) {
-		if (close(index->fd) < 0)
-			index_set_syscall_error(index, "close()");
-		index->fd = -1;
-	}
-
-	if (index->filepath != NULL) {
-		i_free(index->filepath);
-		index->filepath = NULL;
-	}
-
-	if (index->anon_mmap) {
-		if (munmap_anon(index->mmap_base, index->mmap_full_length) < 0)
-			index_set_syscall_error(index, "munmap_anon()");
-		index->anon_mmap = FALSE;
-	} else if (index->mmap_base != NULL) {
-		if (munmap(index->mmap_base, index->mmap_full_length) < 0)
-			index_set_syscall_error(index, "munmap()");
-	}
-	index->mmap_base = NULL;
-
-	if (index->cache != NULL) {
-                mail_cache_free(index->cache);
-		index->cache = NULL;
-	}
-
-	if (index->modifylog != NULL) {
-                mail_modifylog_free(index->modifylog);
-		index->modifylog = NULL;
-	}
-
-	if (index->custom_flags != NULL) {
-		mail_custom_flags_free(index->custom_flags);
-                index->custom_flags = NULL;
-	}
-
-	if (index->error != NULL) {
-		i_free(index->error);
-		index->error = NULL;
-	}
-}
-
-static int mail_index_sync_file(struct mail_index *index)
-{
-	unsigned int i;
-	int failed, fsync_fds[3];
-
-	if (index->anon_mmap)
-		return TRUE;
-
-	for (i = 0; i < sizeof(fsync_fds)/sizeof(fsync_fds[0]); i++)
-		fsync_fds[i] = -1;
-
-	if (msync(index->mmap_base, index->mmap_used_length, MS_SYNC) < 0)
-		return index_set_syscall_error(index, "msync()");
-
-	failed = FALSE;
-
-	if (index->modifylog != NULL) {
-		if (!mail_modifylog_sync_file(index->modifylog, &fsync_fds[2]))
-			failed = TRUE;
-	}
-
-	for (i = 0; i < sizeof(fsync_fds)/sizeof(fsync_fds[0]); i++) {
-		if (fsync_fds[i] != -1 && fdatasync(fsync_fds[i]) < 0)
-			index_set_error(index, "fdatasync(%u) failed: %m", i);
-	}
-
-	if (fdatasync(index->fd) < 0)
-		return index_set_syscall_error(index, "fdatasync()");
-
-	return !failed;
-}
-
-int mail_index_fmdatasync(struct mail_index *index, size_t size)
-{
-	i_assert(index->lock_type == MAIL_LOCK_EXCLUSIVE);
-
-	if (!index->anon_mmap) {
-		if (msync(index->mmap_base, size, MS_SYNC) < 0)
-			return index_set_syscall_error(index, "msync()");
-		if (fdatasync(index->fd) < 0)
-			return index_set_syscall_error(index, "fdatasync()");
-	}
-
-	return TRUE;
-}
-
-static void mail_index_update_header_changes(struct mail_index *index)
-{
-	if (index->set_flags != 0) {
-		index->header->flags |= index->set_flags;
-		index->set_flags = 0;
-	}
-}
-
-static int mail_index_write_header_changes(struct mail_index *index)
-{
-	int failed = FALSE;
-
-	/* use our own locking here so we don't mess up with any other
-	   index states, like inconsistency. */
-	if (!mail_index_wait_lock(index, F_WRLCK))
-		return FALSE;
-
-#ifdef DEBUG
-	mprotect(index->mmap_base, index->mmap_used_length,
-		 PROT_READ|PROT_WRITE);
-#endif
-
-	mail_index_update_header_changes(index);
-
-	if (!index->anon_mmap) {
-		if (msync(index->mmap_base, index->header_size, MS_SYNC) < 0) {
-			index_set_syscall_error(index, "msync()");
-			failed = TRUE;
-		}
-	}
-
-#ifdef DEBUG
-	mprotect(index->mmap_base, index->mmap_used_length, PROT_NONE);
-#endif
-
-	if (!mail_index_wait_lock(index, F_UNLCK))
-		return FALSE;
-
-	return !failed;
-}
-
-static int mail_index_lock_remove(struct mail_index *index)
-{
-	enum mail_lock_type old_lock_type;
-	int ret = TRUE;
-
-	while (index->cache_later_locks > 0) {
-		if (!mail_cache_unlock(index->cache))
-			ret = FALSE;
-		index->cache_later_locks--;
-	}
-
-	if (!mail_index_wait_lock(index, F_UNLCK))
-		return FALSE;
-
-	old_lock_type = index->lock_type;
-	index->lock_type = MAIL_LOCK_UNLOCK;
-
-	if (old_lock_type == MAIL_LOCK_SHARED) {
-		/* releasing shared lock. we may need to update some
-		   flags in header. */
-		unsigned int old_flags;
-
-		old_flags = index->header->flags;
-
-		if ((old_flags | index->set_flags) != old_flags)
-			return mail_index_write_header_changes(index);
-	}
-
-        debug_mprotect(index->mmap_base, index->mmap_full_length, index);
-	return ret;
-}
-
-static int mail_index_lock_change(struct mail_index *index,
-				  enum mail_lock_type lock_type, int try_lock)
-{
-	int ret, fd_lock_type;
-
-	/* shared -> exclusive can deadlock */
-	i_assert(try_lock || lock_type != MAIL_LOCK_EXCLUSIVE ||
-		 index->lock_type != MAIL_LOCK_SHARED);
-
-	/* locking index when cache is locked can deadlock */
-	i_assert(try_lock || index->lock_type == MAIL_LOCK_EXCLUSIVE ||
-		 index->cache == NULL || !mail_cache_is_locked(index->cache));
-
-	if (index->inconsistent) {
-		/* index is in inconsistent state and nothing else than
-		   free() is allowed for it. */
-		if (index->error == NULL) {
-			index->error =
-				i_strdup("Index is in inconsistent state");
-		}
-		return FALSE;
-	}
-
-	fd_lock_type = MAIL_LOCK_TO_FLOCK(lock_type);
-	if (try_lock) {
-		ret = file_try_lock(index->fd, fd_lock_type);
-		if (ret < 0)
-			index_set_syscall_error(index, "file_try_lock()");
-		if (ret <= 0)
-			return FALSE;
-	} else {
-		if (!mail_index_wait_lock(index, fd_lock_type))
-			return FALSE;
-	}
-
-	index->lock_type = lock_type;
-	debug_mprotect(index->mmap_base, index->mmap_full_length, index);
-
-	if (!mail_index_mmap_update(index)) {
-		(void)index->set_lock(index, MAIL_LOCK_UNLOCK);
-		return FALSE;
-	}
-
-	if (index->indexid != index->header->indexid) {
-		/* index was rebuilt, there's no way we can maintain
-		   consistency */
-		index_set_error(index, "Warning: Inconsistency - Index "
-				"%s was rebuilt while we had it open",
-				index->filepath);
-		index->inconsistent = TRUE;
-		return FALSE;
-	}
-
-	if (index->header->flags & MAIL_INDEX_HDR_FLAG_FSCK) {
-		/* someone just partially updated the index, need to fsck it */
-		if (lock_type == MAIL_LOCK_SHARED) {
-			/* we need exclusive lock so fsck()'s set_lock() won't
-			   get us back here */
-			if (!mail_index_lock_remove(index))
-				return FALSE;
-
-			if (!mail_index_wait_lock(index, F_WRLCK))
-				return FALSE;
-			index->lock_type = MAIL_LOCK_EXCLUSIVE;
-
-			debug_mprotect(index->mmap_base,
-				       index->mmap_full_length, index);
-		}
-
-		/* check again, in case it was already fscked while we had
-		   it unlocked for a while */
-		if (index->header->flags & MAIL_INDEX_HDR_FLAG_FSCK) {
-			if (!index->fsck(index))
-				return FALSE;
-		}
-
-		if (lock_type == MAIL_LOCK_SHARED) {
-			/* drop exclusive lock */
-			return index->set_lock(index, lock_type);
-		}
-	}
-
-	if (lock_type == MAIL_LOCK_EXCLUSIVE) {
-		/* while holding exclusive lock, keep the FSCK flag on.
-		   when the lock is released, the FSCK flag will also be
-		   removed. */
-		index->excl_lock_counter++;
-		index->header->flags |= MAIL_INDEX_HDR_FLAG_FSCK;
-		if (!mail_index_fmdatasync(index, index->header_size)) {
-			(void)index->set_lock(index, MAIL_LOCK_UNLOCK);
-			return FALSE;
-		}
-	}
-
-	return TRUE;
-}
-
-static int mail_index_lock_full(struct mail_index *index,
-				enum mail_lock_type lock_type, int try_lock)
-{
-	int keep_fsck;
-
-	if (index->lock_type == lock_type)
-		return TRUE;
-
-	if (index->lock_type == MAIL_LOCK_EXCLUSIVE) {
-		index->excl_lock_counter++;
-		if (index->modifylog != NULL)
-			mail_modifylog_notify_lock_drop(index->modifylog);
-	}
-
-	if (index->anon_mmap) {
-		/* anonymous mmaps are private and don't need any locking */
-#ifdef DEBUG
-		mprotect(index->mmap_base, index->mmap_used_length,
-			 PROT_READ|PROT_WRITE);
-#endif
-		mail_index_update_header_changes(index);
-
-		index->lock_type = lock_type;
-		debug_mprotect(index->mmap_base, index->mmap_full_length,
-			       index);
-		return TRUE;
-	}
-
-	if (index->lock_type == MAIL_LOCK_EXCLUSIVE) {
-		/* dropping exclusive lock (either unlock or to shared) */
-		keep_fsck = (index->set_flags & MAIL_INDEX_HDR_FLAG_FSCK) != 0;
-		mail_index_update_header_changes(index);
-
-		if (index->sync_dirty_stamp == 0) {
-			index->header->sync_stamp = index->sync_stamp;
-			index->header->sync_size = index->sync_size;
-		}
-
-		/* remove the FSCK flag only after successful fsync() */
-		if (mail_index_sync_file(index) && !keep_fsck) {
-			index->header->flags &= ~MAIL_INDEX_HDR_FLAG_FSCK;
-			if (msync(index->mmap_base, index->header_size,
-				  MS_SYNC) < 0) {
-				/* we only failed to remove the fsck flag,
-				   so this isn't fatal. */
-				index_set_syscall_error(index, "msync()");
-			}
-		}
-	}
-
-	if (lock_type == MAIL_LOCK_UNLOCK)
-		return mail_index_lock_remove(index);
-	else
-		return mail_index_lock_change(index, lock_type, try_lock);
-}
-
-int mail_index_set_lock(struct mail_index *index, enum mail_lock_type lock_type)
-{
-	return mail_index_lock_full(index, lock_type, FALSE);
-}
-
-int mail_index_try_lock(struct mail_index *index, enum mail_lock_type lock_type)
-{
-	return mail_index_lock_full(index, lock_type, TRUE);
-}
-
-void mail_index_set_lock_notify_callback(struct mail_index *index,
-					 mail_lock_notify_callback_t callback,
-					 void *context)
-{
-	index->lock_notify_cb = callback;
-	index->lock_notify_context = context;
-}
-
-struct mail_index_header *mail_index_get_header(struct mail_index *index)
-{
-	i_assert(index->lock_type != MAIL_LOCK_UNLOCK);
-
-	return index->header;
-}
-
-void mail_index_mark_flag_changes(struct mail_index *index,
-				  struct mail_index_record *rec,
-				  enum mail_flags old_flags,
-				  enum mail_flags new_flags)
-{
-	if ((old_flags & MAIL_SEEN) == 0 && (new_flags & MAIL_SEEN)) {
-		/* unseen -> seen */
-		index->header->seen_messages_count++;
-		if (index->header->first_unseen_uid_lowwater == rec->uid)
-			index->header->first_unseen_uid_lowwater++;
-	} else if ((old_flags & MAIL_SEEN) && (new_flags & MAIL_SEEN) == 0) {
-		/* seen -> unseen */
-		if (index->header->seen_messages_count ==
-		    index->header->messages_count) {
-			/* this is the first unseen message */
-                        index->header->first_unseen_uid_lowwater = rec->uid;
-		} else if (rec->uid < index->header->first_unseen_uid_lowwater)
-			index->header->first_unseen_uid_lowwater = rec->uid;
-
-		if (index->header->seen_messages_count == 0) {
-			index_set_corrupted(index,
-				"seen_messages_count in header is invalid");
-		} else {
-			index->header->seen_messages_count--;
-		}
-	}
-
-	if ((old_flags & MAIL_DELETED) == 0 && (new_flags & MAIL_DELETED)) {
-		/* undeleted -> deleted */
-		index->header->deleted_messages_count++;
-
-		if (index->header->deleted_messages_count == 1) {
-			/* this is the first deleted message */
-			index->header->first_deleted_uid_lowwater = rec->uid;
-		} else if (rec->uid < index->header->first_deleted_uid_lowwater)
-			index->header->first_deleted_uid_lowwater = rec->uid;
-	} else if ((old_flags & MAIL_DELETED) &&
-		   (new_flags & MAIL_DELETED) == 0) {
-		/* deleted -> undeleted */
-		if (index->header->first_deleted_uid_lowwater == rec->uid)
-			index->header->first_deleted_uid_lowwater++;
-		if (index->header->deleted_messages_count == 0) {
-			index_set_corrupted(index,
-				"deleted_messages_count in header is invalid");
-		} else {
-			index->header->deleted_messages_count--;
-		}
-	}
-}
-
-#define INDEX_NEED_COMPRESS(records, hdr) \
-	((records) > INDEX_MIN_RECORDS_COUNT && \
-	 (records) * (100-INDEX_COMPRESS_PERCENTAGE) / 100 > \
-	 	(hdr)->messages_count)
-
-int mail_index_expunge(struct mail_index *index,
-		       struct mail_index_record *first_rec,
-		       struct mail_index_record *last_rec,
-		       unsigned int first_seq, unsigned int last_seq,
-		       int external_change)
-{
-	unsigned int first_uid, last_uid;
-
-	i_assert(index->lock_type == MAIL_LOCK_EXCLUSIVE);
-	i_assert(first_seq != 0);
-	i_assert(first_seq <= last_seq);
-
-	index->expunge_counter++;
-
-	first_uid = first_rec->uid;
-	last_uid = last_rec->uid;
-
-	if (!mail_index_expunge_record_range(index, first_rec, last_rec))
-		return FALSE;
-
-	if (index->modifylog != NULL) {
-		if (!mail_modifylog_add_expunges(index->modifylog,
-						 first_seq, last_seq,
-						 first_uid, last_uid,
-						 external_change))
-			return FALSE;
-	}
-
-	if (index->header->messages_count == 0) {
-		/* all mail was deleted, truncate cache file */
-		if (!mail_cache_truncate(index->cache))
-			return FALSE;
-	}
-
-	return TRUE;
-}
-
-int mail_index_update_flags(struct mail_index *index,
-			    struct mail_index_record *rec, unsigned int seq,
-			    enum modify_type modify_type, enum mail_flags flags,
-			    int external_change)
-{
-	enum mail_flags new_flags;
-
-	i_assert(index->lock_type == MAIL_LOCK_EXCLUSIVE);
-	i_assert(seq != 0);
-
-	switch (modify_type) {
-	case MODIFY_ADD:
-		new_flags = rec->msg_flags | flags;
-		break;
-	case MODIFY_REMOVE:
-		new_flags = rec->msg_flags & ~flags;
-		break;
-	case MODIFY_REPLACE:
-		new_flags = flags;
-		break;
-	default:
-		new_flags = 0;
-		i_unreached();
-	}
-
-	if (new_flags == rec->msg_flags)
-		return TRUE; /* no changes */
-
-        mail_index_mark_flag_changes(index, rec, rec->msg_flags, new_flags);
-
-	rec->msg_flags = new_flags;
-	return index->modifylog == NULL ? TRUE :
-		mail_modifylog_add_flags(index->modifylog, seq,
-					 rec->uid, external_change);
-}
-
-static int mail_index_grow(struct mail_index *index)
-{
-	uoff_t pos;
-	unsigned int grow_count;
-	void *base;
-
-	grow_count = index->header->messages_count *
-		INDEX_GROW_PERCENTAGE / 100;
-	if (grow_count < 16)
-		grow_count = 16;
-
-	pos = index->mmap_full_length +
-		(grow_count * sizeof(struct mail_index_record));
-	i_assert(pos < OFF_T_MAX);
-
-	if (index->anon_mmap) {
-		i_assert(pos < SSIZE_T_MAX);
-
-		base = mremap_anon(index->mmap_base, index->mmap_full_length,
-				   (size_t)pos, MREMAP_MAYMOVE);
-		if (base == MAP_FAILED)
-			return index_set_syscall_error(index, "mremap_anon()");
-
-		index->mmap_base = base;
-		index->mmap_full_length = (size_t)pos;
-		return mmap_verify(index);
-	}
-
-	if (file_set_size(index->fd, (off_t)pos) < 0)
-		return index_set_syscall_error(index, "file_set_size()");
-
-	/* file size changed, let others know about it too by changing
-	   sync_id in header. */
-	index->header->master_sync_id++;
-
-	if (!mail_index_mmap_update(index))
-		return FALSE;
-
-	return TRUE;
-}
-
-struct mail_index_record *mail_index_append(struct mail_index *index)
-{
-	struct mail_index_record *rec;
-
-	i_assert(index->lock_type == MAIL_LOCK_EXCLUSIVE);
-
-	if (index->header->next_uid == MAX_ALLOWED_UID) {
-		index->set_flags |= MAIL_INDEX_HDR_FLAG_REBUILD;
-		index_set_error(index, "Reached maximum UID in mailbox %s, "
-				"rebuilding index", index->filepath);
-		return NULL;
-	}
-
-	if (index->mmap_used_length == index->mmap_full_length) {
-		if (!mail_index_grow(index))
-			return NULL;
-	}
-
-	i_assert(index->header->used_file_size == index->mmap_used_length);
-	i_assert(index->mmap_used_length + sizeof(struct mail_index_record) <=
-		 index->mmap_full_length);
-
-	index->header->messages_count++;
-
-	rec = (struct mail_index_record *) ((char *) index->mmap_base +
-					    index->mmap_used_length);
-	rec->uid = index->header->next_uid++;
-	rec->msg_flags = 0;
-	rec->cache_offset = 0;
-
-	index->header->used_file_size += sizeof(*rec);
-	index->mmap_used_length += sizeof(*rec);
-
-	return rec;
-}
-
-enum mail_index_error mail_index_get_last_error(struct mail_index *index)
-{
-	if (index->inconsistent)
-		return MAIL_INDEX_ERROR_INCONSISTENT;
-	if (index->nodiskspace)
-		return MAIL_INDEX_ERROR_DISKSPACE;
-	if (index->index_lock_timeout)
-		return MAIL_INDEX_ERROR_INDEX_LOCK_TIMEOUT;
-	if (index->mailbox_lock_timeout)
-		return MAIL_INDEX_ERROR_MAILBOX_LOCK_TIMEOUT;
-
-	if (index->error != NULL)
-		return MAIL_INDEX_ERROR_INTERNAL;
-
-	return MAIL_INDEX_ERROR_NONE;
-}
-
-const char *mail_index_get_last_error_text(struct mail_index *index)
-{
-	return index->error;
-}