# HG changeset patch
# User Timo Sirainen <tss@iki.fi>
# Date 1099874436 -7200
# Node ID bf1e718e7370add38d89c71ad4a0286b08b87060
# Parent  604e9488a61b4fadefb969d5075ed85edc0683a9
Cache file works now with mmap_disable=yes. Still needs a few optimizations.

diff -r 604e9488a61b -r bf1e718e7370 src/lib-index/mail-cache-compress.c
--- a/src/lib-index/mail-cache-compress.c	Sun Nov 07 22:35:26 2004 +0200
+++ b/src/lib-index/mail-cache-compress.c	Mon Nov 08 02:40:36 2004 +0200
@@ -3,6 +3,7 @@
 #include "lib.h"
 #include "buffer.h"
 #include "ostream.h"
+#include "file-cache.h"
 #include "file-set-size.h"
 #include "mail-cache-private.h"
 
@@ -276,6 +277,9 @@
 			mail_cache_file_close(cache);
 			cache->fd = fd;
 
+			if (cache->file_cache != NULL)
+				file_cache_set_fd(cache->file_cache, cache->fd);
+
 			if (mail_cache_map(cache, 0, 0) < 0)
 				ret = -1;
 			else if (mail_cache_header_fields_read(cache) < 0)
diff -r 604e9488a61b -r bf1e718e7370 src/lib-index/mail-cache-fields.c
--- a/src/lib-index/mail-cache-fields.c	Sun Nov 07 22:35:26 2004 +0200
+++ b/src/lib-index/mail-cache-fields.c	Mon Nov 08 02:40:36 2004 +0200
@@ -3,6 +3,7 @@
 #include "lib.h"
 #include "buffer.h"
 #include "hash.h"
+#include "file-cache.h"
 #include "write-full.h"
 #include "mail-cache-private.h"
 
@@ -111,7 +112,7 @@
 				   sizeof(*field_hdr) + CACHE_HDR_PREFETCH) < 0)
 			return -1;
 
-		field_hdr = CONST_PTR_OFFSET(cache->mmap_base, offset);
+		field_hdr = CONST_PTR_OFFSET(cache->data, offset);
 		next_offset =
 			mail_index_offset_to_uint32(field_hdr->next_offset);
 	}
@@ -138,7 +139,7 @@
 		return 0;
 	}
 
-	field_hdr = CONST_PTR_OFFSET(cache->mmap_base, offset);
+	field_hdr = CONST_PTR_OFFSET(cache->data, offset);
 	if (offset + field_hdr->size > cache->mmap_length) {
 		mail_cache_set_corrupted(cache,
 					 "field header points outside file");
@@ -157,7 +158,7 @@
 		if (mail_cache_map(cache, offset, field_hdr->size) < 0)
 			return -1;
 	}
-	field_hdr = CONST_PTR_OFFSET(cache->mmap_base, offset);
+	field_hdr = CONST_PTR_OFFSET(cache->data, offset);
 
 	cache->file_field_map =
 		i_realloc(cache->file_field_map,
@@ -267,6 +268,7 @@
 	int locked = cache->locked;
 	buffer_t *buffer;
 	uint32_t i, offset;
+	size_t size;
 	int ret = 0;
 
 	if (!locked) {
@@ -286,15 +288,18 @@
 	copy_to_buf(cache, buffer,
 		    offsetof(struct mail_cache_field_private, last_used),
 		    sizeof(uint32_t));
-	ret = pwrite_full(cache->fd, buffer_get_data(buffer, NULL),
+	size = buffer->used;
+
+	ret = pwrite_full(cache->fd, buffer->data,
 			  sizeof(uint32_t) * cache->file_fields_count,
 			  offset + MAIL_CACHE_FIELD_LAST_USED());
 	if (ret == 0) {
 		buffer_set_used_size(buffer, 0);
 		copy_to_buf_byte(cache, buffer,
 				 offsetof(struct mail_cache_field, decision));
+		size += buffer->used;
 
-		ret = pwrite_full(cache->fd, buffer_get_data(buffer, NULL),
+		ret = pwrite_full(cache->fd, buffer->data,
 			sizeof(uint8_t) * cache->file_fields_count, offset +
 			MAIL_CACHE_FIELD_DECISION(cache->file_fields_count));
 
@@ -305,8 +310,11 @@
 	}
 	t_pop();
 
-	if (ret == 0)
+	if (ret == 0) {
 		cache->field_header_write_pending = FALSE;
+		if (cache->file_cache != NULL)
+			file_cache_invalidate(cache->file_cache, offset, size);
+	}
 
 	if (!locked)
 		mail_cache_unlock(cache);
diff -r 604e9488a61b -r bf1e718e7370 src/lib-index/mail-cache-lookup.c
--- a/src/lib-index/mail-cache-lookup.c	Sun Nov 07 22:35:26 2004 +0200
+++ b/src/lib-index/mail-cache-lookup.c	Mon Nov 08 02:40:36 2004 +0200
@@ -9,10 +9,10 @@
 
 #define CACHE_PREFETCH 1024
 
-struct mail_cache_record *
+const struct mail_cache_record *
 mail_cache_get_record(struct mail_cache *cache, uint32_t offset)
 {
-	struct mail_cache_record *cache_rec;
+	const struct mail_cache_record *cache_rec;
 
 	if (offset == 0)
 		return NULL;
@@ -401,10 +401,9 @@
 	}
 	lines_count = i;
 
-	/* FIXME: this relies on mmap() too heavily */
 	hdr_data_rec = t_new(struct header_lookup_data_rec, 1);
 	hdr_data_rec->offset = (const char *)&lines[lines_count+1] -
-		(const char *)view->cache->mmap_base;
+		(const char *)view->cache->data;
 	hdr_data_rec->data_size = (uint32_t)data_size;
 
 	for (i = 0; i < lines_count; i++) {
@@ -487,8 +486,7 @@
 
 	/* then start filling dest buffer from the headers */
 	for (i = 0; i < size; i++) {
-		start = CONST_PTR_OFFSET(cache->mmap_base,
-					 data[i].data->offset);
+		start = CONST_PTR_OFFSET(cache->data, data[i].data->offset);
 		end = start + data[i].data->data_size;
 
 		for (p = start; p != end; p++) {
diff -r 604e9488a61b -r bf1e718e7370 src/lib-index/mail-cache-private.h
--- a/src/lib-index/mail-cache-private.h	Sun Nov 07 22:35:26 2004 +0200
+++ b/src/lib-index/mail-cache-private.h	Mon Nov 08 02:40:36 2004 +0200
@@ -33,7 +33,8 @@
 #define MAIL_CACHE_LOCK_IMMEDIATE_TIMEOUT (5*60)
 
 #define CACHE_RECORD(cache, offset) \
-	((struct mail_cache_record *) ((char *) (cache)->mmap_base + offset))
+	((const struct mail_cache_record *) \
+	 ((const char *) (cache)->data + offset))
 
 #define MAIL_CACHE_IS_UNUSABLE(cache) \
 	((cache)->hdr == NULL)
@@ -121,7 +122,9 @@
 	int fd;
 
 	void *mmap_base;
+	const void *data;
 	size_t mmap_length;
+	struct file_cache *file_cache;
 
 	const struct mail_cache_header *hdr;
 	struct mail_cache_header hdr_copy;
@@ -175,7 +178,7 @@
 int mail_cache_header_fields_get_next_offset(struct mail_cache *cache,
 					     uint32_t *offset_r);
 
-struct mail_cache_record *
+const struct mail_cache_record *
 mail_cache_get_record(struct mail_cache *cache, uint32_t offset);
 
 int mail_cache_foreach(struct mail_cache_view *view, uint32_t seq,
diff -r 604e9488a61b -r bf1e718e7370 src/lib-index/mail-cache-sync-update.c
--- a/src/lib-index/mail-cache-sync-update.c	Sun Nov 07 22:35:26 2004 +0200
+++ b/src/lib-index/mail-cache-sync-update.c	Mon Nov 08 02:40:36 2004 +0200
@@ -1,6 +1,7 @@
 /* Copyright (C) 2004 Timo Sirainen */
 
 #include "lib.h"
+#include "file-cache.h"
 #include "mail-cache-private.h"
 #include "mail-index-view-private.h"
 #include "mail-index-sync-private.h"
@@ -96,6 +97,7 @@
 			    void **context)
 {
 	struct mail_index_view *view = sync_ctx->view;
+	struct mail_cache *cache = view->index->cache;
 	struct mail_cache_sync_context *ctx = *context;
 	const uint32_t *old_cache_offset = old_data;
 	const uint32_t *new_cache_offset = new_data;
@@ -108,11 +110,16 @@
 		return 1;
 	}
 
+	if (cache->file_cache != NULL) {
+		file_cache_invalidate(cache->file_cache, *new_cache_offset,
+				      (size_t)-1);
+	}
+
 	if (*old_cache_offset == 0)
 		return 1;
 
 	/* we'll need to link the old and new cache records */
-	ret = mail_cache_handler_init(&ctx, view->index->cache);
+	ret = mail_cache_handler_init(&ctx, cache);
 	*context = ctx;
 	if (ret <= 0)
 		return ret < 0 ? -1 : 1;
@@ -120,13 +127,12 @@
 	if (!get_cache_file_seq(view, &cache_file_seq))
 		return 1;
 
-	if (cache_file_seq != view->index->cache->hdr->file_seq) {
+	if (cache_file_seq != cache->hdr->file_seq) {
 		/* cache has been compressed, don't modify it */
 		return 1;
 	}
 
-	if (mail_cache_link(view->index->cache,
-			    *old_cache_offset, *new_cache_offset) < 0)
+	if (mail_cache_link(cache, *old_cache_offset, *new_cache_offset) < 0)
 		return -1;
 
 	return 1;
diff -r 604e9488a61b -r bf1e718e7370 src/lib-index/mail-cache-transaction.c
--- a/src/lib-index/mail-cache-transaction.c	Sun Nov 07 22:35:26 2004 +0200
+++ b/src/lib-index/mail-cache-transaction.c	Mon Nov 08 02:40:36 2004 +0200
@@ -2,6 +2,7 @@
 
 #include "lib.h"
 #include "buffer.h"
+#include "file-cache.h"
 #include "file-set-size.h"
 #include "read-full.h"
 #include "write-full.h"
@@ -170,6 +171,10 @@
 			mail_cache_set_syscall_error(cache, "pwrite_full()");
 			return FALSE;
 		}
+		if (cache->file_cache != NULL) {
+			file_cache_invalidate(cache->file_cache, prev_offset,
+					      sizeof(hole.next_offset));
+		}
 	}
 	hdr->deleted_space -= hole.size;
 	cache->hdr_modified = TRUE;
@@ -285,6 +290,11 @@
 			return;
 		}
 
+		if (cache->file_cache != NULL) {
+			file_cache_invalidate(cache->file_cache, offset,
+					      sizeof(hole));
+		}
+
 		cache->hdr_copy.deleted_space += size;
 		cache->hdr_copy.hole_offset = offset;
 		cache->hdr_modified = TRUE;
@@ -362,13 +372,62 @@
 	return offset;
 }
 
+static uint32_t
+mail_cache_transaction_update_index(struct mail_cache_transaction_ctx *ctx,
+				    const struct mail_cache_record *rec,
+				    const uint32_t *seq, uint32_t *seq_idx,
+				    uint32_t seq_limit, uint32_t write_offset)
+{
+	struct mail_cache *cache = ctx->cache;
+	uint32_t i, old_offset, orig_write_offset;
+
+	/* write the cache_offsets to index file. records' prev_offset
+	   is updated to point to old cache record when index is being
+	   synced. */
+	orig_write_offset = write_offset;
+	for (i = *seq_idx; i < seq_limit; i++) {
+		mail_index_update_ext(ctx->trans, seq[i], cache->ext_id,
+				      &write_offset, &old_offset);
+		if (old_offset != 0) {
+			/* we added records for this message multiple
+			   times in this same uncommitted transaction.
+			   only the new one will be written to
+			   transaction log, we need to do the linking
+			   ourself here. */
+			if (old_offset > write_offset) {
+				if (mail_cache_link_unlocked(cache, old_offset,
+							     write_offset) < 0)
+					return -1;
+			} else {
+				/* if we're combining multiple transactions,
+				   make sure the one with the smallest offset
+				   is written into index. this is required for
+				   non-file-mmaped cache to work properly. */
+				mail_index_update_ext(ctx->trans, seq[i],
+						      cache->ext_id,
+						      &old_offset, NULL);
+				if (mail_cache_link_unlocked(cache,
+							     write_offset,
+							     old_offset) < 0)
+					return -1;
+			}
+		}
+
+		write_offset += rec->size;
+		rec = CONST_PTR_OFFSET(rec, rec->size);
+	}
+
+	*seq_idx = i;
+	return write_offset - orig_write_offset;
+}
+
 static int
 mail_cache_transaction_flush(struct mail_cache_transaction_ctx *ctx)
 {
 	struct mail_cache *cache = ctx->cache;
 	const struct mail_cache_record *rec, *tmp_rec;
 	const uint32_t *seq;
-	uint32_t write_offset, old_offset, rec_pos, cache_file_seq;
+	uint32_t write_offset, rec_pos, cache_file_seq;
 	size_t size, max_size, seq_idx, seq_limit, seq_count;
 	int commit;
 
@@ -432,29 +491,16 @@
 			mail_cache_set_syscall_error(cache, "pwrite_full()");
 			return -1;
 		}
+		if (cache->file_cache != NULL) {
+			file_cache_invalidate(cache->file_cache,
+					      write_offset, max_size);
+		}
 
-		/* write the cache_offsets to index file. records' prev_offset
-		   is updated to point to old cache record when index is being
-		   synced. */
-		for (; seq_idx < seq_limit; seq_idx++) {
-			mail_index_update_ext(ctx->trans, seq[seq_idx],
-					      cache->ext_id, &write_offset,
-					      &old_offset);
-			if (old_offset != 0) {
-				/* we added records for this message multiple
-				   times in this same uncommitted transaction.
-				   only the new one will be written to
-				   transaction log, we need to do the linking
-				   ourself here. */
-				if (mail_cache_link_unlocked(cache, old_offset,
-							     write_offset) < 0)
-					return -1;
-			}
-
-			write_offset += rec->size;
-			rec_pos += rec->size;
-			rec = CONST_PTR_OFFSET(rec, rec->size);
-		}
+		size = mail_cache_transaction_update_index(ctx, rec, seq,
+							   &seq_idx, seq_limit,
+							   write_offset);
+		rec_pos += size;
+		rec = CONST_PTR_OFFSET(rec, size);
 	}
 
 	/* drop the written data from buffer */
@@ -608,6 +654,11 @@
 		ret = -1;
 	else {
 		/* after it's guaranteed to be in disk, update header offset */
+		if (cache->file_cache != NULL) {
+			file_cache_invalidate(cache->file_cache, offset, size);
+			file_cache_invalidate(cache->file_cache, hdr_offset,
+					      sizeof(offset));
+		}
 		offset = mail_index_uint32_to_offset(offset);
 		if (pwrite_full(cache->fd, &offset, sizeof(offset),
 				hdr_offset) < 0) {
@@ -699,6 +750,10 @@
 		mail_cache_set_syscall_error(cache, "pwrite_full()");
 		return -1;
 	}
+	if (cache->file_cache != NULL) {
+		file_cache_invalidate(cache->file_cache,
+				      new_offset, sizeof(old_offset));
+	}
 	return 0;
 }
 
@@ -728,7 +783,7 @@
 
 int mail_cache_delete(struct mail_cache *cache, uint32_t offset)
 {
-	struct mail_cache_record *cache_rec;
+	const struct mail_cache_record *cache_rec;
 
 	i_assert(cache->locked);
 
diff -r 604e9488a61b -r bf1e718e7370 src/lib-index/mail-cache.c
--- a/src/lib-index/mail-cache.c	Sun Nov 07 22:35:26 2004 +0200
+++ b/src/lib-index/mail-cache.c	Mon Nov 08 02:40:36 2004 +0200
@@ -3,6 +3,7 @@
 #include "lib.h"
 #include "buffer.h"
 #include "hash.h"
+#include "file-cache.h"
 #include "mmap-util.h"
 #include "write-full.h"
 #include "mail-cache-private.h"
@@ -46,7 +47,11 @@
 			mail_cache_set_syscall_error(cache, "munmap()");
 	}
 
+	if (cache->file_cache != NULL)
+		file_cache_set_fd(cache->file_cache, -1);
+
 	cache->mmap_base = NULL;
+	cache->data = NULL;
 	cache->hdr = NULL;
 	cache->mmap_length = 0;
 
@@ -78,6 +83,9 @@
 		return -1;
 	}
 
+	if (cache->file_cache != NULL)
+		file_cache_set_fd(cache->file_cache, cache->fd);
+
 	if (mail_cache_map(cache, 0, 0) < 0)
 		return -1;
 
@@ -100,16 +108,15 @@
 	return 1;
 }
 
-static int mmap_verify_header(struct mail_cache *cache)
+static int mail_cache_verify_header(struct mail_cache *cache)
 {
-	const struct mail_cache_header *hdr;
+	const struct mail_cache_header *hdr = cache->data;
 
 	/* check that the header is still ok */
 	if (cache->mmap_length < sizeof(struct mail_cache_header)) {
 		mail_cache_set_corrupted(cache, "File too small");
 		return FALSE;
 	}
-	cache->hdr = hdr = cache->mmap_base;
 
 	if (cache->hdr->version != MAIL_CACHE_VERSION) {
 		/* version changed - upgrade silently */
@@ -135,7 +142,8 @@
 		return FALSE;
 	}
 
-	if (hdr->used_file_size > cache->mmap_length) {
+	if (cache->mmap_base != NULL &&
+	    hdr->used_file_size > cache->mmap_length) {
 		mail_cache_set_corrupted(cache, "used_file_size too large");
 		return FALSE;
 	}
@@ -144,9 +152,33 @@
 
 int mail_cache_map(struct mail_cache *cache, size_t offset, size_t size)
 {
+	ssize_t ret;
+
 	if (size == 0)
 		size = sizeof(struct mail_cache_header);
 
+	if (cache->file_cache != NULL) {
+		cache->data = NULL;
+		cache->hdr = NULL;
+
+		ret = file_cache_read(cache->file_cache, offset, size);
+		if (ret < 0) {
+			// FIXME: ESTALE
+			mail_cache_set_syscall_error(cache, "read()");
+			return -1;
+		}
+
+		cache->data = file_cache_get_map(cache->file_cache,
+						 &cache->mmap_length);
+		cache->hdr = cache->data;
+
+		if (offset == 0 && !mail_cache_verify_header(cache)) {
+			cache->need_compress = TRUE;
+			return -1;
+		}
+		return 0;
+	}
+
 	if (offset < cache->mmap_length &&
 	    size <= cache->mmap_length - offset) {
 		/* already mapped */
@@ -171,11 +203,14 @@
 	cache->mmap_base = mmap_ro_file(cache->fd, &cache->mmap_length);
 	if (cache->mmap_base == MAP_FAILED) {
 		cache->mmap_base = NULL;
+		cache->data = NULL;
 		mail_cache_set_syscall_error(cache, "mmap()");
 		return -1;
 	}
+	cache->data = cache->mmap_base;
+	cache->hdr = cache->mmap_base;
 
-	if (!mmap_verify_header(cache)) {
+	if (!mail_cache_verify_header(cache)) {
 		cache->need_compress = TRUE;
 		return -1;
 	}
@@ -188,6 +223,9 @@
 	cache->filepath = i_strconcat(cache->index->filepath,
 				      MAIL_CACHE_FILE_PREFIX, NULL);
 
+	if (cache->index->mmap_disable || cache->index->mmap_no_write)
+		cache->file_cache = file_cache_new(-1);
+
 	cache->fd = open(cache->filepath, O_RDWR);
 	if (cache->fd == -1) {
 		if (errno == ENOENT) {
@@ -199,6 +237,9 @@
 		return -1;
 	}
 
+	if (cache->file_cache != NULL)
+		file_cache_set_fd(cache->file_cache, cache->fd);
+
 	if (mail_cache_map(cache, 0, sizeof(struct mail_cache_header)) < 0)
 		return -1;
 
@@ -217,12 +258,10 @@
 		hash_create(default_pool, cache->field_pool, 0,
 			    strcase_hash, (hash_cmp_callback_t *)strcasecmp);
 
-	if (!index->mmap_disable && !index->mmap_no_write) {
-		if (mail_cache_open_and_verify(cache) < 0) {
-			/* failed for some reason - doesn't really matter,
-			   it's disabled for now. */
-			mail_cache_file_close(cache);
-		}
+	if (mail_cache_open_and_verify(cache) < 0) {
+		/* failed for some reason - doesn't really matter,
+		   it's disabled for now. */
+		mail_cache_file_close(cache);
 	}
 
 	cache->ext_id =
@@ -237,6 +276,11 @@
 
 void mail_cache_free(struct mail_cache *cache)
 {
+	if (cache->file_cache != NULL) {
+		file_cache_free(cache->file_cache);
+		cache->file_cache = NULL;
+	}
+
 	mail_cache_file_close(cache);
 
 	hash_destroy(cache->field_name_hash);
@@ -299,8 +343,12 @@
 		ret = 0;
 	}
 
-	if (ret > 0)
+	if (ret > 0) {
+		/* make sure our header is up to date */
+		if (mail_cache_map(cache, 0, 0) < 0)
+			ret = -1;
 		cache->hdr_copy = *cache->hdr;
+	}
 
 	mail_index_view_close(view);
 	return ret;
@@ -342,7 +390,11 @@
 		if (pwrite_full(cache->fd, &cache->hdr_copy,
 				sizeof(cache->hdr_copy), 0) < 0)
 			mail_cache_set_syscall_error(cache, "pwrite_full()");
-                mail_cache_update_need_compress(cache);
+		if (cache->file_cache != NULL) {
+			file_cache_invalidate(cache->file_cache, 0,
+					      sizeof(cache->hdr_copy));
+		}
+		mail_cache_update_need_compress(cache);
 	}
 
 	if (mail_index_lock_fd(cache->index, cache->fd, F_UNLCK, 0) <= 0) {
diff -r 604e9488a61b -r bf1e718e7370 src/lib/Makefile.am
--- a/src/lib/Makefile.am	Sun Nov 07 22:35:26 2004 +0200
+++ b/src/lib/Makefile.am	Mon Nov 08 02:40:36 2004 +0200
@@ -10,6 +10,7 @@
 	failures.c \
 	fd-close-on-exec.c \
 	fdpass.c \
+	file-cache.c \
 	file-dotlock.c \
 	file-lock.c \
 	file-set-size.c \
@@ -79,6 +80,7 @@
 	failures.h \
 	fd-close-on-exec.h \
 	fdpass.h \
+	file-cache.h \
 	file-dotlock.h \
 	file-lock.h \
 	file-set-size.h \
diff -r 604e9488a61b -r bf1e718e7370 src/lib/file-cache.c
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/src/lib/file-cache.c	Mon Nov 08 02:40:36 2004 +0200
@@ -0,0 +1,196 @@
+/* Copyright (c) 2004 Timo Sirainen */
+
+#include "lib.h"
+#include "buffer.h"
+#include "mmap-util.h"
+#include "file-cache.h"
+
+#include <sys/stat.h>
+
+struct file_cache {
+	int fd;
+	buffer_t *page_bitmask;
+
+	void *mmap_base;
+	size_t mmap_length;
+	size_t read_highwater;
+};
+
+struct file_cache *file_cache_new(int fd)
+{
+	struct file_cache *cache;
+
+	cache = i_new(struct file_cache, 1);
+	cache->fd = fd;
+	cache->page_bitmask = buffer_create_dynamic(default_pool, 128);
+	return cache;
+}
+
+void file_cache_free(struct file_cache *cache)
+{
+	if (cache->mmap_base != NULL) {
+		if (munmap_anon(cache->mmap_base, cache->mmap_length) < 0)
+			i_error("munmap_anon() failed: %m");
+	}
+	buffer_free(cache->page_bitmask);
+	i_free(cache);
+}
+
+void file_cache_set_fd(struct file_cache *cache, int fd)
+{
+	cache->fd = fd;
+	file_cache_invalidate(cache, 0, cache->mmap_length);
+}
+
+ssize_t file_cache_read(struct file_cache *cache, uoff_t offset, size_t size)
+{
+	size_t page_size = mmap_get_page_size();
+	size_t poffset, psize, mmap_needed, dest_offset, dest_size;
+	unsigned char *bits, *dest;
+	ssize_t ret;
+
+	i_assert(size < INT_MAX);
+
+	if (offset + size > cache->mmap_length &&
+	    offset + size - cache->mmap_length > 1024*1024) {
+		/* growing more than a megabyte, make sure that the
+		   file is large enough so we don't allocate memory
+		   more than needed */
+		struct stat st;
+
+		if (fstat(cache->fd, &st) < 0) {
+			i_error("fstat(file_cache) failed: %m");
+			return -1;
+		}
+
+		if (offset + size > (uoff_t)st.st_size) {
+			if (offset >= (uoff_t)st.st_size)
+				return 0;
+			size = (uoff_t)st.st_size - offset;
+		}
+	}
+
+	poffset = offset / page_size;
+	psize = (offset + size + page_size-1) / page_size - poffset;
+	i_assert(psize > 0);
+
+	mmap_needed = (poffset + psize) * page_size;
+	if (mmap_needed > cache->mmap_length) {
+		/* grow mmaping */
+		if (cache->mmap_base == NULL) {
+			cache->mmap_base = mmap_anon(mmap_needed);
+			if (cache->mmap_base == MAP_FAILED) {
+				i_error("mmap_anon(%"PRIuSIZE_T") failed: %m",
+					mmap_needed);
+				return -1;
+			}
+		} else {
+			cache->mmap_base = mremap_anon(cache->mmap_base,
+						       cache->mmap_length,
+						       mmap_needed,
+						       MREMAP_MAYMOVE);
+			if (cache->mmap_base == MAP_FAILED) {
+				i_error("mremap_anon(%"PRIuSIZE_T") failed: %m",
+					mmap_needed);
+				return -1;
+			}
+		}
+		cache->mmap_length = mmap_needed;
+	}
+
+	bits = buffer_get_space_unsafe(cache->page_bitmask, poffset / CHAR_BIT,
+				       (psize + CHAR_BIT - 1) / CHAR_BIT);
+
+	dest_offset = poffset * page_size;
+	dest = PTR_OFFSET(cache->mmap_base, dest_offset);
+	dest_size = page_size;
+
+	poffset %= CHAR_BIT;
+	while (psize > 0) {
+		if (bits[poffset / CHAR_BIT] & (1 << (poffset % CHAR_BIT))) {
+			/* page is already in cache */
+			psize--; poffset++;
+			dest += page_size;
+			dest_offset += page_size;
+			continue;
+		}
+
+		ret = pread(cache->fd, dest, dest_size, dest_offset);
+		if (ret <= 0) {
+			if (ret < 0)
+				return -1;
+
+			/* EOF */
+			/* FIXME: we should mark the last block cached and
+			   invalidate it only when trying to read past the
+			   file */
+			return dest_offset <= offset ? 0 :
+				dest_offset - offset < size ?
+				dest_offset - offset : size;
+		}
+
+		dest += ret;
+		dest_offset += ret;
+
+		if (cache->read_highwater < dest_offset)
+			cache->read_highwater = dest_offset;
+
+		if ((size_t)ret != dest_size) {
+			/* partial read - probably EOF but make sure. */
+			dest_size -= ret;
+			continue;
+		}
+
+		bits[poffset / CHAR_BIT] |= 1 << (poffset % CHAR_BIT);
+		dest_size = page_size;
+		psize--; poffset++;
+	}
+
+	return size;
+}
+
+const void *file_cache_get_map(struct file_cache *cache, size_t *size_r)
+{
+	*size_r = cache->read_highwater;
+	return cache->mmap_base;
+}
+
+void file_cache_invalidate(struct file_cache *cache, uoff_t offset, size_t size)
+{
+	size_t page_size = mmap_get_page_size();
+	unsigned char *bits, mask;
+	unsigned int i;
+
+	if (offset >= cache->read_highwater)
+		return;
+
+	if (size > cache->read_highwater - offset)
+		size = cache->read_highwater - offset;
+
+	size = (offset + size + page_size-1) / page_size;
+	offset /= page_size;
+	size -= offset;
+
+	bits = buffer_get_space_unsafe(cache->page_bitmask, offset / CHAR_BIT,
+				       (size + CHAR_BIT - 1) / CHAR_BIT);
+
+	/* set the first byte */
+	for (i = offset % CHAR_BIT, mask = 0; i < CHAR_BIT && size > 0; i++) {
+		mask |= 1 << i;
+		size--;
+	}
+	*bits++ &= ~mask;
+
+	/* set the middle bytes */
+	memset(bits, 0, size / CHAR_BIT);
+	bits += size / CHAR_BIT;
+	size %= CHAR_BIT;
+
+	/* set the last byte */
+	if (size > 0) {
+		mask = 0;
+		for (i = 0, mask = 0; i < size; i++)
+			mask |= 1 << i;
+		*bits &= ~mask;
+	}
+}
diff -r 604e9488a61b -r bf1e718e7370 src/lib/file-cache.h
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/src/lib/file-cache.h	Mon Nov 08 02:40:36 2004 +0200
@@ -0,0 +1,26 @@
+#ifndef __FILE_CACHE_H
+#define __FILE_CACHE_H
+
+/* Create a new file cache. It works very much like file-backed mmap()ed
+   memory, but it works more nicely with remote filesystems (no SIGBUS). */
+struct file_cache *file_cache_new(int fd);
+void file_cache_free(struct file_cache *cache);
+
+/* Change cached file descriptor. Invalidates the whole cache. */
+void file_cache_set_fd(struct file_cache *cache, int fd);
+
+/* Read data from file, returns how many bytes was actually read or -1 if
+   error occured. */
+ssize_t file_cache_read(struct file_cache *cache, uoff_t offset, size_t size);
+
+/* Returns pointer to beginning of cached file. Only parts of the returned
+   memory that are valid are the ones that have been file_cache_read().
+   Note that the pointer may become invalid after calling file_cache_read(). */
+const void *file_cache_get_map(struct file_cache *cache, size_t *size_r);
+
+/* Invalidate cached memory area. It will be read again next time it's tried
+   to be accessed. */
+void file_cache_invalidate(struct file_cache *cache,
+			   uoff_t offset, size_t size);
+
+#endif