# HG changeset patch
# User Timo Sirainen <tss@iki.fi>
# Date 1265147372 -7200
# Node ID c609104198619a183719edbad434064c53e63385
# Parent  55cce06818b83aeb0404b465b33b83597ff435f2
mdbox: Purging now also moves mails to alt storage (if it's used).
mdbox_altmove setting specifies how old files should be moved.

diff -r 55cce06818b8 -r c60910419861 doc/example-config/conf.d/mail.conf
--- a/doc/example-config/conf.d/mail.conf	Tue Feb 02 23:24:52 2010 +0200
+++ b/doc/example-config/conf.d/mail.conf	Tue Feb 02 23:49:32 2010 +0200
@@ -310,3 +310,6 @@
 # Maximum dbox file age until it's rotated. Typically in days. Day begins
 # from midnight, so 1d = today, 2d = yesterday, etc. 0 = check disabled.
 #mdbox_rotate_interval = 1d
+
+# When purging, move unchanged files to alt storage after this much time.
+#mdbox_altmove = 1w
diff -r 55cce06818b8 -r c60910419861 src/lib-storage/index/dbox-common/dbox-file.c
--- a/src/lib-storage/index/dbox-common/dbox-file.c	Tue Feb 02 23:24:52 2010 +0200
+++ b/src/lib-storage/index/dbox-common/dbox-file.c	Tue Feb 02 23:49:32 2010 +0200
@@ -155,7 +155,7 @@
 	return ret;
 }
 
-static int dbox_file_open_fd(struct dbox_file *file)
+static int dbox_file_open_fd(struct dbox_file *file, bool try_altpath)
 {
 	const char *path;
 	bool alt = FALSE;
@@ -169,7 +169,7 @@
 			return -1;
 		}
 
-		if (file->alt_path == NULL || alt) {
+		if (file->alt_path == NULL || alt || !try_altpath) {
 			/* not found */
 			return 0;
 		}
@@ -182,22 +182,23 @@
 	return 1;
 }
 
-int dbox_file_open(struct dbox_file *file, bool *deleted_r)
+static int dbox_file_open_full(struct dbox_file *file, bool try_altpath,
+			       bool *notfound_r)
 {
 	int ret;
 
-	*deleted_r = FALSE;
+	*notfound_r = FALSE;
 	if (file->input != NULL)
 		return 1;
 
 	if (file->fd == -1) {
 		T_BEGIN {
-			ret = dbox_file_open_fd(file);
+			ret = dbox_file_open_fd(file, try_altpath);
 		} T_END;
 		if (ret <= 0) {
 			if (ret < 0)
 				return -1;
-			*deleted_r = TRUE;
+			*notfound_r = TRUE;
 			return 1;
 		}
 	}
@@ -207,6 +208,16 @@
 	return dbox_file_read_header(file);
 }
 
+int dbox_file_open(struct dbox_file *file, bool *deleted_r)
+{
+	return dbox_file_open_full(file, TRUE, deleted_r);
+}
+
+int dbox_file_open_primary(struct dbox_file *file, bool *notfound_r)
+{
+	return dbox_file_open_full(file, FALSE, notfound_r);
+}
+
 int dbox_file_header_write(struct dbox_file *file, struct ostream *output)
 {
 	string_t *hdr;
diff -r 55cce06818b8 -r c60910419861 src/lib-storage/index/dbox-common/dbox-file.h
--- a/src/lib-storage/index/dbox-common/dbox-file.h	Tue Feb 02 23:24:52 2010 +0200
+++ b/src/lib-storage/index/dbox-common/dbox-file.h	Tue Feb 02 23:49:32 2010 +0200
@@ -125,6 +125,8 @@
 /* Open the file. Returns 1 if ok, 0 if file header is corrupted, -1 if error.
    If file is deleted, deleted_r=TRUE and 1 is returned. */
 int dbox_file_open(struct dbox_file *file, bool *deleted_r);
+/* Try to open file only from primary path. */
+int dbox_file_open_primary(struct dbox_file *file, bool *notfound_r);
 /* Close the file handle from the file, but don't free it. */
 void dbox_file_close(struct dbox_file *file);
 
diff -r 55cce06818b8 -r c60910419861 src/lib-storage/index/dbox-multi/mdbox-file-purge.c
--- a/src/lib-storage/index/dbox-multi/mdbox-file-purge.c	Tue Feb 02 23:24:52 2010 +0200
+++ b/src/lib-storage/index/dbox-multi/mdbox-file-purge.c	Tue Feb 02 23:49:32 2010 +0200
@@ -95,6 +95,7 @@
 	ARRAY_TYPE(uint32_t) copied_map_uids;
 	unsigned int i, count;
 	uoff_t offset, physical_size, msg_size;
+	enum dbox_map_append_flags append_flags = 0;
 	int ret;
 
 	if ((ret = dbox_file_try_lock(file)) <= 0)
@@ -112,6 +113,11 @@
 		return -1;
 	}
 
+	if (dstorage->set->mdbox_altmove > 0 &&
+	    st.st_mtime + dstorage->set->mdbox_altmove < ioloop_time &&
+	    dstorage->alt_storage_dir != NULL)
+		append_flags |= DBOX_MAP_APPEND_FLAG_ALT;
+
 	i_array_init(&msgs_arr, 128);
 	if (dbox_map_get_file_msgs(dstorage->map,
 				   ((struct mdbox_file *)file)->file_id,
@@ -124,7 +130,7 @@
 	array_sort(&msgs_arr, mdbox_map_file_msg_offset_cmp);
 
 	msgs = array_get(&msgs_arr, &count);
-	append_ctx = dbox_map_append_begin(dstorage->map);
+	append_ctx = dbox_map_append_begin(dstorage->map, append_flags);
 	i_array_init(&copied_map_uids, I_MIN(count, 1));
 	i_array_init(&expunged_map_uids, I_MIN(count, 1));
 	offset = file->file_header_size;
diff -r 55cce06818b8 -r c60910419861 src/lib-storage/index/dbox-multi/mdbox-file.c
--- a/src/lib-storage/index/dbox-multi/mdbox-file.c	Tue Feb 02 23:24:52 2010 +0200
+++ b/src/lib-storage/index/dbox-multi/mdbox-file.c	Tue Feb 02 23:49:32 2010 +0200
@@ -103,10 +103,12 @@
 
 static int mdbox_file_create(struct dbox_file *file)
 {
+	bool create_parents;
 	int ret;
 
+	create_parents = dbox_file_is_in_alt(file);
 	file->fd = file->storage->v.
-		file_create_fd(file, file->primary_path, FALSE);
+		file_create_fd(file, file->cur_path, create_parents);
 
 	/* even though we don't need it locked while writing to it, by the
 	   time we rename() it it needs to be locked. so we might as well do
@@ -122,8 +124,9 @@
 	return 0;
 }
 
-struct dbox_file *
-mdbox_file_init(struct mdbox_storage *storage, uint32_t file_id)
+static struct dbox_file *
+mdbox_file_init_full(struct mdbox_storage *storage,
+		     uint32_t file_id, bool alt_dir)
 {
 	struct mdbox_file *file;
 	const char *fname;
@@ -150,6 +153,8 @@
 		t_strdup_printf(MDBOX_MAIL_FILE_FORMAT, file_id);
 	mdbox_file_init_paths(file, fname);
 	dbox_file_init(&file->file);
+	if (alt_dir)
+		file->file.cur_path = file->file.alt_path;
 
 	if (file_id != 0)
 		array_append(&storage->open_files, &file, 1);
@@ -158,18 +163,31 @@
 	return &file->file;
 }
 
+struct dbox_file *
+mdbox_file_init(struct mdbox_storage *storage, uint32_t file_id)
+{
+	return mdbox_file_init_full(storage, file_id, FALSE);
+}
+
+struct dbox_file *
+mdbox_file_init_new_alt(struct mdbox_storage *storage)
+{
+	return mdbox_file_init_full(storage, 0, TRUE);
+}
+
 int mdbox_file_assign_file_id(struct mdbox_file *file, uint32_t file_id)
 {
 	const char *old_path;
-	const char *new_fname, *new_path;
+	const char *new_dir, *new_fname, *new_path;
 
 	i_assert(file->file_id == 0);
 	i_assert(file_id != 0);
 
 	old_path = file->file.cur_path;
 	new_fname = t_strdup_printf(MDBOX_MAIL_FILE_FORMAT, file_id);
-	new_path = t_strdup_printf("%s/%s", file->storage->storage_dir,
-				   new_fname);
+	new_dir = !dbox_file_is_in_alt(&file->file) ?
+		file->storage->storage_dir : file->storage->alt_storage_dir;
+	new_path = t_strdup_printf("%s/%s", new_dir, new_fname);
 	if (rename(old_path, new_path) < 0) {
 		mail_storage_set_critical(&file->storage->storage.storage,
 					  "rename(%s, %s) failed: %m",
diff -r 55cce06818b8 -r c60910419861 src/lib-storage/index/dbox-multi/mdbox-file.h
--- a/src/lib-storage/index/dbox-multi/mdbox-file.h	Tue Feb 02 23:24:52 2010 +0200
+++ b/src/lib-storage/index/dbox-multi/mdbox-file.h	Tue Feb 02 23:49:32 2010 +0200
@@ -12,6 +12,8 @@
 
 struct dbox_file *
 mdbox_file_init(struct mdbox_storage *storage, uint32_t file_id);
+struct dbox_file *
+mdbox_file_init_new_alt(struct mdbox_storage *storage);
 
 /* Assign file ID for a newly created file. */
 int mdbox_file_assign_file_id(struct mdbox_file *file, uint32_t file_id);
diff -r 55cce06818b8 -r c60910419861 src/lib-storage/index/dbox-multi/mdbox-map-private.h
--- a/src/lib-storage/index/dbox-multi/mdbox-map-private.h	Tue Feb 02 23:24:52 2010 +0200
+++ b/src/lib-storage/index/dbox-multi/mdbox-map-private.h	Tue Feb 02 23:49:32 2010 +0200
@@ -19,7 +19,6 @@
 	uint32_t created_uid_validity;
 
 	uint32_t map_ext_id, ref_ext_id;
-	ARRAY_TYPE(seq_range) ref0_file_ids;
 
 	mode_t create_mode, create_dir_mode;
 	gid_t create_gid;
@@ -33,6 +32,7 @@
 
 struct dbox_map_append_context {
 	struct dbox_map *map;
+	enum dbox_map_append_flags flags;
 
 	struct mail_index_sync_ctx *sync_ctx;
 	struct mail_index_view *sync_view;
@@ -43,7 +43,6 @@
 	ARRAY_DEFINE(appends, struct dbox_map_append);
 
 	uint32_t first_new_file_id;
-	uint32_t orig_next_uid;
 
 	unsigned int files_nonappendable_count;
 
diff -r 55cce06818b8 -r c60910419861 src/lib-storage/index/dbox-multi/mdbox-map.c
--- a/src/lib-storage/index/dbox-multi/mdbox-map.c	Tue Feb 02 23:24:52 2010 +0200
+++ b/src/lib-storage/index/dbox-multi/mdbox-map.c	Tue Feb 02 23:49:32 2010 +0200
@@ -9,6 +9,9 @@
 #include "mdbox-file.h"
 #include "mdbox-map-private.h"
 
+#include <stdlib.h>
+#include <dirent.h>
+
 #define MAX_BACKWARDS_LOOKUPS 10
 
 #define DBOX_FORCE_PURGE_MIN_BYTES (1024*1024*10)
@@ -75,8 +78,6 @@
 
 	*_map = NULL;
 
-	if (array_is_created(&map->ref0_file_ids))
-		array_free(&map->ref0_file_ids);
 	if (map->view != NULL)
 		mail_index_view_close(&map->view);
 	mail_index_free(&map->index);
@@ -264,12 +265,8 @@
 	return 0;
 }
 
-struct dbox_file_size {
-	uoff_t file_size;
-	uoff_t ref0_size;
-};
-
-const ARRAY_TYPE(seq_range) *dbox_map_get_zero_ref_files(struct dbox_map *map)
+int dbox_map_get_zero_ref_files(struct dbox_map *map,
+				ARRAY_TYPE(seq_range) *file_ids_r)
 {
 	const struct mail_index_header *hdr;
 	const struct dbox_map_mail_index_record *rec;
@@ -278,16 +275,12 @@
 	uint32_t seq;
 	bool expunged;
 
-	if (array_is_created(&map->ref0_file_ids))
-		array_clear(&map->ref0_file_ids);
-	else
-		i_array_init(&map->ref0_file_ids, 64);
-
 	if (dbox_map_open(map, FALSE) < 0) {
 		/* some internal error */
-		return &map->ref0_file_ids;
+		return -1;
 	}
-	(void)dbox_map_refresh(map);
+	if (dbox_map_refresh(map) < 0)
+		return -1;
 
 	hdr = mail_index_get_header(map->view);
 	for (seq = 1; seq <= hdr->messages_count; seq++) {
@@ -303,11 +296,10 @@
 				      &data, &expunged);
 		if (data != NULL && !expunged) {
 			rec = data;
-			seq_range_array_add(&map->ref0_file_ids, 0,
-					    rec->file_id);
+			seq_range_array_add(file_ids_r, 0, rec->file_id);
 		}
 	}
-	return &map->ref0_file_ids;
+	return 0;
 }
 
 struct dbox_map_transaction_context *
@@ -481,12 +473,13 @@
 }
 
 struct dbox_map_append_context *
-dbox_map_append_begin(struct dbox_map *map)
+dbox_map_append_begin(struct dbox_map *map, enum dbox_map_append_flags flags)
 {
 	struct dbox_map_append_context *ctx;
 
 	ctx = i_new(struct dbox_map_append_context, 1);
 	ctx->map = map;
+	ctx->flags = flags;
 	ctx->first_new_file_id = (uint32_t)-1;
 	i_array_init(&ctx->file_appends, 64);
 	i_array_init(&ctx->files, 64);
@@ -533,6 +526,34 @@
 	return stamp - (interval - unit);
 }
 
+static bool dbox_try_open(struct dbox_map_append_context *ctx,
+			  struct dbox_file *file)
+{
+	bool want_altpath, notfound;
+
+	want_altpath = (ctx->flags & DBOX_MAP_APPEND_FLAG_ALT) != 0;
+	if (want_altpath) {
+		if (dbox_file_open(file, &notfound) <= 0)
+			return FALSE;
+	} else {
+		if (dbox_file_open_primary(file, &notfound) <= 0)
+			return FALSE;
+	}
+	if (notfound)
+		return FALSE;
+
+	if (file->lock != NULL) {
+		/* already locked, we're possibly in the middle of purging it
+		   in which case we really don't want to write there. */
+		return FALSE;
+	}
+	if (dbox_file_is_in_alt(file) != want_altpath) {
+		/* different alt location than what we want, can't use it */
+		return FALSE;
+	}
+	return TRUE;
+}
+
 static bool
 dbox_map_file_try_append(struct dbox_map_append_context *ctx,
 			 uint32_t file_id, time_t stamp, uoff_t mail_size,
@@ -544,7 +565,7 @@
 	struct dbox_file *file;
 	struct dbox_file_append_context *file_append;
 	struct stat st;
-	bool deleted, file_too_old = FALSE;
+	bool file_too_old = FALSE;
 	int ret;
 
 	*file_append_r = NULL;
@@ -552,13 +573,7 @@
 	*retry_later_r = FALSE;
 
 	file = mdbox_file_init(storage, file_id);
-	if (dbox_file_open(file, &deleted) <= 0 || deleted) {
-		dbox_file_unref(&file);
-		return TRUE;
-	}
-	if (file->lock != NULL) {
-		/* already locked, we're possibly in the middle of purging it
-		   in which case we really don't want to write there. */
+	if (!dbox_try_open(ctx, file)) {
 		dbox_file_unref(&file);
 		return TRUE;
 	}
@@ -611,50 +626,132 @@
 	return FALSE;
 }
 
-static int
-dbox_map_find_appendable_file(struct dbox_map_append_context *ctx,
-			      uoff_t mail_size,
-			      struct dbox_file_append_context **file_append_r,
-			      struct ostream **output_r, bool *existing_r)
+static struct dbox_file_append_context *
+dbox_map_find_existing_append(struct dbox_map_append_context *ctx,
+			      uoff_t mail_size, struct ostream **output_r)
 {
 	struct dbox_map *map = ctx->map;
-	ARRAY_TYPE(seq_range) checked_file_ids;
 	struct dbox_file_append_context *const *file_appends;
-	const struct mail_index_header *hdr;
-	unsigned int i, count, backwards_lookup_count;
-	uint32_t seq, seq1, uid, file_id;
-	uoff_t offset, append_offset, size;
-	time_t stamp;
-	bool retry_later;
-
-	*existing_r = FALSE;
+	unsigned int i, count;
+	uoff_t append_offset;
 
 	if (mail_size >= map->set->mdbox_rotate_size)
-		return 0;
+		return NULL;
 
 	/* first try to use files already used in this append */
 	file_appends = array_get(&ctx->file_appends, &count);
 	for (i = count; i > ctx->files_nonappendable_count; i--) {
 		append_offset = file_appends[i-1]->output->offset;
 		if (append_offset + mail_size <= map->set->mdbox_rotate_size &&
-		    dbox_file_get_append_stream(file_appends[i-1], output_r) > 0) {
-			*file_append_r = file_appends[i-1];
-			*existing_r = TRUE;
-			return 1;
-		}
+		    dbox_file_get_append_stream(file_appends[i-1], output_r) > 0)
+			return file_appends[i-1];
+
 		/* can't append to this file anymore. we could close it
 		   otherwise, except that would also lose our lock too early. */
 	}
 	ctx->files_nonappendable_count = count;
+	return NULL;
+}
+
+static int
+dbox_map_find_first_alt(struct dbox_map_append_context *ctx,
+			uint32_t *min_file_id_r, uint32_t *seq_r)
+{
+	struct mdbox_storage *dstorage = ctx->map->storage;
+	struct mail_storage *storage = &dstorage->storage.storage;
+	DIR *dir;
+	struct dirent *d;
+	const struct mail_index_header *hdr;
+	uint32_t seq, file_id, min_file_id = -1U;
+	uoff_t offset, size;
+	int ret = 0;
+
+	/* we want to quickly find the latest alt file, but we also want to
+	   avoid accessing the alt storage as much as possible. so we'll do
+	   this by finding the lowest numbered file (n) from primary storage.
+	   hopefully one of n-[1..m] is appendable in alt storage. */
+	dir = opendir(dstorage->storage_dir);
+	if (dir == NULL) {
+		mail_storage_set_critical(storage,
+			"opendir(%s) failed: %m", dstorage->storage_dir);
+		return -1;
+	}
+
+	for (errno = 0; (d = readdir(dir)) != NULL; errno = 0) {
+		if (strncmp(d->d_name, MDBOX_MAIL_FILE_PREFIX,
+			    strlen(MDBOX_MAIL_FILE_PREFIX)) != 0)
+			continue;
+
+		file_id = strtoul(d->d_name + strlen(MDBOX_MAIL_FILE_PREFIX),
+				  NULL, 10);
+		if (min_file_id > file_id)
+			min_file_id = file_id;
+	}
+	if (errno != 0) {
+		mail_storage_set_critical(storage,
+			"readdir(%s) failed: %m", dstorage->storage_dir);
+		ret = -1;
+	}
+	if (closedir(dir) < 0) {
+		mail_storage_set_critical(storage,
+			"closedir(%s) failed: %m", dstorage->storage_dir);
+		ret = -1;
+	}
+	if (ret < 0)
+		return -1;
+
+	/* find the newest message in alt storage from map view */
+	hdr = mail_index_get_header(ctx->map->view);
+	for (seq = hdr->messages_count; seq > 0; seq--) {
+		if (dbox_map_lookup_seq(ctx->map, seq, &file_id,
+					&offset, &size) < 0)
+			return -1;
+
+		if (file_id < min_file_id)
+			break;
+	}
+
+	*min_file_id_r = min_file_id;
+	*seq_r = seq;
+	return 0;
+}
+
+static int
+dbox_map_find_appendable_file(struct dbox_map_append_context *ctx,
+			      uoff_t mail_size,
+			      struct dbox_file_append_context **file_append_r,
+			      struct ostream **output_r)
+{
+	struct dbox_map *map = ctx->map;
+	ARRAY_TYPE(seq_range) checked_file_ids;
+	const struct mail_index_header *hdr;
+	unsigned int backwards_lookup_count;
+	uint32_t seq, seq1, uid, file_id, min_file_id;
+	uoff_t offset, size;
+	time_t stamp;
+	bool retry_later;
+
+	if (mail_size >= map->set->mdbox_rotate_size)
+		return 0;
 
 	/* try to find an existing appendable file */
 	stamp = day_begin_stamp(map->set->mdbox_rotate_interval);
 	hdr = mail_index_get_header(map->view);
 
-	ctx->orig_next_uid = hdr->next_uid;
 	backwards_lookup_count = 0;
 	t_array_init(&checked_file_ids, 16);
-	for (seq = hdr->messages_count; seq > 0; seq--) {
+
+	if ((ctx->flags & DBOX_MAP_APPEND_FLAG_ALT) == 0)
+		seq = hdr->messages_count;
+	else {
+		/* we want to save to alt storage. */
+		if (dbox_map_find_first_alt(ctx, &min_file_id, &seq) < 0)
+			return -1;
+		seq_range_array_add_range(&checked_file_ids,
+					  min_file_id, (uint32_t)-1);
+	}
+
+	for (; seq > 0; seq--) {
 		if (dbox_map_lookup_seq(map, seq, &file_id, &offset, &size) < 0)
 			return -1;
 
@@ -712,15 +809,24 @@
 	if (ctx->failed)
 		return -1;
 
-	ret = dbox_map_find_appendable_file(ctx, mail_size, &file_append,
-					    output_r, &existing);
+	file_append = dbox_map_find_existing_append(ctx, mail_size, output_r);
+	if (file_append != NULL) {
+		ret = 1;
+		existing = TRUE;
+	} else {
+		ret = dbox_map_find_appendable_file(ctx, mail_size,
+						    &file_append, output_r);
+		existing = FALSE;
+	}
 	if (ret > 0)
 		file = file_append->file;
 	else if (ret < 0)
 		return -1;
 	else {
 		/* create a new file */
-		file = mdbox_file_init(ctx->map->storage, 0);
+		file = (ctx->flags & DBOX_MAP_APPEND_FLAG_ALT) == 0 ?
+			mdbox_file_init(ctx->map->storage, 0) :
+			mdbox_file_init_new_alt(ctx->map->storage);
 		file_append = dbox_file_append_init(file);
 
 		ret = dbox_file_get_append_stream(file_append, output_r);
diff -r 55cce06818b8 -r c60910419861 src/lib-storage/index/dbox-multi/mdbox-map.h
--- a/src/lib-storage/index/dbox-multi/mdbox-map.h	Tue Feb 02 23:24:52 2010 +0200
+++ b/src/lib-storage/index/dbox-multi/mdbox-map.h	Tue Feb 02 23:49:32 2010 +0200
@@ -7,6 +7,10 @@
 struct dbox_file_append_context;
 struct mdbox_storage;
 
+enum dbox_map_append_flags {
+	DBOX_MAP_APPEND_FLAG_ALT	= 0x01
+};
+
 struct dbox_map_mail_index_header {
 	uint32_t highest_file_id;
 };
@@ -56,10 +60,11 @@
 int dbox_map_remove_file_id(struct dbox_map *map, uint32_t file_id);
 
 /* Return all files containing messages with zero refcount. */
-const ARRAY_TYPE(seq_range) *dbox_map_get_zero_ref_files(struct dbox_map *map);
+int dbox_map_get_zero_ref_files(struct dbox_map *map,
+				ARRAY_TYPE(seq_range) *file_ids_r);
 
 struct dbox_map_append_context *
-dbox_map_append_begin(struct dbox_map *map);
+dbox_map_append_begin(struct dbox_map *map, enum dbox_map_append_flags flags);
 /* Request file for saving a new message with given size (if available). If an
    existing file can be used, the record is locked and updated in index.
    Returns 0 if ok, -1 if error. */
diff -r 55cce06818b8 -r c60910419861 src/lib-storage/index/dbox-multi/mdbox-save.c
--- a/src/lib-storage/index/dbox-multi/mdbox-save.c	Tue Feb 02 23:24:52 2010 +0200
+++ b/src/lib-storage/index/dbox-multi/mdbox-save.c	Tue Feb 02 23:49:32 2010 +0200
@@ -85,7 +85,7 @@
 	ctx->ctx.ctx.transaction = t;
 	ctx->ctx.trans = it->trans;
 	ctx->mbox = mbox;
-	ctx->append_ctx = dbox_map_append_begin(mbox->storage->map);
+	ctx->append_ctx = dbox_map_append_begin(mbox->storage->map, 0);
 	i_array_init(&ctx->mails, 32);
 	t->save_ctx = &ctx->ctx.ctx;
 	return t->save_ctx;
diff -r 55cce06818b8 -r c60910419861 src/lib-storage/index/dbox-multi/mdbox-settings.c
--- a/src/lib-storage/index/dbox-multi/mdbox-settings.c	Tue Feb 02 23:24:52 2010 +0200
+++ b/src/lib-storage/index/dbox-multi/mdbox-settings.c	Tue Feb 02 23:49:32 2010 +0200
@@ -17,6 +17,7 @@
 static const struct setting_define mdbox_setting_defines[] = {
 	DEF(SET_SIZE, mdbox_rotate_size),
 	DEF(SET_TIME, mdbox_rotate_interval),
+	DEF(SET_TIME, mdbox_altmove),
 	DEF(SET_UINT, mdbox_max_open_files),
 
 	SETTING_DEFINE_LIST_END
@@ -25,6 +26,7 @@
 static const struct mdbox_settings mdbox_default_settings = {
 	.mdbox_rotate_size = 2*1024*1024,
 	.mdbox_rotate_interval = 0,
+	.mdbox_altmove = 3600*24*7,
 	.mdbox_max_open_files = 64
 };
 
diff -r 55cce06818b8 -r c60910419861 src/lib-storage/index/dbox-multi/mdbox-settings.h
--- a/src/lib-storage/index/dbox-multi/mdbox-settings.h	Tue Feb 02 23:24:52 2010 +0200
+++ b/src/lib-storage/index/dbox-multi/mdbox-settings.h	Tue Feb 02 23:49:32 2010 +0200
@@ -4,6 +4,7 @@
 struct mdbox_settings {
 	uoff_t mdbox_rotate_size;
 	unsigned int mdbox_rotate_interval;
+	unsigned int mdbox_altmove;
 	unsigned int mdbox_max_open_files;
 };
 
diff -r 55cce06818b8 -r c60910419861 src/lib-storage/index/dbox-multi/mdbox-sync.c
--- a/src/lib-storage/index/dbox-multi/mdbox-sync.c	Tue Feb 02 23:24:52 2010 +0200
+++ b/src/lib-storage/index/dbox-multi/mdbox-sync.c	Tue Feb 02 23:49:32 2010 +0200
@@ -11,6 +11,9 @@
 #include "mdbox-file.h"
 #include "mdbox-sync.h"
 
+#include <stdlib.h>
+#include <dirent.h>
+
 #define DBOX_REBUILD_COUNT 3
 
 static int
@@ -311,10 +314,75 @@
 	return index_mailbox_sync_init(box, flags, ret < 0);
 }
 
+static int mdbox_sync_altmove_add_files(struct mdbox_storage *dstorage,
+					ARRAY_TYPE(seq_range) *file_ids)
+{
+	struct mail_storage *storage = &dstorage->storage.storage;
+	DIR *dir;
+	struct dirent *d;
+	struct stat st;
+	time_t altmove_mtime;
+	string_t *path;
+	unsigned int file_id, dir_len;
+	int ret = 0;
+
+	if (dstorage->set->mdbox_altmove == 0 ||
+	    dstorage->alt_storage_dir == NULL)
+		return 0;
+
+	altmove_mtime = ioloop_time - dstorage->set->mdbox_altmove;
+
+	/* we want to quickly find the latest alt file, but we also want to
+	   avoid accessing the alt storage as much as possible. so we'll do
+	   this by finding the lowest numbered file (n) from primary storage.
+	   hopefully one of n-[1..m] is appendable in alt storage. */
+	dir = opendir(dstorage->storage_dir);
+	if (dir == NULL) {
+		mail_storage_set_critical(storage,
+			"opendir(%s) failed: %m", dstorage->storage_dir);
+		return -1;
+	}
+
+	path = t_str_new(256);
+	str_append(path, dstorage->storage_dir);
+	str_append_c(path, '/');
+	dir_len = str_len(path);
+
+	for (errno = 0; (d = readdir(dir)) != NULL; errno = 0) {
+		if (strncmp(d->d_name, MDBOX_MAIL_FILE_PREFIX,
+			    strlen(MDBOX_MAIL_FILE_PREFIX)) != 0)
+			continue;
+
+		str_truncate(path, dir_len);
+		str_append(path, d->d_name);
+
+		file_id = strtoul(d->d_name + strlen(MDBOX_MAIL_FILE_PREFIX),
+				  NULL, 10);
+
+		if (stat(str_c(path), &st) < 0) {
+			mail_storage_set_critical(storage,
+				"stat(%s) failed: %m", str_c(path));
+		} else if (st.st_mtime < altmove_mtime) {
+			seq_range_array_add(file_ids, 0, file_id);
+		}
+	}
+	if (errno != 0) {
+		mail_storage_set_critical(storage,
+			"readdir(%s) failed: %m", dstorage->storage_dir);
+		ret = -1;
+	}
+	if (closedir(dir) < 0) {
+		mail_storage_set_critical(storage,
+			"closedir(%s) failed: %m", dstorage->storage_dir);
+		ret = -1;
+	}
+	return ret;
+}
+
 int mdbox_sync_purge(struct mail_storage *_storage)
 {
 	struct mdbox_storage *storage = (struct mdbox_storage *)_storage;
-	const ARRAY_TYPE(seq_range) *ref0_file_ids;
+	ARRAY_TYPE(seq_range) ref0_file_ids;
 	struct dbox_file *file;
 	struct seq_range_iter iter;
 	unsigned int i = 0;
@@ -322,8 +390,15 @@
 	bool deleted;
 	int ret = 0;
 
-	ref0_file_ids = dbox_map_get_zero_ref_files(storage->map);
-	seq_range_array_iter_init(&iter, ref0_file_ids); i = 0;
+	i_array_init(&ref0_file_ids, 64);
+	if (dbox_map_get_zero_ref_files(storage->map, &ref0_file_ids) < 0)
+		ret = -1;
+
+	/* add also files that can be altmoved */
+	if (mdbox_sync_altmove_add_files(storage, &ref0_file_ids) < 0)
+		ret = -1;
+
+	seq_range_array_iter_init(&iter, &ref0_file_ids); i = 0;
 	while (seq_range_array_iter_nth(&iter, i++, &file_id)) T_BEGIN {
 		file = mdbox_file_init(storage, file_id);
 		if (dbox_file_open(file, &deleted) > 0 && !deleted) {
@@ -334,5 +409,6 @@
 		}
 		dbox_file_unref(&file);
 	} T_END;
+	array_free(&ref0_file_ids);
 	return ret;
 }