# HG changeset patch
# User Timo Sirainen <tss@iki.fi>
# Date 1083525875 -10800
# Node ID d6941cd8afdc7e190e1e6cfb281a3392c9d33ef8
# Parent  0f0128b4af5dbda47693054366dc413eb3e98f27
Added support for setting dirty flags for messages (TODO: undirty..)

s/mail_index_record_flag/mail_cache_record_flag/

diff -r 0f0128b4af5d -r d6941cd8afdc src/lib-index/mail-cache.c
--- a/src/lib-index/mail-cache.c	Sun May 02 21:42:28 2004 +0300
+++ b/src/lib-index/mail-cache.c	Sun May 02 22:24:35 2004 +0300
@@ -13,7 +13,7 @@
 #include <sys/stat.h>
 
 unsigned int mail_cache_field_sizes[32] = {
-	sizeof(enum mail_index_record_flag),
+	sizeof(enum mail_cache_record_flag),
 	sizeof(uoff_t),
 	16,
 	sizeof(struct mail_sent_date),
@@ -640,14 +640,14 @@
 			     enum mail_cache_field fields) {}
 
 /* Return index flags. */
-enum mail_index_record_flag
-mail_cache_get_index_flags(struct mail_cache_view *view, uint32_t seq)
+enum mail_cache_record_flag
+mail_cache_get_record_flags(struct mail_cache_view *view, uint32_t seq)
 {return 0;}
 
 /* Update index flags. The cache file must be locked and the flags must be
    already inserted to the record. */
-int mail_cache_update_index_flags(struct mail_cache_view *view, uint32_t seq,
-				  enum mail_index_record_flag flags)
+int mail_cache_update_record_flags(struct mail_cache_view *view, uint32_t seq,
+				   enum mail_cache_record_flag flags)
 {return 0;}
 
 /* Update location offset. External locking is assumed to take care of locking
diff -r 0f0128b4af5d -r d6941cd8afdc src/lib-index/mail-cache.h
--- a/src/lib-index/mail-cache.h	Sun May 02 21:42:28 2004 +0300
+++ b/src/lib-index/mail-cache.h	Sun May 02 22:24:35 2004 +0300
@@ -11,6 +11,20 @@
 struct mail_cache_view;
 struct mail_cache_transaction_ctx;
 
+enum mail_cache_record_flag {
+	/* If binary flags are set, it's not checked whether mail is
+	   missing CRs. So this flag may be set as an optimization for
+	   regular non-binary mails as well if it's known that it contains
+	   valid CR+LF line breaks. */
+	MAIL_INDEX_FLAG_BINARY_HEADER		= 0x0001,
+	MAIL_INDEX_FLAG_BINARY_BODY		= 0x0002,
+
+	/* Mail header or body is known to contain NUL characters. */
+	MAIL_INDEX_FLAG_HAS_NULS		= 0x0004,
+	/* Mail header or body is known to not contain NUL characters. */
+	MAIL_INDEX_FLAG_HAS_NO_NULS		= 0x0008
+};
+
 enum mail_cache_field {
 	/* fixed size fields */
 	MAIL_CACHE_INDEX_FLAGS		= 0x00000001,
@@ -144,14 +158,14 @@
 void mail_cache_mark_missing(struct mail_cache_view *view,
 			     enum mail_cache_field fields);
 
-/* Return index flags. */
-enum mail_index_record_flag
-mail_cache_get_index_flags(struct mail_cache_view *view, uint32_t seq);
+/* Return record flags. */
+enum mail_cache_record_flag
+mail_cache_get_record_flags(struct mail_cache_view *view, uint32_t seq);
 
-/* Update index flags. The cache file must be locked and the flags must be
+/* Update record flags. The cache file must be locked and the flags must be
    already inserted to the record. */
-int mail_cache_update_index_flags(struct mail_cache_view *view, uint32_t seq,
-				  enum mail_index_record_flag flags);
+int mail_cache_update_record_flags(struct mail_cache_view *view, uint32_t seq,
+				   enum mail_cache_record_flag flags);
 
 /* Update location offset. External locking is assumed to take care of locking
    readers out to prevent race conditions. */
diff -r 0f0128b4af5d -r d6941cd8afdc src/lib-index/mail-index-sync-private.h
--- a/src/lib-index/mail-index-sync-private.h	Sun May 02 21:42:28 2004 +0300
+++ b/src/lib-index/mail-index-sync-private.h	Sun May 02 22:24:35 2004 +0300
@@ -17,9 +17,10 @@
 	size_t expunge_idx, update_idx;
 	uint32_t next_seq;
 
-	unsigned int lock_id;
+	unsigned int lock_id, dirty_lock_id;
 
 	unsigned int sync_appends:1;
+	unsigned int have_dirty:1;
 };
 
 int mail_index_sync_update_index(struct mail_index_sync_ctx *sync_ctx,
diff -r 0f0128b4af5d -r d6941cd8afdc src/lib-index/mail-index-sync-update.c
--- a/src/lib-index/mail-index-sync-update.c	Sun May 02 21:42:28 2004 +0300
+++ b/src/lib-index/mail-index-sync-update.c	Sun May 02 22:24:35 2004 +0300
@@ -175,7 +175,7 @@
 	struct mail_index_sync_rec rec;
 	const struct mail_index_record *appends;
 	unsigned int append_count;
-	uint32_t count, file_seq, src_idx, dest_idx;
+	uint32_t count, file_seq, src_idx, dest_idx, dirty_flag;
 	uoff_t file_offset;
 	unsigned int lock_id;
 	int ret, changed;
@@ -192,6 +192,12 @@
 	ctx.hdr = *index->hdr;
 	ctx.log_view = sync_ctx->view->log_view;
 
+	dirty_flag = sync_ctx->have_dirty ? MAIL_INDEX_HDR_FLAG_HAVE_DIRTY : 0;
+	if ((ctx.hdr.flags & MAIL_INDEX_HDR_FLAG_HAVE_DIRTY) != dirty_flag) {
+		ctx.hdr.flags ^= MAIL_INDEX_HDR_FLAG_HAVE_DIRTY;
+		changed = TRUE;
+	}
+
 	/* see if we need to update sync headers */
 	if (ctx.hdr.sync_stamp != sync_stamp && sync_stamp != 0) {
 		ctx.hdr.sync_stamp = sync_stamp;
diff -r 0f0128b4af5d -r d6941cd8afdc src/lib-index/mail-index-sync.c
--- a/src/lib-index/mail-index-sync.c	Sun May 02 21:42:28 2004 +0300
+++ b/src/lib-index/mail-index-sync.c	Sun May 02 22:24:35 2004 +0300
@@ -397,6 +397,20 @@
 		ctx->sync_appends;
 }
 
+int mail_index_sync_set_dirty(struct mail_index_sync_ctx *ctx, uint32_t seq)
+{
+	if (ctx->dirty_lock_id == 0) {
+		if (mail_index_lock_exclusive(ctx->index,
+					      &ctx->dirty_lock_id) < 0)
+			return -1;
+	}
+
+	i_assert(seq <= ctx->view->map->records_count);
+	ctx->view->map->records[seq-1].flags |= MAIL_INDEX_MAIL_FLAG_DIRTY;
+	ctx->have_dirty = TRUE;
+	return 0;
+}
+
 int mail_index_sync_end(struct mail_index_sync_ctx *ctx,
 			uint32_t sync_stamp, uint64_t sync_size)
 {
@@ -421,11 +435,15 @@
 
 	if (ret == 0) {
 		mail_index_sync_read_and_sort(ctx, TRUE);
+
 		if (mail_index_sync_update_index(ctx, sync_stamp,
 						 sync_size) < 0)
 			ret = -1;
 	}
 
+	if (ctx->dirty_lock_id == 0) 
+		mail_index_unlock(ctx->index, ctx->dirty_lock_id);
+
 	mail_index_unlock(ctx->index, ctx->lock_id);
 	mail_transaction_log_sync_unlock(ctx->index->log);
 	mail_index_view_close(ctx->view);
diff -r 0f0128b4af5d -r d6941cd8afdc src/lib-index/mail-index.h
--- a/src/lib-index/mail-index.h	Sun May 02 21:42:28 2004 +0300
+++ b/src/lib-index/mail-index.h	Sun May 02 22:24:35 2004 +0300
@@ -34,21 +34,12 @@
 
 enum mail_index_header_flag {
 	/* Index file is corrupted, reopen or recreate it. */
-	MAIL_INDEX_HDR_FLAG_CORRUPTED		= 0x0001
+	MAIL_INDEX_HDR_FLAG_CORRUPTED		= 0x0001,
+	MAIL_INDEX_HDR_FLAG_HAVE_DIRTY		= 0x0002
 };
 
-enum mail_index_record_flag {
-	/* If binary flags are set, it's not checked whether mail is
-	   missing CRs. So this flag may be set as an optimization for
-	   regular non-binary mails as well if it's known that it contains
-	   valid CR+LF line breaks. */
-	MAIL_INDEX_FLAG_BINARY_HEADER		= 0x0001,
-	MAIL_INDEX_FLAG_BINARY_BODY		= 0x0002,
-
-	/* Mail header or body is known to contain NUL characters. */
-	MAIL_INDEX_FLAG_HAS_NULS		= 0x0004,
-	/* Mail header or body is known to not contain NUL characters. */
-	MAIL_INDEX_FLAG_HAS_NO_NULS		= 0x0008
+enum mail_index_mail_flags {
+	MAIL_INDEX_MAIL_FLAG_DIRTY = 0x80
 };
 
 enum mail_index_error {
@@ -204,6 +195,10 @@
 			 struct mail_index_sync_rec *sync_rec);
 /* Returns 1 if there's more to sync, 0 if not. */
 int mail_index_sync_have_more(struct mail_index_sync_ctx *ctx);
+/* Mark given message to be dirty, ie. we couldn't temporarily change the
+   message flags in storage. Dirty messages are tried to be synced again in
+   next sync. */
+int mail_index_sync_set_dirty(struct mail_index_sync_ctx *ctx, uint32_t seq);
 /* End synchronization by unlocking the index and closing the view.
    sync_stamp/sync_size in header is updated to given values. */
 int mail_index_sync_end(struct mail_index_sync_ctx *ctx,
diff -r 0f0128b4af5d -r d6941cd8afdc src/lib-storage/index/index-mail.c
--- a/src/lib-storage/index/index-mail.c	Sun May 02 21:42:28 2004 +0300
+++ b/src/lib-storage/index/index-mail.c	Sun May 02 22:24:35 2004 +0300
@@ -297,7 +297,7 @@
 static void index_mail_parse_body(struct index_mail *mail)
 {
 	struct index_mail_data *data = &mail->data;
-        enum mail_index_record_flag index_flags;
+        enum mail_cache_record_flag cache_flags;
 	buffer_t *buffer;
 	const void *buf_data;
 	size_t buf_size;
@@ -335,16 +335,16 @@
 	if (!index_mail_cache_transaction_begin(mail))
 		return;
 
-	/* update index_flags */
-	index_flags = mail_cache_get_index_flags(mail->ibox->cache_view,
-						 mail->data.seq);
+	/* update cache_flags */
+	cache_flags = mail_cache_get_record_flags(mail->ibox->cache_view,
+						  mail->data.seq);
 	if (mail->mail.has_nuls)
-		index_flags |= MAIL_INDEX_FLAG_HAS_NULS;
+		cache_flags |= MAIL_INDEX_FLAG_HAS_NULS;
 	else
-		index_flags |= MAIL_INDEX_FLAG_HAS_NO_NULS;
+		cache_flags |= MAIL_INDEX_FLAG_HAS_NO_NULS;
 
-	if (!mail_cache_update_index_flags(mail->ibox->cache_view,
-					   mail->data.seq, index_flags))
+	if (!mail_cache_update_record_flags(mail->ibox->cache_view,
+					    mail->data.seq, cache_flags))
 		return;
 
 	if (index_mail_cache_can_add(mail, MAIL_CACHE_MESSAGEPART)) {
@@ -517,7 +517,7 @@
 		    uint32_t seq, int delay_open)
 {
 	struct index_mail_data *data = &mail->data;
-        enum mail_index_record_flag index_flags;
+        enum mail_cache_record_flag cache_flags;
 	int ret, open_mail;
 
 	t_push();
@@ -528,12 +528,12 @@
 
 	data->cached_fields =
 		mail_cache_get_fields(mail->ibox->cache_view, seq);
-	index_flags = (data->cached_fields & MAIL_CACHE_INDEX_FLAGS) == 0 ? 0 :
-		mail_cache_get_index_flags(mail->ibox->cache_view, seq);
+	cache_flags = (data->cached_fields & MAIL_CACHE_INDEX_FLAGS) == 0 ? 0 :
+		mail_cache_get_record_flags(mail->ibox->cache_view, seq);
 
-	mail->mail.has_nuls = (index_flags & MAIL_INDEX_FLAG_HAS_NULS) != 0;
+	mail->mail.has_nuls = (cache_flags & MAIL_INDEX_FLAG_HAS_NULS) != 0;
 	mail->mail.has_no_nuls =
-		(index_flags & MAIL_INDEX_FLAG_HAS_NO_NULS) != 0;
+		(cache_flags & MAIL_INDEX_FLAG_HAS_NO_NULS) != 0;
 
 	data->rec = rec;
 	data->seq = seq;
diff -r 0f0128b4af5d -r d6941cd8afdc src/lib-storage/index/maildir/maildir-sync.c
--- a/src/lib-storage/index/maildir/maildir-sync.c	Sun May 02 21:42:28 2004 +0300
+++ b/src/lib-storage/index/maildir/maildir-sync.c	Sun May 02 22:24:35 2004 +0300
@@ -195,6 +195,15 @@
         struct maildir_uidlist_sync_ctx *uidlist_sync_ctx;
 };
 
+struct maildir_index_sync_context {
+        struct index_mailbox *ibox;
+        struct mail_index_view *view;
+	struct mail_index_sync_ctx *sync_ctx;
+
+	struct mail_index_sync_rec sync_rec;
+	uint32_t seq;
+};
+
 static int maildir_expunge(struct index_mailbox *ibox, const char *path,
 			   void *context __attr_unused__)
 {
@@ -213,7 +222,7 @@
 static int maildir_sync_flags(struct index_mailbox *ibox, const char *path,
 			      void *context)
 {
-	struct mail_index_sync_rec *syncrec = context;
+        struct maildir_index_sync_context *ctx = context;
 	const char *newpath;
 	enum mail_flags flags;
 	uint8_t flags8;
@@ -222,7 +231,7 @@
 	(void)maildir_filename_get_flags(path, &flags, custom_flags);
 
 	flags8 = flags;
-	mail_index_sync_flags_apply(syncrec, &flags8, custom_flags);
+	mail_index_sync_flags_apply(&ctx->sync_rec, &flags8, custom_flags);
 
 	newpath = maildir_filename_set_flags(path, flags8, custom_flags);
 	if (rename(path, newpath) == 0) {
@@ -232,22 +241,29 @@
 	if (errno == ENOENT)
 		return 0;
 
+	if (ENOSPACE(errno)) {
+		if (mail_index_sync_set_dirty(ctx->sync_ctx, ctx->seq) < 0)
+			return -1;
+		return 1;
+	}
+
 	mail_storage_set_critical(ibox->box.storage,
 				  "rename(%s, %s) failed: %m", path, newpath);
 	return -1;
 }
 
 static int maildir_sync_record(struct index_mailbox *ibox,
-			       struct mail_index_view *view,
-			       struct mail_index_sync_rec *syncrec)
+                               struct maildir_index_sync_context *ctx)
 {
+	struct mail_index_sync_rec *sync_rec = &ctx->sync_rec;
+	struct mail_index_view *view = ctx->view;
 	uint32_t seq, uid;
 
-	switch (syncrec->type) {
+	switch (sync_rec->type) {
 	case MAIL_INDEX_SYNC_TYPE_APPEND:
 		break;
 	case MAIL_INDEX_SYNC_TYPE_EXPUNGE:
-		for (seq = syncrec->seq1; seq <= syncrec->seq2; seq++) {
+		for (seq = sync_rec->seq1; seq <= sync_rec->seq2; seq++) {
 			if (mail_index_lookup_uid(view, seq, &uid) < 0)
 				return -1;
 			if (maildir_file_do(ibox, uid, maildir_expunge,
@@ -256,11 +272,12 @@
 		}
 		break;
 	case MAIL_INDEX_SYNC_TYPE_FLAGS:
-		for (seq = syncrec->seq1; seq <= syncrec->seq2; seq++) {
-			if (mail_index_lookup_uid(view, seq, &uid) < 0)
+                ctx->seq = sync_rec->seq1;
+		for (; ctx->seq <= sync_rec->seq2; ctx->seq++) {
+			if (mail_index_lookup_uid(view, ctx->seq, &uid) < 0)
 				return -1;
 			if (maildir_file_do(ibox, uid, maildir_sync_flags,
-					    syncrec) < 0)
+					    ctx) < 0)
 				return -1;
 		}
 		break;
@@ -271,25 +288,27 @@
 
 int maildir_sync_last_commit(struct index_mailbox *ibox)
 {
-        struct mail_index_view *view;
-	struct mail_index_sync_ctx *sync_ctx;
-	struct mail_index_sync_rec sync_rec;
+	struct maildir_index_sync_context ctx;
 	int ret;
 
 	if (ibox->commit_log_file_seq == 0)
 		return 0;
 
-	ret = mail_index_sync_begin(ibox->index, &sync_ctx, &view,
+	memset(&ctx, 0, sizeof(ctx));
+	ctx.ibox = ibox;
+
+	ret = mail_index_sync_begin(ibox->index, &ctx.sync_ctx, &ctx.view,
 				    ibox->commit_log_file_seq,
 				    ibox->commit_log_file_offset);
 	if (ret > 0) {
-		while ((ret = mail_index_sync_next(sync_ctx, &sync_rec)) > 0) {
-			if (maildir_sync_record(ibox, view, &sync_rec) < 0) {
+		while ((ret = mail_index_sync_next(ctx.sync_ctx,
+						   &ctx.sync_rec)) > 0) {
+			if (maildir_sync_record(ibox, &ctx) < 0) {
 				ret = -1;
 				break;
 			}
 		}
-		if (mail_index_sync_end(sync_ctx, 0, 0) < 0)
+		if (mail_index_sync_end(ctx.sync_ctx, 0, 0) < 0)
 			ret = -1;
 	}
 
@@ -476,8 +495,7 @@
 static int maildir_sync_index(struct maildir_sync_context *ctx)
 {
 	struct index_mailbox *ibox = ctx->ibox;
-	struct mail_index_sync_ctx *sync_ctx;
-	struct mail_index_sync_rec sync_rec;
+	struct maildir_index_sync_context sync_ctx;
 	struct maildir_uidlist_iter_ctx *iter;
 	struct mail_index_transaction *trans;
 	struct mail_index_view *view;
@@ -491,11 +509,15 @@
 	uint32_t sync_stamp;
 	int ret;
 
-	if (mail_index_sync_begin(ibox->index, &sync_ctx, &view,
+	memset(&sync_ctx, 0, sizeof(sync_ctx));
+	sync_ctx.ibox = ibox;
+
+	if (mail_index_sync_begin(ibox->index, &sync_ctx.sync_ctx, &view,
 				  (uint32_t)-1, (uoff_t)-1) <= 0) {
 		mail_storage_set_index_error(ibox);
 		return -1;
 	}
+	sync_ctx.view = view;
 
 	ret = mail_index_get_header(view, &hdr);
 	i_assert(ret == 0); /* view is locked, can't happen */
@@ -544,6 +566,12 @@
 			break;
 		}
 
+		if ((rec->flags & MAIL_INDEX_MAIL_FLAG_DIRTY) != 0) {
+			/* we haven't been able to update maildir with this
+			   record's flag changes. don't sync them. */
+			continue;
+		}
+
 		maildir_filename_get_flags(filename, &flags, custom_flags);
 		if ((uint8_t)flags != (rec->flags & MAIL_FLAGS_MASK) ||
 		    memcmp(custom_flags, rec->custom_flags,
@@ -575,15 +603,16 @@
 	}
 
 	/* now, sync the index */
-	while ((ret = mail_index_sync_next(sync_ctx, &sync_rec)) > 0) {
-		if (maildir_sync_record(ibox, view, &sync_rec) < 0) {
+	while ((ret = mail_index_sync_next(sync_ctx.sync_ctx,
+					   &sync_ctx.sync_rec)) > 0) {
+		if (maildir_sync_record(ibox, &sync_ctx) < 0) {
 			ret = -1;
 			break;
 		}
 	}
 
 	sync_stamp = ibox->dirty_cur_time != 0 ? 0 : ibox->last_cur_mtime;
-	if (mail_index_sync_end(sync_ctx, sync_stamp, 0) < 0)
+	if (mail_index_sync_end(sync_ctx.sync_ctx, sync_stamp, 0) < 0)
 		ret = -1;
 
 	if (ret == 0) {