changeset 8924:a0b72838b441 HEAD

dbox: Save optimizations.
author Timo Sirainen <tss@iki.fi>
date Tue, 24 Feb 2009 21:39:59 -0500
parents 47076db1f911
children a1d920a81adc
files src/lib-storage/index/dbox/dbox-file.c src/lib-storage/index/dbox/dbox-file.h src/lib-storage/index/dbox/dbox-map.c src/lib-storage/index/dbox/dbox-save.c
diffstat 4 files changed, 51 insertions(+), 59 deletions(-) [+]
line wrap: on
line diff
--- a/src/lib-storage/index/dbox/dbox-file.c	Tue Feb 24 21:39:41 2009 -0500
+++ b/src/lib-storage/index/dbox/dbox-file.c	Tue Feb 24 21:39:59 2009 -0500
@@ -515,6 +515,8 @@
 {
 	if (file->lock != NULL)
 		file_unlock(&file->lock);
+	if (file->input != NULL)
+		i_stream_sync(file->input);
 }
 
 static int
@@ -730,16 +732,27 @@
 {
 	if (ftruncate(file->fd, append_offset) < 0)
 		dbox_file_set_syscall_error(file, "ftruncate()");
-	if (file->input != NULL)
-		i_stream_sync(file->input);
 
 	o_stream_seek(file->output, append_offset);
 }
 
-void dbox_file_finish_append(struct dbox_file *file)
+int dbox_file_flush_append(struct dbox_file *file)
 {
-	if (file->input != NULL)
-		i_stream_sync(file->input);
+	i_assert(file->output != NULL);
+
+	if (o_stream_flush(file->output) < 0) {
+		dbox_file_set_syscall_error(file, "write()");
+		return -1;
+	}
+
+	if ((file->storage->storage.flags &
+	     MAIL_STORAGE_FLAG_FSYNC_DISABLE) == 0) {
+		if (fdatasync(file->fd) < 0) {
+			dbox_file_set_syscall_error(file, "fdatasync()");
+			return -1;
+		}
+	}
+	return 0;
 }
 
 static uoff_t
--- a/src/lib-storage/index/dbox/dbox-file.h	Tue Feb 24 21:39:41 2009 -0500
+++ b/src/lib-storage/index/dbox/dbox-file.h	Tue Feb 24 21:39:59 2009 -0500
@@ -164,8 +164,8 @@
 uoff_t dbox_file_get_next_append_offset(struct dbox_file *file);
 /* Truncate file to append_offset */
 void dbox_file_cancel_append(struct dbox_file *file, uoff_t append_offset);
-/* Finish appending the current mail. */
-void dbox_file_finish_append(struct dbox_file *file);
+/* Flush writes to dbox file. */
+int dbox_file_flush_append(struct dbox_file *file);
 
 /* Seek to given metadata block. Returns 1 if ok, 0 if file/offset is
    corrupted, -1 if I/O error. If message has already been expunged,
--- a/src/lib-storage/index/dbox/dbox-map.c	Tue Feb 24 21:39:41 2009 -0500
+++ b/src/lib-storage/index/dbox/dbox-map.c	Tue Feb 24 21:39:59 2009 -0500
@@ -294,8 +294,6 @@
 			      struct ostream **output_r, bool *existing_r)
 {
 	struct dbox_map *map = ctx->map;
-	bool fsync_disable = (map->storage->storage.flags &
-			      MAIL_STORAGE_FLAG_FSYNC_DISABLE) != 0;
 	struct dbox_file *const *files;
 	const struct mail_index_header *hdr;
 	unsigned int i, count, backwards_lookup_count;
@@ -323,13 +321,8 @@
 		/* can't append to this file anymore */
 		if (files[i-1]->fd != -1) {
 			/* avoid wasting fds by closing the file */
-			if (!fsync_disable) {
-				if (fdatasync(files[i-1]->fd) < 0) {
-					dbox_file_set_syscall_error(files[i-1],
-						"fdatasync()");
-					return -1;
-				}
-			}
+			if (dbox_file_flush_append(files[i-1]) < 0)
+				return -1;
 			dbox_file_unlock(files[i-1]);
 			dbox_file_close(files[i-1]);
 		}
@@ -431,9 +424,10 @@
 		append->file = file;
 		append->offset = (*output_r)->offset;
 	}
-	if (!existing)
+	if (!existing) {
+		i_assert(file->output != NULL);
 		array_append(&ctx->files, &file, 1);
-
+	}
 	*file_r = file;
 	return 0;
 }
@@ -467,8 +461,6 @@
 				    uint32_t *first_map_uid_r,
 				    uint32_t *last_map_uid_r)
 {
-	bool fsync_disable = (ctx->map->storage->storage.flags &
-			      MAIL_STORAGE_FLAG_FSYNC_DISABLE) != 0;
 	struct dbox_file *const *files;
 	const struct dbox_map_append *appends;
 	struct mail_index_sync_ctx *sync_ctx;
@@ -508,11 +500,8 @@
 		if (files[i]->single_mbox != NULL)
 			continue;
 
-		if (!fsync_disable && files[i]->single_mbox == NULL &&
-		    files[i]->fd != -1) {
-			if (fdatasync(files[i]->fd) < 0) {
-				dbox_file_set_syscall_error(files[i],
-							    "fdatasync()");
+		if (files[i]->single_mbox == NULL && files[i]->output != NULL) {
+			if (dbox_file_flush_append(files[i]) < 0) {
 				ret = -1;
 				break;
 			}
@@ -631,6 +620,11 @@
 	for (i = 0; i < count; i++) {
 		file = files[i];
 
+		if (file->output != NULL) {
+			/* flush before truncating */
+			(void)o_stream_flush(file->output);
+		}
+
 		if (file->file_id != 0) {
 			/* FIXME: truncate? */
 		} else {
--- a/src/lib-storage/index/dbox/dbox-save.c	Tue Feb 24 21:39:41 2009 -0500
+++ b/src/lib-storage/index/dbox/dbox-save.c	Tue Feb 24 21:39:59 2009 -0500
@@ -78,13 +78,11 @@
 	struct dbox_save_mail *save_mail;
 	struct istream *crlf_input;
 	enum mail_flags save_flags;
-	const struct stat *st;
 	uoff_t mail_size;
 
 	/* get the size of the mail to be saved, if possible */
-	st = i_stream_stat(input, TRUE);
-	mail_size = st == NULL || st->st_size == -1 ? 0 : st->st_size;
-
+	if (i_stream_get_size(input, TRUE, &mail_size) <= 0)
+		mail_size = 0;
 	if (dbox_map_append_next(ctx->append_ctx, mail_size,
 				 &ctx->cur_file, &ctx->cur_output) < 0) {
 		ctx->failed = TRUE;
@@ -217,26 +215,24 @@
 	o_stream_send(ctx->cur_output, str_data(str), str_len(str));
 }
 
-static int dbox_save_mail_write_header(struct dbox_save_mail *mail)
+static int dbox_save_mail_write_metadata(struct dbox_save_context *ctx,
+					 struct dbox_save_mail *mail)
 {
 	struct dbox_message_header dbox_msg_hdr;
 
 	i_assert(mail->file->msg_header_size == sizeof(dbox_msg_hdr));
 
+	dbox_save_write_metadata(ctx);
 	dbox_msg_header_fill(&dbox_msg_hdr, mail->message_size);
-	if (pwrite_full(mail->file->fd, &dbox_msg_hdr,
-			sizeof(dbox_msg_hdr), mail->append_offset) < 0) {
-		dbox_file_set_syscall_error(mail->file, "write()");
+	if (o_stream_pwrite(ctx->cur_output, &dbox_msg_hdr,
+			    sizeof(dbox_msg_hdr), mail->append_offset) < 0) {
+		dbox_file_set_syscall_error(mail->file, "pwrite()");
 		return -1;
 	}
-	/* we're done writing to single-files now, so fsync them here. */
-	if ((mail->file->storage->storage.flags &
-	     MAIL_STORAGE_FLAG_FSYNC_DISABLE) == 0 &&
-	    mail->file->single_mbox != NULL) {
-		if (fdatasync(mail->file->fd) < 0) {
-			dbox_file_set_syscall_error(mail->file, "fdatasync()");
+	if (mail->file->single_mbox != NULL) {
+		/* we're done writing to single-files now */
+		if (dbox_file_flush_append(mail->file) < 0)
 			return -1;
-		}
 	}
 	return 0;
 }
@@ -244,9 +240,7 @@
 static int dbox_save_finish_write(struct mail_save_context *_ctx)
 {
 	struct dbox_save_context *ctx = (struct dbox_save_context *)_ctx;
-	struct mail_storage *storage = &ctx->mbox->storage->storage;
 	struct dbox_save_mail *save_mail;
-	uoff_t metadata_offset = 0;
 	unsigned int count;
 
 	ctx->finished = TRUE;
@@ -256,30 +250,21 @@
 	index_mail_cache_parse_deinit(_ctx->dest_mail,
 				      _ctx->received_date, !ctx->failed);
 
+	count = array_count(&ctx->mails);
+	save_mail = array_idx_modifiable(&ctx->mails, count - 1);
+
 	if (!ctx->failed) T_BEGIN {
-		metadata_offset = ctx->cur_output->offset;
-		dbox_save_write_metadata(ctx);
-		if (o_stream_flush(ctx->cur_output) < 0) {
-			mail_storage_set_critical(storage,
-				"o_stream_flush(%s) failed: %m",
-				ctx->cur_file->current_path);
+		save_mail->message_size = ctx->cur_output->offset -
+			save_mail->append_offset -
+			save_mail->file->msg_header_size;
+
+		if (dbox_save_mail_write_metadata(ctx, save_mail) < 0)
 			ctx->failed = TRUE;
-		}
 	} T_END;
 
 	o_stream_unref(&ctx->cur_output);
 	i_stream_unref(&ctx->input);
 
-	count = array_count(&ctx->mails);
-	save_mail = array_idx_modifiable(&ctx->mails, count - 1);
-	if (!ctx->failed) {
-		dbox_file_finish_append(save_mail->file);
-		save_mail->message_size = metadata_offset -
-			save_mail->append_offset -
-			save_mail->file->msg_header_size;
-		if (dbox_save_mail_write_header(save_mail) < 0)
-			ctx->failed = TRUE;
-	}
 	if (ctx->failed) {
 		dbox_file_cancel_append(save_mail->file,
 					save_mail->append_offset);