changeset 6977:038467bffcbd HEAD

When saving multiple mails, close some of the files temporarily so we don't use all available fds.
author Timo Sirainen <tss@iki.fi>
date Sun, 09 Dec 2007 15:48:38 +0200
parents 7cedc391e6c5
children 4fb1b0f43d60
files src/lib-storage/index/dbox/dbox-file.c src/lib-storage/index/dbox/dbox-file.h src/lib-storage/index/dbox/dbox-save.c
diffstat 3 files changed, 76 insertions(+), 41 deletions(-) [+]
line wrap: on
line diff
--- a/src/lib-storage/index/dbox/dbox-file.c	Sun Dec 09 15:47:03 2007 +0200
+++ b/src/lib-storage/index/dbox/dbox-file.c	Sun Dec 09 15:48:38 2007 +0200
@@ -85,15 +85,7 @@
 
 	if (file->metadata_pool != NULL)
 		pool_unref(&file->metadata_pool);
-	if (file->input != NULL)
-		i_stream_unref(&file->input);
-	if (file->output != NULL)
-		o_stream_unref(&file->output);
-	if (file->fd != -1) {
-		if (close(file->fd) < 0)
-			dbox_file_set_syscall_error(file, "close");
-		file->fd = -1;
-	}
+	dbox_file_close(file);
 	i_free(file->current_path);
 	i_free(file->fname);
 	i_free(file);
@@ -508,6 +500,38 @@
 		return dbox_file_open(file, read_header, deleted_r);
 }
 
+int dbox_file_open_if_needed(struct dbox_file *file)
+{
+	const char *path;
+	int ret;
+
+	if (file->fd != -1)
+		return 0;
+
+	T_FRAME(
+		ret = dbox_file_open_fd(file);
+	);
+	if (ret == 0) {
+		path = t_strdup_printf("%s/%s", file->mbox->path, file->fname);
+		mail_storage_set_critical(file->mbox->ibox.box.storage,
+					  "open(%s) failed: %m", path);
+	}
+	return ret <= 0 ? -1 : 0;
+}
+
+void dbox_file_close(struct dbox_file *file)
+{
+	if (file->input != NULL)
+		i_stream_unref(&file->input);
+	if (file->output != NULL)
+		o_stream_unref(&file->output);
+	if (file->fd != -1) {
+		if (close(file->fd) < 0)
+			dbox_file_set_syscall_error(file, "close");
+		file->fd = -1;
+	}
+}
+
 const char *dbox_file_get_path(struct dbox_file *file)
 {
 	i_assert(file->current_path != NULL);
--- a/src/lib-storage/index/dbox/dbox-file.h	Sun Dec 09 15:47:03 2007 +0200
+++ b/src/lib-storage/index/dbox/dbox-file.h	Sun Dec 09 15:48:38 2007 +0200
@@ -153,6 +153,10 @@
    deleted, deleted_r=TRUE and 1 is returned. */
 int dbox_file_open_or_create(struct dbox_file *file, bool read_header,
 			     bool *deleted_r);
+/* Open the file's fd if it's currently closed. Assumes that the file exists. */
+int dbox_file_open_if_needed(struct dbox_file *file);
+/* Close the file handle from the file, but don't free it. */
+void dbox_file_close(struct dbox_file *file);
 
 /* Returns the current fulle path for an opened/created file. It's an error to
    call this function for a non-opened file. */
--- a/src/lib-storage/index/dbox/dbox-save.c	Sun Dec 09 15:47:03 2007 +0200
+++ b/src/lib-storage/index/dbox/dbox-save.c	Sun Dec 09 15:48:38 2007 +0200
@@ -18,7 +18,7 @@
 
 struct dbox_save_mail {
 	struct dbox_file *file;
-	uint32_t seq;
+	uint32_t seq, uid;
 	uint32_t append_offset;
 	uoff_t message_size;
 };
@@ -250,6 +250,12 @@
 	i_stream_unref(&ctx->input);
 
 	count = array_count(&ctx->mails);
+	if (count >= ctx->mbox->max_open_files) {
+		/* too many open files, close one of them */
+		save_mail = array_idx_modifiable(&ctx->mails, count -
+						 ctx->mbox->max_open_files);
+		dbox_file_close(save_mail->file);
+	}
 	save_mail = array_idx_modifiable(&ctx->mails, count - 1);
 	if (ctx->failed) {
 		dbox_file_cancel_append(save_mail->file,
@@ -273,28 +279,21 @@
 	(void)dbox_save_finish(_ctx);
 }
 
-static int
-dbox_save_mail_write_header(struct dbox_save_mail *mail, uint32_t uid)
+static int dbox_save_mail_write_header(struct dbox_save_mail *mail)
 {
 	struct dbox_message_header dbox_msg_hdr;
-	struct ostream *output = mail->file->output;
-	uoff_t orig_offset;
-	int ret = 0;
 
 	i_assert(mail->file->msg_header_size == sizeof(dbox_msg_hdr));
 
-	mail->file->last_append_uid = uid;
-	dbox_msg_header_fill(&dbox_msg_hdr, uid, mail->message_size);
+	mail->file->last_append_uid = mail->uid;
+	dbox_msg_header_fill(&dbox_msg_hdr, mail->uid, mail->message_size);
 
-	orig_offset = output->offset;
-	o_stream_seek(output, mail->append_offset);
-	if (o_stream_send(output, &dbox_msg_hdr, sizeof(dbox_msg_hdr)) < 0 ||
-	    o_stream_flush(output) < 0) {
+	if (pwrite_full(mail->file->fd, &dbox_msg_hdr,
+			sizeof(dbox_msg_hdr), mail->append_offset) < 0) {
 		dbox_file_set_syscall_error(mail->file, "write");
-		ret = -1;
+		return -1;
 	}
-	o_stream_seek(output, orig_offset);
-	return ret;
+	return 0;
 }
 
 static int
@@ -350,33 +349,41 @@
 	struct dbox_mail_index_record rec;
 	struct dbox_save_mail *mails;
 	unsigned int i, count;
+	int ret = 0;
 
-	/* first write updated mail headers and collect all files we wrote to */
+	/* assign UIDs to mails */
 	mails = array_get_modifiable(&ctx->mails, &count);
-	for (i = 0; i < count; i++) {
-		if (dbox_save_mail_write_header(&mails[i], first_uid++) < 0)
-			return -1;
-	}
+	for (i = 0; i < count; i++)
+		mails[i].uid = first_uid++;
 
-	/* update append offsets in file headers */
+	/* update headers */
 	qsort(mails, count, sizeof(*mails), dbox_save_mail_file_cmp);
 	for (i = 0; i < count; i++) {
-		if (i > 0 && mails[i].file == mails[i-1].file) {
-			/* already written */
-			continue;
+		if (dbox_file_open_if_needed(mails[i].file) < 0 ||
+		    dbox_save_mail_write_header(&mails[i]) < 0) {
+			ret = -1;
+			break;
 		}
 
-		if (dbox_save_file_commit_header(&mails[i]) < 0) {
-			/* have to uncommit all changes so far */
-			for (; i > 0; i--) {
-				if (i > 1 &&
-				    mails[i-2].file == mails[i-1].file)
-					continue;
-				dbox_save_file_uncommit_header(&mails[i-1]);
+		/* write file header only once after all mails headers
+		   have been written */
+		if (i+1 == count || mails[i].file != mails[i+1].file) {
+			if (dbox_save_file_commit_header(&mails[i]) < 0) {
+				ret = -1;
+				break;
 			}
-			return -1;
+			dbox_file_close(mails[i].file);
 		}
 	}
+	if (ret < 0) {
+		/* have to uncommit all written changes */
+		for (; i > 0; i--) {
+			if (i > 1 && mails[i-2].file == mails[i-1].file)
+				continue;
+			dbox_save_file_uncommit_header(&mails[i-1]);
+		}
+		return -1;
+	}
 
 	/* set file_id / offsets to records */
 	if (dbox_index_append_assign_file_ids(ctx->append_ctx) < 0)