Mercurial > dovecot > original-hg > dovecot-1.2
changeset 1280:2ea9661542ee HEAD
UIDs are now saved into mbox file. added a few rewriting optimizations so
that we don't always have to rewrite the whole file when updating messages
at the beginning of file.
author | Timo Sirainen <tss@iki.fi> |
---|---|
date | Thu, 06 Mar 2003 21:23:44 +0200 |
parents | b14d1f375039 |
children | 043b71a06568 |
files | src/lib-index/mail-index-open.c src/lib-index/mail-index-update.c src/lib-index/mail-index.c src/lib-index/mail-index.h src/lib-index/maildir/maildir-index.c src/lib-index/mbox/mbox-append.c src/lib-index/mbox/mbox-index.c src/lib-index/mbox/mbox-index.h src/lib-index/mbox/mbox-rebuild.c src/lib-index/mbox/mbox-rewrite.c src/lib-index/mbox/mbox-sync-full.c src/lib-index/mbox/mbox-sync.c |
diffstat | 12 files changed, 393 insertions(+), 151 deletions(-) [+] |
line wrap: on
line diff
--- a/src/lib-index/mail-index-open.c Wed Mar 05 22:54:17 2003 +0200 +++ b/src/lib-index/mail-index-open.c Thu Mar 06 21:23:44 2003 +0200 @@ -126,6 +126,7 @@ may happen because of this. */ if (!index->sync_and_lock(index, MAIL_LOCK_SHARED, NULL)) return FALSE; + index->inconsistent = FALSE; /* we never want to keep shared lock if syncing happens to set it. either exclusive or nothing (NOTE: drop it directly, not through
--- a/src/lib-index/mail-index-update.c Wed Mar 05 22:54:17 2003 +0200 +++ b/src/lib-index/mail-index-update.c Thu Mar 06 21:23:44 2003 +0200 @@ -306,6 +306,11 @@ return !failed; } +void mail_index_update_abort(struct mail_index_update *update) +{ + pool_unref(update->pool); +} + static void update_field_full(struct mail_index_update *update, enum mail_data_field field, const void *value, size_t size,
--- a/src/lib-index/mail-index.c Wed Mar 05 22:54:17 2003 +0200 +++ b/src/lib-index/mail-index.c Thu Mar 06 21:23:44 2003 +0200 @@ -330,6 +330,10 @@ if (index->inconsistent) { /* index is in inconsistent state and nothing else than free() is allowed for it. */ + if (index->error == NULL) { + index->error = + i_strdup("Index is in inconsistent state"); + } return FALSE; } @@ -1029,7 +1033,7 @@ index->header->used_file_size - sizeof(*rec)) { /* we can just rollback */ index->header->used_file_size -= sizeof(*rec); - index->mmap_used_length += sizeof(*rec); + index->mmap_used_length -= sizeof(*rec); } else { /* mark it deleted */ update_first_hole(index, rec);
--- a/src/lib-index/mail-index.h Wed Mar 05 22:54:17 2003 +0200 +++ b/src/lib-index/mail-index.h Thu Mar 06 21:23:44 2003 +0200 @@ -343,6 +343,7 @@ (*update_begin)(struct mail_index *index, struct mail_index_record *rec); int (*update_end)(struct mail_index_update *update); + void (*update_abort)(struct mail_index_update *update); void (*update_field)(struct mail_index_update *update, enum mail_data_field field, @@ -420,6 +421,7 @@ unsigned int nodiskspace:1; unsigned int index_lock_timeout:1; unsigned int allow_new_custom_flags:1; + unsigned int mailbox_readonly:1; unsigned int mailbox_lock_timeout:1; }; @@ -436,7 +438,7 @@ 0, 0, 0, 0, 0, 0, { 0, 0, 0 }, 0, 0, \ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, \ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, \ - 0, 0, 0, 0 + 0, 0, 0, 0, 0 #endif /* defaults - same as above but prefixed with mail_index_. */ @@ -481,6 +483,7 @@ mail_index_update_begin(struct mail_index *index, struct mail_index_record *rec); int mail_index_update_end(struct mail_index_update *update); +void mail_index_update_abort(struct mail_index_update *update); void mail_index_update_field(struct mail_index_update *update, enum mail_data_field field, const char *value, size_t extra_space);
--- a/src/lib-index/maildir/maildir-index.c Wed Mar 05 22:54:17 2003 +0200 +++ b/src/lib-index/maildir/maildir-index.c Thu Mar 06 21:23:44 2003 +0200 @@ -266,6 +266,7 @@ mail_index_append_abort, mail_index_update_begin, mail_index_update_end, + mail_index_update_abort, mail_index_update_field, mail_index_update_field_raw, mail_index_get_last_error,
--- a/src/lib-index/mbox/mbox-append.c Wed Mar 05 22:54:17 2003 +0200 +++ b/src/lib-index/mbox/mbox-append.c Thu Mar 06 21:23:44 2003 +0200 @@ -19,7 +19,7 @@ const unsigned char *data; unsigned char md5_digest[16]; size_t size, pos; - int failed; + int ret; /* get the From-line */ pos = 0; @@ -41,7 +41,7 @@ "From-line not found where expected", index->mailbox_path); index->set_flags |= MAIL_INDEX_FLAG_FSCK; - return FALSE; + return -1; } /* parse the From-line */ @@ -60,7 +60,7 @@ /* add message to index */ rec = index->append_begin(index); if (rec == NULL) - return FALSE; + return -1; update = index->update_begin(index, rec); @@ -89,31 +89,75 @@ i_stream_seek(input, input->v_limit); i_stream_set_read_limit(input, 0); - /* save MD5 */ - md5_final(&ctx.md5, md5_digest); - index->update_field_raw(update, DATA_FIELD_MD5, - md5_digest, sizeof(md5_digest)); + ret = 1; + if (index->header->messages_count == 0 && + ctx.uid_validity != index->header->messages_count) { + /* UID validity is different */ + if (ctx.uid_validity == 0) { + /* we have to write it to mbox */ + if (index->mbox_lock_type != MAIL_LOCK_EXCLUSIVE) { + /* try again */ + ret = 0; + } else { + index->header->flags |= + MAIL_INDEX_FLAG_DIRTY_MESSAGES; + rec->index_flags |= INDEX_MAIL_FLAG_DIRTY; + } + } else { + /* change it in index */ + index->header->uid_validity = ctx.uid_validity; + index->header->next_uid = 1; + index->header->last_nonrecent_uid = 0; + index->inconsistent = TRUE; + } + } - if (!index->update_end(update)) { + if (ctx.uid >= index->header->next_uid) { + /* X-UID header looks ok */ + if (ret != 0) + index->header->next_uid = ctx.uid; + } else if (!index->mailbox_readonly) { + /* Write X-UID for it */ + if (index->mbox_lock_type != MAIL_LOCK_EXCLUSIVE) { + /* try again */ + ret = 0; + } else { + index->header->flags |= MAIL_INDEX_FLAG_DIRTY_MESSAGES; + rec->index_flags |= INDEX_MAIL_FLAG_DIRTY; + } + } else { + /* save MD5 */ + md5_final(&ctx.md5, md5_digest); + index->update_field_raw(update, DATA_FIELD_MD5, + md5_digest, sizeof(md5_digest)); + } + + if (ret <= 0) { + index->update_abort(update); index->append_abort(index, rec); - failed = TRUE; } else { - /* save message flags */ - rec->msg_flags = ctx.flags; - mail_index_mark_flag_changes(index, rec, 0, rec->msg_flags); - failed = FALSE; + if (!index->update_end(update)) { + index->append_abort(index, rec); + ret = -1; + } else { + /* save message flags */ + rec->msg_flags = ctx.flags; + mail_index_mark_flag_changes(index, rec, 0, + rec->msg_flags); + ret = 1; - if (!index->append_end(index, rec)) - failed = TRUE; + if (!index->append_end(index, rec)) + ret = -1; + } } mbox_header_free_context(&ctx); - - return !failed; + return ret; } int mbox_index_append(struct mail_index *index, struct istream *input) { + uoff_t offset; int ret; if (input->v_offset == input->v_size) { @@ -124,7 +168,8 @@ if (!index->set_lock(index, MAIL_LOCK_EXCLUSIVE)) return FALSE; - for (;;) { + do { + offset = input->v_offset; if (input->start_offset + input->v_offset != 0) { /* we're at the [\r]\n before the From-line, skip it */ @@ -139,16 +184,26 @@ } } - if (input->v_offset == input->v_size) + if (input->v_offset == input->v_size) { + ret = 1; break; + } t_push(); ret = mbox_index_append_next(index, input); t_pop(); - if (!ret) - return FALSE; + if (ret == 0) { + /* we want to rescan this message with exclusive + locking */ + i_stream_seek(input, offset); + } + } while (ret > 0); + + if (index->mbox_lock_type == MAIL_LOCK_EXCLUSIVE) { + /* Write missing X-IMAPbase and new/changed X-UID headers */ + return mbox_index_rewrite(index); } - return TRUE; + return ret >= 0; }
--- a/src/lib-index/mbox/mbox-index.c Wed Mar 05 22:54:17 2003 +0200 +++ b/src/lib-index/mbox/mbox-index.c Thu Mar 06 21:23:44 2003 +0200 @@ -9,6 +9,7 @@ #include "mail-index-data.h" #include "mail-custom-flags.h" +#include <stdlib.h> #include <unistd.h> #include <fcntl.h> #include <sys/stat.h> @@ -191,32 +192,33 @@ return flags; } -static int mbox_parse_imapbase(const unsigned char *value, size_t len, - struct mbox_header_context *ctx) +static void mbox_parse_imapbase(const unsigned char *value, size_t len, + struct mbox_header_context *ctx) { - const char **flag; + const char **flag, *str; + char *end; buffer_t *buf; size_t pos, start; enum mail_flags flags; unsigned int count; - int ret, spaces; - - /* skip <uid validity> and <last uid> fields */ - spaces = 0; - for (pos = 0; pos < len; pos++) { - if (value[pos] == ' ' && (pos == 0 || value[pos-1] != ' ')) { - if (++spaces == 2) - break; - } - } - - while (pos < len && value[pos] == ' ') pos++; - - if (pos == len) - return TRUE; + int ret; t_push(); + /* <uid validity> <last uid> */ + str = t_strndup(value, len); + ctx->uid_validity = strtoul(str, &end, 10); + ctx->uid_last = strtoul(end, &end, 10); + pos = end - str; + + while (pos < len && value[pos] == ' ') + pos++; + + if (pos == len) { + t_pop(); + return; + } + /* we're at the 3rd field now, which begins the list of custom flags */ buf = buffer_create_dynamic(data_stack_pool, MAIL_CUSTOM_FLAGS_COUNT, MAX_CUSTOM_FLAGS); @@ -243,8 +245,6 @@ flag, count); t_pop(); - - return ret > 0; } void mbox_header_cb(struct message_part *part __attr_unused__, @@ -348,6 +348,14 @@ ID's but don't blindly trust this header alone as it could just as easily come from the remote. */ fixed = memcasecmp(name, "X-Delivery-ID:", 13) == 0; + } else if (name_len == 5 && + memcasecmp(name, "X-UID", 5) == 0) { + ctx->uid = 0; + for (i = 0; i < value_len; i++) { + if (value[i] < '0' || value[i] > '9') + break; + ctx->uid = ctx->uid * 10 + (value[i]-'0'); + } } else if (name_len == 8 && memcasecmp(name, "X-Status", 8) == 0) { /* update message flags */ @@ -359,8 +367,7 @@ ctx->custom_flags); } else if (name_len == 10 && memcasecmp(name, "X-IMAPbase", 10) == 0) { - /* update list of custom message flags */ - (void)mbox_parse_imapbase(value, value_len, ctx); + mbox_parse_imapbase(value, value_len, ctx); } break; } @@ -799,6 +806,18 @@ return TRUE; } +static int mbox_index_append_end(struct mail_index *index, + struct mail_index_record *rec) +{ + if (!mail_index_append_end(index, rec)) + return FALSE; + + /* update last_uid in X-IMAPbase */ + index->header->flags |= MAIL_INDEX_FLAG_DIRTY_MESSAGES | + MAIL_INDEX_FLAG_DIRTY_CUSTOMFLAGS; + return TRUE; +} + struct mail_index mbox_index = { mail_index_open, mbox_index_free, @@ -820,10 +839,11 @@ mbox_index_expunge, mbox_index_update_flags, mail_index_append_begin, - mail_index_append_end, + mbox_index_append_end, mail_index_append_abort, mail_index_update_begin, mail_index_update_end, + mail_index_update_abort, mail_index_update_field, mail_index_update_field_raw, mail_index_get_last_error,
--- a/src/lib-index/mbox/mbox-index.h Wed Mar 05 22:54:17 2003 +0200 +++ b/src/lib-index/mbox/mbox-index.h Thu Mar 06 21:23:44 2003 +0200 @@ -11,6 +11,8 @@ struct md5_context md5; int received; + unsigned int uid_validity, uid_last, uid; + struct istream *input; uoff_t content_length; int set_read_limit;
--- a/src/lib-index/mbox/mbox-rebuild.c Wed Mar 05 22:54:17 2003 +0200 +++ b/src/lib-index/mbox/mbox-rebuild.c Thu Mar 06 21:23:44 2003 +0200 @@ -1,22 +1,17 @@ /* Copyright (C) 2002 Timo Sirainen */ #include "lib.h" -#include "istream.h" #include "mbox-index.h" -#include "mbox-lock.h" #include "mail-index-data.h" #include "mail-index-util.h" #include <unistd.h> -#include <fcntl.h> #include <sys/stat.h> #include <sys/mman.h> int mbox_index_rebuild(struct mail_index *index) { - struct istream *input; struct stat st; - int failed; i_assert(index->lock_type != MAIL_LOCK_SHARED); @@ -34,7 +29,8 @@ /* update indexid, which also means that our state has completely changed */ index->indexid = index->header->indexid; - index->inconsistent = TRUE; + if (index->opened) + index->inconsistent = TRUE; if (msync(index->mmap_base, sizeof(struct mail_index_header), MS_SYNC) < 0) @@ -44,16 +40,7 @@ if (!mail_index_data_reset(index->data)) return FALSE; - input = mbox_get_stream(index, 0, MAIL_LOCK_SHARED); - if (input == NULL) - return FALSE; - - mbox_skip_empty_lines(input); - failed = !mbox_index_append(index, input); - - i_stream_unref(input); - - if (failed) + if (!mbox_sync_full(index)) return FALSE; /* update sync stamp */ @@ -64,5 +51,6 @@ /* rebuild is complete - remove the flag */ index->header->flags &= ~(MAIL_INDEX_FLAG_REBUILD|MAIL_INDEX_FLAG_FSCK); + index->set_flags &= ~MAIL_INDEX_FLAG_REBUILD; return TRUE; }
--- a/src/lib-index/mbox/mbox-rewrite.c Wed Mar 05 22:54:17 2003 +0200 +++ b/src/lib-index/mbox/mbox-rewrite.c Thu Mar 06 21:23:44 2003 +0200 @@ -4,6 +4,7 @@ #include "ioloop.h" #include "istream.h" #include "ostream.h" +#include "file-set-size.h" #include "str.h" #include "write-full.h" #include "mbox-index.h" @@ -15,13 +16,14 @@ #include <stdlib.h> #include <unistd.h> #include <fcntl.h> +#include <sys/stat.h> struct mbox_rewrite_context { struct ostream *output; int failed; uoff_t content_length; - unsigned int seq; + unsigned int seq, uid; unsigned int msg_flags; const char **custom_flags; @@ -30,6 +32,7 @@ unsigned int ximapbase_found:1; unsigned int xkeywords_found:1; + unsigned int xuid_found:1; unsigned int status_found:1; unsigned int xstatus_found:1; unsigned int content_length_found:1; @@ -105,6 +108,18 @@ return TRUE; } +static int mbox_write_xuid(struct mbox_rewrite_context *ctx) +{ + const char *str; + + str = t_strdup_printf("X-UID: %u\n", ctx->uid); + + if (o_stream_send_str(ctx->output, str) < 0) + return FALSE; + + return TRUE; +} + static int mbox_write_xkeywords(struct mbox_rewrite_context *ctx, const char *x_keywords) { @@ -250,7 +265,6 @@ { struct mbox_rewrite_context *ctx = context; const char *str; - char *end; if (ctx->failed) return; @@ -269,20 +283,12 @@ (void)mbox_write_xkeywords(ctx, str); } else if (name_len == 10 && memcasecmp(name, "X-IMAPbase", 10) == 0) { if (ctx->seq == 1) { - /* temporarily copy the value to make sure we - don't overflow it */ - const char *str; - - t_push(); - str = t_strndup(value, value_len); - ctx->uid_validity = strtoul(str, &end, 10); - while (*end == ' ') end++; - ctx->uid_last = strtoul(end, &end, 10); - t_pop(); - ctx->ximapbase_found = TRUE; (void)mbox_write_ximapbase(ctx); } + } else if (name_len == 5 && memcasecmp(name, "X-UID", 5) == 0) { + ctx->xuid_found = TRUE; + (void)mbox_write_xuid(ctx); } else if (name_len == 14 && memcasecmp(name, "Content-Length", 14) == 0) { ctx->content_length_found = TRUE; @@ -331,10 +337,12 @@ /* parse the header, write the fields we don't want to change */ memset(&ctx, 0, sizeof(ctx)); ctx.output = output; + ctx.content_length = body_size; ctx.seq = seq; - ctx.content_length = body_size; + ctx.uid = rec->uid; ctx.msg_flags = rec->msg_flags; - ctx.uid_validity = index->header->uid_validity-1; + ctx.uid_validity = index->header->uid_validity; + ctx.uid_last = index->header->next_uid-1; ctx.custom_flags = mail_custom_flags_list_get(index->custom_flags); i_stream_set_read_limit(input, input->v_offset + hdr_size); @@ -349,12 +357,14 @@ (void)mbox_write_ximapbase(&ctx); } - if (!ctx.xkeywords_found) - (void)mbox_write_xkeywords(&ctx, NULL); if (!ctx.status_found) (void)mbox_write_status(&ctx, NULL); if (!ctx.xstatus_found) (void)mbox_write_xstatus(&ctx, NULL); + if (!ctx.xkeywords_found) + (void)mbox_write_xkeywords(&ctx, NULL); + if (!ctx.xuid_found) + (void)mbox_write_xuid(&ctx); if (!ctx.content_length_found) (void)mbox_write_content_length(&ctx); @@ -366,31 +376,50 @@ return TRUE; } -static int fd_copy(int in_fd, int out_fd, uoff_t out_offset) +static int fd_copy(struct mail_index *index, int in_fd, int out_fd, + uoff_t out_offset, uoff_t size) { struct istream *input; struct ostream *output; + struct stat st; int ret; i_assert(out_offset <= OFF_T_MAX); - if (lseek(out_fd, (off_t)out_offset, SEEK_SET) < 0) + /* first grow the file to wanted size, to make sure we don't run out + of disk space */ + if (fstat(out_fd, &st) < 0) { + mbox_set_syscall_error(index, "fstat()"); return -1; + } + + if ((uoff_t)st.st_size < out_offset + size) { + if (file_set_size(out_fd, (off_t)(out_offset + size)) < 0) { + mbox_set_syscall_error(index, "file_set_size()"); + (void)ftruncate(out_fd, st.st_size); + return -1; + } + } + + if (lseek(out_fd, (off_t)out_offset, SEEK_SET) < 0) { + mbox_set_syscall_error(index, "lseek()"); + (void)ftruncate(out_fd, st.st_size); + return -1; + } t_push(); input = i_stream_create_mmap(in_fd, data_stack_pool, 1024*256, 0, 0, FALSE); + i_stream_set_read_limit(input, size); + output = o_stream_create_file(out_fd, data_stack_pool, 1024, 0, FALSE); o_stream_set_blocking(output, 60000, NULL, NULL); ret = o_stream_send_istream(output, input); - if (ret < 0) + if (ret < 0) { errno = output->stream_errno; - else { - /* we may have shrinked the file */ - i_assert(out_offset + input->v_size <= OFF_T_MAX); - ret = ftruncate(out_fd, (off_t) (out_offset + input->v_size)); + mbox_set_syscall_error(index, "o_stream_send_istream()"); } o_stream_unref(output); @@ -400,6 +429,47 @@ return ret; } +static int dirty_flush(struct mail_index *index, uoff_t dirty_offset, + struct ostream *output, int output_fd) +{ + if (output->offset == 0) + return TRUE; + + if (o_stream_flush(output) < 0) { + mbox_set_syscall_error(index, "o_stream_flush()"); + return FALSE; + } + + /* POSSIBLE DATA LOSS HERE. We're writing to the mbox file, + so if we get killed here before finished, we'll lose some + bytes. I can't really think of any way to fix this, + rename() is problematic too especially because of file + locking issues (new mail could be lost). + + Usually we're moving the data by just a few bytes, so + the data loss should never be more than those few bytes.. + If we moved more, we could have written the file from end + to beginning in blocks (it'd be a bit slow to do it in + blocks of ~1-10 bytes which is the usual case, so we don't + bother). + + Also, we might as well be shrinking the file, in which + case we can't lose data. */ + if (fd_copy(index, output_fd, index->mbox_fd, + dirty_offset, output->offset) < 0) + return FALSE; + + /* All ok. Just make sure the timestamps of index and + mbox differ, so index will be updated at next sync */ + index->file_sync_stamp = ioloop_time-61; + + if (o_stream_seek(output, 0) < 0) { + mbox_set_syscall_error(index, "o_stream_seek()"); + return FALSE; + } + return TRUE; +} + #define INDEX_DIRTY_FLAGS \ (MAIL_INDEX_FLAG_DIRTY_MESSAGES | MAIL_INDEX_FLAG_DIRTY_CUSTOMFLAGS) @@ -415,18 +485,25 @@ uoff_t offset, hdr_size, body_size, dirty_offset; const char *path; unsigned int seq; - int tmp_fd, failed, dirty_found, rewrite; + int tmp_fd, failed, dirty_found, rewrite, no_locking; + + i_assert(index->lock_type == MAIL_LOCK_UNLOCK || + (index->lock_type == MAIL_LOCK_EXCLUSIVE && + index->mbox_lock_type == MAIL_LOCK_EXCLUSIVE)); - i_assert(index->lock_type == MAIL_LOCK_UNLOCK); - - if (!index->set_lock(index, MAIL_LOCK_SHARED)) - return FALSE; + no_locking = index->mbox_lock_type == MAIL_LOCK_EXCLUSIVE; + if (!no_locking) { + if (!index->set_lock(index, MAIL_LOCK_SHARED)) + return FALSE; + } rewrite = (index->header->flags & INDEX_DIRTY_FLAGS) && index->header->messages_count > 0; - if (!index->set_lock(index, MAIL_LOCK_UNLOCK)) - return FALSE; + if (!no_locking) { + if (!index->set_lock(index, MAIL_LOCK_UNLOCK)) + return FALSE; + } if (!rewrite) { /* no need to rewrite */ @@ -436,11 +513,14 @@ tmp_fd = -1; input = NULL; failed = TRUE; rewrite = FALSE; do { - if (!index->set_lock(index, MAIL_LOCK_EXCLUSIVE)) - break; + if (!no_locking) { + if (!index->set_lock(index, MAIL_LOCK_EXCLUSIVE)) + break; - if (!index->sync_and_lock(index, MAIL_LOCK_EXCLUSIVE, NULL)) - break; + if (!index->sync_and_lock(index, MAIL_LOCK_EXCLUSIVE, + NULL)) + break; + } input = mbox_get_stream(index, 0, MAIL_LOCK_EXCLUSIVE); if (input == NULL) @@ -461,8 +541,10 @@ } while (0); if (!rewrite) { - if (!index->set_lock(index, MAIL_LOCK_UNLOCK)) - failed = TRUE; + if (!no_locking) { + if (!index->set_lock(index, MAIL_LOCK_UNLOCK)) + failed = TRUE; + } if (input != NULL) i_stream_unref(input); return !failed; @@ -532,11 +614,22 @@ break; } - /* write body */ - offset += body_size; - if (!mbox_write(index, input, output, offset)) { - failed = TRUE; - break; + if (dirty_found && + offset - dirty_offset == output->offset) { + /* no need to write more, flush */ + if (!dirty_flush(index, dirty_offset, + output, tmp_fd)) { + failed = TRUE; + break; + } + dirty_found = FALSE; + } else { + /* write body */ + offset += body_size; + if (!mbox_write(index, input, output, offset)) { + failed = TRUE; + break; + } } } @@ -544,14 +637,8 @@ rec = index->next(index, rec); } - if (!dirty_found) { - index_set_error(index, "Expected dirty messages not found " - "from mbox file %s", index->mailbox_path); - failed = TRUE; - } - - if (!failed) { - /* always end with a \n */ + if (!failed && dirty_found) { + /* end with \n */ (void)o_stream_send(output, "\n", 1); } @@ -561,39 +648,33 @@ failed = TRUE; } + if (!failed && dirty_found) { + uoff_t dirty_size = output->offset; + + if (!dirty_flush(index, dirty_offset, output, tmp_fd)) + failed = TRUE; + else { + /* we may have shrinked the file */ + i_assert(dirty_offset + dirty_size <= OFF_T_MAX); + if (ftruncate(index->mbox_fd, + (off_t)(dirty_offset + dirty_size)) < 0) { + mbox_set_syscall_error(index, "ftruncate()"); + failed = TRUE; + } + } + } + + if (!failed) + reset_dirty_flags(index); + i_stream_unref(input); o_stream_unref(output); - if (!failed) { - /* POSSIBLE DATA LOSS HERE. We're writing to the mbox file, - so if we get killed here before finished, we'll lose some - bytes. I can't really think of any way to fix this, - rename() is problematic too especially because of file - locking issues (new mail could be lost). - - Usually we're moving the data by just a few bytes, so - the data loss should never be more than those few bytes.. - If we moved more, we could have written the file from end - to beginning in blocks (it'd be a bit slow to do it in - blocks of ~1-10 bytes which is the usual case, so we don't - bother). - - Also, we might as well be shrinking the file, in which - case we can't lose data. */ - if (fd_copy(tmp_fd, index->mbox_fd, dirty_offset) == 0) { - /* All ok. Just make sure the timestamps of index and - mbox differ, so index will be updated at next sync */ - index->file_sync_stamp = ioloop_time-61; - reset_dirty_flags(index); - } else { - mbox_set_syscall_error(index, "fd_copy()"); + if (!no_locking) { + if (!index->set_lock(index, MAIL_LOCK_UNLOCK)) failed = TRUE; - } } - if (!index->set_lock(index, MAIL_LOCK_UNLOCK)) - failed = TRUE; - (void)unlink(path); if (close(tmp_fd) < 0)
--- a/src/lib-index/mbox/mbox-sync-full.c Wed Mar 05 22:54:17 2003 +0200 +++ b/src/lib-index/mbox/mbox-sync-full.c Thu Mar 06 21:23:44 2003 +0200 @@ -11,6 +11,7 @@ #include <unistd.h> #include <fcntl.h> +#include <sys/stat.h> static void skip_line(struct istream *input) { @@ -29,17 +30,20 @@ } } -static int verify_header_md5sum(struct mail_index *index, - struct mail_index_record *rec, - unsigned char current_digest[16]) +static int verify_header(struct mail_index *index, + struct mail_index_record *rec, + unsigned int uid, unsigned char current_digest[16]) { const unsigned char *old_digest; size_t size; /* MD5 sums must match */ old_digest = index->lookup_field_raw(index, rec, DATA_FIELD_MD5, &size); - return old_digest != NULL && size >= 16 && - memcmp(old_digest, current_digest, 16) == 0; + if (old_digest == NULL) + return uid == rec->uid; + + return size >= 16 && memcmp(old_digest, current_digest, 16) == 0 && + (uid == 0 || uid == rec->uid); } static int mail_update_header_size(struct mail_index *index, @@ -95,6 +99,26 @@ return TRUE; } +static int mbox_check_uidvalidity(struct mail_index *index, + unsigned int uid_validity) +{ + if (uid_validity == index->header->uid_validity) + return TRUE; + + index->header->flags |= MAIL_INDEX_FLAG_DIRTY_MESSAGES | + MAIL_INDEX_FLAG_DIRTY_CUSTOMFLAGS; + + if (uid_validity == 0) { + /* X-IMAPbase header isn't written yet */ + } else { + /* UID validity has changed - rebuild whole index */ + index->set_flags |= MAIL_INDEX_FLAG_REBUILD; + return FALSE; + } + + return TRUE; +} + static int match_next_record(struct mail_index *index, struct mail_index_record *rec, unsigned int seq, struct istream *input, @@ -139,13 +163,27 @@ mbox_header_cb, &ctx); md5_final(&ctx.md5, current_digest); + if (seq == 1) { + if (!mbox_check_uidvalidity(index, + ctx.uid_validity)) { + /* uidvalidity changed, abort */ + break; + } + + if (ctx.uid_last >= index->header->next_uid) { + /* last_uid larger than ours */ + index->header->next_uid = + ctx.uid_last+1; + } + } + mbox_header_free_context(&ctx); i_stream_set_read_limit(input, 0); body_offset = input->v_offset; } - if (verify_header_md5sum(index, rec, current_digest) && + if (verify_header(index, rec, ctx.uid, current_digest) && mbox_verify_end_of_body(input, body_offset + body_size)) { /* valid message */ update = index->update_begin(index, rec); @@ -211,7 +249,7 @@ /* first make sure we start with a "From " line. If file is too small, we'll just treat it as empty mbox file. */ if (i_stream_read_data(input, &data, &size, 5) > 0 && - strncmp((const char *) data, "From ", 5) != 0) { + memcmp(data, "From ", 5) != 0) { index_set_error(index, "File isn't in mbox format: %s", index->mailbox_path); return FALSE; @@ -269,11 +307,12 @@ } if (!dirty && (index->header->flags & MAIL_INDEX_FLAG_DIRTY_MESSAGES)) { - /* no flags were dirty anymore, no need to rewrite */ + /* no flags are dirty anymore, no need to rewrite */ index->header->flags &= ~MAIL_INDEX_FLAG_DIRTY_MESSAGES; } - if (input->v_offset == input->v_size) + if (input->v_offset == input->v_size || + (index->set_flags & MAIL_INDEX_FLAG_REBUILD)) return TRUE; else return mbox_index_append(index, input); @@ -282,6 +321,8 @@ int mbox_sync_full(struct mail_index *index) { struct istream *input; + struct stat orig_st, st; + uoff_t continue_offset; int failed; i_assert(index->lock_type == MAIL_LOCK_EXCLUSIVE); @@ -290,8 +331,43 @@ if (input == NULL) return FALSE; - failed = !mbox_sync_from_stream(index, input); - i_stream_unref(input); + if (fstat(index->mbox_fd, &orig_st) < 0) { + mbox_set_syscall_error(index, "fstat()"); + continue_offset = (uoff_t)-1; + failed = TRUE; + } else { + failed = !mbox_sync_from_stream(index, input); + continue_offset = failed || input->v_offset == input->v_size || + (index->set_flags & MAIL_INDEX_FLAG_REBUILD) ? + (uoff_t)-1 : input->v_offset; + i_stream_unref(input); + } + + if (continue_offset != (uoff_t)-1) { + /* mbox_index_append() stopped, which means that it wants + write access to mbox. if mbox hasn't changed after + unlock+lock, we should be able to safely continue where we + were left off last time. otherwise do full resync. */ + if (!mbox_unlock(index)) + return FALSE; + + input = mbox_get_stream(index, 0, MAIL_LOCK_EXCLUSIVE); + if (input == NULL) + return FALSE; + + if (fstat(index->mbox_fd, &st) < 0) { + mbox_set_syscall_error(index, "fstat()"); + failed = TRUE; + } else if (st.st_mtime == orig_st.st_mtime && + st.st_size == orig_st.st_size) { + i_stream_seek(input, continue_offset); + failed = !mbox_index_append(index, input); + } else { + failed = !mbox_sync_from_stream(index, input); + } + + i_stream_unref(input); + } return !failed; }
--- a/src/lib-index/mbox/mbox-sync.c Wed Mar 05 22:54:17 2003 +0200 +++ b/src/lib-index/mbox/mbox-sync.c Thu Mar 06 21:23:44 2003 +0200 @@ -134,6 +134,12 @@ if (!mbox_lock_and_sync_full(index, data_lock_type)) return FALSE; + if ((index->set_flags & MAIL_INDEX_FLAG_REBUILD) != 0) { + /* uidvalidity probably changed, rebuild */ + if (!index->rebuild(index)) + return FALSE; + } + index->mbox_size = filesize; }