Mercurial > dovecot > original-hg > dovecot-1.2
changeset 7515:c14a2b0a3126 HEAD
Keep track of dovecot-uidlist mtime+size in index file. If it's up-to-date
(and unless indexes weren't updated at some point, it always is) and we're
saving a new message, don't bother reading the uidlist contents.
author | Timo Sirainen <tss@iki.fi> |
---|---|
date | Thu, 15 May 2008 07:01:26 +0300 |
parents | 3b818654abfa |
children | 7a7cf6662302 |
files | src/lib-storage/index/maildir/maildir-save.c src/lib-storage/index/maildir/maildir-storage.h src/lib-storage/index/maildir/maildir-sync-index.c src/lib-storage/index/maildir/maildir-sync.c src/lib-storage/index/maildir/maildir-sync.h src/lib-storage/index/maildir/maildir-uidlist.c src/lib-storage/index/maildir/maildir-uidlist.h |
diffstat | 7 files changed, 223 insertions(+), 102 deletions(-) [+] |
line wrap: on
line diff
--- a/src/lib-storage/index/maildir/maildir-save.c Thu May 15 06:05:39 2008 +0300 +++ b/src/lib-storage/index/maildir/maildir-save.c Thu May 15 07:01:26 2008 +0300 @@ -609,6 +609,11 @@ if (maildir_sync_index_begin(mbox, NULL, &ctx->sync_ctx) < 0) return -1; + if (maildir_sync_header_refresh(mbox) < 0) + return -1; + if (maildir_uidlist_refresh_fast_init(mbox->uidlist) < 0) + return 1; + ctx->keywords_sync_ctx = maildir_sync_get_keywords_sync_ctx(ctx->sync_ctx); @@ -659,7 +664,8 @@ i_assert(ctx->output == NULL); i_assert(ctx->finished); - sync_flags = MAILDIR_UIDLIST_SYNC_PARTIAL; + sync_flags = MAILDIR_UIDLIST_SYNC_PARTIAL | + MAILDIR_UIDLIST_SYNC_NOREFRESH; /* if we want to assign UIDs or keywords, we require uidlist lock */ if ((t->ictx.flags & MAILBOX_TRANSACTION_FLAG_ASSIGN_UIDS) == 0 &&
--- a/src/lib-storage/index/maildir/maildir-storage.h Thu May 15 06:05:39 2008 +0300 +++ b/src/lib-storage/index/maildir/maildir-storage.h Thu May 15 07:01:26 2008 +0300 @@ -57,6 +57,7 @@ struct maildir_index_header { uint32_t new_check_time, new_mtime, new_mtime_nsecs; uint32_t cur_check_time, cur_mtime, cur_mtime_nsecs; + uint32_t uidlist_mtime, uidlist_mtime_nsecs, uidlist_size; }; struct maildir_list_index_record {
--- a/src/lib-storage/index/maildir/maildir-sync-index.c Thu May 15 06:05:39 2008 +0300 +++ b/src/lib-storage/index/maildir/maildir-sync-index.c Thu May 15 07:01:26 2008 +0300 @@ -186,6 +186,46 @@ return 0; } +static bool +maildir_index_header_has_changed(const struct maildir_index_header *old_hdr, + const struct maildir_index_header *new_hdr) +{ +#define DIR_DELAYED_REFRESH(hdr, name) \ + ((hdr)->name ## _check_time <= \ + (hdr)->name ## _mtime + MAILDIR_SYNC_SECS) + + if (old_hdr->new_mtime != new_hdr->new_mtime || + old_hdr->new_mtime_nsecs != new_hdr->new_mtime_nsecs || + old_hdr->cur_mtime != new_hdr->cur_mtime || + old_hdr->cur_mtime_nsecs != new_hdr->cur_mtime_nsecs || + old_hdr->uidlist_mtime != new_hdr->uidlist_mtime || + old_hdr->uidlist_mtime_nsecs != new_hdr->uidlist_mtime_nsecs || + old_hdr->uidlist_size != new_hdr->uidlist_size) + return TRUE; + + return DIR_DELAYED_REFRESH(old_hdr, new) != + DIR_DELAYED_REFRESH(new_hdr, new) || + DIR_DELAYED_REFRESH(old_hdr, cur) != + DIR_DELAYED_REFRESH(new_hdr, cur); +} + +static void +maildir_sync_index_update_ext_header(struct maildir_index_sync_context *ctx) +{ + struct maildir_mailbox *mbox = ctx->mbox; + const void *data; + size_t data_size; + + mail_index_get_header_ext(mbox->ibox.view, mbox->maildir_ext_id, + &data, &data_size); + if (data_size != sizeof(mbox->maildir_hdr) || + maildir_index_header_has_changed(data, &mbox->maildir_hdr)) { + mail_index_update_header_ext(ctx->trans, mbox->maildir_ext_id, + 0, &mbox->maildir_hdr, + sizeof(mbox->maildir_hdr)); + } +} + int maildir_sync_index_finish(struct maildir_index_sync_context **_ctx, bool failed, bool cancel) { @@ -198,6 +238,8 @@ if (ret < 0 || cancel) mail_index_sync_rollback(&ctx->sync_ctx); else { + maildir_sync_index_update_ext_header(ctx); + /* Set syncing_commit=TRUE so that if any sync callbacks try to access mails which got lost (eg. expunge callback trying to open the file which was just unlinked) we don't try to @@ -219,43 +261,6 @@ return ret; } -static bool -maildir_index_header_has_changed(const struct maildir_index_header *old_hdr, - const struct maildir_index_header *new_hdr) -{ -#define DIR_DELAYED_REFRESH(hdr, name) \ - ((hdr)->name ## _check_time <= \ - (hdr)->name ## _mtime + MAILDIR_SYNC_SECS) - - if (old_hdr->new_mtime != new_hdr->new_mtime || - old_hdr->cur_mtime != new_hdr->cur_mtime || - old_hdr->new_mtime_nsecs != new_hdr->new_mtime_nsecs || - old_hdr->cur_mtime_nsecs != new_hdr->cur_mtime_nsecs) - return TRUE; - - return DIR_DELAYED_REFRESH(old_hdr, new) != - DIR_DELAYED_REFRESH(new_hdr, new) || - DIR_DELAYED_REFRESH(old_hdr, cur) != - DIR_DELAYED_REFRESH(new_hdr, cur); -} - -static void -maildir_index_update_ext_header(struct maildir_mailbox *mbox, - struct mail_index_transaction *trans) -{ - const void *data; - size_t data_size; - - mail_index_get_header_ext(mbox->ibox.view, mbox->maildir_ext_id, - &data, &data_size); - if (data_size != sizeof(mbox->maildir_hdr) || - maildir_index_header_has_changed(data, &mbox->maildir_hdr)) { - mail_index_update_header_ext(trans, mbox->maildir_ext_id, 0, - &mbox->maildir_hdr, - sizeof(mbox->maildir_hdr)); - } -} - int maildir_sync_index(struct maildir_index_sync_context *ctx, bool partial) { @@ -461,7 +466,6 @@ if (ctx->changed) mbox->maildir_hdr.cur_mtime = time(NULL); - maildir_index_update_ext_header(mbox, trans); if (uid_validity == 0) { uid_validity = hdr->uid_validity != 0 ?
--- a/src/lib-storage/index/maildir/maildir-sync.c Thu May 15 06:05:39 2008 +0300 +++ b/src/lib-storage/index/maildir/maildir-sync.c Thu May 15 07:01:26 2008 +0300 @@ -512,7 +512,7 @@ (move_count <= MAILDIR_RENAME_RESCAN_COUNT ? 0 : 1); } -static int maildir_header_refresh(struct maildir_mailbox *mbox) +int maildir_sync_header_refresh(struct maildir_mailbox *mbox) { const void *data; size_t data_size; @@ -529,10 +529,8 @@ return 0; } - if (data_size != sizeof(mbox->maildir_hdr)) - i_warning("Maildir %s: Invalid header record size", mbox->path); - else - memcpy(&mbox->maildir_hdr, data, sizeof(mbox->maildir_hdr)); + memcpy(&mbox->maildir_hdr, data, + I_MIN(sizeof(mbox->maildir_hdr), data_size)); return 0; } @@ -562,7 +560,7 @@ bool refreshed = FALSE, check_new = FALSE, check_cur = FALSE; if (mbox->maildir_hdr.new_mtime == 0) { - if (maildir_header_refresh(mbox) < 0) + if (maildir_sync_header_refresh(mbox) < 0) return -1; if (mbox->maildir_hdr.new_mtime == 0) { /* first sync */ @@ -577,7 +575,7 @@ if (DIR_DELAYED_REFRESH(hdr, new) || DIR_DELAYED_REFRESH(hdr, cur)) { /* refresh index and try again */ - if (maildir_header_refresh(mbox) < 0) + if (maildir_sync_header_refresh(mbox) < 0) return -1; refreshed = TRUE; @@ -610,7 +608,7 @@ break; /* refresh index and try again */ - if (maildir_header_refresh(mbox) < 0) + if (maildir_sync_header_refresh(mbox) < 0) return -1; refreshed = TRUE; }
--- a/src/lib-storage/index/maildir/maildir-sync.h Thu May 15 06:05:39 2008 +0300 +++ b/src/lib-storage/index/maildir/maildir-sync.h Thu May 15 07:01:26 2008 +0300 @@ -24,6 +24,8 @@ maildir_storage_sync_init(struct mailbox *box, enum mailbox_sync_flags flags); int maildir_storage_sync_force(struct maildir_mailbox *mbox, uint32_t uid); +int maildir_sync_header_refresh(struct maildir_mailbox *mbox); + int maildir_sync_index_begin(struct maildir_mailbox *mbox, struct maildir_sync_context *maildir_sync_ctx, struct maildir_index_sync_context **ctx_r);
--- a/src/lib-storage/index/maildir/maildir-uidlist.c Thu May 15 06:05:39 2008 +0300 +++ b/src/lib-storage/index/maildir/maildir-uidlist.c Thu May 15 07:01:26 2008 +0300 @@ -49,6 +49,7 @@ /* how many seconds to wait before overriding uidlist.lock */ #define UIDLIST_LOCK_STALE_TIMEOUT (60*2) +#define UIDLIST_VERSION 3 #define UIDLIST_COMPRESS_PERCENTAGE 75 #define UIDLIST_IS_LOCKED(uidlist) \ @@ -90,6 +91,7 @@ unsigned int recreate:1; unsigned int initial_read:1; + unsigned int initial_hdr_read:1; unsigned int initial_sync:1; }; @@ -124,7 +126,7 @@ struct maildir_uidlist_rec **rec_r); static int maildir_uidlist_lock_timeout(struct maildir_uidlist *uidlist, - bool nonblock) + bool nonblock, bool refresh) { struct mailbox *box = &uidlist->ibox->box; const char *control_dir, *path; @@ -170,22 +172,25 @@ uidlist->lock_count++; - /* make sure we have the latest changes before changing anything */ - if (maildir_uidlist_refresh(uidlist) < 0) { - maildir_uidlist_unlock(uidlist); - return -1; + if (refresh) { + /* make sure we have the latest changes before + changing anything */ + if (maildir_uidlist_refresh(uidlist) < 0) { + maildir_uidlist_unlock(uidlist); + return -1; + } } return 1; } int maildir_uidlist_lock(struct maildir_uidlist *uidlist) { - return maildir_uidlist_lock_timeout(uidlist, FALSE); + return maildir_uidlist_lock_timeout(uidlist, FALSE, TRUE); } int maildir_uidlist_try_lock(struct maildir_uidlist *uidlist) { - return maildir_uidlist_lock_timeout(uidlist, TRUE); + return maildir_uidlist_lock_timeout(uidlist, TRUE, TRUE); } int maildir_uidlist_lock_touch(struct maildir_uidlist *uidlist) @@ -293,6 +298,32 @@ (*rec1)->uid > (*rec2)->uid ? 1 : 0; } +static void ATTR_FORMAT(2, 3) +maildir_uidlist_set_corrupted(struct maildir_uidlist *uidlist, + const char *fmt, ...) +{ + struct mail_storage *storage = uidlist->ibox->box.storage; + va_list args; + + va_start(args, fmt); + mail_storage_set_critical(storage, "Broken file %s line %u: %s", + uidlist->path, uidlist->read_line_count, + t_strdup_vprintf(fmt, args)); + va_end(args); +} + +static void maildir_uidlist_update_hdr(struct maildir_uidlist *uidlist, + const struct stat *st) +{ + struct maildir_index_header *mhdr = &uidlist->mbox->maildir_hdr; + + mhdr->uidlist_mtime = st->st_mtime; +#ifdef HAVE_STAT_TV_NSEC + mhdr->uidlist_mtime_nsecs = st->st_mtim.tv_nsec; +#endif + mhdr->uidlist_size = st->st_size; +} + static void maildir_uidlist_records_array_delete(struct maildir_uidlist *uidlist, struct maildir_uidlist_rec *rec) @@ -341,20 +372,6 @@ return TRUE; } -static void ATTR_FORMAT(2, 3) -maildir_uidlist_set_corrupted(struct maildir_uidlist *uidlist, - const char *fmt, ...) -{ - struct mail_storage *storage = uidlist->ibox->box.storage; - va_list args; - - va_start(args, fmt); - mail_storage_set_critical(storage, "Broken file %s line %u: %s", - uidlist->path, uidlist->read_line_count, - t_strdup_vprintf(fmt, args)); - va_end(args); -} - static int maildir_uidlist_next(struct maildir_uidlist *uidlist, const char *line) { @@ -400,7 +417,7 @@ while (*line == ' ') line++; - if (uidlist->version == 3) { + if (uidlist->version == UIDLIST_VERSION) { /* read extended fields */ bool ret; @@ -474,7 +491,7 @@ return 0; } break; - case 3: + case UIDLIST_VERSION: ext_hdr = uidlist->hdr_extensions; str_truncate(ext_hdr, 0); while (*line != '\0') T_BEGIN { @@ -623,6 +640,7 @@ uidlist->fd_ino = st.st_ino; uidlist->fd_size = st.st_size; uidlist->last_read_offset = input->v_offset; + maildir_uidlist_update_hdr(uidlist, &st); } else { /* I/O error */ if (input->stream_errno == ESTALE && try_retry) @@ -643,18 +661,15 @@ } static int -maildir_uidlist_has_changed(struct maildir_uidlist *uidlist, bool *recreated_r) +maildir_uidlist_stat(struct maildir_uidlist *uidlist, struct stat *st_r) { struct mail_storage *storage = uidlist->ibox->box.storage; - struct stat st; - - *recreated_r = FALSE; if ((storage->flags & MAIL_STORAGE_FLAG_NFS_FLUSH_STORAGE) != 0) { nfs_flush_file_handle_cache(uidlist->path); nfs_flush_attr_cache_unlocked(uidlist->path); } - if (nfs_safe_stat(uidlist->path, &st) < 0) { + if (nfs_safe_stat(uidlist->path, st_r) < 0) { if (errno != ENOENT) { mail_storage_set_critical(storage, "stat(%s) failed: %m", uidlist->path); @@ -662,6 +677,20 @@ } return 0; } + return 1; +} + +static int +maildir_uidlist_has_changed(struct maildir_uidlist *uidlist, bool *recreated_r) +{ + struct mail_storage *storage = uidlist->ibox->box.storage; + struct stat st; + int ret; + + *recreated_r = FALSE; + + if ((ret = maildir_uidlist_stat(uidlist, &st)) <= 0) + return ret; if (st.st_ino != uidlist->fd_ino || !CMP_DEV_T(st.st_dev, uidlist->fd_dev)) { @@ -717,11 +746,43 @@ /* ESTALE - try reopening and rereading */ maildir_uidlist_close(uidlist); } - if (ret >= 0) + if (ret >= 0) { uidlist->initial_read = TRUE; + uidlist->initial_hdr_read = TRUE; + } return ret; } +int maildir_uidlist_refresh_fast_init(struct maildir_uidlist *uidlist) +{ + const struct maildir_index_header *mhdr = &uidlist->mbox->maildir_hdr; + const struct mail_index_header *hdr; + struct stat st; + int ret; + + if (uidlist->fd != -1) + return maildir_uidlist_refresh(uidlist); + + if ((ret = maildir_uidlist_stat(uidlist, &st)) < 0) + return ret; + + if (st.st_size == mhdr->uidlist_size && + st.st_mtime == mhdr->uidlist_mtime && +#ifdef HAVE_STAT_TV_NSEC + st.st_mtim.tv_nsec == mhdr->uidlist_mtime_nsecs +#endif + ) { + /* index is up-to-date */ + hdr = mail_index_get_header(uidlist->mbox->ibox.view); + uidlist->uid_validity = hdr->uid_validity; + uidlist->next_uid = hdr->next_uid; + uidlist->initial_hdr_read = TRUE; + return 1; + } else { + return maildir_uidlist_refresh(uidlist); + } +} + static struct maildir_uidlist_rec * maildir_uidlist_lookup_rec(struct maildir_uidlist *uidlist, uint32_t uid, unsigned int *idx_r) @@ -823,7 +884,7 @@ uint32_t maildir_uidlist_get_next_uid(struct maildir_uidlist *uidlist) { - return !uidlist->initial_read ? 0 : uidlist->next_uid; + return !uidlist->initial_hdr_read ? 0 : uidlist->next_uid; } void maildir_uidlist_set_uid_validity(struct maildir_uidlist *uidlist, @@ -917,7 +978,7 @@ if (output->offset == 0) { i_assert(first_idx == 0); - uidlist->version = 3; + uidlist->version = UIDLIST_VERSION; i_assert(uidlist->uid_validity != 0); i_assert(uidlist->next_uid > 0); @@ -984,6 +1045,8 @@ uoff_t file_size; int i, fd, ret; + i_assert(uidlist->initial_read); + control_dir = mailbox_list_get_path(box->storage->list, box->name, MAILBOX_LIST_PATH_TYPE_CONTROL); temp_path = t_strconcat(control_dir, @@ -1031,12 +1094,14 @@ "unlink(%s) failed: %m", temp_path); } } else if (fstat(fd, &st) < 0) { - i_error("fstat(%s) failed: %m", temp_path); + mail_storage_set_critical(box->storage, + "fstat(%s) failed: %m", temp_path); (void)close(fd); ret = -1; } else if (file_size != (uoff_t)st.st_size) { i_assert(!file_dotlock_is_locked(uidlist->dotlock)); - i_error("Maildir uidlist dotlock overridden: %s", + mail_storage_set_critical(box->storage, + "Maildir uidlist dotlock overridden: %s", uidlist->path); (void)close(fd); ret = -1; @@ -1048,6 +1113,7 @@ uidlist->fd_size = st.st_size; uidlist->last_read_offset = st.st_size; uidlist->recreate = FALSE; + maildir_uidlist_update_hdr(uidlist, &st); } return ret; } @@ -1066,9 +1132,29 @@ return ret; } +static bool maildir_uidlist_want_recreate(struct maildir_uidlist_sync_ctx *ctx) +{ + struct maildir_uidlist *uidlist = ctx->uidlist; + unsigned int min_rewrite_count; + + if (!uidlist->initial_read) + return FALSE; + + if (uidlist->recreate || uidlist->fd == -1 || + uidlist->version != UIDLIST_VERSION || + ctx->finish_change_counter != uidlist->change_counter) + return TRUE; + + min_rewrite_count = + (uidlist->read_records_count + ctx->new_files_count) * + UIDLIST_COMPRESS_PERCENTAGE / 100; + return min_rewrite_count >= array_count(&uidlist->records); +} + static int maildir_uidlist_sync_update(struct maildir_uidlist_sync_ctx *ctx) { struct maildir_uidlist *uidlist = ctx->uidlist; + struct stat st; uoff_t file_size; if (uidlist->uid_validity == 0) { @@ -1080,13 +1166,22 @@ hdr->uid_validity : (uint32_t)ioloop_time; } - if (ctx->uidlist->recreate || uidlist->fd == -1 || - uidlist->version != 3 || - ctx->finish_change_counter != ctx->uidlist->change_counter || - (uidlist->read_records_count + ctx->new_files_count) * - UIDLIST_COMPRESS_PERCENTAGE / 100 >= array_count(&uidlist->records)) + + if (maildir_uidlist_want_recreate(ctx)) return maildir_uidlist_recreate(uidlist); + if (uidlist->fd == -1) { + /* NOREFRESH flag used. we're just appending some messages. */ + i_assert(uidlist->initial_hdr_read); + + uidlist->fd = nfs_safe_open(uidlist->path, O_RDWR); + if (uidlist->fd == -1) { + mail_storage_set_critical(uidlist->ibox->box.storage, + "open(%s) failed: %m", uidlist->path); + return -1; + } + } + i_assert(ctx->first_unwritten_pos != (unsigned int)-1); if (lseek(uidlist->fd, 0, SEEK_END) < 0) { @@ -1099,7 +1194,19 @@ ctx->first_unwritten_pos, &file_size) < 0) return -1; - uidlist->last_read_offset = file_size; + if (fstat(uidlist->fd, &st) < 0) { + mail_storage_set_critical(uidlist->ibox->box.storage, + "fstat(%s) failed: %m", uidlist->path); + return -1; + } + if ((uoff_t)st.st_size != file_size) { + i_warning("%s: file size changed unexpectedly after write", + uidlist->path); + } else { + uidlist->fd_size = st.st_size; + uidlist->last_read_offset = st.st_size; + maildir_uidlist_update_hdr(uidlist, &st); + } return 0; } @@ -1124,27 +1231,26 @@ struct maildir_uidlist_sync_ctx **sync_ctx_r) { struct maildir_uidlist_sync_ctx *ctx; - bool locked; + bool nonblock, refresh, locked; int ret; - if ((sync_flags & MAILDIR_UIDLIST_SYNC_TRYLOCK) == 0) { - if ((ret = maildir_uidlist_lock(uidlist)) <= 0) + nonblock = (sync_flags & MAILDIR_UIDLIST_SYNC_TRYLOCK) != 0; + refresh = (sync_flags & MAILDIR_UIDLIST_SYNC_NOREFRESH) == 0; + + ret = maildir_uidlist_lock_timeout(uidlist, nonblock, refresh); + if (ret <= 0) { + if (ret < 0 || !nonblock) return ret; - } else { - if ((ret = maildir_uidlist_try_lock(uidlist)) < 0) - return -1; - if (ret == 0) { - /* couldn't lock it */ - if ((sync_flags & MAILDIR_UIDLIST_SYNC_FORCE) == 0) - return 0; - /* forcing the lock */ - } - } - locked = ret > 0; - if (!locked) { + /* couldn't lock it */ + if ((sync_flags & MAILDIR_UIDLIST_SYNC_FORCE) == 0) + return 0; + /* forcing the sync anyway */ if (maildir_uidlist_refresh(uidlist) < 0) return -1; + locked = FALSE; + } else { + locked = TRUE; } *sync_ctx_r = ctx = i_new(struct maildir_uidlist_sync_ctx, 1);
--- a/src/lib-storage/index/maildir/maildir-uidlist.h Thu May 15 06:05:39 2008 +0300 +++ b/src/lib-storage/index/maildir/maildir-uidlist.h Thu May 15 07:01:26 2008 +0300 @@ -11,7 +11,8 @@ MAILDIR_UIDLIST_SYNC_PARTIAL = 0x01, MAILDIR_UIDLIST_SYNC_KEEP_STATE = 0x02, MAILDIR_UIDLIST_SYNC_FORCE = 0x04, - MAILDIR_UIDLIST_SYNC_TRYLOCK = 0x08 + MAILDIR_UIDLIST_SYNC_TRYLOCK = 0x08, + MAILDIR_UIDLIST_SYNC_NOREFRESH = 0x10 }; enum maildir_uidlist_rec_flag { @@ -55,6 +56,9 @@ and storage has NFS_FLUSH flag set, the NFS attribute cache is flushed to make sure that we see the latest uidlist file. */ int maildir_uidlist_refresh(struct maildir_uidlist *uidlist); +/* Like maildir_uidlist_refresh(), but if uidlist isn't opened yet, try to + fill in the uidvalidity/nextuid from index file instead. */ +int maildir_uidlist_refresh_fast_init(struct maildir_uidlist *uidlist); /* Returns uidlist record for given filename, or NULL if not found. */ const char *