Mercurial > dovecot > original-hg > dovecot-1.2
changeset 8876:88aeadb32151 HEAD
Added maildir_very_dirty_syncs setting.
| author | Timo Sirainen <tss@iki.fi> |
|---|---|
| date | Mon, 30 Mar 2009 20:27:14 -0400 |
| parents | 629004d13ec4 |
| children | e5658894cd13 |
| files | src/lib-storage/index/maildir/maildir-keywords.c src/lib-storage/index/maildir/maildir-keywords.h src/lib-storage/index/maildir/maildir-storage.c src/lib-storage/index/maildir/maildir-storage.h src/lib-storage/index/maildir/maildir-sync-index.c src/lib-storage/index/maildir/maildir-sync.c src/lib-storage/index/maildir/maildir-uidlist.c src/lib-storage/index/maildir/maildir-uidlist.h src/lib-storage/index/maildir/maildir-util.c src/master/mail-process.c src/master/master-settings-defs.c src/master/master-settings.c src/master/master-settings.h |
| diffstat | 13 files changed, 150 insertions(+), 26 deletions(-) [+] |
line wrap: on
line diff
--- a/src/lib-storage/index/maildir/maildir-keywords.c Mon Mar 30 20:27:02 2009 -0400 +++ b/src/lib-storage/index/maildir/maildir-keywords.c Mon Mar 30 20:27:14 2009 -0400 @@ -47,6 +47,7 @@ const ARRAY_TYPE(keywords) *keywords; ARRAY_DEFINE(idx_to_chr, char); unsigned int chridx_to_idx[MAILDIR_MAX_KEYWORDS]; + bool readonly; }; struct maildir_keywords *maildir_keywords_init(struct maildir_mailbox *mbox) @@ -404,6 +405,17 @@ return ctx; } +struct maildir_keywords_sync_ctx * +maildir_keywords_sync_init_readonly(struct maildir_keywords *mk, + struct mail_index *index) +{ + struct maildir_keywords_sync_ctx *ctx; + + ctx = maildir_keywords_sync_init(mk, index); + ctx->readonly = TRUE; + return ctx; +} + void maildir_keywords_sync_deinit(struct maildir_keywords_sync_ctx **_ctx) { struct maildir_keywords_sync_ctx *ctx = *_ctx; @@ -455,13 +467,17 @@ const char *const *name_p; char *chr_p; unsigned int chridx; + int ret; chr_p = array_idx_modifiable(&ctx->idx_to_chr, idx); if (*chr_p != '\0') return *chr_p; name_p = array_idx(ctx->keywords, idx); - if (maildir_keywords_lookup_or_create(ctx->mk, *name_p, &chridx) <= 0) + ret = !ctx->readonly ? + maildir_keywords_lookup_or_create(ctx->mk, *name_p, &chridx) : + maildir_keywords_lookup(ctx->mk, *name_p, &chridx); + if (ret <= 0) return '\0'; *chr_p = chridx + MAILDIR_KEYWORD_FIRST;
--- a/src/lib-storage/index/maildir/maildir-keywords.h Mon Mar 30 20:27:02 2009 -0400 +++ b/src/lib-storage/index/maildir/maildir-keywords.h Mon Mar 30 20:27:14 2009 -0400 @@ -19,6 +19,10 @@ struct maildir_keywords_sync_ctx * maildir_keywords_sync_init(struct maildir_keywords *mk, struct mail_index *index); +/* Don't try to add any nonexisting keywords */ +struct maildir_keywords_sync_ctx * +maildir_keywords_sync_init_readonly(struct maildir_keywords *mk, + struct mail_index *index); void maildir_keywords_sync_deinit(struct maildir_keywords_sync_ctx **ctx); /* Returns keyword index. */
--- a/src/lib-storage/index/maildir/maildir-storage.c Mon Mar 30 20:27:02 2009 -0400 +++ b/src/lib-storage/index/maildir/maildir-storage.c Mon Mar 30 20:27:14 2009 -0400 @@ -442,6 +442,7 @@ index = index_storage_alloc(&storage->storage, name, flags, MAILDIR_INDEX_PREFIX); mbox->ibox.index = index; + mbox->very_dirty_syncs = getenv("MAILDIR_VERY_DIRTY_SYNCS") != NULL; /* for shared mailboxes get the create mode from the permissions of dovecot-shared file. */ @@ -869,6 +870,8 @@ timeout_remove(&mbox->keep_lock_to); } + if (mbox->flags_view != NULL) + mail_index_view_close(&mbox->flags_view); if (mbox->keywords != NULL) maildir_keywords_deinit(&mbox->keywords); maildir_uidlist_deinit(&mbox->uidlist);
--- a/src/lib-storage/index/maildir/maildir-storage.h Mon Mar 30 20:27:02 2009 -0400 +++ b/src/lib-storage/index/maildir/maildir-storage.h Mon Mar 30 20:27:14 2009 -0400 @@ -82,6 +82,7 @@ struct maildir_mailbox { struct index_mailbox ibox; struct maildir_storage *storage; + struct mail_index_view *flags_view; const char *path; struct timeout *keep_lock_to; @@ -94,6 +95,7 @@ uint32_t maildir_ext_id; unsigned int syncing_commit:1; + unsigned int very_dirty_syncs:1; }; struct maildir_transaction_context {
--- a/src/lib-storage/index/maildir/maildir-sync-index.c Mon Mar 30 20:27:02 2009 -0400 +++ b/src/lib-storage/index/maildir/maildir-sync-index.c Mon Mar 30 20:27:14 2009 -0400 @@ -387,6 +387,8 @@ uint32_t first_uid; unsigned int changes = 0; int ret = 0; + time_t time_before_sync; + struct stat st; bool expunged, full_rescan = FALSE; i_assert(!mbox->syncing_commit); @@ -412,6 +414,7 @@ } hdr_next_uid = hdr->next_uid; + time_before_sync = time(NULL); mbox->syncing_commit = TRUE; seq = prev_uid = 0; first_recent_uid = I_MAX(hdr->first_recent_uid, 1); t_array_init(&ctx->keywords, MAILDIR_MAX_KEYWORDS); @@ -560,8 +563,12 @@ if (mbox->ibox.box.v.sync_notify != NULL) mbox->ibox.box.v.sync_notify(&mbox->ibox.box, 0, 0); - if (ctx->changed) - mbox->maildir_hdr.cur_mtime = time(NULL); + if (stat(t_strconcat(mbox->path, "/cur", NULL), &st) == 0) { + mbox->maildir_hdr.new_check_time = + I_MAX(st.st_mtime, time_before_sync); + mbox->maildir_hdr.cur_mtime = st.st_mtime; + mbox->maildir_hdr.cur_mtime_nsecs = ST_MTIME_NSEC(st); + } if (uid_validity == 0) { uid_validity = hdr->uid_validity != 0 ? hdr->uid_validity :
--- a/src/lib-storage/index/maildir/maildir-sync.c Mon Mar 30 20:27:02 2009 -0400 +++ b/src/lib-storage/index/maildir/maildir-sync.c Mon Mar 30 20:27:14 2009 -0400 @@ -500,10 +500,23 @@ } if (dir_changed) { - if (new_dir) - ctx->mbox->maildir_hdr.new_mtime = now; - else - ctx->mbox->maildir_hdr.cur_mtime = now; + /* save the exact new times. the new mtimes should be >= + "now", but just in case something weird happens and mtime + doesn't update, use "now". */ + if (stat(ctx->new_dir, &st) == 0) { + ctx->mbox->maildir_hdr.new_check_time = + I_MAX(st.st_mtime, now); + ctx->mbox->maildir_hdr.new_mtime = st.st_mtime; + ctx->mbox->maildir_hdr.new_mtime_nsecs = + ST_MTIME_NSEC(st); + } + if (stat(ctx->cur_dir, &st) == 0) { + ctx->mbox->maildir_hdr.new_check_time = + I_MAX(st.st_mtime, now); + ctx->mbox->maildir_hdr.cur_mtime = st.st_mtime; + ctx->mbox->maildir_hdr.cur_mtime_nsecs = + ST_MTIME_NSEC(st); + } } return ret < 0 ? -1 : @@ -564,7 +577,7 @@ /* try to avoid stat()ing by first checking delayed changes */ if (DIR_DELAYED_REFRESH(hdr, new) || - DIR_DELAYED_REFRESH(hdr, cur)) { + (DIR_DELAYED_REFRESH(hdr, cur) && !mbox->very_dirty_syncs)) { /* refresh index and try again */ if (maildir_sync_header_refresh(mbox) < 0) return -1; @@ -572,7 +585,7 @@ if (DIR_DELAYED_REFRESH(hdr, new)) *new_changed_r = TRUE; - if (DIR_DELAYED_REFRESH(hdr, cur)) + if (DIR_DELAYED_REFRESH(hdr, cur) && !mbox->very_dirty_syncs) *cur_changed_r = TRUE; if (*new_changed_r && *cur_changed_r) return 0; @@ -901,6 +914,32 @@ } } + if (mbox->very_dirty_syncs) { + struct mail_index_view_sync_ctx *sync_ctx; + bool b; + + if (mbox->flags_view == NULL) { + mbox->flags_view = + mail_index_view_open(mbox->ibox.index); + } + sync_ctx = mail_index_view_sync_begin(mbox->flags_view, + MAIL_INDEX_VIEW_SYNC_FLAG_FIX_INCONSISTENT); + if (mail_index_view_sync_commit(&sync_ctx, &b) < 0) { + mail_storage_set_index_error(&mbox->ibox); + ret = -1; + } + /* make sure the map stays in private memory */ + if (mbox->flags_view->map->refcount > 1) { + struct mail_index_map *map; + + map = mail_index_map_clone(mbox->flags_view->map); + mail_index_unmap(&mbox->flags_view->map); + mbox->flags_view->map = map; + } + mail_index_record_map_move_to_private(mbox->flags_view->map); + mail_index_map_move_to_memory(mbox->flags_view->map); + maildir_uidlist_set_all_nonsynced(mbox->uidlist); + } return index_mailbox_sync_init(box, flags, ret < 0); }
--- a/src/lib-storage/index/maildir/maildir-uidlist.c Mon Mar 30 20:27:02 2009 -0400 +++ b/src/lib-storage/index/maildir/maildir-uidlist.c Mon Mar 30 20:27:14 2009 -0400 @@ -1338,6 +1338,11 @@ return 1; } +void maildir_uidlist_set_all_nonsynced(struct maildir_uidlist *uidlist) +{ + maildir_uidlist_mark_all(uidlist, TRUE); +} + int maildir_uidlist_sync_init(struct maildir_uidlist *uidlist, enum maildir_uidlist_sync_flags sync_flags, struct maildir_uidlist_sync_ctx **sync_ctx_r)
--- a/src/lib-storage/index/maildir/maildir-uidlist.h Mon Mar 30 20:27:02 2009 -0400 +++ b/src/lib-storage/index/maildir/maildir-uidlist.h Mon Mar 30 20:27:14 2009 -0400 @@ -93,6 +93,7 @@ maildir_uidlist_set_ext() */ int maildir_uidlist_update(struct maildir_uidlist *uidlist); +void maildir_uidlist_set_all_nonsynced(struct maildir_uidlist *uidlist); /* Sync uidlist with what's actually on maildir. Returns same as maildir_uidlist_lock(). */ int maildir_uidlist_sync_init(struct maildir_uidlist *uidlist,
--- a/src/lib-storage/index/maildir/maildir-util.c Mon Mar 30 20:27:02 2009 -0400 +++ b/src/lib-storage/index/maildir/maildir-util.c Mon Mar 30 20:27:14 2009 -0400 @@ -1,11 +1,14 @@ /* Copyright (c) 2004-2009 Dovecot authors, see the included COPYING file */ #include "lib.h" +#include "array.h" #include "ioloop.h" #include "str.h" #include "mkdir-parents.h" #include "maildir-storage.h" #include "maildir-uidlist.h" +#include "maildir-keywords.h" +#include "maildir-filename.h" #include "maildir-sync.h" #include <unistd.h> @@ -16,35 +19,69 @@ #define MAILDIR_RESYNC_RETRY_COUNT 10 +static const char * +maildir_filename_guess(struct maildir_mailbox *mbox, uint32_t uid, + const char *fname, bool *have_flags_r) + +{ + struct mail_index_view *view = mbox->flags_view; + struct maildir_keywords_sync_ctx *kw_ctx; + enum mail_flags flags; + ARRAY_TYPE(keyword_indexes) keywords; + uint32_t seq; + + if (view == NULL || !mail_index_lookup_seq(view, uid, &seq)) { + *have_flags_r = FALSE; + return fname; + } + + t_array_init(&keywords, 32); + mail_index_lookup_view_flags(view, seq, &flags, &keywords); + if (array_count(&keywords) == 0) { + *have_flags_r = (flags & MAIL_FLAGS_NONRECENT) != 0; + fname = maildir_filename_set_flags(NULL, fname, flags, NULL); + } else { + *have_flags_r = TRUE; + kw_ctx = maildir_keywords_sync_init_readonly(mbox->keywords, + mbox->ibox.index); + fname = maildir_filename_set_flags(kw_ctx, fname, + flags, &keywords); + maildir_keywords_sync_deinit(&kw_ctx); + } + return fname; +} + static int maildir_file_do_try(struct maildir_mailbox *mbox, uint32_t uid, maildir_file_do_func *callback, void *context) { - const char *fname; - enum maildir_uidlist_rec_flag flags; + const char *path, *fname; + enum maildir_uidlist_rec_flag flags; + bool have_flags; int ret; fname = maildir_uidlist_lookup(mbox->uidlist, uid, &flags); if (fname == NULL) return -2; /* expunged */ + if ((flags & MAILDIR_UIDLIST_REC_FLAG_NONSYNCED) != 0) { + /* let's see if we can guess the filename based on index */ + fname = maildir_filename_guess(mbox, uid, fname, &have_flags); + if (have_flags) { + /* don't even bother looking into new/ dir */ + flags &= MAILDIR_UIDLIST_REC_FLAG_NEW_DIR; + } + } + if ((flags & MAILDIR_UIDLIST_REC_FLAG_NEW_DIR) != 0) { /* probably in new/ dir */ - T_BEGIN { - const char *path; - - path = t_strconcat(mbox->path, "/new/", fname, NULL); - ret = callback(mbox, path, context); - } T_END; + path = t_strconcat(mbox->path, "/new/", fname, NULL); + ret = callback(mbox, path, context); if (ret != 0) return ret; } - T_BEGIN { - const char *path; - - path = t_strconcat(mbox->path, "/cur/", fname, NULL); - ret = callback(mbox, path, context); - } T_END; + path = t_strconcat(mbox->path, "/cur/", fname, NULL); + ret = callback(mbox, path, context); return ret; } @@ -71,7 +108,9 @@ { int i, ret; - ret = maildir_file_do_try(mbox, uid, callback, context); + T_BEGIN { + ret = maildir_file_do_try(mbox, uid, callback, context); + } T_END; for (i = 0; i < MAILDIR_RESYNC_RETRY_COUNT && ret == 0; i++) { /* file is either renamed or deleted. sync the maildir and see which one. if file appears to be renamed constantly, @@ -79,11 +118,14 @@ if (maildir_storage_sync_force(mbox, uid) < 0) return -1; - ret = maildir_file_do_try(mbox, uid, callback, context); + T_BEGIN { + ret = maildir_file_do_try(mbox, uid, callback, context); + } T_END; } - if (i == MAILDIR_RESYNC_RETRY_COUNT) + if (i == MAILDIR_RESYNC_RETRY_COUNT) T_BEGIN { ret = maildir_file_do_try(mbox, uid, do_racecheck, context); + } T_END; return ret == -2 ? 0 : ret; }
--- a/src/master/mail-process.c Mon Mar 30 20:27:02 2009 -0400 +++ b/src/master/mail-process.c Mon Mar 30 20:27:14 2009 -0400 @@ -366,6 +366,8 @@ env_put("MAILDIR_COPY_WITH_HARDLINKS=1"); if (set->maildir_copy_preserve_filename) env_put("MAILDIR_COPY_PRESERVE_FILENAME=1"); + if (set->maildir_very_dirty_syncs) + env_put("MAILDIR_VERY_DIRTY_SYNCS=1"); if (set->mail_debug) env_put("DEBUG=1"); if (set->mail_full_filesystem_access)
--- a/src/master/master-settings-defs.c Mon Mar 30 20:27:02 2009 -0400 +++ b/src/master/master-settings-defs.c Mon Mar 30 20:27:14 2009 -0400 @@ -87,6 +87,7 @@ DEF_BOOL(maildir_stat_dirs), DEF_BOOL(maildir_copy_with_hardlinks), DEF_BOOL(maildir_copy_preserve_filename), + DEF_BOOL(maildir_very_dirty_syncs), DEF_STR(mbox_read_locks), DEF_STR(mbox_write_locks), DEF_INT(mbox_lock_timeout),
--- a/src/master/master-settings.c Mon Mar 30 20:27:02 2009 -0400 +++ b/src/master/master-settings.c Mon Mar 30 20:27:14 2009 -0400 @@ -253,6 +253,7 @@ MEMBER(maildir_stat_dirs) FALSE, MEMBER(maildir_copy_with_hardlinks) TRUE, MEMBER(maildir_copy_preserve_filename) FALSE, + MEMBER(maildir_very_dirty_syncs) FALSE, MEMBER(mbox_read_locks) "fcntl", MEMBER(mbox_write_locks) "dotlock fcntl", MEMBER(mbox_lock_timeout) 300,
--- a/src/master/master-settings.h Mon Mar 30 20:27:02 2009 -0400 +++ b/src/master/master-settings.h Mon Mar 30 20:27:14 2009 -0400 @@ -99,6 +99,7 @@ bool maildir_stat_dirs; bool maildir_copy_with_hardlinks; bool maildir_copy_preserve_filename; + bool maildir_very_dirty_syncs; const char *mbox_read_locks; const char *mbox_write_locks; unsigned int mbox_lock_timeout;