Mercurial > dovecot > original-hg > dovecot-1.2
changeset 5920:00c5e3cbeaf0 HEAD
Moved index syncing code to its own file.
author | Timo Sirainen <tss@iki.fi> |
---|---|
date | Sun, 08 Jul 2007 23:28:22 +0300 |
parents | cc439e4e99cb |
children | 00bdb1f546d3 |
files | src/lib-storage/index/maildir/Makefile.am src/lib-storage/index/maildir/maildir-save.c src/lib-storage/index/maildir/maildir-sync-index.c src/lib-storage/index/maildir/maildir-sync.c src/lib-storage/index/maildir/maildir-sync.h |
diffstat | 5 files changed, 509 insertions(+), 490 deletions(-) [+] |
line wrap: on
line diff
--- a/src/lib-storage/index/maildir/Makefile.am Sun Jul 08 23:20:34 2007 +0300 +++ b/src/lib-storage/index/maildir/Makefile.am Sun Jul 08 23:28:22 2007 +0300 @@ -16,6 +16,7 @@ maildir-save.c \ maildir-storage.c \ maildir-sync.c \ + maildir-sync-index.c \ maildir-transaction.c \ maildir-uidlist.c \ maildir-util.c
--- a/src/lib-storage/index/maildir/maildir-save.c Sun Jul 08 23:20:34 2007 +0300 +++ b/src/lib-storage/index/maildir/maildir-save.c Sun Jul 08 23:28:22 2007 +0300 @@ -604,7 +604,7 @@ } /* Start syncing so that keywords_sync_ctx gets set.. */ - if (maildir_sync_index_begin(ctx->mbox, &ctx->sync_ctx) < 0) { + if (maildir_sync_index_begin(ctx->mbox, NULL, &ctx->sync_ctx) < 0) { maildir_transaction_save_rollback(ctx); return -1; }
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-storage/index/maildir/maildir-sync-index.c Sun Jul 08 23:28:22 2007 +0300 @@ -0,0 +1,493 @@ +/* Copyright (C) 2007 Timo Sirainen */ + +#include "lib.h" +#include "ioloop.h" +#include "array.h" +#include "maildir-storage.h" +#include "index-sync-changes.h" +#include "maildir-uidlist.h" +#include "maildir-keywords.h" +#include "maildir-filename.h" +#include "maildir-sync.h" + +#include <stdio.h> +#include <unistd.h> + +struct maildir_index_sync_context { + struct maildir_mailbox *mbox; + struct maildir_sync_context *maildir_sync_ctx; + + struct mail_index_view *view; + struct mail_index_sync_ctx *sync_ctx; + struct maildir_keywords_sync_ctx *keywords_sync_ctx; + struct mail_index_transaction *trans; + + struct index_sync_changes_context *sync_changes; + enum mail_flags flags; + ARRAY_TYPE(keyword_indexes) keywords; + + uint32_t seq, uid; + + bool changed; +}; + +struct maildir_keywords_sync_ctx * +maildir_sync_get_keywords_sync_ctx(struct maildir_index_sync_context *ctx) +{ + return ctx->keywords_sync_ctx; +} + +static int maildir_expunge(struct maildir_mailbox *mbox, const char *path, + struct maildir_index_sync_context *ctx) +{ + struct mailbox *box = &mbox->ibox.box; + + if (unlink(path) == 0) { + if (box->v.sync_notify != NULL) { + box->v.sync_notify(box, ctx->uid, + MAILBOX_SYNC_TYPE_EXPUNGE); + } + mail_index_expunge(ctx->trans, ctx->seq); + ctx->changed = TRUE; + return 1; + } + if (errno == ENOENT) + return 0; + + mail_storage_set_critical(&mbox->storage->storage, + "unlink(%s) failed: %m", path); + return -1; +} + +static int maildir_sync_flags(struct maildir_mailbox *mbox, const char *path, + struct maildir_index_sync_context *ctx) +{ + struct mailbox *box = &mbox->ibox.box; + const char *dir, *fname, *newfname, *newpath; + enum mailbox_sync_type sync_type = 0; + uint8_t flags8; + + fname = strrchr(path, '/'); + i_assert(fname != NULL); + fname++; + dir = t_strdup_until(path, fname); + + /* get the current flags and keywords */ + maildir_filename_get_flags(ctx->keywords_sync_ctx, + fname, &ctx->flags, &ctx->keywords); + + /* apply changes */ + flags8 = ctx->flags; + index_sync_changes_apply(ctx->sync_changes, NULL, + &flags8, &ctx->keywords, &sync_type); + ctx->flags = flags8; + + /* and try renaming with the new name */ + newfname = maildir_filename_set_flags(ctx->keywords_sync_ctx, fname, + ctx->flags, &ctx->keywords); + newpath = t_strconcat(dir, newfname, NULL); + if (rename(path, newpath) == 0) { + if (box->v.sync_notify != NULL) + box->v.sync_notify(box, ctx->uid, sync_type); + + ctx->changed = TRUE; + return 1; + } + if (errno == ENOENT) + return 0; + + if (!ENOSPACE(errno) && errno != EACCES) { + mail_storage_set_critical(&mbox->storage->storage, + "rename(%s, %s) failed: %m", path, newpath); + } + return -1; +} + +int maildir_sync_index_begin(struct maildir_mailbox *mbox, + struct maildir_sync_context *maildir_sync_ctx, + struct maildir_index_sync_context **ctx_r) +{ + struct maildir_index_sync_context *ctx; + struct mail_index_sync_ctx *sync_ctx; + struct mail_index_view *view; + struct mail_index_transaction *trans; + + if (mail_index_sync_begin(mbox->ibox.index, &sync_ctx, &view, &trans, + (uint32_t)-1, (uoff_t)-1, 0) <= 0) { + mail_storage_set_index_error(&mbox->ibox); + return -1; + } + + ctx = i_new(struct maildir_index_sync_context, 1); + ctx->mbox = mbox; + ctx->maildir_sync_ctx = maildir_sync_ctx; + ctx->sync_ctx = sync_ctx; + ctx->view = view; + ctx->trans = trans; + ctx->keywords_sync_ctx = + maildir_keywords_sync_init(mbox->keywords, mbox->ibox.index); + + ctx->sync_changes = index_sync_changes_init(&mbox->ibox, ctx->sync_ctx, + ctx->view, ctx->trans, + mbox->ibox.readonly); + + *ctx_r = ctx; + return 0; +} + +int maildir_sync_index_finish(struct maildir_index_sync_context **_ctx, + bool failed, bool cancel) +{ + struct maildir_index_sync_context *ctx = *_ctx; + struct maildir_mailbox *mbox = ctx->mbox; + int ret = failed ? -1 : 0; + + *_ctx = NULL; + + if (ret < 0 || cancel) + mail_index_sync_rollback(&ctx->sync_ctx); + else { + /* Set syncing_commit=TRUE so that if any sync callbacks try + to access mails which got lost (eg. expunge callback trying + to open the file which was just unlinked) we don't try to + start a second index sync and crash. */ + mbox->syncing_commit = TRUE; + if (mail_index_sync_commit(&ctx->sync_ctx) < 0) { + mail_storage_set_index_error(&mbox->ibox); + ret = -1; + } else { + mbox->ibox.commit_log_file_seq = 0; + mbox->ibox.commit_log_file_offset = 0; + } + mbox->syncing_commit = FALSE; + } + + maildir_keywords_sync_deinit(ctx->keywords_sync_ctx); + ctx->keywords_sync_ctx = NULL; + + index_sync_changes_deinit(&ctx->sync_changes); + i_free(ctx); + return ret; +} + +int maildir_sync_index(struct maildir_index_sync_context *ctx, + bool partial) +{ + struct maildir_mailbox *mbox = ctx->mbox; + struct mail_index_view *view = ctx->view; + struct maildir_uidlist_iter_ctx *iter; + struct mail_index_transaction *trans = ctx->trans; + const struct mail_index_header *hdr; + struct mail_index_header empty_hdr; + const struct mail_index_record *rec; + uint32_t seq, uid, prev_uid; + enum maildir_uidlist_rec_flag uflags; + const char *filename; + ARRAY_TYPE(keyword_indexes) idx_keywords; + uint32_t uid_validity, next_uid; + uint64_t value; + unsigned int changes = 0; + int ret = 0; + bool expunged, full_rescan = FALSE; + + i_assert(!mbox->syncing_commit); + i_assert(maildir_uidlist_is_locked(ctx->mbox->uidlist)); + + hdr = mail_index_get_header(view); + uid_validity = maildir_uidlist_get_uid_validity(mbox->uidlist); + if (uid_validity != hdr->uid_validity && + uid_validity != 0 && hdr->uid_validity != 0) { + /* uidvalidity changed and mailbox isn't being initialized, + reset mailbox so we can add all messages as new */ + i_warning("Maildir %s: UIDVALIDITY changed (%u -> %u)", + mbox->path, hdr->uid_validity, uid_validity); + mail_index_reset(trans); + + memset(&empty_hdr, 0, sizeof(empty_hdr)); + empty_hdr.next_uid = 1; + hdr = &empty_hdr; + } + + mbox->syncing_commit = TRUE; + seq = prev_uid = 0; + t_array_init(&ctx->keywords, MAILDIR_MAX_KEYWORDS); + t_array_init(&idx_keywords, MAILDIR_MAX_KEYWORDS); + iter = maildir_uidlist_iter_init(mbox->uidlist); + while (maildir_uidlist_iter_next(iter, &uid, &uflags, &filename)) { + maildir_filename_get_flags(ctx->keywords_sync_ctx, filename, + &ctx->flags, &ctx->keywords); + + i_assert(uid > prev_uid); + prev_uid = uid; + + /* the private flags are kept only in indexes. don't use them + at all even for newly seen mails */ + ctx->flags &= ~mbox->private_flags_mask; + + if ((uflags & MAILDIR_UIDLIST_REC_FLAG_RECENT) != 0 && + (uflags & MAILDIR_UIDLIST_REC_FLAG_NEW_DIR) != 0 && + (uflags & MAILDIR_UIDLIST_REC_FLAG_MOVED) == 0) { + /* mail is recent for next session as well */ + ctx->flags |= MAIL_RECENT; + } + + __again: + ctx->seq = ++seq; + ctx->uid = uid; + + if (seq > hdr->messages_count) { + if (uid < hdr->next_uid) { + /* most likely a race condition: we read the + maildir, then someone else expunged messages + and committed changes to index. so, this + message shouldn't actually exist. mark it + racy and check in next sync. + + the difference between this and the later + check is that this one happens when messages + are expunged from the end */ + if ((uflags & + MAILDIR_UIDLIST_REC_FLAG_NONSYNCED) != 0) { + /* partial syncing */ + continue; + } + if ((uflags & + MAILDIR_UIDLIST_REC_FLAG_RACING) != 0) { + mail_storage_set_critical( + &mbox->storage->storage, + "Maildir %s sync: " + "UID < next_uid " + "(%u < %u, file = %s)", + mbox->path, uid, hdr->next_uid, + filename); + mail_index_mark_corrupted( + mbox->ibox.index); + ret = -1; + break; + } + mbox->dirty_cur_time = ioloop_time; + maildir_uidlist_add_flags(mbox->uidlist, + filename, + MAILDIR_UIDLIST_REC_FLAG_RACING); + + seq--; + continue; + } + + mail_index_append(trans, uid, &seq); + mail_index_update_flags(trans, seq, MODIFY_REPLACE, + ctx->flags); + + if (array_count(&ctx->keywords) > 0) { + struct mail_keywords *kw; + + kw = mail_index_keywords_create_from_indexes( + trans, &ctx->keywords); + mail_index_update_keywords(trans, seq, + MODIFY_REPLACE, kw); + mail_index_keywords_free(&kw); + } + continue; + } + + if (mail_index_lookup(view, seq, &rec) < 0) { + mail_storage_set_index_error(&mbox->ibox); + ret = -1; + break; + } + + if (rec->uid < uid) { + /* expunged */ + mail_index_expunge(trans, seq); + goto __again; + } + + if (rec->uid > uid) { + /* most likely a race condition: we read the + maildir, then someone else expunged messages and + committed changes to index. so, this message + shouldn't actually exist. mark it racy and check + in next sync. */ + if ((uflags & + MAILDIR_UIDLIST_REC_FLAG_NONSYNCED) != 0) { + /* partial syncing */ + seq--; + continue; + } + if ((uflags & MAILDIR_UIDLIST_REC_FLAG_RACING) != 0) { + mail_storage_set_critical( + &mbox->storage->storage, + "Maildir %s sync: " + "UID inserted in the middle of mailbox " + "(%u > %u, file = %s)", + mbox->path, rec->uid, uid, filename); + mail_index_mark_corrupted(mbox->ibox.index); + ret = -1; + break; + } + + mbox->dirty_cur_time = ioloop_time; + maildir_uidlist_add_flags(mbox->uidlist, filename, + MAILDIR_UIDLIST_REC_FLAG_RACING); + + seq--; + continue; + } + + if (index_sync_changes_read(ctx->sync_changes, rec->uid, + &expunged) < 0) { + ret = -1; + break; + } + + if (expunged) { + if (maildir_file_do(ctx->mbox, ctx->uid, + maildir_expunge, ctx) >= 0) { + /* successful expunge */ + mail_index_expunge(trans, ctx->seq); + } + if ((++changes % MAILDIR_SLOW_MOVE_COUNT) == 0) + maildir_sync_notify(ctx->maildir_sync_ctx); + continue; + } + + /* the private flags are stored only in indexes, keep them */ + ctx->flags |= rec->flags & mbox->private_flags_mask; + + if ((rec->flags & MAIL_RECENT) != 0) { + index_mailbox_set_recent(&mbox->ibox, seq); + if (mbox->ibox.keep_recent) { + ctx->flags |= MAIL_RECENT; + } else { + mail_index_update_flags(trans, seq, + MODIFY_REMOVE, + MAIL_RECENT); + } + } + + if ((uflags & MAILDIR_UIDLIST_REC_FLAG_NONSYNCED) != 0) { + /* partial syncing */ + if ((ctx->flags & MAIL_RECENT) != 0) { + /* we last saw this mail in new/, but it's + not there anymore. possibly expunged, + make sure. */ + full_rescan = TRUE; + } + continue; + } + + if (index_sync_changes_have(ctx->sync_changes)) { + /* apply flag changes to maildir */ + if (maildir_file_do(ctx->mbox, ctx->uid, + maildir_sync_flags, ctx) < 0) + ctx->flags |= MAIL_INDEX_MAIL_FLAG_DIRTY; + if ((++changes % MAILDIR_SLOW_MOVE_COUNT) == 0) + maildir_sync_notify(ctx->maildir_sync_ctx); + } + + if ((rec->flags & MAIL_INDEX_MAIL_FLAG_DIRTY) != 0) { + /* we haven't been able to update maildir with this + record's flag changes. don't sync them. */ + continue; + } + + if ((ctx->flags & ~MAIL_RECENT) != + (rec->flags & (MAIL_FLAGS_MASK^MAIL_RECENT))) { + /* FIXME: this is wrong if there's pending changes in + transaction log already. it gets fixed in next sync + however.. */ + mail_index_update_flags(trans, seq, MODIFY_REPLACE, + ctx->flags); + } else if ((ctx->flags & MAIL_RECENT) == 0 && + (rec->flags & MAIL_RECENT) != 0) { + /* just remove recent flag */ + mail_index_update_flags(trans, seq, MODIFY_REMOVE, + MAIL_RECENT); + } + + /* update keywords if they have changed */ + if (mail_index_lookup_keywords(view, seq, &idx_keywords) < 0) { + mail_storage_set_index_error(&mbox->ibox); + ret = -1; + break; + } + if (!index_keyword_array_cmp(&ctx->keywords, &idx_keywords)) { + struct mail_keywords *kw; + + kw = mail_index_keywords_create_from_indexes( + trans, &ctx->keywords); + mail_index_update_keywords(trans, seq, + MODIFY_REPLACE, kw); + mail_index_keywords_free(&kw); + } + } + maildir_uidlist_iter_deinit(iter); + mbox->syncing_commit = FALSE; + + if (mbox->ibox.box.v.sync_notify != NULL) + mbox->ibox.box.v.sync_notify(&mbox->ibox.box, 0, 0); + + if (!partial) { + /* expunge the rest */ + for (seq++; seq <= hdr->messages_count; seq++) + mail_index_expunge(trans, seq); + + /* next_uid must be updated only in non-partial syncs since + partial syncs don't add the new mails to index. also we'll + have to do it here before syncing index records, since after + that the uidlist's next_uid value may have changed. */ + next_uid = maildir_uidlist_get_next_uid(mbox->uidlist); + i_assert(next_uid > prev_uid); + if (hdr->next_uid < next_uid) { + mail_index_update_header(trans, + offsetof(struct mail_index_header, next_uid), + &next_uid, sizeof(next_uid), FALSE); + } + } + + if (ctx->changed) + mbox->dirty_cur_time = ioloop_time; + if (mbox->dirty_cur_time != 0) + mbox->last_dirty_flags |= MAILDIR_DIRTY_CUR; + + if (mbox->last_cur_mtime != (time_t)hdr->sync_stamp) { + uint32_t sync_stamp = mbox->last_cur_mtime; + + mail_index_update_header(trans, + offsetof(struct mail_index_header, sync_stamp), + &sync_stamp, sizeof(sync_stamp), TRUE); + } + + /* FIXME: use a header extension instead of sync_size.. */ + value = mbox->last_new_mtime | + ((uint64_t)mbox->last_dirty_flags << 32); + if (value != hdr->sync_size) { + mail_index_update_header(trans, + offsetof(struct mail_index_header, sync_size), + &value, sizeof(value), TRUE); + } + + if (hdr->uid_validity == 0) { + /* get the initial uidvalidity */ + if (maildir_uidlist_refresh(mbox->uidlist) < 0) + ret = -1; + uid_validity = maildir_uidlist_get_uid_validity(mbox->uidlist); + if (uid_validity == 0) { + uid_validity = ioloop_time; + maildir_uidlist_set_uid_validity(mbox->uidlist, + uid_validity, 0); + } + } else if (uid_validity == 0) { + maildir_uidlist_set_uid_validity(mbox->uidlist, + hdr->uid_validity, + hdr->next_uid); + } + + if (uid_validity != hdr->uid_validity && uid_validity != 0) { + mail_index_update_header(trans, + offsetof(struct mail_index_header, uid_validity), + &uid_validity, sizeof(uid_validity), TRUE); + } + + return ret < 0 ? -1 : (full_rescan ? 0 : 1); +}
--- a/src/lib-storage/index/maildir/maildir-sync.c Sun Jul 08 23:20:34 2007 +0300 +++ b/src/lib-storage/index/maildir/maildir-sync.c Sun Jul 08 23:28:22 2007 +0300 @@ -177,9 +177,7 @@ #include "hash.h" #include "str.h" #include "maildir-storage.h" -#include "index-sync-changes.h" #include "maildir-uidlist.h" -#include "maildir-keywords.h" #include "maildir-filename.h" #include "maildir-sync.h" @@ -200,13 +198,6 @@ bet here, but I guess 5 will do just fine too. */ #define MAILDIR_RENAME_RESCAN_COUNT 5 -/* After moving 100 mails from new/ to cur/, check if we need to touch the - uidlist lock. */ -#define MAILDIR_SLOW_MOVE_COUNT 100 -/* readdir() should be pretty fast to do, but check anyway every 10000 mails - to see if we need to touch the uidlist lock. */ -#define MAILDIR_SLOW_CHECK_COUNT 10000 - /* This is mostly to avoid infinite looping when rename() destination already exists as the hard link of the file itself. */ #define MAILDIR_SCAN_DIR_MAX_COUNT 5 @@ -224,97 +215,7 @@ struct maildir_index_sync_context *index_sync_ctx; }; -struct maildir_index_sync_context { - struct maildir_mailbox *mbox; - struct maildir_sync_context *maildir_sync_ctx; - - struct mail_index_view *view; - struct mail_index_sync_ctx *sync_ctx; - struct maildir_keywords_sync_ctx *keywords_sync_ctx; - struct mail_index_transaction *trans; - - struct index_sync_changes_context *sync_changes; - enum mail_flags flags; - ARRAY_TYPE(keyword_indexes) keywords; - - uint32_t seq, uid; - - bool changed; -}; - -struct maildir_keywords_sync_ctx * -maildir_sync_get_keywords_sync_ctx(struct maildir_index_sync_context *ctx) -{ - return ctx->keywords_sync_ctx; -} - -static int maildir_expunge(struct maildir_mailbox *mbox, const char *path, - struct maildir_index_sync_context *ctx) -{ - struct mailbox *box = &mbox->ibox.box; - - if (unlink(path) == 0) { - if (box->v.sync_notify != NULL) { - box->v.sync_notify(box, ctx->uid, - MAILBOX_SYNC_TYPE_EXPUNGE); - } - mail_index_expunge(ctx->trans, ctx->seq); - ctx->changed = TRUE; - return 1; - } - if (errno == ENOENT) - return 0; - - mail_storage_set_critical(&mbox->storage->storage, - "unlink(%s) failed: %m", path); - return -1; -} - -static int maildir_sync_flags(struct maildir_mailbox *mbox, const char *path, - struct maildir_index_sync_context *ctx) -{ - struct mailbox *box = &mbox->ibox.box; - const char *dir, *fname, *newfname, *newpath; - enum mailbox_sync_type sync_type = 0; - uint8_t flags8; - - fname = strrchr(path, '/'); - i_assert(fname != NULL); - fname++; - dir = t_strdup_until(path, fname); - - /* get the current flags and keywords */ - maildir_filename_get_flags(ctx->keywords_sync_ctx, - fname, &ctx->flags, &ctx->keywords); - - /* apply changes */ - flags8 = ctx->flags; - index_sync_changes_apply(ctx->sync_changes, NULL, - &flags8, &ctx->keywords, &sync_type); - ctx->flags = flags8; - - /* and try renaming with the new name */ - newfname = maildir_filename_set_flags(ctx->keywords_sync_ctx, fname, - ctx->flags, &ctx->keywords); - newpath = t_strconcat(dir, newfname, NULL); - if (rename(path, newpath) == 0) { - if (box->v.sync_notify != NULL) - box->v.sync_notify(box, ctx->uid, sync_type); - - ctx->changed = TRUE; - return 1; - } - if (errno == ENOENT) - return 0; - - if (!ENOSPACE(errno) && errno != EACCES) { - mail_storage_set_critical(&mbox->storage->storage, - "rename(%s, %s) failed: %m", path, newpath); - } - return -1; -} - -static void maildir_sync_notify(struct maildir_sync_context *ctx) +void maildir_sync_notify(struct maildir_sync_context *ctx) { time_t now; @@ -663,393 +564,6 @@ return 0; } -int maildir_sync_index_begin(struct maildir_mailbox *mbox, - struct maildir_index_sync_context **ctx_r) -{ - struct maildir_index_sync_context *ctx; - struct mail_index_sync_ctx *sync_ctx; - struct mail_index_view *view; - struct mail_index_transaction *trans; - - if (mail_index_sync_begin(mbox->ibox.index, &sync_ctx, &view, &trans, - (uint32_t)-1, (uoff_t)-1, 0) <= 0) { - mail_storage_set_index_error(&mbox->ibox); - return -1; - } - - ctx = i_new(struct maildir_index_sync_context, 1); - ctx->mbox = mbox; - ctx->sync_ctx = sync_ctx; - ctx->view = view; - ctx->trans = trans; - ctx->keywords_sync_ctx = - maildir_keywords_sync_init(mbox->keywords, mbox->ibox.index); - - ctx->sync_changes = index_sync_changes_init(&mbox->ibox, ctx->sync_ctx, - ctx->view, ctx->trans, - mbox->ibox.readonly); - - *ctx_r = ctx; - return 0; -} - -int maildir_sync_index_finish(struct maildir_index_sync_context **_ctx, - bool failed, bool cancel) -{ - struct maildir_index_sync_context *ctx = *_ctx; - struct maildir_mailbox *mbox = ctx->mbox; - int ret = failed ? -1 : 0; - - *_ctx = NULL; - - if (ret < 0 || cancel) - mail_index_sync_rollback(&ctx->sync_ctx); - else { - /* Set syncing_commit=TRUE so that if any sync callbacks try - to access mails which got lost (eg. expunge callback trying - to open the file which was just unlinked) we don't try to - start a second index sync and crash. */ - mbox->syncing_commit = TRUE; - if (mail_index_sync_commit(&ctx->sync_ctx) < 0) { - mail_storage_set_index_error(&mbox->ibox); - ret = -1; - } else { - mbox->ibox.commit_log_file_seq = 0; - mbox->ibox.commit_log_file_offset = 0; - } - mbox->syncing_commit = FALSE; - } - - maildir_keywords_sync_deinit(ctx->keywords_sync_ctx); - ctx->keywords_sync_ctx = NULL; - - index_sync_changes_deinit(&ctx->sync_changes); - i_free(ctx); - return ret; -} - -int maildir_sync_index(struct maildir_index_sync_context *ctx, - bool partial) -{ - struct maildir_mailbox *mbox = ctx->mbox; - struct mail_index_view *view = ctx->view; - struct maildir_uidlist_iter_ctx *iter; - struct mail_index_transaction *trans = ctx->trans; - const struct mail_index_header *hdr; - struct mail_index_header empty_hdr; - const struct mail_index_record *rec; - uint32_t seq, uid, prev_uid; - enum maildir_uidlist_rec_flag uflags; - const char *filename; - ARRAY_TYPE(keyword_indexes) idx_keywords; - uint32_t uid_validity, next_uid; - uint64_t value; - unsigned int changes = 0; - int ret = 0; - bool expunged, full_rescan = FALSE; - - i_assert(!mbox->syncing_commit); - i_assert(maildir_uidlist_is_locked(ctx->mbox->uidlist)); - - hdr = mail_index_get_header(view); - uid_validity = maildir_uidlist_get_uid_validity(mbox->uidlist); - if (uid_validity != hdr->uid_validity && - uid_validity != 0 && hdr->uid_validity != 0) { - /* uidvalidity changed and mailbox isn't being initialized, - reset mailbox so we can add all messages as new */ - i_warning("Maildir %s: UIDVALIDITY changed (%u -> %u)", - mbox->path, hdr->uid_validity, uid_validity); - mail_index_reset(trans); - - memset(&empty_hdr, 0, sizeof(empty_hdr)); - empty_hdr.next_uid = 1; - hdr = &empty_hdr; - } - - mbox->syncing_commit = TRUE; - seq = prev_uid = 0; - t_array_init(&ctx->keywords, MAILDIR_MAX_KEYWORDS); - t_array_init(&idx_keywords, MAILDIR_MAX_KEYWORDS); - iter = maildir_uidlist_iter_init(mbox->uidlist); - while (maildir_uidlist_iter_next(iter, &uid, &uflags, &filename)) { - maildir_filename_get_flags(ctx->keywords_sync_ctx, filename, - &ctx->flags, &ctx->keywords); - - i_assert(uid > prev_uid); - prev_uid = uid; - - /* the private flags are kept only in indexes. don't use them - at all even for newly seen mails */ - ctx->flags &= ~mbox->private_flags_mask; - - if ((uflags & MAILDIR_UIDLIST_REC_FLAG_RECENT) != 0 && - (uflags & MAILDIR_UIDLIST_REC_FLAG_NEW_DIR) != 0 && - (uflags & MAILDIR_UIDLIST_REC_FLAG_MOVED) == 0) { - /* mail is recent for next session as well */ - ctx->flags |= MAIL_RECENT; - } - - __again: - ctx->seq = ++seq; - ctx->uid = uid; - - if (seq > hdr->messages_count) { - if (uid < hdr->next_uid) { - /* most likely a race condition: we read the - maildir, then someone else expunged messages - and committed changes to index. so, this - message shouldn't actually exist. mark it - racy and check in next sync. - - the difference between this and the later - check is that this one happens when messages - are expunged from the end */ - if ((uflags & - MAILDIR_UIDLIST_REC_FLAG_NONSYNCED) != 0) { - /* partial syncing */ - continue; - } - if ((uflags & - MAILDIR_UIDLIST_REC_FLAG_RACING) != 0) { - mail_storage_set_critical( - &mbox->storage->storage, - "Maildir %s sync: " - "UID < next_uid " - "(%u < %u, file = %s)", - mbox->path, uid, hdr->next_uid, - filename); - mail_index_mark_corrupted( - mbox->ibox.index); - ret = -1; - break; - } - mbox->dirty_cur_time = ioloop_time; - maildir_uidlist_add_flags(mbox->uidlist, - filename, - MAILDIR_UIDLIST_REC_FLAG_RACING); - - seq--; - continue; - } - - mail_index_append(trans, uid, &seq); - mail_index_update_flags(trans, seq, MODIFY_REPLACE, - ctx->flags); - - if (array_count(&ctx->keywords) > 0) { - struct mail_keywords *kw; - - kw = mail_index_keywords_create_from_indexes( - trans, &ctx->keywords); - mail_index_update_keywords(trans, seq, - MODIFY_REPLACE, kw); - mail_index_keywords_free(&kw); - } - continue; - } - - if (mail_index_lookup(view, seq, &rec) < 0) { - mail_storage_set_index_error(&mbox->ibox); - ret = -1; - break; - } - - if (rec->uid < uid) { - /* expunged */ - mail_index_expunge(trans, seq); - goto __again; - } - - if (rec->uid > uid) { - /* most likely a race condition: we read the - maildir, then someone else expunged messages and - committed changes to index. so, this message - shouldn't actually exist. mark it racy and check - in next sync. */ - if ((uflags & - MAILDIR_UIDLIST_REC_FLAG_NONSYNCED) != 0) { - /* partial syncing */ - seq--; - continue; - } - if ((uflags & MAILDIR_UIDLIST_REC_FLAG_RACING) != 0) { - mail_storage_set_critical( - &mbox->storage->storage, - "Maildir %s sync: " - "UID inserted in the middle of mailbox " - "(%u > %u, file = %s)", - mbox->path, rec->uid, uid, filename); - mail_index_mark_corrupted(mbox->ibox.index); - ret = -1; - break; - } - - mbox->dirty_cur_time = ioloop_time; - maildir_uidlist_add_flags(mbox->uidlist, filename, - MAILDIR_UIDLIST_REC_FLAG_RACING); - - seq--; - continue; - } - - if (index_sync_changes_read(ctx->sync_changes, rec->uid, - &expunged) < 0) { - ret = -1; - break; - } - - if (expunged) { - if (maildir_file_do(ctx->mbox, ctx->uid, - maildir_expunge, ctx) >= 0) { - /* successful expunge */ - mail_index_expunge(trans, ctx->seq); - } - if ((++changes % MAILDIR_SLOW_MOVE_COUNT) == 0) - maildir_sync_notify(ctx->maildir_sync_ctx); - continue; - } - - /* the private flags are stored only in indexes, keep them */ - ctx->flags |= rec->flags & mbox->private_flags_mask; - - if ((rec->flags & MAIL_RECENT) != 0) { - index_mailbox_set_recent(&mbox->ibox, seq); - if (mbox->ibox.keep_recent) { - ctx->flags |= MAIL_RECENT; - } else { - mail_index_update_flags(trans, seq, - MODIFY_REMOVE, - MAIL_RECENT); - } - } - - if ((uflags & MAILDIR_UIDLIST_REC_FLAG_NONSYNCED) != 0) { - /* partial syncing */ - if ((ctx->flags & MAIL_RECENT) != 0) { - /* we last saw this mail in new/, but it's - not there anymore. possibly expunged, - make sure. */ - full_rescan = TRUE; - } - continue; - } - - if (index_sync_changes_have(ctx->sync_changes)) { - /* apply flag changes to maildir */ - if (maildir_file_do(ctx->mbox, ctx->uid, - maildir_sync_flags, ctx) < 0) - ctx->flags |= MAIL_INDEX_MAIL_FLAG_DIRTY; - if ((++changes % MAILDIR_SLOW_MOVE_COUNT) == 0) - maildir_sync_notify(ctx->maildir_sync_ctx); - } - - if ((rec->flags & MAIL_INDEX_MAIL_FLAG_DIRTY) != 0) { - /* we haven't been able to update maildir with this - record's flag changes. don't sync them. */ - continue; - } - - if ((ctx->flags & ~MAIL_RECENT) != - (rec->flags & (MAIL_FLAGS_MASK^MAIL_RECENT))) { - /* FIXME: this is wrong if there's pending changes in - transaction log already. it gets fixed in next sync - however.. */ - mail_index_update_flags(trans, seq, MODIFY_REPLACE, - ctx->flags); - } else if ((ctx->flags & MAIL_RECENT) == 0 && - (rec->flags & MAIL_RECENT) != 0) { - /* just remove recent flag */ - mail_index_update_flags(trans, seq, MODIFY_REMOVE, - MAIL_RECENT); - } - - /* update keywords if they have changed */ - if (mail_index_lookup_keywords(view, seq, &idx_keywords) < 0) { - mail_storage_set_index_error(&mbox->ibox); - ret = -1; - break; - } - if (!index_keyword_array_cmp(&ctx->keywords, &idx_keywords)) { - struct mail_keywords *kw; - - kw = mail_index_keywords_create_from_indexes( - trans, &ctx->keywords); - mail_index_update_keywords(trans, seq, - MODIFY_REPLACE, kw); - mail_index_keywords_free(&kw); - } - } - maildir_uidlist_iter_deinit(iter); - mbox->syncing_commit = FALSE; - - if (mbox->ibox.box.v.sync_notify != NULL) - mbox->ibox.box.v.sync_notify(&mbox->ibox.box, 0, 0); - - if (!partial) { - /* expunge the rest */ - for (seq++; seq <= hdr->messages_count; seq++) - mail_index_expunge(trans, seq); - - /* next_uid must be updated only in non-partial syncs since - partial syncs don't add the new mails to index. also we'll - have to do it here before syncing index records, since after - that the uidlist's next_uid value may have changed. */ - next_uid = maildir_uidlist_get_next_uid(mbox->uidlist); - i_assert(next_uid > prev_uid); - if (hdr->next_uid < next_uid) { - mail_index_update_header(trans, - offsetof(struct mail_index_header, next_uid), - &next_uid, sizeof(next_uid), FALSE); - } - } - - if (ctx->changed) - mbox->dirty_cur_time = ioloop_time; - if (mbox->dirty_cur_time != 0) - mbox->last_dirty_flags |= MAILDIR_DIRTY_CUR; - - if (mbox->last_cur_mtime != (time_t)hdr->sync_stamp) { - uint32_t sync_stamp = mbox->last_cur_mtime; - - mail_index_update_header(trans, - offsetof(struct mail_index_header, sync_stamp), - &sync_stamp, sizeof(sync_stamp), TRUE); - } - - /* FIXME: use a header extension instead of sync_size.. */ - value = mbox->last_new_mtime | - ((uint64_t)mbox->last_dirty_flags << 32); - if (value != hdr->sync_size) { - mail_index_update_header(trans, - offsetof(struct mail_index_header, sync_size), - &value, sizeof(value), TRUE); - } - - if (hdr->uid_validity == 0) { - /* get the initial uidvalidity */ - if (maildir_uidlist_refresh(mbox->uidlist) < 0) - ret = -1; - uid_validity = maildir_uidlist_get_uid_validity(mbox->uidlist); - if (uid_validity == 0) { - uid_validity = ioloop_time; - maildir_uidlist_set_uid_validity(mbox->uidlist, - uid_validity, 0); - } - } else if (uid_validity == 0) { - maildir_uidlist_set_uid_validity(mbox->uidlist, - hdr->uid_validity, - hdr->next_uid); - } - - if (uid_validity != hdr->uid_validity && uid_validity != 0) { - mail_index_update_header(trans, - offsetof(struct mail_index_header, uid_validity), - &uid_validity, sizeof(uid_validity), TRUE); - } - - return ret < 0 ? -1 : (full_rescan ? 0 : 1); -} - static int maildir_sync_context(struct maildir_sync_context *ctx, bool forced, bool sync_last_commit) { @@ -1126,10 +640,9 @@ } if (!ctx->mbox->syncing_commit) { - if (maildir_sync_index_begin(ctx->mbox, + if (maildir_sync_index_begin(ctx->mbox, ctx, &ctx->index_sync_ctx) < 0) return -1; - ctx->index_sync_ctx->maildir_sync_ctx = ctx; } if (new_changed || cur_changed) {
--- a/src/lib-storage/index/maildir/maildir-sync.h Sun Jul 08 23:20:34 2007 +0300 +++ b/src/lib-storage/index/maildir/maildir-sync.h Sun Jul 08 23:28:22 2007 +0300 @@ -1,10 +1,20 @@ #ifndef __MAILDIR_SYNC_H #define __MAILDIR_SYNC_H +/* All systems accessing the filesystem must have their clock less than this + many seconds apart from each others. 0 works only for local filesystems. */ #define MAILDIR_SYNC_SECS 1 +/* After moving this many mails from new/ to cur/, check if we need to touch + the uidlist lock. */ +#define MAILDIR_SLOW_MOVE_COUNT 100 +/* readdir() should be pretty fast to do, but check anyway every n files + to see if we need to touch the uidlist lock. */ +#define MAILDIR_SLOW_CHECK_COUNT 10000 + struct maildir_mailbox; +struct maildir_sync_context; struct maildir_keywords_sync_ctx; struct maildir_index_sync_context; @@ -15,6 +25,7 @@ int maildir_storage_sync_force(struct maildir_mailbox *mbox); int maildir_sync_index_begin(struct maildir_mailbox *mbox, + struct maildir_sync_context *maildir_sync_ctx, struct maildir_index_sync_context **ctx_r); int maildir_sync_index(struct maildir_index_sync_context *sync_ctx, bool partial); @@ -25,5 +36,6 @@ struct maildir_keywords_sync_ctx * maildir_sync_get_keywords_sync_ctx(struct maildir_index_sync_context *ctx); +void maildir_sync_notify(struct maildir_sync_context *ctx); #endif