# HG changeset patch # User Timo Sirainen # Date 1083521244 -10800 # Node ID 2f6e137cdc44e4f186d0b389e44224fb1856dc61 # Parent a1c00aa3a078439f3863d0c28130e75d202dc29a Syncing optimizations. diff -r a1c00aa3a078 -r 2f6e137cdc44 src/lib-index/mail-index-sync-private.h --- a/src/lib-index/mail-index-sync-private.h Sun May 02 16:05:53 2004 +0300 +++ b/src/lib-index/mail-index-sync-private.h Sun May 02 21:07:24 2004 +0300 @@ -22,7 +22,8 @@ unsigned int sync_appends:1; }; -int mail_index_sync_update_index(struct mail_index_sync_ctx *sync_ctx); +int mail_index_sync_update_index(struct mail_index_sync_ctx *sync_ctx, + uint32_t sync_stamp, uint64_t sync_size); void mail_index_header_update_counts(struct mail_index_header *hdr, uint8_t old_flags, uint8_t new_flags); diff -r a1c00aa3a078 -r 2f6e137cdc44 src/lib-index/mail-index-sync-update.c --- a/src/lib-index/mail-index-sync-update.c Sun May 02 16:05:53 2004 +0300 +++ b/src/lib-index/mail-index-sync-update.c Sun May 02 21:07:24 2004 +0300 @@ -166,7 +166,8 @@ return 0; } -int mail_index_sync_update_index(struct mail_index_sync_ctx *sync_ctx) +int mail_index_sync_update_index(struct mail_index_sync_ctx *sync_ctx, + uint32_t sync_stamp, uint64_t sync_size) { struct mail_index *index = sync_ctx->index; struct mail_index_map *map = index->map; @@ -177,14 +178,31 @@ uint32_t count, file_seq, src_idx, dest_idx; uoff_t file_offset; unsigned int lock_id; - int ret; + int ret, changed; /* rewind */ sync_ctx->update_idx = sync_ctx->expunge_idx = 0; sync_ctx->sync_appends = buffer_get_used_size(sync_ctx->appends_buf) != 0; - if (!mail_index_sync_have_more(sync_ctx)) { + changed = mail_index_sync_have_more(sync_ctx); + + memset(&ctx, 0, sizeof(ctx)); + ctx.index = index; + ctx.hdr = *index->hdr; + ctx.log_view = sync_ctx->view->log_view; + + /* see if we need to update sync headers */ + if (ctx.hdr.sync_stamp != sync_stamp && sync_stamp != 0) { + ctx.hdr.sync_stamp = sync_stamp; + changed = TRUE; + } + if (ctx.hdr.sync_size != sync_size && sync_size != 0) { + ctx.hdr.sync_size = sync_size; + changed = TRUE; + } + + if (!changed) { /* nothing to sync */ return 0; } @@ -195,11 +213,6 @@ if (MAIL_INDEX_MAP_IS_IN_MEMORY(map)) map->write_to_disk = TRUE; - memset(&ctx, 0, sizeof(ctx)); - ctx.index = index; - ctx.hdr = *index->hdr; - ctx.log_view = sync_ctx->view->log_view; - src_idx = dest_idx = 0; append_count = 0; appends = NULL; while (mail_index_sync_next(sync_ctx, &rec) > 0) { diff -r a1c00aa3a078 -r 2f6e137cdc44 src/lib-index/mail-index-sync.c --- a/src/lib-index/mail-index-sync.c Sun May 02 16:05:53 2004 +0300 +++ b/src/lib-index/mail-index-sync.c Sun May 02 21:07:24 2004 +0300 @@ -176,7 +176,7 @@ index->hdr->log_file_offset, seq, offset, MAIL_TRANSACTION_TYPE_MASK) < 0) { - mail_index_sync_end(ctx); + mail_index_sync_end(ctx, 0, 0); return -1; } @@ -189,7 +189,7 @@ ctx->appends_buf = buffer_create_dynamic(default_pool, 1024, (size_t)-1); if (mail_index_sync_read_and_sort(ctx, FALSE) < 0) { - mail_index_sync_end(ctx); + mail_index_sync_end(ctx, 0, 0); return -1; } @@ -397,7 +397,8 @@ ctx->sync_appends; } -int mail_index_sync_end(struct mail_index_sync_ctx *ctx) +int mail_index_sync_end(struct mail_index_sync_ctx *ctx, + uint32_t sync_stamp, uint64_t sync_size) { const struct mail_index_header *hdr; uint32_t seq; @@ -420,7 +421,8 @@ if (ret == 0) { mail_index_sync_read_and_sort(ctx, TRUE); - if (mail_index_sync_update_index(ctx) < 0) + if (mail_index_sync_update_index(ctx, sync_stamp, + sync_size) < 0) ret = -1; } diff -r a1c00aa3a078 -r 2f6e137cdc44 src/lib-index/mail-index.h --- a/src/lib-index/mail-index.h Sun May 02 16:05:53 2004 +0300 +++ b/src/lib-index/mail-index.h Sun May 02 21:07:24 2004 +0300 @@ -33,13 +33,8 @@ }; enum mail_index_header_flag { - /* Corrupted-flag should be set while anything dangerous is done to - index file, such as when expunging messages. Once the operation - is finished, the corrupted-flag is removed. If this flag is noticed - unexpectedly, the index must be assumed to be corrupted and must - not be used. */ - MAIL_INDEX_HDR_FLAG_CORRUPTED = 0x0001, - MAIL_INDEX_HDR_FLAG_COMPRESS_CACHE = 0x0002 + /* Index file is corrupted, reopen or recreate it. */ + MAIL_INDEX_HDR_FLAG_CORRUPTED = 0x0001 }; enum mail_index_record_flag { @@ -209,8 +204,10 @@ struct mail_index_sync_rec *sync_rec); /* Returns 1 if there's more to sync, 0 if not. */ int mail_index_sync_have_more(struct mail_index_sync_ctx *ctx); -/* End synchronization by unlocking the index and closing the view. */ -int mail_index_sync_end(struct mail_index_sync_ctx *ctx); +/* End synchronization by unlocking the index and closing the view. + sync_stamp/sync_size in header is updated to given values. */ +int mail_index_sync_end(struct mail_index_sync_ctx *ctx, + uint32_t sync_stamp, uint64_t sync_size); /* Mark index file corrupted. Invalidates all views. */ void mail_index_mark_corrupted(struct mail_index *index); diff -r a1c00aa3a078 -r 2f6e137cdc44 src/lib-storage/index/index-storage.h --- a/src/lib-storage/index/index-storage.h Sun May 02 16:05:53 2004 +0300 +++ b/src/lib-storage/index/index-storage.h Sun May 02 21:07:24 2004 +0300 @@ -88,6 +88,7 @@ unsigned int mail_read_mmaped:1; unsigned int maildir_keep_new:1; + unsigned int last_cur_dirty:1; }; struct index_transaction_context { diff -r a1c00aa3a078 -r 2f6e137cdc44 src/lib-storage/index/maildir/maildir-sync.c --- a/src/lib-storage/index/maildir/maildir-sync.c Sun May 02 16:05:53 2004 +0300 +++ b/src/lib-storage/index/maildir/maildir-sync.c Sun May 02 21:07:24 2004 +0300 @@ -30,6 +30,7 @@ struct maildir_sync_context { struct index_mailbox *ibox; const char *new_dir, *cur_dir; + int partial; struct maildir_uidlist_sync_ctx *uidlist_sync_ctx; }; @@ -124,7 +125,7 @@ break; } } - if (mail_index_sync_end(sync_ctx) < 0) + if (mail_index_sync_end(sync_ctx, 0, 0) < 0) ret = -1; } @@ -261,6 +262,7 @@ static int maildir_sync_quick_check(struct maildir_sync_context *ctx, int *new_changed_r, int *cur_changed_r) { + const struct mail_index_header *hdr; struct index_mailbox *ibox = ctx->ibox; struct stat st; time_t new_mtime, cur_mtime; @@ -281,19 +283,27 @@ } cur_mtime = st.st_mtime; + if (ibox->last_cur_mtime == 0) { + /* first sync in this session, get cur stamp from index */ + if (mail_index_get_header(ibox->view, &hdr) == 0) + ibox->last_cur_mtime = hdr->sync_stamp; + } + if (new_mtime != ibox->last_new_mtime || new_mtime >= ibox->last_sync - MAILDIR_SYNC_SECS) { *new_changed_r = TRUE; ibox->last_new_mtime = new_mtime; } + if (cur_mtime != ibox->last_cur_mtime || - (cur_mtime >= ibox->last_sync - MAILDIR_SYNC_SECS && + (ibox->last_cur_dirty && ioloop_time - ibox->last_sync > MAILDIR_SYNC_SECS)) { /* cur/ changed, or delayed cur/ check */ *cur_changed_r = TRUE; ibox->last_cur_mtime = cur_mtime; } ibox->last_sync = ioloop_time; + ibox->last_cur_dirty = cur_mtime >= ibox->last_sync - MAILDIR_SYNC_SECS; return 0; } @@ -308,10 +318,12 @@ struct mail_index_view *view; const struct mail_index_header *hdr; const struct mail_index_record *rec; - uint32_t seq, uid, uflags; + uint32_t seq, uid; + enum maildir_uidlist_rec_flag uflags; const char *filename; enum mail_flags flags; custom_flags_mask_t custom_flags; + uint32_t sync_stamp; int ret; if (mail_index_sync_begin(ibox->index, &sync_ctx, &view, @@ -332,6 +344,11 @@ __again: seq++; + if ((uflags & MAILDIR_UIDLIST_REC_FLAG_NONSYNCED) != 0) { + /* partial syncing */ + continue; + } + if (seq > hdr->messages_count) { mail_index_append(trans, uid, &seq); mail_index_update_flags(trans, seq, MODIFY_REPLACE, @@ -372,6 +389,12 @@ } maildir_uidlist_iter_deinit(iter); + if (!ctx->partial) { + /* expunge the rest */ + for (seq++; seq <= hdr->messages_count; seq++) + mail_index_expunge(trans, seq); + } + if (ret < 0) mail_index_transaction_rollback(trans); else { @@ -393,7 +416,9 @@ break; } } - if (mail_index_sync_end(sync_ctx) < 0) + + sync_stamp = ibox->last_cur_dirty ? 0 : ibox->last_cur_mtime; + if (mail_index_sync_end(sync_ctx, sync_stamp, 0) < 0) ret = -1; if (ret == 0) { @@ -416,13 +441,16 @@ if (!new_changed && !cur_changed) return 0; - // FIXME: don't sync cur/ directory if not needed - ctx->uidlist_sync_ctx = maildir_uidlist_sync_init(ctx->ibox->uidlist); + ctx->partial = !cur_changed; + ctx->uidlist_sync_ctx = + maildir_uidlist_sync_init(ctx->ibox->uidlist, ctx->partial); if (maildir_scan_dir(ctx, TRUE) < 0) return -1; - if (maildir_scan_dir(ctx, FALSE) < 0) - return -1; + if (cur_changed) { + if (maildir_scan_dir(ctx, FALSE) < 0) + return -1; + } ret = maildir_uidlist_sync_deinit(ctx->uidlist_sync_ctx); ctx->uidlist_sync_ctx = NULL; @@ -436,7 +464,8 @@ { int ret; - ctx->uidlist_sync_ctx = maildir_uidlist_sync_init(ctx->ibox->uidlist); + ctx->uidlist_sync_ctx = + maildir_uidlist_sync_init(ctx->ibox->uidlist, FALSE); if (maildir_scan_dir(ctx, TRUE) < 0) return -1; diff -r a1c00aa3a078 -r 2f6e137cdc44 src/lib-storage/index/maildir/maildir-uidlist.c --- a/src/lib-storage/index/maildir/maildir-uidlist.c Sun May 02 16:05:53 2004 +0300 +++ b/src/lib-storage/index/maildir/maildir-uidlist.c Sun May 02 21:07:24 2004 +0300 @@ -51,6 +51,9 @@ struct hash_table *files; buffer_t *new_record_buf; + unsigned int partial_new_pos; + + unsigned int partial:1; unsigned int new_files:1; unsigned int synced:1; unsigned int failed:1; @@ -204,6 +207,7 @@ } rec = buffer_append_space_unsafe(uidlist->record_buf, sizeof(*rec)); + memset(rec, 0, sizeof(*rec)); rec->uid = uid; rec->flags = flags; rec->filename = p_strdup(uidlist->filename_pool, line); @@ -308,7 +312,11 @@ unsigned int idx, left_idx, right_idx; size_t size; - i_assert(uidlist->last_mtime != 0); + if (uidlist->last_mtime == 0) { + /* first time we need to read uidlist */ + if (maildir_uidlist_update(uidlist) < 0) + return NULL; + } rec = buffer_get_data(uidlist->record_buf, &size); size /= sizeof(*rec); @@ -378,7 +386,7 @@ rec = buffer_get_data(uidlist->record_buf, &size); size /= sizeof(*rec); - maildir_uidlist_lookup(uidlist, uidlist->first_recent_uid, &idx); + maildir_uidlist_lookup_rec(uidlist, uidlist->first_recent_uid, &idx); for (count = 0; idx < size; idx++) { if ((rec[idx].flags & MAILDIR_UIDLIST_REC_FLAG_RECENT) != 0) count++; @@ -393,7 +401,8 @@ struct maildir_uidlist_iter_ctx *iter; struct utimbuf ut; string_t *str; - uint32_t uid, flags; + uint32_t uid; + enum maildir_uidlist_rec_flag flags; const char *filename, *flags_str; int ret = 0; @@ -485,26 +494,115 @@ return ret; } +static void maildir_uidlist_mark_all(struct maildir_uidlist *uidlist, + int nonsynced) +{ + struct maildir_uidlist_rec *rec; + size_t i, size; + + rec = buffer_get_modifyable_data(uidlist->record_buf, &size); + size /= sizeof(*rec); + + if (nonsynced) { + for (i = 0; i < size; i++) + rec[i].flags |= MAILDIR_UIDLIST_REC_FLAG_NONSYNCED; + } else { + for (i = 0; i < size; i++) + rec[i].flags &= ~MAILDIR_UIDLIST_REC_FLAG_NONSYNCED; + } +} + struct maildir_uidlist_sync_ctx * -maildir_uidlist_sync_init(struct maildir_uidlist *uidlist) +maildir_uidlist_sync_init(struct maildir_uidlist *uidlist, int partial) { struct maildir_uidlist_sync_ctx *ctx; ctx = i_new(struct maildir_uidlist_sync_ctx, 1); ctx->uidlist = uidlist; + ctx->partial = partial; + + if (partial) { + /* initially mark all nonsynced */ + maildir_uidlist_mark_all(uidlist, TRUE); + return ctx; + } + ctx->filename_pool = pool_alloconly_create("maildir_uidlist_sync", 16384); ctx->new_record_buf = buffer_create_dynamic(default_pool, 512, (size_t)-1); ctx->files = hash_create(default_pool, ctx->filename_pool, 4096, maildir_hash, maildir_cmp); + return ctx; +} - if (uidlist->last_mtime == 0) { - /* uidlist not read yet, do it */ - if (maildir_uidlist_update(uidlist) < 0) +static int maildir_uidlist_sync_uidlist(struct maildir_uidlist_sync_ctx *ctx) +{ + int ret; + + if (ctx->uidlist->last_mtime == 0) { + /* first time reading the uidlist, + no locking yet */ + if (maildir_uidlist_update(ctx->uidlist) < 0) { ctx->failed = TRUE; + return -1; + } + return 0; } - return ctx; + + /* lock and update uidlist to see if it's just been added */ + ret = maildir_uidlist_try_lock(ctx->uidlist); + if (ret <= 0) { + if (ret == 0) + return 1; // FIXME: does it work right? + ctx->failed = TRUE; + return -1; + } + if (maildir_uidlist_update(ctx->uidlist) < 0) { + ctx->failed = TRUE; + return -1; + } + + ctx->synced = TRUE; + return 1; +} + +static int +maildir_uidlist_sync_next_partial(struct maildir_uidlist_sync_ctx *ctx, + const char *filename, + enum maildir_uidlist_rec_flag flags) +{ + struct maildir_uidlist *uidlist = ctx->uidlist; + struct maildir_uidlist_rec *rec; + int ret; + + /* we'll update uidlist directly */ + rec = hash_lookup(uidlist->files, filename); + if (rec == NULL && !ctx->synced) { + ret = maildir_uidlist_sync_uidlist(ctx); + if (ret < 0) + return -1; + if (ret == 0) { + return maildir_uidlist_sync_next_partial(ctx, filename, + flags); + } + rec = hash_lookup(uidlist->files, filename); + } + + if (rec == NULL) { + ctx->new_files = TRUE; + ctx->partial_new_pos = + buffer_get_used_size(uidlist->record_buf) / + sizeof(*rec); + rec = buffer_append_space_unsafe(uidlist->record_buf, + sizeof(*rec)); + memset(rec, 0, sizeof(*rec)); + } + + rec->flags = (rec->flags | flags) & ~MAILDIR_UIDLIST_REC_FLAG_NONSYNCED; + rec->filename = p_strdup(uidlist->filename_pool, filename); + hash_insert(uidlist->files, rec->filename, rec); + return 1; } int maildir_uidlist_sync_next(struct maildir_uidlist_sync_ctx *ctx, @@ -512,12 +610,14 @@ enum maildir_uidlist_rec_flag flags) { struct maildir_uidlist_rec *rec; - char *fname; int ret; if (ctx->failed) return -1; + if (ctx->partial) + return maildir_uidlist_sync_next_partial(ctx, filename, flags); + rec = hash_lookup(ctx->files, filename); if (rec != NULL) { if ((rec->flags & (MAILDIR_UIDLIST_REC_FLAG_NEW_DIR | @@ -531,21 +631,13 @@ } else { rec = hash_lookup(ctx->uidlist->files, filename); if (rec == NULL && !ctx->synced) { - /* lock and update uidlist to see if it's just - been added */ - ret = maildir_uidlist_try_lock(ctx->uidlist); - if (ret <= 0) { - if (ret == 0) - return 1; // FIXME: does it work right? - ctx->failed = TRUE; + ret = maildir_uidlist_sync_uidlist(ctx); + if (ret < 0) return -1; + if (ret == 0) { + return maildir_uidlist_sync_next(ctx, filename, + flags); } - if (maildir_uidlist_update(ctx->uidlist) < 0) { - ctx->failed = TRUE; - return -1; - } - - ctx->synced = TRUE; rec = hash_lookup(ctx->uidlist->files, filename); } @@ -558,11 +650,8 @@ } rec->flags |= flags; - - fname = p_strdup(ctx->filename_pool, filename); - if (rec->filename == NULL) - rec->filename = fname; - hash_insert(ctx->files, fname, rec); + rec->filename = p_strdup(ctx->filename_pool, filename); + hash_insert(ctx->files, rec->filename, rec); return 1; } @@ -587,6 +676,31 @@ return t1 < t2 ? -1 : t1 > t2 ? 1 : 0; } +static void maildir_uidlist_assign_uids(struct maildir_uidlist *uidlist, + unsigned int first_new_pos) +{ + struct maildir_uidlist_rec *rec; + unsigned int dest; + size_t size; + + rec = buffer_get_modifyable_data(uidlist->record_buf, &size); + size /= sizeof(*rec); + + /* sort new files and assign UIDs for them */ + qsort(rec + first_new_pos, size - first_new_pos, + sizeof(*rec), maildir_time_cmp); + for (dest = first_new_pos; dest < size; dest++) { + i_assert(rec[dest].uid == 0); + rec[dest].uid = uidlist->next_uid++; + rec[dest].flags &= ~MAILDIR_UIDLIST_REC_FLAG_MOVED; + + if ((rec[dest].flags & MAILDIR_UIDLIST_REC_FLAG_RECENT) != 0) { + maildir_uidlist_mark_recent(uidlist, + rec[dest].uid); + } + } +} + static void maildir_uidlist_swap(struct maildir_uidlist_sync_ctx *ctx) { struct maildir_uidlist *uidlist = ctx->uidlist; @@ -620,21 +734,7 @@ buffer_append_buf(uidlist->record_buf, ctx->new_record_buf, 0, (size_t)-1); - - rec = buffer_get_modifyable_data(uidlist->record_buf, &size); - size /= sizeof(*rec); - - /* sort new files and assign UIDs for them */ - qsort(rec + dest, size - dest, sizeof(*rec), maildir_time_cmp); - for (; dest < size; dest++) { - rec[dest].uid = uidlist->next_uid++; - rec[dest].flags &= ~MAILDIR_UIDLIST_REC_FLAG_MOVED; - - if ((rec[dest].flags & MAILDIR_UIDLIST_REC_FLAG_RECENT) != 0) { - maildir_uidlist_mark_recent(ctx->uidlist, - rec[dest].uid); - } - } + maildir_uidlist_assign_uids(uidlist, dest); hash_destroy(uidlist->files); uidlist->files = ctx->files; @@ -653,7 +753,16 @@ if (ctx->failed) ret = -1; else { - maildir_uidlist_swap(ctx); + if (!ctx->partial) + maildir_uidlist_swap(ctx); + else { + if (ctx->new_files) { + maildir_uidlist_assign_uids(ctx->uidlist, + ctx->partial_new_pos); + } + maildir_uidlist_mark_all(ctx->uidlist, FALSE); + } + if (!ctx->new_files) ret = 0; else @@ -667,7 +776,8 @@ hash_destroy(ctx->files); if (ctx->filename_pool != NULL) pool_unref(ctx->filename_pool); - buffer_free(ctx->new_record_buf); + if (ctx->new_record_buf != NULL) + buffer_free(ctx->new_record_buf); i_free(ctx); return ret; } @@ -686,7 +796,8 @@ } int maildir_uidlist_iter_next(struct maildir_uidlist_iter_ctx *ctx, - uint32_t *uid_r, uint32_t *flags_r, + uint32_t *uid_r, + enum maildir_uidlist_rec_flag *flags_r, const char **filename_r) { if (ctx->next == ctx->end) diff -r a1c00aa3a078 -r 2f6e137cdc44 src/lib-storage/index/maildir/maildir-uidlist.h --- a/src/lib-storage/index/maildir/maildir-uidlist.h Sun May 02 16:05:53 2004 +0300 +++ b/src/lib-storage/index/maildir/maildir-uidlist.h Sun May 02 21:07:24 2004 +0300 @@ -6,7 +6,8 @@ enum maildir_uidlist_rec_flag { MAILDIR_UIDLIST_REC_FLAG_NEW_DIR = 0x01, MAILDIR_UIDLIST_REC_FLAG_MOVED = 0x02, - MAILDIR_UIDLIST_REC_FLAG_RECENT = 0x04 + MAILDIR_UIDLIST_REC_FLAG_RECENT = 0x04, + MAILDIR_UIDLIST_REC_FLAG_NONSYNCED = 0x08 }; int maildir_uidlist_try_lock(struct maildir_uidlist *uidlist); @@ -29,7 +30,7 @@ /* Sync uidlist with what's actually on maildir. */ struct maildir_uidlist_sync_ctx * -maildir_uidlist_sync_init(struct maildir_uidlist *uidlist); +maildir_uidlist_sync_init(struct maildir_uidlist *uidlist, int partial); int maildir_uidlist_sync_next(struct maildir_uidlist_sync_ctx *ctx, const char *filename, enum maildir_uidlist_rec_flag flags); @@ -39,7 +40,8 @@ struct maildir_uidlist_iter_ctx * maildir_uidlist_iter_init(struct maildir_uidlist *uidlist); int maildir_uidlist_iter_next(struct maildir_uidlist_iter_ctx *ctx, - uint32_t *uid_r, uint32_t *flags_r, + uint32_t *uid_r, + enum maildir_uidlist_rec_flag *flags_r, const char **filename_r); void maildir_uidlist_iter_deinit(struct maildir_uidlist_iter_ctx *ctx);