Mercurial > dovecot > original-hg > dovecot-1.2
changeset 7627:4fd41f9467ea HEAD
Left out from initial CONDSTORE commit.
author | Timo Sirainen <tss@iki.fi> |
---|---|
date | Sat, 15 Mar 2008 15:23:35 +0200 |
parents | 96207583aaa0 |
children | af2441dc6de6 |
files | src/lib-index/mail-index-modseq.c src/lib-index/mail-index-modseq.h |
diffstat | 2 files changed, 654 insertions(+), 0 deletions(-) [+] |
line wrap: on
line diff
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-index/mail-index-modseq.c Sat Mar 15 15:23:35 2008 +0200 @@ -0,0 +1,610 @@ +/* Copyright (c) 2008 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "mail-index-private.h" +#include "mail-index-modseq.h" +#include "mail-index-sync-private.h" + +#define MAIL_INDEX_MODSEQ_EXT_NAME "modseq" + +ARRAY_DEFINE_TYPE(modseqs, uint64_t); + +enum modseq_metadata_idx { + /* must be in the same order as enum mail_flags */ + METADATA_MODSEQ_IDX_ANSWERED = 0, + METADATA_MODSEQ_IDX_FLAGGED, + METADATA_MODSEQ_IDX_DELETED, + METADATA_MODSEQ_IDX_SEEN, + METADATA_MODSEQ_IDX_DRAFT, + + METADATA_MODSEQ_IDX_KEYWORD_START +}; + +struct mail_index_modseq_header { + /* highest used modseq */ + uint64_t highest_modseq; + /* last tracked log file position */ + uint32_t log_seq; + uint32_t log_offset; +}; + +struct metadata_modseqs { + ARRAY_TYPE(modseqs) modseqs; +}; + +struct mail_index_map_modseq { + /* indexes use enum modseq_metadata_idx */ + ARRAY_DEFINE(metadata_modseqs, struct metadata_modseqs); +}; + +struct mail_index_modseq_sync { + struct mail_index_sync_map_ctx *sync_map_ctx; + struct mail_index_view *view; + struct mail_transaction_log_view *log_view; + struct mail_index_map_modseq *mmap; + + uint64_t highest_modseq; + uint32_t log_seq; + uoff_t log_offset; +}; + +void mail_index_modseq_init(struct mail_index *index) +{ + index->modseq_ext_id = + mail_index_ext_register(index, MAIL_INDEX_MODSEQ_EXT_NAME, + sizeof(struct mail_index_modseq_header), + sizeof(uint64_t), sizeof(uint64_t)); +} + +static uint64_t mail_index_modseq_get_head(struct mail_index_map *map) +{ + return map->hdr.log_file_head_offset | + ((uint64_t)(map->hdr.indexid + map->hdr.log_file_seq) << 32); +} + +void mail_index_modseq_enable(struct mail_index *index) +{ + struct mail_index_transaction *trans; + struct mail_index_view *view; + struct mail_index_modseq_header hdr; + uint32_t ext_map_idx, log_seq; + uoff_t log_offset; + + if (index->modseqs_enabled) + return; + + if (!mail_index_map_get_ext_idx(index->map, index->modseq_ext_id, + &ext_map_idx)) { + /* modseqs not enabled to the index yet, add them. */ + view = mail_index_view_open(index); + trans = mail_index_transaction_begin(view, 0); + + memset(&hdr, 0, sizeof(hdr)); + hdr.highest_modseq = mail_index_modseq_get_head(index->map); + mail_index_update_header_ext(trans, index->modseq_ext_id, + 0, &hdr, sizeof(hdr)); + + /* commit also refreshes the index, which syncs the modseqs */ + (void)mail_index_transaction_commit(&trans, + &log_seq, &log_offset); + mail_index_view_close(&view); + + /* get the modseq extension to index map */ + if (!mail_index_map_get_ext_idx(index->map, + index->modseq_ext_id, + &ext_map_idx)) { + /* didn't work for some reason */ + return; + } + } + index->modseqs_enabled = TRUE; +} + +uint64_t mail_index_modseq_get_highest(struct mail_index_view *view) +{ + const struct mail_index_modseq_header *modseq_hdr; + const void *data; + size_t size; + + mail_index_get_header_ext(view, view->index->modseq_ext_id, + &data, &size); + if (size == sizeof(*modseq_hdr)) { + modseq_hdr = data; + if (modseq_hdr->highest_modseq != 0) + return modseq_hdr->highest_modseq; + } + /* fallback to returning the log head */ + return mail_index_modseq_get_head(view->map); +} + +static struct mail_index_map_modseq * +mail_index_map_modseq(struct mail_index_view *view) +{ + struct mail_index_map_modseq *mmap = view->map->rec_map->modseq; + uint32_t ext_map_idx; + + if (mmap != NULL) + return mmap; + + /* don't start tracking until we've seen modseq extension intro */ + if (!mail_index_map_get_ext_idx(view->map, view->index->modseq_ext_id, + &ext_map_idx)) + return NULL; + + mmap = i_new(struct mail_index_map_modseq, 1); + i_array_init(&mmap->metadata_modseqs, + METADATA_MODSEQ_IDX_KEYWORD_START + + array_count(&view->index->keywords)); + view->map->rec_map->modseq = mmap; + return mmap; +} + +uint64_t mail_index_modseq_lookup(struct mail_index_view *view, uint32_t seq) +{ + struct mail_index_map_modseq *mmap = mail_index_map_modseq(view); + struct mail_index_map *map; + const struct mail_index_ext *ext; + const struct mail_index_record *rec; + const uint64_t *modseqp; + uint32_t ext_map_idx; + + if (mmap == NULL) + return mail_index_modseq_get_head(view->map); + + rec = mail_index_lookup_full(view, seq, &map); + if (!mail_index_map_get_ext_idx(map, view->index->modseq_ext_id, + &ext_map_idx)) { + /* not enabled yet */ + return mail_index_modseq_get_head(view->map); + } + + ext = array_idx(&map->extensions, ext_map_idx); + modseqp = CONST_PTR_OFFSET(rec, ext->record_offset); + if (*modseqp == 0) { + /* If we're here because we just enabled modseqs, we'll return + the same modseq (initial highestmodseq) for all messages. + The next sync will change these zeros to initial + highestmodseq or higher. + + If we're here because a message got appended but modseq + wasn't set (older Dovecot?), we'll again use the current + highest modseq. This isn't exactly correct, but it gets + fixed after the next sync and this situation shouldn't + normally happen anyway. */ + return mail_index_modseq_get_highest(view); + } + return *modseqp; +} + +static uint64_t +modseq_idx_lookup(struct mail_index_map_modseq *mmap, + unsigned int idx, uint32_t seq) +{ + const struct metadata_modseqs *metadata; + const uint64_t *modseqs; + unsigned int count; + + metadata = array_get(&mmap->metadata_modseqs, &count); + if (idx >= count || !array_is_created(&metadata[idx].modseqs)) + return 0; + + modseqs = array_get(&metadata[idx].modseqs, &count); + return seq > count ? 0 : modseqs[seq-1]; +} + +uint64_t mail_index_modseq_lookup_flags(struct mail_index_view *view, + enum mail_flags flags_mask, + uint32_t seq) +{ + struct mail_index_map_modseq *mmap = mail_index_map_modseq(view); + unsigned int i; + uint64_t modseq, highest_modseq = 0; + + if (mmap != NULL) { + /* first try to find a specific match */ + for (i = 0; i < METADATA_MODSEQ_IDX_KEYWORD_START; i++) { + if ((flags_mask & (1 << i)) != 0) { + modseq = modseq_idx_lookup(mmap, i, seq); + if (highest_modseq < modseq) + highest_modseq = modseq; + } + } + } + + if (highest_modseq == 0) { + /* no specific matches, fallback to using the highest */ + highest_modseq = mail_index_modseq_lookup(view, seq); + } + return highest_modseq; +} + +uint64_t mail_index_modseq_lookup_keywords(struct mail_index_view *view, + const struct mail_keywords *keywords, + uint32_t seq) +{ + struct mail_index_map_modseq *mmap = mail_index_map_modseq(view); + unsigned int i, metadata_idx; + uint64_t modseq, highest_modseq = 0; + + if (mmap != NULL) { + /* first try to find a specific match */ + for (i = 0; i < keywords->count; i++) { + metadata_idx = METADATA_MODSEQ_IDX_KEYWORD_START + + keywords->idx[i]; + + modseq = modseq_idx_lookup(mmap, metadata_idx, seq); + if (highest_modseq < modseq) + highest_modseq = modseq; + } + } + + if (highest_modseq == 0) { + /* no specific matches, fallback to using the highest */ + highest_modseq = mail_index_modseq_lookup(view, seq); + } + return highest_modseq; +} + +static uint64_t get_cur_modseq(struct mail_index_modseq_sync *ctx) +{ + mail_transaction_log_view_get_prev_pos(ctx->log_view, + &ctx->log_seq, &ctx->log_offset); + i_assert(ctx->log_offset <= (uint32_t)-1); + + return ctx->log_offset | + ((uint64_t)(ctx->view->map->hdr.indexid + ctx->log_seq) << 32); +} + +static void +mail_index_modseq_update(struct mail_index_modseq_sync *ctx, + uint64_t modseq, bool nonzeros, + uint32_t seq1, uint32_t seq2) +{ + const struct mail_index_ext *ext; + const struct mail_index_record *rec; + uint32_t ext_map_idx; + uint64_t *modseqp; + + if (!mail_index_map_get_ext_idx(ctx->view->map, + ctx->view->index->modseq_ext_id, + &ext_map_idx)) + return; + + if (modseq > ctx->highest_modseq) + ctx->highest_modseq = modseq; + + ext = array_idx(&ctx->view->map->extensions, ext_map_idx); + for (; seq1 <= seq2; seq1++) { + rec = MAIL_INDEX_MAP_IDX(ctx->view->map, seq1-1); + modseqp = PTR_OFFSET(rec, ext->record_offset); + if (*modseqp == 0 || (nonzeros && *modseqp < modseq)) + *modseqp = modseq; + } +} + +static bool +mail_index_modseq_update_highest(struct mail_index_modseq_sync *ctx, + uint32_t seq1, uint32_t seq2) +{ + if (ctx->mmap == NULL) + return FALSE; + + mail_index_modseq_update(ctx, get_cur_modseq(ctx), TRUE, seq1, seq2); + return TRUE; +} + +static void +mail_index_modseq_update_old_rec(struct mail_index_modseq_sync *ctx, + const struct mail_transaction_header *thdr, + const void *tdata) +{ + ARRAY_TYPE(seq_range) uids = ARRAY_INIT; + const struct seq_range *rec; + buffer_t *uid_buf; + unsigned int i, count; + uint32_t seq1, seq2; + + switch (thdr->type & MAIL_TRANSACTION_TYPE_MASK) { + case MAIL_TRANSACTION_APPEND: { + const struct mail_index_record *appends = tdata; + + count = thdr->size / sizeof(*appends); + for (i = 0; i < count; i++) { + if (mail_index_lookup_seq(ctx->view, + appends[i].uid, &seq1)) { + mail_index_modseq_update_highest(ctx, seq1, + seq1); + } + } + return; + } + case MAIL_TRANSACTION_FLAG_UPDATE: { + uid_buf = buffer_create_const_data(pool_datastack_create(), + tdata, thdr->size); + array_create_from_buffer(&uids, uid_buf, + sizeof(struct mail_transaction_flag_update)); + break; + } + case MAIL_TRANSACTION_KEYWORD_UPDATE: { + const struct mail_transaction_keyword_update *rec = tdata; + unsigned int seqset_offset; + + seqset_offset = sizeof(*rec) + rec->name_size; + if ((seqset_offset % 4) != 0) + seqset_offset += 4 - (seqset_offset % 4); + + uid_buf = buffer_create_const_data(pool_datastack_create(), + CONST_PTR_OFFSET(tdata, seqset_offset), + thdr->size - seqset_offset); + array_create_from_buffer(&uids, uid_buf, sizeof(uint32_t)*2); + break; + } + case MAIL_TRANSACTION_KEYWORD_RESET: + uid_buf = buffer_create_const_data(pool_datastack_create(), + tdata, thdr->size); + array_create_from_buffer(&uids, uid_buf, + sizeof(struct mail_transaction_keyword_reset)); + break; + default: + return; + } + + /* update modseqs */ + count = array_count(&uids); + for (i = 0; i < count; i++) { + rec = array_idx(&uids, i); + if (mail_index_lookup_seq_range(ctx->view, rec->seq1, rec->seq2, + &seq1, &seq2)) + mail_index_modseq_update_highest(ctx, seq1, seq2); + } +} + +static void mail_index_modseq_sync_init(struct mail_index_modseq_sync *ctx) +{ + struct mail_index_map *map = ctx->view->map; + const struct mail_index_ext *ext; + const struct mail_index_modseq_header *hdr; + const struct mail_transaction_header *thdr; + const void *tdata; + uint32_t ext_map_idx; + uint32_t end_seq; + uoff_t end_offset; + uint64_t cur_modseq; + bool reset; + int ret; + + if (!mail_index_map_get_ext_idx(map, ctx->view->index->modseq_ext_id, + &ext_map_idx)) + i_unreached(); + ext = array_idx(&map->extensions, ext_map_idx); + + /* get the current highest_modseq. don't change any modseq below it. */ + hdr = CONST_PTR_OFFSET(map->hdr_base, ext->hdr_offset); + ctx->highest_modseq = hdr->highest_modseq; + + /* Scan logs for updates between ext_hdr.log_* .. view position. + There are two reasons why there could be any: + + 1) We just enabled modseqs and we're filling the initial values. + 2) A non-modseq-aware Dovecot version added new messages and wrote + dovecot.index file. */ + mail_transaction_log_view_get_prev_pos(ctx->view->log_view, + &end_seq, &end_offset); + if (end_seq <= hdr->log_seq || + (end_seq == hdr->log_seq && end_offset <= hdr->log_offset)) { + /* modseqs are up to date */ + return; + } + + ctx->log_view = mail_transaction_log_view_open(ctx->view->index->log); + ret = mail_transaction_log_view_set(ctx->log_view, + I_MAX(1, hdr->log_seq), + hdr->log_offset, + end_seq, end_offset, &reset); + if (ret == 0) { + /* missing files - try with only the last file */ + ret = mail_transaction_log_view_set(ctx->log_view, end_seq, 0, + end_seq, end_offset, + &reset); + /* since we don't know if we skipped some changes, set all + modseqs to beginning of the latest file. */ + cur_modseq = get_cur_modseq(ctx); + if (cur_modseq < hdr->highest_modseq) { + /* should happen only when setting initial modseqs. + we may already have returned highest_modseq as + some messages' modseq value. don't shrink it. */ + cur_modseq = hdr->highest_modseq; + } + mail_index_modseq_update(ctx, cur_modseq, TRUE, 1, + map->hdr.messages_count); + } else { + /* we have all the logs. replace zero modseqs with the current + highest modseq (we may have already returned it for them). */ + mail_index_modseq_update(ctx, hdr->highest_modseq, FALSE, 1, + map->hdr.messages_count); + } + if (ret > 0) { + while (mail_transaction_log_view_next(ctx->log_view, + &thdr, &tdata) > 0) { + T_BEGIN { + mail_index_modseq_update_old_rec(ctx, thdr, + tdata); + } T_END; + } + } + mail_index_sync_write_seq_update(ctx->sync_map_ctx, 1, + map->hdr.messages_count); + mail_transaction_log_view_close(&ctx->log_view); +} + +struct mail_index_modseq_sync * +mail_index_modseq_sync_begin(struct mail_index_sync_map_ctx *sync_map_ctx) +{ + struct mail_index_modseq_sync *ctx; + + ctx = i_new(struct mail_index_modseq_sync, 1); + ctx->sync_map_ctx = sync_map_ctx; + ctx->view = sync_map_ctx->view; + ctx->mmap = mail_index_map_modseq(ctx->view); + if (ctx->mmap != NULL) { + mail_index_modseq_sync_init(ctx); + ctx->log_view = ctx->view->log_view; + } + return ctx; +} + +static void mail_index_modseq_update_header(struct mail_index_modseq_sync *ctx) +{ + struct mail_index_map *map = ctx->view->map; + const struct mail_index_ext *ext; + const struct mail_index_modseq_header *old_modseq_hdr; + struct mail_index_modseq_header new_modseq_hdr; + uint32_t ext_map_idx; + + if (!mail_index_map_get_ext_idx(map, ctx->view->index->modseq_ext_id, + &ext_map_idx)) + return; + + ext = array_idx(&map->extensions, ext_map_idx); + old_modseq_hdr = CONST_PTR_OFFSET(map->hdr_base, ext->hdr_offset); + + if (old_modseq_hdr->log_seq < ctx->log_seq || + (old_modseq_hdr->log_seq == ctx->log_seq && + old_modseq_hdr->log_offset < ctx->log_offset)) { + new_modseq_hdr.highest_modseq = ctx->highest_modseq; + new_modseq_hdr.log_seq = ctx->log_seq; + new_modseq_hdr.log_offset = ctx->log_offset; + + buffer_write(map->hdr_copy_buf, ext->hdr_offset, + &new_modseq_hdr, sizeof(new_modseq_hdr)); + map->hdr_base = map->hdr_copy_buf->data; + map->write_ext_header = TRUE; + } +} + +void mail_index_modseq_sync_end(struct mail_index_modseq_sync **_ctx) +{ + struct mail_index_modseq_sync *ctx = *_ctx; + + *_ctx = NULL; + if (ctx->mmap != NULL) + mail_index_modseq_update_header(ctx); + i_free(ctx); +} + +void mail_index_modseq_hdr_update(struct mail_index_modseq_sync *ctx) +{ + if (ctx->mmap == NULL) { + ctx->mmap = mail_index_map_modseq(ctx->view); + i_assert(ctx->mmap != NULL); + mail_index_modseq_sync_init(ctx); + ctx->log_view = ctx->view->log_view; + } +} + +void mail_index_modseq_append(struct mail_index_modseq_sync *ctx, uint32_t seq) +{ + mail_index_modseq_update_highest(ctx, seq, seq); +} + +void mail_index_modseq_expunge(struct mail_index_modseq_sync *ctx, + uint32_t seq1, uint32_t seq2) +{ + struct metadata_modseqs *metadata; + unsigned int i, count; + uint64_t modseq; + + if (ctx->mmap == NULL) + return; + + seq1--; + metadata = array_get_modifiable(&ctx->mmap->metadata_modseqs, &count); + for (i = 0; i < count; i++) { + if (array_is_created(&metadata->modseqs)) + array_delete(&metadata->modseqs, seq1, seq2-seq1); + } + + modseq = get_cur_modseq(ctx); + if (ctx->highest_modseq < modseq) + ctx->highest_modseq = modseq; +} + +static void +modseqs_update(ARRAY_TYPE(modseqs) *array, uint32_t seq1, uint32_t seq2, + uint64_t value) +{ + for (; seq1 <= seq2; seq1++) + array_idx_set(array, seq1-1, &value); +} + +static void +modseqs_idx_update(struct mail_index_modseq_sync *ctx, unsigned int idx, + uint32_t seq1, uint32_t seq2) +{ + struct metadata_modseqs *metadata; + + if (!ctx->view->index->modseqs_enabled) { + /* we want to keep permanent modseqs updated, but don't bother + updating in-memory per-flag updates */ + return; + } + + metadata = array_idx_modifiable(&ctx->mmap->metadata_modseqs, idx); + if (!array_is_created(&metadata->modseqs)) + i_array_init(&metadata->modseqs, seq2 + 16); + modseqs_update(&metadata->modseqs, seq1, seq2, ctx->highest_modseq); +} + +void mail_index_modseq_update_flags(struct mail_index_modseq_sync *ctx, + enum mail_flags flags_mask, + uint32_t seq1, uint32_t seq2) +{ + unsigned int i; + + if (!mail_index_modseq_update_highest(ctx, seq1, seq2)) + return; + + for (i = 0; i < METADATA_MODSEQ_IDX_KEYWORD_START; i++) { + if ((flags_mask & (1 << i)) != 0) + modseqs_idx_update(ctx, i, seq1, seq2); + } +} + +void mail_index_modseq_update_keyword(struct mail_index_modseq_sync *ctx, + unsigned int keyword_idx, + uint32_t seq1, uint32_t seq2) +{ + if (!mail_index_modseq_update_highest(ctx, seq1, seq2)) + return; + + modseqs_idx_update(ctx, METADATA_MODSEQ_IDX_KEYWORD_START + keyword_idx, + seq1, seq2); +} + +void mail_index_modseq_reset_keywords(struct mail_index_modseq_sync *ctx, + uint32_t seq1, uint32_t seq2) +{ + unsigned int i, count; + + if (!mail_index_modseq_update_highest(ctx, seq1, seq2)) + return; + + count = array_count(&ctx->mmap->metadata_modseqs); + for (i = METADATA_MODSEQ_IDX_KEYWORD_START; i < count; i++) + modseqs_idx_update(ctx, i, seq1, seq2); +} + +void mail_index_map_modseq_free(struct mail_index_map_modseq *mmap) +{ + struct metadata_modseqs *metadata; + unsigned int i, count; + + metadata = array_get_modifiable(&mmap->metadata_modseqs, &count); + for (i = 0; i < count; i++) { + if (array_is_created(&metadata->modseqs)) + array_free(&metadata->modseqs); + } + array_free(&mmap->metadata_modseqs); + i_free(mmap); +}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-index/mail-index-modseq.h Sat Mar 15 15:23:35 2008 +0200 @@ -0,0 +1,44 @@ +#ifndef MAIL_INDEX_MODSEQ_H +#define MAIL_INDEX_MODSEQ_H + +enum mail_flags; +struct mail_keywords; +struct mail_index; +struct mail_index_view; +struct mail_index_modseq; +struct mail_index_map_modseq; +struct mail_index_sync_map_ctx; + +void mail_index_modseq_init(struct mail_index *index); + +void mail_index_modseq_enable(struct mail_index *index); +uint64_t mail_index_modseq_get_highest(struct mail_index_view *view); + +uint64_t mail_index_modseq_lookup(struct mail_index_view *view, uint32_t seq); +uint64_t mail_index_modseq_lookup_flags(struct mail_index_view *view, + enum mail_flags flags_mask, + uint32_t seq); +uint64_t mail_index_modseq_lookup_keywords(struct mail_index_view *view, + const struct mail_keywords *keywords, + uint32_t seq); + +struct mail_index_modseq_sync * +mail_index_modseq_sync_begin(struct mail_index_sync_map_ctx *sync_map_ctx); +void mail_index_modseq_sync_end(struct mail_index_modseq_sync **ctx); + +void mail_index_modseq_hdr_update(struct mail_index_modseq_sync *ctx); +void mail_index_modseq_append(struct mail_index_modseq_sync *ctx, uint32_t seq); +void mail_index_modseq_expunge(struct mail_index_modseq_sync *ctx, + uint32_t seq1, uint32_t seq2); +void mail_index_modseq_update_flags(struct mail_index_modseq_sync *ctx, + enum mail_flags flags_mask, + uint32_t seq1, uint32_t seq2); +void mail_index_modseq_update_keyword(struct mail_index_modseq_sync *ctx, + unsigned int keyword_idx, + uint32_t seq1, uint32_t seq2); +void mail_index_modseq_reset_keywords(struct mail_index_modseq_sync *ctx, + uint32_t seq1, uint32_t seq2); + +void mail_index_map_modseq_free(struct mail_index_map_modseq *mmap); + +#endif