Mercurial > dovecot > original-hg > dovecot-1.2
changeset 7931:502cfdcc5650 HEAD
Keep modseqs as 1 until the first modseq ext intro record enables them.
Performance should be better again when modseqs are disabled.
author | Timo Sirainen <tss@iki.fi> |
---|---|
date | Sat, 21 Jun 2008 15:06:46 +0300 |
parents | b4286cde68d0 |
children | 91758686277a |
files | src/lib-index/mail-index-modseq.c src/lib-index/mail-index-modseq.h src/lib-index/mail-transaction-log-append.c src/lib-index/mail-transaction-log-file.c src/lib-index/mail-transaction-log-private.h src/lib-index/mail-transaction-log-view.c src/util/logview.c |
diffstat | 7 files changed, 134 insertions(+), 75 deletions(-) [+] |
line wrap: on
line diff
--- a/src/lib-index/mail-index-modseq.c Sat Jun 21 14:03:59 2008 +0300 +++ b/src/lib-index/mail-index-modseq.c Sat Jun 21 15:06:46 2008 +0300 @@ -7,8 +7,6 @@ #include "mail-index-sync-private.h" #include "mail-index-modseq.h" -#define MAIL_INDEX_MODSEQ_EXT_NAME "modseq" - ARRAY_DEFINE_TYPE(modseqs, uint64_t); enum modseq_metadata_idx { @@ -51,7 +49,7 @@ static uint64_t mail_index_modseq_get_head(struct mail_index *index) { return index->log->head == NULL ? 1 : - index->log->head->sync_highest_modseq; + I_MAX(index->log->head->sync_highest_modseq, 1); } void mail_index_modseq_enable(struct mail_index *index)
--- a/src/lib-index/mail-index-modseq.h Sat Jun 21 14:03:59 2008 +0300 +++ b/src/lib-index/mail-index-modseq.h Sat Jun 21 15:06:46 2008 +0300 @@ -1,6 +1,8 @@ #ifndef MAIL_INDEX_MODSEQ_H #define MAIL_INDEX_MODSEQ_H +#define MAIL_INDEX_MODSEQ_EXT_NAME "modseq" + enum mail_flags; struct mail_keywords; struct mail_index;
--- a/src/lib-index/mail-transaction-log-append.c Sat Jun 21 14:03:59 2008 +0300 +++ b/src/lib-index/mail-transaction-log-append.c Sat Jun 21 15:06:46 2008 +0300 @@ -6,6 +6,7 @@ #include "write-full.h" #include "mail-index-private.h" #include "mail-index-view-private.h" +#include "mail-index-modseq.h" #include "mail-index-transaction-private.h" #include "mail-transaction-log-private.h" @@ -14,7 +15,7 @@ struct mail_index_transaction *trans; buffer_t *output; - unsigned int modseq_change_count; + uint64_t modseq; uint32_t first_append_size; bool sync_includes_this; }; @@ -25,6 +26,7 @@ { struct mail_transaction_header hdr; uint32_t hdr_size; + size_t hdr_pos; i_assert((type & MAIL_TRANSACTION_TYPE_MASK) != 0); i_assert((buf->used % 4) == 0); @@ -39,26 +41,31 @@ hdr.type |= MAIL_TRANSACTION_EXPUNGE_PROT; if ((ctx->trans->flags & MAIL_INDEX_TRANSACTION_FLAG_EXTERNAL) != 0) hdr.type |= MAIL_TRANSACTION_EXTERNAL; - - if (mail_transaction_header_has_modseq(&hdr)) - ctx->modseq_change_count++; + hdr.size = sizeof(hdr) + buf->used + + (hdr_buf == NULL ? 0 : hdr_buf->used); - hdr_size = mail_index_uint32_to_offset(sizeof(hdr) + buf->used + - (hdr_buf == NULL ? 0 : - hdr_buf->used)); + hdr_pos = ctx->output->used; + buffer_append(ctx->output, &hdr, sizeof(hdr)); + if (hdr_buf != NULL) + buffer_append(ctx->output, hdr_buf->data, hdr_buf->used); + buffer_append(ctx->output, buf->data, buf->used); + + if (mail_transaction_header_has_modseq(buf->data, + CONST_PTR_OFFSET(buf->data, sizeof(hdr)), ctx->modseq)) + ctx->modseq++; + + /* update the size */ + hdr_size = mail_index_uint32_to_offset(hdr.size); if (!MAIL_TRANSACTION_LOG_FILE_IN_MEMORY(ctx->file) && ctx->first_append_size == 0) { /* size will be written later once everything is in disk */ ctx->first_append_size = hdr_size; + hdr.size = 0; } else { hdr.size = hdr_size; } - - buffer_append(ctx->output, &hdr, sizeof(hdr)); - if (hdr_buf != NULL) - buffer_append(ctx->output, hdr_buf->data, hdr_buf->used); - buffer_append(ctx->output, buf->data, buf->used); + buffer_write(ctx->output, hdr_pos, &hdr, sizeof(hdr)); } static int log_buffer_move_to_memory(struct log_append_context *ctx) @@ -280,10 +287,15 @@ /* new extension, reset_id defaults to 0 */ } buffer_append(buf, rext->name, intro->name_size); - if ((buf->used % 4) != 0) buffer_append_zero(buf, 4 - (buf->used % 4)); + if (ctx->file->sync_highest_modseq == 0 && + strcmp(rext->name, MAIL_INDEX_MODSEQ_EXT_NAME) == 0) { + /* modseq tracking started */ + ctx->file->sync_highest_modseq = 1; + } + log_append_buffer(ctx, buf, NULL, MAIL_TRANSACTION_EXT_INTRO); } @@ -601,6 +613,7 @@ ctx.file = file; ctx.trans = t; ctx.output = buffer_create_dynamic(default_pool, 1024); + ctx.modseq = file->sync_highest_modseq; /* send all extension introductions and resizes before appends to avoid resize overhead as much as possible */ @@ -674,7 +687,7 @@ buffer_free(&ctx.output); return -1; } - file->sync_highest_modseq += ctx.modseq_change_count; + file->sync_highest_modseq = ctx.modseq; buffer_free(&ctx.output); if ((t->flags & MAIL_INDEX_TRANSACTION_FLAG_HIDE) != 0) {
--- a/src/lib-index/mail-transaction-log-file.c Sat Jun 21 14:03:59 2008 +0300 +++ b/src/lib-index/mail-transaction-log-file.c Sat Jun 21 15:06:46 2008 +0300 @@ -102,45 +102,62 @@ } static void -mail_transaction_log_file_add_to_list(struct mail_transaction_log_file *file) +mail_transaction_log_file_skip_to_head(struct mail_transaction_log_file *file) { struct mail_transaction_log *log = file->log; - struct mail_transaction_log_file **p; struct mail_index_map *map = log->index->map; const struct mail_index_modseq_header *modseq_hdr; + uoff_t head_offset; - if (map != NULL && file->hdr.file_seq == map->hdr.log_file_seq && - map->hdr.log_file_head_offset != 0) { - /* we can get a valid log offset from index file. initialize - sync_offset from it so we don't have to read the whole log - file from beginning. */ - uoff_t head_offset = map->hdr.log_file_head_offset; + if (map == NULL || file->hdr.file_seq != map->hdr.log_file_seq || + map->hdr.log_file_head_offset == 0) + return; + + /* we can get a valid log offset from index file. initialize + sync_offset from it so we don't have to read the whole log + file from beginning. */ + head_offset = map->hdr.log_file_head_offset; - modseq_hdr = mail_index_map_get_modseq_header(map); - if (head_offset < file->hdr.hdr_size) { - mail_index_set_error(log->index, - "%s: log_file_head_offset too small", - log->index->filepath); - file->sync_offset = file->hdr.hdr_size; - file->sync_highest_modseq = file->hdr.initial_modseq; - } else if (modseq_hdr == NULL || - modseq_hdr->log_seq != file->hdr.file_seq || - modseq_hdr->log_offset != head_offset) { - /* highest_modseq not synced, start from beginning */ - file->sync_offset = file->hdr.hdr_size; - file->sync_highest_modseq = file->hdr.initial_modseq; - } else { - file->sync_offset = head_offset; - file->sync_highest_modseq = modseq_hdr->highest_modseq; - } - file->saved_tail_offset = map->hdr.log_file_tail_offset; - } else { + modseq_hdr = mail_index_map_get_modseq_header(map); + if (head_offset < file->hdr.hdr_size) { + mail_index_set_error(log->index, + "%s: log_file_head_offset too small", + log->index->filepath); + file->sync_offset = file->hdr.hdr_size; + file->sync_highest_modseq = file->hdr.initial_modseq; + } else if (modseq_hdr == NULL && file->hdr.initial_modseq == 0) { + /* modseqs not used yet */ + file->sync_offset = head_offset; + file->sync_highest_modseq = 0; + } else if (modseq_hdr->log_seq != file->hdr.file_seq) { + /* highest_modseq not synced, start from beginning */ file->sync_offset = file->hdr.hdr_size; file->sync_highest_modseq = file->hdr.initial_modseq; + } else if (modseq_hdr->log_offset > head_offset) { + mail_index_set_error(log->index, + "%s: modseq_hdr.log_offset too large", + log->index->filepath); + file->sync_offset = file->hdr.hdr_size; + file->sync_highest_modseq = file->hdr.initial_modseq; + } else { + /* start from where we last stopped tracking modseqs */ + file->sync_offset = modseq_hdr->log_offset; + file->sync_highest_modseq = modseq_hdr->highest_modseq; } + file->saved_tail_offset = log->index->map->hdr.log_file_tail_offset; +} + +static void +mail_transaction_log_file_add_to_list(struct mail_transaction_log_file *file) +{ + struct mail_transaction_log_file **p; + + file->sync_offset = file->hdr.hdr_size; + file->sync_highest_modseq = file->hdr.initial_modseq; + mail_transaction_log_file_skip_to_head(file); /* insert it to correct position */ - for (p = &log->files; *p != NULL; p = &(*p)->next) { + for (p = &file->log->files; *p != NULL; p = &(*p)->next) { if ((*p)->hdr.file_seq > file->hdr.file_seq) break; i_assert((*p)->hdr.file_seq < file->hdr.file_seq); @@ -179,7 +196,8 @@ hdr->prev_file_seq = index->map->hdr.log_file_seq; hdr->prev_file_offset = index->map->hdr.log_file_head_offset; hdr->file_seq = index->map->hdr.log_file_seq + 1; - hdr->initial_modseq = + hdr->initial_modseq = log->head == NULL || + log->head->sync_highest_modseq == 0 ? 0 : mail_index_map_modseq_get_highest(index->map); } else { hdr->file_seq = 1; @@ -368,8 +386,6 @@ shouldn't have filled */ memset(PTR_OFFSET(&file->hdr, file->hdr.hdr_size), 0, sizeof(file->hdr) - file->hdr.hdr_size); - if (file->hdr.minor_version == 0) - file->hdr.initial_modseq = 1; } if (file->hdr.indexid == 0) { @@ -524,7 +540,7 @@ if (reset) { file->hdr.prev_file_seq = 0; file->hdr.prev_file_offset = 0; - file->hdr.initial_modseq = 1; + file->hdr.initial_modseq = 0; } if (write_full(new_fd, &file->hdr, sizeof(file->hdr)) < 0) { @@ -724,8 +740,31 @@ } bool -mail_transaction_header_has_modseq(const struct mail_transaction_header *hdr) +mail_transaction_header_has_modseq(const struct mail_transaction_header *hdr, + const void *data, + uint64_t cur_modseq) { + if (cur_modseq != 0) { + /* tracking modseqs */ + } else if ((hdr->type & MAIL_TRANSACTION_TYPE_MASK) == + MAIL_TRANSACTION_EXT_INTRO) { + /* modseqs not tracked yet. see if this is a modseq + extension introduction. */ + const struct mail_transaction_ext_intro *intro = data; + const unsigned int modseq_ext_len = + strlen(MAIL_INDEX_MODSEQ_EXT_NAME); + + if (intro->name_size == modseq_ext_len && + memcmp(intro + 1, MAIL_INDEX_MODSEQ_EXT_NAME, + modseq_ext_len) == 0) { + /* modseq tracking started */ + return TRUE; + } + } else { + /* not tracking modseqs */ + return FALSE; + } + switch (hdr->type & MAIL_TRANSACTION_TYPE_MASK) { case MAIL_TRANSACTION_EXPUNGE | MAIL_TRANSACTION_EXPUNGE_PROT: if ((hdr->type & MAIL_TRANSACTION_EXTERNAL) == 0) { @@ -882,7 +921,8 @@ while (cur_offset < offset) { if (log_get_synced_record(file, &cur_offset, &hdr) < 0) return- 1; - if (mail_transaction_header_has_modseq(hdr)) + if (mail_transaction_header_has_modseq(hdr, hdr + 1, + cur_modseq)) cur_modseq++; } @@ -943,7 +983,8 @@ prev_offset = cur_offset; if (log_get_synced_record(file, &cur_offset, &hdr) < 0) return -1; - if (mail_transaction_header_has_modseq(hdr)) { + if (mail_transaction_header_has_modseq(hdr, hdr + 1, + cur_modseq)) { if (++cur_modseq == modseq) break; } @@ -973,11 +1014,12 @@ const struct mail_transaction_header *hdr, unsigned int trans_size) { + const void *data = hdr + 1; int ret; - if (mail_transaction_header_has_modseq(hdr)) + if (mail_transaction_header_has_modseq(hdr, hdr + 1, + file->sync_highest_modseq)) file->sync_highest_modseq++; - if ((hdr->type & MAIL_TRANSACTION_EXTERNAL) == 0) return 0; @@ -985,7 +1027,7 @@ if ((hdr->type & MAIL_TRANSACTION_TYPE_MASK) == MAIL_TRANSACTION_HEADER_UPDATE) { /* see if this updates mailbox_sync_offset */ - ret = log_file_track_mailbox_sync_offset_hdr(file, hdr + 1, + ret = log_file_track_mailbox_sync_offset_hdr(file, data, trans_size - sizeof(*hdr)); if (ret != 0) @@ -1209,13 +1251,6 @@ } } - if (start_offset > file->sync_offset) { - /* although we could just skip over the unwanted data, we have - to sync everything so that modseqs are calculated - correctly */ - start_offset = file->sync_offset; - } - if (file->buffer != NULL && file->buffer_offset > start_offset) { /* we have to insert missing data to beginning of buffer */ ret = mail_transaction_log_file_insert_read(file, start_offset); @@ -1419,6 +1454,15 @@ end_offset); } + if (start_offset > file->sync_offset) + mail_transaction_log_file_skip_to_head(file); + if (start_offset > file->sync_offset) { + /* although we could just skip over the unwanted data, we have + to sync everything so that modseqs are calculated + correctly */ + start_offset = file->sync_offset; + } + if (!index->mmap_disable) ret = mail_transaction_log_file_map_mmap(file, start_offset); else {
--- a/src/lib-index/mail-transaction-log-private.h Sat Jun 21 14:03:59 2008 +0300 +++ b/src/lib-index/mail-transaction-log-private.h Sat Jun 21 15:06:46 2008 +0300 @@ -122,7 +122,9 @@ void mail_transaction_log_file_unlock(struct mail_transaction_log_file *file); bool -mail_transaction_header_has_modseq(const struct mail_transaction_header *hdr); +mail_transaction_header_has_modseq(const struct mail_transaction_header *hdr, + const void *data, + uint64_t cur_modseq); int mail_transaction_log_file_get_highest_modseq_at( struct mail_transaction_log_file *file, uoff_t offset, uint64_t *highest_modseq_r);
--- a/src/lib-index/mail-transaction-log-view.c Sat Jun 21 14:03:59 2008 +0300 +++ b/src/lib-index/mail-transaction-log-view.c Sat Jun 21 15:06:46 2008 +0300 @@ -21,12 +21,7 @@ struct mail_transaction_log_file *cur, *head, *tail; uoff_t cur_offset; - /* prev_modseq doesn't contain correct values until we know that - caller is really interested in modseqs. so the prev_modseq begins - from 0 and it's relative to prev_modseq_start_offset. when - prev_modseq_initialized=TRUE prev_modseq contains a correct value */ uint64_t prev_modseq; - uint32_t prev_file_seq; uoff_t prev_file_offset; @@ -614,7 +609,8 @@ ret = log_view_is_record_valid(file, hdr, data) ? 1 : -1; } T_END; if (ret > 0) { - if (mail_transaction_header_has_modseq(hdr)) + if (mail_transaction_header_has_modseq(hdr, data, + view->prev_modseq)) view->prev_modseq++; *hdr_r = hdr; *data_r = data;
--- a/src/util/logview.c Sat Jun 21 14:03:59 2008 +0300 +++ b/src/util/logview.c Sat Jun 21 15:06:46 2008 +0300 @@ -45,7 +45,7 @@ printf("create stamp = %u\n", hdr.create_stamp); printf("initial modseq = %llu\n", (unsigned long long)hdr.initial_modseq); - *modseq_r = I_MAX(hdr.initial_modseq, 1); + *modseq_r = hdr.initial_modseq; } static bool @@ -220,7 +220,7 @@ } static void log_record_print(const struct mail_transaction_header *hdr, - const void *data) + const void *data, uint64_t *modseq) { unsigned int size = hdr->size - sizeof(*hdr); @@ -275,8 +275,12 @@ printf(" - flags = %u\n", intro->flags); printf(" - name_size = %u\n", intro->name_size); if (intro->name_size > 0) { - printf(" - name = '%.*s'\n", - intro->name_size, (const char *)(intro+1)); + const char *name = (const char *)(intro+1); + + printf(" - name = '%.*s'\n", intro->name_size, name); + if (*modseq == 0 && intro->name_size == 6 && + memcmp(name, "modseq", 6) == 0) + *modseq = 1; } break; } @@ -372,7 +376,7 @@ printf("record: offset=%"PRIuUOFF_T", type=%s, size=%u", offset, log_record_type(hdr.type), hdr.size); - if (mail_transaction_header_has_modseq(&hdr)) { + if (*modseq > 0 && mail_transaction_header_has_modseq(&hdr)) { *modseq += 1; printf(", modseq=%llu", (unsigned long long)*modseq); } @@ -386,7 +390,7 @@ i_fatal("rec data read() %"PRIuSIZE_T" != %"PRIuSIZE_T, ret, hdr.size - sizeof(hdr)); } - log_record_print(&hdr, buf); + log_record_print(&hdr, buf, modseq); } else { lseek(fd, hdr.size - sizeof(hdr), SEEK_CUR); }