Mercurial > dovecot > original-hg > dovecot-1.2
changeset 2885:1880f191278a HEAD
Avoid re-reading index file with mmap_disabled=yes. Instead sync it by
reading transaction log file.
author | Timo Sirainen <tss@iki.fi> |
---|---|
date | Sun, 14 Nov 2004 03:23:21 +0200 |
parents | 173a0296e81d |
children | 634cf6aa34d1 |
files | src/lib-index/mail-index-private.h src/lib-index/mail-index-sync.c src/lib-index/mail-index-view-sync.c src/lib-index/mail-index.c src/lib-index/mail-transaction-log-view.c |
diffstat | 5 files changed, 181 insertions(+), 29 deletions(-) [+] |
line wrap: on
line diff
--- a/src/lib-index/mail-index-private.h Sun Nov 14 01:08:07 2004 +0200 +++ b/src/lib-index/mail-index-private.h Sun Nov 14 03:23:21 2004 +0200 @@ -107,6 +107,11 @@ struct dotlock dotlock; enum mail_index_lock_method lock_method; + /* These are typically same as map->hdr->log_file_*, but with + mmap_disable we may have synced more than index */ + uint32_t sync_log_file_seq; + uoff_t sync_log_file_offset; + unsigned int last_grow_count; char *error;
--- a/src/lib-index/mail-index-sync.c Sun Nov 14 01:08:07 2004 +0200 +++ b/src/lib-index/mail-index-sync.c Sun Nov 14 03:23:21 2004 +0200 @@ -310,10 +310,29 @@ return -1; } - if (mail_index_map(index, FALSE) <= 0) { - mail_transaction_log_sync_unlock(index->log); - mail_index_unlock(index, lock_id); - return -1; + if (index->mmap_disable) { + if (index->sync_log_file_seq != seq || + index->sync_log_file_offset != offset) { + /* we may have synced our internal view more than what + is synced in index. re-read the whole index if our + sync seq/offset doesn't match what is in index's + header. */ + if (mail_index_map(index, TRUE) <= 0) { + mail_transaction_log_sync_unlock(index->log); + mail_index_unlock(index, lock_id); + return -1; + } + } else { + /* the whole log file is synced already. */ + i_assert(index->map->hdr.log_file_seq == seq); + i_assert(index->map->hdr.log_file_ext_offset == offset); + } + } else { + if (mail_index_map(index, FALSE) <= 0) { + mail_transaction_log_sync_unlock(index->log); + mail_index_unlock(index, lock_id); + return -1; + } } if (!mail_index_need_lock(index, sync_recent, @@ -504,6 +523,7 @@ int mail_index_sync_commit(struct mail_index_sync_ctx *ctx) { + struct mail_index *index = ctx->index; const struct mail_index_header *hdr; uint32_t seq, seq2; uoff_t offset, offset2; @@ -515,9 +535,9 @@ /* we have had the transaction log locked since the beginning of sync, so only external changes could have been committed. write them to the index here as well. */ - mail_transaction_log_get_head(ctx->index->log, &seq, &offset); + mail_transaction_log_get_head(index->log, &seq, &offset); - hdr = ctx->index->hdr; + hdr = index->hdr; if (ret == 0 && (hdr->log_file_seq != seq || hdr->log_file_int_offset != offset || hdr->log_file_ext_offset != offset)) { @@ -529,12 +549,12 @@ ret = -1; } - if (ret == 0 && mail_cache_need_compress(ctx->index->cache)) { - if (mail_cache_compress(ctx->index->cache, ctx->view) < 0) + if (ret == 0 && mail_cache_need_compress(index->cache)) { + if (mail_cache_compress(index->cache, ctx->view) < 0) ret = -1; else { /* cache_offsets have changed, sync them */ - mail_transaction_log_get_head(ctx->index->log, + mail_transaction_log_get_head(index->log, &seq2, &offset2); if (mail_transaction_log_view_set(ctx->view->log_view, seq, offset, seq2, offset2, @@ -545,6 +565,9 @@ } } + index->sync_log_file_seq = index->map->hdr.log_file_seq; + index->sync_log_file_offset = index->map->hdr.log_file_int_offset; + mail_index_sync_end(ctx); return ret; }
--- a/src/lib-index/mail-index-view-sync.c Sun Nov 14 01:08:07 2004 +0200 +++ b/src/lib-index/mail-index-view-sync.c Sun Nov 14 03:23:21 2004 +0200 @@ -342,6 +342,9 @@ i_assert(view->syncing); + if (ctx->sync_map_update) + mail_index_sync_map_deinit(&ctx->sync_map_ctx); + if (view->log_syncs != NULL && !ctx->skipped_some) buffer_set_used_size(view->log_syncs, 0);
--- a/src/lib-index/mail-index.c Sun Nov 14 01:08:07 2004 +0200 +++ b/src/lib-index/mail-index.c Sun Nov 14 03:23:21 2004 +0200 @@ -6,6 +6,7 @@ #include "read-full.h" #include "write-full.h" #include "mail-index-private.h" +#include "mail-index-sync-private.h" #include "mail-transaction-log.h" #include "mail-cache.h" @@ -432,6 +433,25 @@ return 1; } +static int mail_index_read_header(struct mail_index *index, + struct mail_index_header *hdr, size_t *pos_r) +{ + size_t pos; + int ret; + + memset(hdr, 0, sizeof(*hdr)); + + ret = 1; + for (pos = 0; ret > 0 && pos < sizeof(*hdr); ) { + ret = pread(index->fd, PTR_OFFSET(hdr, pos), + sizeof(*hdr) - pos, pos); + if (ret > 0) + pos += ret; + } + *pos_r = pos; + return ret; +} + static int mail_index_read_map(struct mail_index *index, struct mail_index_map *map, int *retry_r) { @@ -443,15 +463,7 @@ i_assert(map->mmap_base == NULL); *retry_r = FALSE; - memset(&hdr, 0, sizeof(hdr)); - - ret = 1; - for (pos = 0; ret > 0 && pos < sizeof(hdr); ) { - ret = pread(index->fd, PTR_OFFSET(&hdr, pos), - sizeof(hdr) - pos, pos); - if (ret > 0) - pos += ret; - } + ret = mail_index_read_header(index, &hdr, &pos); if (pos > (ssize_t)offsetof(struct mail_index_header, major_version) && hdr.major_version != MAIL_INDEX_MAJOR_VERSION) { @@ -523,14 +535,114 @@ memcpy(&map->hdr, &hdr, hdr.base_header_size); } map->hdr_base = map->hdr_copy_buf->data; + + index->sync_log_file_seq = hdr.log_file_seq; + index->sync_log_file_offset = hdr.log_file_int_offset; return 1; } +static int mail_index_sync_from_transactions(struct mail_index *index, + struct mail_index_map *map, + int sync_to_index) +{ + struct mail_index_view *view; + struct mail_transaction_log_view *log_view; + struct mail_index_sync_map_ctx sync_map_ctx; + struct mail_index_header hdr; + const struct mail_transaction_header *thdr; + const void *tdata; + uint32_t max_seq; + uoff_t max_offset; + size_t pos; + int ret, skipped; + + if (sync_to_index) { + /* read the real log position where we are supposed to be + synced */ + ret = mail_index_read_header(index, &hdr, &pos); + if (ret < 0 && errno != ESTALE) { + mail_index_set_syscall_error(index, "pread()"); + return -1; + } + if (pos < MAIL_INDEX_HEADER_MIN_SIZE) + return 0; + + if (map->hdr.log_file_seq == hdr.log_file_seq && + map->hdr.log_file_int_offset == hdr.log_file_int_offset) { + /* nothing to do */ + return 1; + } + + if (map->hdr.log_file_seq > hdr.log_file_seq || + (map->hdr.log_file_seq == hdr.log_file_seq && + map->hdr.log_file_int_offset > hdr.log_file_int_offset)) { + /* we went too far, have to re-read the file */ + return 0; + } + if (map->hdr.log_file_ext_offset != + map->hdr.log_file_int_offset || + hdr.log_file_ext_offset != hdr.log_file_int_offset) { + /* too much trouble to get this right. */ + return 0; + } + max_seq = hdr.log_file_seq; + max_offset = hdr.log_file_int_offset; + } else { + /* sync everything there is */ + max_seq = (uint32_t)-1; + max_offset = (uoff_t)-1; + } + + log_view = mail_transaction_log_view_open(index->log); + if (mail_transaction_log_view_set(log_view, + map->hdr.log_file_seq, + map->hdr.log_file_int_offset, + max_seq, max_offset, + MAIL_TRANSACTION_TYPE_MASK) < 0) { + mail_transaction_log_view_close(log_view); + return 0; + } + + index->map = map; + + view = mail_index_view_open(index); + mail_index_sync_map_init(&sync_map_ctx, view, + MAIL_INDEX_SYNC_HANDLER_VIEW); + + while ((ret = mail_transaction_log_view_next(log_view, &thdr, &tdata, + &skipped)) > 0) { + if (mail_index_sync_record(&sync_map_ctx, thdr, tdata) < 0) { + ret = -1; + break; + } + } + + mail_index_sync_map_deinit(&sync_map_ctx); + mail_index_view_close(view); + mail_transaction_log_view_close(log_view); + + index->map = NULL; + return ret < 0 ? -1 : 1; +} + static int mail_index_read_map_with_retry(struct mail_index *index, - struct mail_index_map *map) + struct mail_index_map *map, + int sync_to_index) { int i, ret, retry; + if (map->hdr.indexid != 0) { + /* sync this as a view from transaction log. */ + ret = mail_index_sync_from_transactions(index, map, + sync_to_index); + if (ret != 0) + return ret; + + /* transaction log lost/broken, fallback to re-reading it */ + /* FIXME: file cache need to be reset (except not really with + sync_to_index if we were just rewinding..) */ + } + for (i = 0; i < MAIL_INDEX_ESTALE_RETRY_COUNT; i++) { ret = mail_index_read_map(index, map, &retry); if (ret != 0 || !retry) @@ -598,24 +710,27 @@ if (index->map != NULL && index->map->refcount > 1) { /* this map is already used by some views and they may have pointers into it. leave them and create a new mapping. */ + if (!index->mmap_disable) { + map = NULL; + } else { + /* create a copy of the mapping instead so we don't + have to re-read it */ + map = mail_index_map_to_memory(index->map, + index->map->hdr.record_size); + } index->map->refcount--; index->map = NULL; + } else { + map = index->map; } - map = index->map; if (map == NULL) { map = i_new(struct mail_index_map, 1); map->refcount = 1; map->hdr_copy_buf = buffer_create_dynamic(default_pool, sizeof(map->hdr)); } else if (MAIL_INDEX_MAP_IS_IN_MEMORY(map)) { - if (map->write_to_disk) { - /* we have modified this mapping and it's waiting to - be written to disk once we drop exclusive lock. - mapping couldn't have changed, so do nothing. */ - return 1; - } - /* FIXME: we need to re-read header */ + i_assert(!map->write_to_disk); } else if (map->mmap_base != NULL) { i_assert(map->buffer == NULL); if (munmap(map->mmap_base, map->mmap_size) < 0) @@ -628,8 +743,8 @@ if (!index->mmap_disable) ret = mail_index_mmap(index, map); - else - ret = mail_index_read_map_with_retry(index, map); + else + ret = mail_index_read_map_with_retry(index, map, force); if (ret <= 0) { mail_index_unmap_forced(index, map); return ret;
--- a/src/lib-index/mail-transaction-log-view.c Sun Nov 14 01:08:07 2004 +0200 +++ b/src/lib-index/mail-transaction-log-view.c Sun Nov 14 03:23:21 2004 +0200 @@ -153,6 +153,12 @@ for (seq = min_file_seq+1; seq <= max_file_seq; seq++) { file = file->next; if (file == NULL || file->hdr.file_seq != seq) { + if (file == NULL && max_file_seq == (uint32_t)-1) { + /* we just wanted to sync everything */ + max_file_seq = seq-1; + break; + } + mail_index_set_error(view->log->index, "Lost transaction log file %s seq %u", view->log->tail->filepath, seq);