Mercurial > dovecot > core-2.2
changeset 2275:c68a3c9f6d73 HEAD
Cache file compression works now and compressed cache file is reopened.
Several other cleanups related to opening - cache file isn't created
immediately anymore.
author | Timo Sirainen <tss@iki.fi> |
---|---|
date | Sun, 04 Jul 2004 14:50:48 +0300 |
parents | 97da1d911484 |
children | 5f374049abdb |
files | src/lib-index/mail-cache-compress.c src/lib-index/mail-cache-decisions.c src/lib-index/mail-cache-lookup.c src/lib-index/mail-cache-private.h src/lib-index/mail-cache-transaction.c src/lib-index/mail-cache.c src/lib-index/mail-cache.h src/lib-index/mail-index-sync.c src/lib-storage/index/index-mail.c src/lib-storage/index/index-storage.h |
diffstat | 10 files changed, 242 insertions(+), 370 deletions(-) [+] |
line wrap: on
line diff
--- a/src/lib-index/mail-cache-compress.c Sat Jul 03 23:17:37 2004 +0300 +++ b/src/lib-index/mail-cache-compress.c Sun Jul 04 14:50:48 2004 +0300 @@ -4,6 +4,7 @@ #include "buffer.h" #include "byteorder.h" #include "ostream.h" +#include "file-set-size.h" #include "mail-cache-private.h" static unsigned char null4[4] = { 0, 0, 0, 0 }; @@ -100,39 +101,46 @@ removed. */ if (mail_index_get_header(view, &idx_hdr) < 0) return -1; - if (mail_index_lookup_uid_range(view, idx_hdr->day_first_uid[7], - (uint32_t)-1, &first_new_seq, - &message_count) < 0) - return -1; - if (first_new_seq == 0) - first_new_seq = message_count+1; + if (idx_hdr->day_first_uid[7] == 0) { + first_new_seq = 1; + message_count = mail_index_view_get_message_count(view); + } else { + if (mail_index_lookup_uid_range(view, idx_hdr->day_first_uid[7], + (uint32_t)-1, &first_new_seq, + &message_count) < 0) + return -1; + if (first_new_seq == 0) + first_new_seq = message_count+1; + } cache_view = mail_cache_view_open(cache, view); t = mail_index_transaction_begin(view, FALSE); output = o_stream_create_file(fd, default_pool, 0, FALSE); memset(&hdr, 0, sizeof(hdr)); - hdr.indexid = cache->hdr->indexid; - hdr.file_seq = cache->hdr->file_seq + 1; + hdr.indexid = idx_hdr->indexid; + hdr.file_seq = idx_hdr->cache_file_seq + 1; - memcpy(hdr.field_usage_decision_type, - cache->hdr->field_usage_decision_type, - sizeof(hdr.field_usage_decision_type)); - memcpy(hdr.field_usage_last_used, - cache->hdr->field_usage_last_used, - sizeof(hdr.field_usage_last_used)); + if (cache->hdr != NULL) { + memcpy(hdr.field_usage_decision_type, + cache->hdr->field_usage_decision_type, + sizeof(hdr.field_usage_decision_type)); + memcpy(hdr.field_usage_last_used, + cache->hdr->field_usage_last_used, + sizeof(hdr.field_usage_last_used)); - keep_fields = temp_fields = 0; - for (i = 0; i < 32; i++) { - if (cache->hdr->field_usage_decision_type[i] & - MAIL_CACHE_DECISION_YES) - keep_fields |= 1 << i; - else if (cache->hdr->field_usage_decision_type[i] & - MAIL_CACHE_DECISION_TEMP) - temp_fields |= 1 << i; + keep_fields = temp_fields = 0; + for (i = 0; i < 32; i++) { + if (cache->hdr->field_usage_decision_type[i] & + MAIL_CACHE_DECISION_YES) + keep_fields |= 1 << i; + else if (cache->hdr->field_usage_decision_type[i] & + MAIL_CACHE_DECISION_TEMP) + temp_fields |= 1 << i; + } } - offset = sizeof(hdr); + o_stream_send(output, &hdr, sizeof(hdr)); /* merge all the header pieces into one. if some message doesn't have all the required pieces, we'll just have to drop them all. */ @@ -145,7 +153,8 @@ if (str == NULL) header_idx = -1; else { - hdr.header_offsets[0] = mail_cache_uint32_to_offset(offset); + hdr.header_offsets[0] = + mail_cache_uint32_to_offset(output->offset); header_idx = i; size = strlen(str) + 1; @@ -198,8 +207,25 @@ } hdr.used_file_size = uint32_to_nbo(output->offset); + o_stream_seek(output, 0); + o_stream_send(output, &hdr, sizeof(hdr)); + + mail_cache_view_close(cache_view); + + if (o_stream_flush(output) < 0) { + errno = output->stream_errno; + mail_cache_set_syscall_error(cache, "o_stream_flush()"); + (void)mail_index_transaction_rollback(t); + o_stream_unref(output); + return -1; + } + + if (output->offset < MAIL_CACHE_INITIAL_SIZE) { + /* grow the file some more. doesn't matter if it fails */ + (void)file_set_size(fd, MAIL_CACHE_INITIAL_SIZE); + } + o_stream_unref(output); - mail_cache_view_close(cache_view); if (fdatasync(fd) < 0) { mail_cache_set_syscall_error(cache, "fdatasync()"); @@ -212,12 +238,13 @@ int mail_cache_compress(struct mail_cache *cache, struct mail_index_view *view) { - int fd, ret; + int fd, ret, locked; i_assert(cache->trans_ctx == NULL); - if ((ret = mail_cache_lock(cache, TRUE)) <= 0) - return ret; + if ((ret = mail_cache_lock(cache, TRUE)) < 0) + return -1; + locked = ret > 0; #ifdef DEBUG i_warning("Compressing cache file %s", cache->filepath); @@ -232,6 +259,7 @@ return -1; } + ret = 0; if (mail_cache_copy(cache, view, fd) < 0) { (void)file_dotlock_delete(cache->filepath, NULL, fd); ret = -1; @@ -246,7 +274,7 @@ mail_cache_file_close(cache); cache->fd = fd; - if (mail_cache_mmap_update(cache, 0, 0) < 0) + if (mail_cache_map(cache, 0, 0) < 0) ret = -1; } } @@ -255,8 +283,10 @@ memset(cache->split_offsets, 0, sizeof(cache->split_offsets)); memset(cache->split_headers, 0, sizeof(cache->split_headers)); - if (mail_cache_unlock(cache) < 0) - return -1; + if (locked) { + if (mail_cache_unlock(cache) < 0) + return -1; + } if (ret == 0) cache->need_compress = FALSE; @@ -265,5 +295,5 @@ int mail_cache_need_compress(struct mail_cache *cache) { - return FALSE; //FIXME:cache->need_compress; + return cache->need_compress; }
--- a/src/lib-index/mail-cache-decisions.c Sat Jul 03 23:17:37 2004 +0300 +++ b/src/lib-index/mail-cache-decisions.c Sun Jul 04 14:50:48 2004 +0300 @@ -132,6 +132,9 @@ unsigned int idx; uint32_t uid; + if (MAIL_CACHE_IS_UNUSABLE(view->cache)) + return; + idx = mail_cache_field_index(field); if (view->cache->hdr->field_usage_decision_type[idx] != MAIL_CACHE_DECISION_NO) {
--- a/src/lib-index/mail-cache-lookup.c Sat Jul 03 23:17:37 2004 +0300 +++ b/src/lib-index/mail-cache-lookup.c Sun Jul 04 14:50:48 2004 +0300 @@ -13,7 +13,7 @@ uint32_t offset, data_size; unsigned char *buf; - if (cache->disabled) + if (MAIL_CACHE_IS_UNUSABLE(cache)) return NULL; offset = mail_cache_offset_to_uint32(cache->hdr->header_offsets[idx]); @@ -21,7 +21,7 @@ if (offset == 0) return NULL; - if (mail_cache_mmap_update(cache, offset, CACHE_PREFETCH) < 0) + if (mail_cache_map(cache, offset, CACHE_PREFETCH) < 0) return NULL; if (offset + sizeof(data_size) > cache->mmap_length) { @@ -42,7 +42,7 @@ } if (data_size + sizeof(data_size) > CACHE_PREFETCH) { - if (mail_cache_mmap_update(cache, offset, data_size) < 0) + if (mail_cache_map(cache, offset, data_size) < 0) return NULL; } @@ -93,7 +93,7 @@ i_assert(idx < MAIL_CACHE_HEADERS_COUNT); - if (view->cache->disabled) + if (MAIL_CACHE_IS_UNUSABLE(view->cache)) return NULL; /* t_strsplit() is a bit slow, so we cache it */ @@ -127,8 +127,8 @@ if (offset == 0) return NULL; - if (mail_cache_mmap_update(cache, offset, - sizeof(*cache_rec) + CACHE_PREFETCH) < 0) + if (mail_cache_map(cache, offset, + sizeof(*cache_rec) + CACHE_PREFETCH) < 0) return NULL; if (offset + sizeof(*cache_rec) > cache->mmap_length) { @@ -143,7 +143,7 @@ return NULL; } if (size > CACHE_PREFETCH) { - if (mail_cache_mmap_update(cache, offset, size) < 0) + if (mail_cache_map(cache, offset, size) < 0) return NULL; } @@ -154,42 +154,48 @@ return cache_rec; } -struct mail_cache_record * -mail_cache_get_next_record(struct mail_cache *cache, - struct mail_cache_record *rec) +int mail_cache_lookup_offset(struct mail_cache_view *view, uint32_t seq, + uint32_t *offset_r, int skip_expunged) { - struct mail_cache_record *next; + const struct mail_index_record *rec; + struct mail_index_map *map; + int i, ret; - next = mail_cache_get_record(cache, rec->next_offset, FALSE); - if (next != NULL && next <= rec) { - mail_cache_set_corrupted(cache, "next_offset points backwards"); - return NULL; + for (i = 0; i < 2; i++) { + ret = mail_index_lookup_full(view->view, seq, &map, &rec); + if (ret < 0) + return -1; + if (ret == 0 && skip_expunged) + return 0; + + if (map->hdr->cache_file_seq == view->cache->hdr->file_seq) { + *offset_r = rec->cache_offset; + return 1; + } + + if ((ret = mail_cache_reopen(view->cache)) <= 0) + return ret; } - return next; + + return 0; } struct mail_cache_record * mail_cache_lookup(struct mail_cache_view *view, uint32_t seq, enum mail_cache_field fields) { - const struct mail_index_record *rec; - struct mail_index_map *map; + uint32_t offset; if (mail_cache_transaction_autocommit(view, seq, fields) < 0) return NULL; - if (view->cache->disabled) - return NULL; - - if (mail_index_lookup_full(view->view, seq, &map, &rec) < 0) + if (MAIL_CACHE_IS_UNUSABLE(view->cache)) return NULL; - if (map->hdr->cache_file_seq != view->cache->hdr->file_seq) { - /* FIXME: we should check if newer file is available? */ + if (mail_cache_lookup_offset(view, seq, &offset, FALSE) <= 0) return NULL; - } - return mail_cache_get_record(view->cache, rec->cache_offset, TRUE); + return mail_cache_get_record(view->cache, offset, TRUE); } enum mail_cache_field @@ -201,7 +207,9 @@ cache_rec = mail_cache_lookup(view, seq, 0); while (cache_rec != NULL) { fields |= cache_rec->fields; - cache_rec = mail_cache_get_next_record(view->cache, cache_rec); + cache_rec = mail_cache_get_record(view->cache, + cache_rec->next_offset, + FALSE); } return fields; @@ -280,7 +288,9 @@ return cache_get_field(view->cache, cache_rec, field, data_r, size_r); } - cache_rec = mail_cache_get_next_record(view->cache, cache_rec); + cache_rec = mail_cache_get_record(view->cache, + cache_rec->next_offset, + FALSE); } return FALSE;
--- a/src/lib-index/mail-cache-private.h Sat Jul 03 23:17:37 2004 +0300 +++ b/src/lib-index/mail-cache-private.h Sun Jul 04 14:50:48 2004 +0300 @@ -27,6 +27,9 @@ #define CACHE_RECORD(cache, offset) \ ((struct mail_cache_record *) ((char *) (cache)->mmap_base + offset)) +#define MAIL_CACHE_IS_UNUSABLE(cache) \ + ((cache)->hdr == NULL) + enum mail_cache_decision_type { /* Not needed currently */ MAIL_CACHE_DECISION_NO = 0x00, @@ -83,10 +86,7 @@ struct mail_cache_transaction_ctx *trans_ctx; unsigned int locks; - unsigned int mmap_refresh:1; unsigned int need_compress:1; - unsigned int silent:1; - unsigned int disabled:1; }; struct mail_cache_view { @@ -111,10 +111,9 @@ struct mail_cache_record * mail_cache_get_record(struct mail_cache *cache, uint32_t offset, int index_offset); -struct mail_cache_record * -mail_cache_get_next_record(struct mail_cache *cache, - struct mail_cache_record *rec); +int mail_cache_lookup_offset(struct mail_cache_view *view, uint32_t seq, + uint32_t *offset, int skip_expunged); struct mail_cache_record * mail_cache_lookup(struct mail_cache_view *view, uint32_t seq, enum mail_cache_field fields); @@ -123,9 +122,9 @@ mail_cache_transaction_autocommit(struct mail_cache_view *view, uint32_t seq, enum mail_cache_field fields); -int mail_cache_mmap_update(struct mail_cache *cache, - size_t offset, size_t size); +int mail_cache_map(struct mail_cache *cache, size_t offset, size_t size); void mail_cache_file_close(struct mail_cache *cache); +int mail_cache_reopen(struct mail_cache *cache); void mail_cache_handle_decisions(struct mail_cache_view *view, uint32_t seq, enum mail_cache_field field);
--- a/src/lib-index/mail-cache-transaction.c Sat Jul 03 23:17:37 2004 +0300 +++ b/src/lib-index/mail-cache-transaction.c Sun Jul 04 14:50:48 2004 +0300 @@ -7,6 +7,7 @@ #include "mmap-util.h" #include "mail-cache-private.h" +#include <stddef.h> #include <sys/stat.h> struct mail_cache_transaction_ctx { @@ -174,7 +175,7 @@ if (ctx->used_file_size + size <= (uoff_t)st.st_size) { /* no need to grow, just update mmap */ - if (mail_cache_mmap_update(cache, 0, 0) < 0) + if (mail_cache_map(cache, 0, (size_t)st.st_size) < 0) return -1; i_assert(cache->mmap_length >= (uoff_t)st.st_size); @@ -186,7 +187,7 @@ return -1; } - return mail_cache_mmap_update(cache, 0, 0); + return mail_cache_map(cache, 0, (size_t)new_fsize); } static uint32_t mail_cache_append_space(struct mail_cache_transaction_ctx *ctx, @@ -217,10 +218,8 @@ static int mail_cache_write(struct mail_cache_transaction_ctx *ctx) { struct mail_cache *cache = ctx->cache; - struct mail_cache_record *cache_rec, *next; - const struct mail_index_record *rec; - struct mail_index_map *map; - uint32_t write_offset, update_offset; + struct mail_cache_record *cache_rec; + uint32_t offset, write_offset; const void *buf; size_t size, buf_size; int ret; @@ -230,16 +229,10 @@ size = sizeof(*cache_rec) + buf_size; ctx->cache_rec.size = uint32_to_nbo(size); - ret = mail_index_lookup_full(ctx->view->view, ctx->prev_seq, - &map, &rec); + ret = mail_cache_lookup_offset(ctx->view, ctx->prev_seq, &offset, TRUE); if (ret < 0) return -1; - if (map->hdr->cache_file_seq != cache->hdr->file_seq) { - /* FIXME: we should check if newer file is available? */ - ret = 0; - } - if (ret == 0) { /* it's been expunged already, do nothing */ } else { @@ -247,22 +240,25 @@ if (write_offset == 0) return -1; - cache_rec = mail_cache_get_record(cache, rec->cache_offset, - TRUE); + cache_rec = mail_cache_get_record(cache, offset, TRUE); if (cache_rec == NULL) { /* first cache record - update offset in index file */ mail_index_update_cache(ctx->trans, ctx->prev_seq, write_offset); } else { - /* find the last cache record */ - while ((next = mail_cache_get_next_record(cache, - cache_rec)) != NULL) - cache_rec = next; + /* find offset to last cache record */ + for (;;) { + cache_rec = mail_cache_get_record(cache, offset, + FALSE); + if (cache_rec == NULL) + break; + offset = cache_rec->next_offset; + } /* mark next_offset to be updated later */ - update_offset = (char *) &cache_rec->next_offset - - (char *) cache->mmap_base; - mark_update(&ctx->cache_marks, update_offset, + offset = mail_cache_offset_to_uint32(offset) + + offsetof(struct mail_cache_record, next_offset); + mark_update(&ctx->cache_marks, offset, mail_cache_uint32_to_offset(write_offset)); } @@ -285,7 +281,7 @@ { int ret = 0; - if (ctx->cache->disabled) { + if (MAIL_CACHE_IS_UNUSABLE(ctx->cache)) { mail_cache_transaction_flush(ctx); return 0; } @@ -541,7 +537,8 @@ do { deleted_space -= nbo_to_uint32(cache_rec->size); - cache_rec = mail_cache_get_next_record(cache, cache_rec); + cache_rec = mail_cache_get_record(cache, cache_rec->next_offset, + FALSE); } while (cache_rec != NULL); /* see if we've reached the max. deleted space in file */
--- a/src/lib-index/mail-cache.c Sat Jul 03 23:17:37 2004 +0300 +++ b/src/lib-index/mail-cache.c Sun Jul 04 14:50:48 2004 +0300 @@ -9,7 +9,6 @@ #include "mail-cache-private.h" #include <unistd.h> -#include <sys/stat.h> unsigned int mail_cache_field_sizes[32] = { sizeof(enum mail_cache_record_flag), @@ -92,10 +91,8 @@ { va_list va; - (void)mail_cache_reset(cache); - - if (cache->silent) - return; + (void)unlink(cache->filepath); + mail_cache_file_close(cache); va_start(va, fmt); t_push(); @@ -123,20 +120,37 @@ } } -static int mail_cache_file_reopen(struct mail_cache *cache) +int mail_cache_reopen(struct mail_cache *cache) { - int fd; - - fd = open(cache->filepath, O_RDWR); - if (fd == -1) { - mail_cache_set_syscall_error(cache, "open()"); - return -1; + if (MAIL_CACHE_IS_UNUSABLE(cache) && cache->need_compress) { + /* unusable, we're just waiting for compression */ + return 0; } mail_cache_file_close(cache); - cache->fd = fd; - return 0; + cache->fd = open(cache->filepath, O_RDWR); + if (cache->fd == -1) { + if (errno == ENOENT) + cache->need_compress = TRUE; + else + mail_cache_set_syscall_error(cache, "open()"); + return -1; + } + + if (mail_cache_map(cache, 0, 0) < 0) + return -1; + + if (cache->hdr->file_seq != cache->index->hdr->cache_file_seq) { + /* still different - maybe a race condition or maybe the + file_seq really is corrupted. either way, this shouldn't + happen often so we'll just mark cache to be compressed + later which fixes this. */ + cache->need_compress = TRUE; + return 0; + } + + return 1; } static int mmap_verify_header(struct mail_cache *cache) @@ -147,75 +161,52 @@ /* check that the header is still ok */ if (cache->mmap_length < sizeof(struct mail_cache_header)) { mail_cache_set_corrupted(cache, "File too small"); - return 0; + return FALSE; } cache->hdr = hdr = cache->mmap_base; if (cache->hdr->indexid != cache->index->indexid) { /* index id changed */ - if (cache->hdr->indexid != 0) - mail_cache_set_corrupted(cache, "indexid changed"); - return 0; + mail_cache_set_corrupted(cache, "indexid changed"); + return FALSE; } if (cache->trans_ctx != NULL) { /* we've updated used_file_size, do nothing */ - return 1; + return TRUE; } /* only check the header if we're locked */ if (cache->locks == 0) - return 1; + return TRUE; used_file_size = nbo_to_uint32(hdr->used_file_size); if (used_file_size < sizeof(struct mail_cache_header)) { mail_cache_set_corrupted(cache, "used_file_size too small"); - return 0; + return FALSE; } if ((used_file_size % sizeof(uint32_t)) != 0) { mail_cache_set_corrupted(cache, "used_file_size not aligned"); - return 0; + return FALSE; } if (used_file_size > cache->mmap_length) { mail_cache_set_corrupted(cache, "used_file_size too large"); - return 0; + return FALSE; } - return 1; + return TRUE; } -static int mmap_update_nocheck(struct mail_cache *cache, - size_t offset, size_t size) +int mail_cache_map(struct mail_cache *cache, size_t offset, size_t size) { - struct stat st; - -#if 0 // FIXME - /* if sequence has changed, the file has to be reopened. - note that if main index isn't locked, it may change again */ - if (cache->hdr->file_seq != cache->index->hdr->cache_file_seq && - cache->mmap_base != NULL) { - if (mail_cache_file_reopen(cache) < 0) - return -1; - } -#endif + if (size == 0) + size = sizeof(struct mail_cache_header); if (offset < cache->mmap_length && - size <= cache->mmap_length - offset && - !cache->mmap_refresh) { + size <= cache->mmap_length - offset) { /* already mapped */ - if (size != 0) - return 1; - - /* requesting the whole file - see if we need to - re-mmap */ - if (fstat(cache->fd, &st) < 0) { - mail_cache_set_syscall_error(cache, "fstat()"); - return -1; - } - if ((uoff_t)st.st_size == cache->mmap_length) - return 1; + return 0; } - cache->mmap_refresh = FALSE; if (cache->mmap_base != NULL) { if (cache->locks != 0) { @@ -229,10 +220,14 @@ if (munmap(cache->mmap_base, cache->mmap_length) < 0) mail_cache_set_syscall_error(cache, "munmap()"); + } else { + if (cache->fd == -1) { + /* unusable, waiting for compression */ + i_assert(cache->need_compress); + return -1; + } } - i_assert(cache->fd != -1); - /* map the whole file */ cache->hdr = NULL; cache->mmap_length = 0; @@ -244,156 +239,48 @@ return -1; } - /* re-mmaped, check header */ + if (!mmap_verify_header(cache)) { + cache->need_compress = TRUE; + return -1; + } + return 0; } -int mail_cache_mmap_update(struct mail_cache *cache, size_t offset, size_t size) +static int mail_cache_open_and_verify(struct mail_cache *cache) { - int synced, ret; - - ret = mmap_update_nocheck(cache, offset, size); - if (ret > 0) - return 0; - if (ret < 0) - return -1; - - if (mmap_verify_header(cache) <= 0) - return -1; - -#if 0 // FIXME - /* see if cache file was rebuilt - do it only once to avoid - infinite looping */ - if (cache->hdr->file_seq == cache->index->hdr->cache_file_seq || - synced) - break; - - if (mail_cache_file_reopen(cache) < 0) - return -1; -#endif - return 0; -} - -static int mail_cache_open_and_verify(struct mail_cache *cache, int silent) -{ - struct stat st; - int ret; - - mail_cache_file_close(cache); + cache->filepath = i_strconcat(cache->index->filepath, + MAIL_CACHE_FILE_PREFIX, NULL); cache->fd = open(cache->filepath, O_RDWR); if (cache->fd == -1) { - if (errno == ENOENT) + if (errno == ENOENT) { + cache->need_compress = TRUE; return 0; + } mail_cache_set_syscall_error(cache, "open()"); return -1; } - if (fstat(cache->fd, &st) < 0) { - mail_cache_set_syscall_error(cache, "fstat()"); - return -1; - } - - if (st.st_size < sizeof(struct mail_cache_header)) - return 0; - - cache->mmap_refresh = TRUE; - if (mmap_update_nocheck(cache, 0, sizeof(struct mail_cache_header)) < 0) - return -1; - - /* verify that this really is the cache for wanted index */ - cache->silent = silent; - if ((ret = mmap_verify_header(cache)) <= 0) { - cache->silent = FALSE; - return ret; - } - - cache->silent = FALSE; - return 1; -} - -static int mail_cache_open_or_create_file(struct mail_cache *cache, - struct mail_cache_header *hdr) -{ - int ret, fd; - - cache->filepath = i_strconcat(cache->index->filepath, - MAIL_CACHE_FILE_PREFIX, NULL); - - ret = mail_cache_open_and_verify(cache, FALSE); - if (ret != 0) - return ret < 0 ? -1 : 0; - - /* maybe a rebuild.. */ - fd = file_dotlock_open(cache->filepath, NULL, NULL, - MAIL_CACHE_LOCK_TIMEOUT, - MAIL_CACHE_LOCK_CHANGE_TIMEOUT, - MAIL_CACHE_LOCK_IMMEDIATE_TIMEOUT, NULL, NULL); - if (fd == -1) { - mail_cache_set_syscall_error(cache, "file_dotlock_open()"); - return -1; - } - - /* see if someone else just created the cache file */ - ret = mail_cache_open_and_verify(cache, TRUE); - if (ret != 0) { - (void)file_dotlock_delete(cache->filepath, NULL, fd); - return ret < 0 ? -1 : 0; - } - - /* rebuild then */ - if (write_full(fd, hdr, sizeof(*hdr)) < 0) { - mail_cache_set_syscall_error(cache, "write_full()"); - (void)file_dotlock_delete(cache->filepath, NULL, fd); - return -1; - } - if (file_set_size(fd, MAIL_CACHE_INITIAL_SIZE) < 0) { - mail_cache_set_syscall_error(cache, "file_set_size()"); - (void)file_dotlock_delete(cache->filepath, NULL, fd); - return -1; - } - - mail_cache_file_close(cache); - cache->fd = dup(fd); - - if (file_dotlock_replace(cache->filepath, NULL, fd, FALSE) < 0) { - mail_cache_set_syscall_error(cache, "file_dotlock_replace()"); - return -1; - } - - if (mail_cache_mmap_update(cache, 0, - sizeof(struct mail_cache_header)) < 0) - return -1; - - return 0; + return mail_cache_map(cache, 0, sizeof(struct mail_cache_header)); } struct mail_cache *mail_cache_open_or_create(struct mail_index *index) { - struct mail_cache_header hdr; struct mail_cache *cache; - memset(&hdr, 0, sizeof(hdr)); - hdr.indexid = index->indexid; - hdr.file_seq = index->hdr->cache_file_seq + 1; - hdr.used_file_size = uint32_to_nbo(sizeof(hdr)); - cache = i_new(struct mail_cache, 1); cache->index = index; cache->fd = -1; cache->split_header_pool = pool_alloconly_create("Headers", 512); - if (index->mmap_disable || - mail_cache_open_or_create_file(cache, &hdr) < 0) { - /* failed for some reason - doesn't really matter, - just disable caching. */ - mail_cache_file_close(cache); - - i_free(cache->filepath); - cache->filepath = i_strdup_printf("(disabled cache for %s)", - index->filepath); - cache->disabled = TRUE; + if (!index->mmap_disable) { + if (mail_cache_open_and_verify(cache) < 0) { + /* failed for some reason - doesn't really matter, + it's disabled for now. */ + mail_cache_file_close(cache); + } } return cache; @@ -418,83 +305,51 @@ cache->never_cache_fields = never_cache_fields; } -int mail_cache_reset(struct mail_cache *cache) -{ - struct mail_cache_header hdr; - int fd; - - memset(&hdr, 0, sizeof(hdr)); - hdr.indexid = cache->index->indexid; - hdr.file_seq = cache->index->hdr->cache_file_seq + 1; - hdr.used_file_size = uint32_to_nbo(sizeof(hdr)); - - fd = file_dotlock_open(cache->filepath, NULL, NULL, - MAIL_CACHE_LOCK_TIMEOUT, - MAIL_CACHE_LOCK_CHANGE_TIMEOUT, - MAIL_CACHE_LOCK_IMMEDIATE_TIMEOUT, NULL, NULL); - if (fd == -1) { - mail_cache_set_syscall_error(cache, "file_dotlock_open()"); - return -1; - } - - if (write_full(fd, &hdr, sizeof(hdr)) < 0) { - mail_cache_set_syscall_error(cache, "write_full()"); - (void)file_dotlock_delete(cache->filepath, NULL, fd); - return -1; - } - if (file_set_size(fd, MAIL_CACHE_INITIAL_SIZE) < 0) { - mail_cache_set_syscall_error(cache, "file_set_size()"); - (void)file_dotlock_delete(cache->filepath, NULL, fd); - return -1; - } - - mail_cache_file_close(cache); - cache->fd = dup(fd); - - if (file_dotlock_replace(cache->filepath, NULL, fd, FALSE) < 0) { - mail_cache_set_syscall_error(cache, "file_dotlock_replace()"); - return -1; - } - - cache->mmap_refresh = TRUE; - if (mail_cache_mmap_update(cache, 0, - sizeof(struct mail_cache_header)) < 0) - return -1; - - return 0; -} - int mail_cache_lock(struct mail_cache *cache, int nonblock) { - int ret; + int i, ret; if (cache->locks != 0) return 1; - if (cache->disabled) + if (MAIL_CACHE_IS_UNUSABLE(cache)) return 0; - if (nonblock) { - ret = file_try_lock(cache->fd, F_WRLCK); - if (ret < 0) - mail_cache_set_syscall_error(cache, "file_try_lock()"); - } else { - ret = file_wait_lock(cache->fd, F_WRLCK); - if (ret <= 0) - mail_cache_set_syscall_error(cache, "file_wait_lock()"); + if (cache->hdr->file_seq != cache->index->hdr->cache_file_seq) { + /* we want the latest cache file */ + if ((ret = mail_cache_reopen(cache)) <= 0) + return ret; } - if (ret > 0) { - if (mail_cache_mmap_update(cache, 0, 0) < 0) { - (void)mail_cache_unlock(cache); - return -1; + for (i = 0; i < 3; i++) { + if (nonblock) { + ret = file_try_lock(cache->fd, F_WRLCK); + if (ret < 0) { + mail_cache_set_syscall_error(cache, + "file_try_lock()"); + } + } else { + ret = file_wait_lock(cache->fd, F_WRLCK); + if (ret <= 0) { + mail_cache_set_syscall_error(cache, + "file_wait_lock()"); + } } - if (cache->hdr->file_seq != cache->index->hdr->cache_file_seq) { - mail_cache_unlock(cache); - return 0; + if (ret <= 0) + break; + + if (cache->hdr->file_seq == cache->index->hdr->cache_file_seq) { + /* got it */ + cache->locks++; + break; } - cache->locks++; + + /* okay, so it was just compressed. try again. */ + mail_cache_unlock(cache); + if ((ret = mail_cache_reopen(cache)) <= 0) + return ret; + ret = 0; } return ret; } @@ -517,19 +372,6 @@ return cache->locks > 0; } -int mail_cache_need_reset(struct mail_cache *cache, uint32_t *new_file_seq_r) -{ - if (!cache->disabled && - cache->hdr->file_seq != cache->index->hdr->cache_file_seq) { - if (mail_cache_lock(cache, TRUE) == 0) { - *new_file_seq_r = cache->hdr->file_seq; - return TRUE; - } - } - - return FALSE; -} - struct mail_cache_view * mail_cache_view_open(struct mail_cache *cache, struct mail_index_view *iview) {
--- a/src/lib-index/mail-cache.h Sat Jul 03 23:17:37 2004 +0300 +++ b/src/lib-index/mail-cache.h Sun Jul 04 14:50:48 2004 +0300 @@ -83,20 +83,14 @@ /* Compress cache file. */ int mail_cache_compress(struct mail_cache *cache, struct mail_index_view *view); -/* Reset the cache file, clearing all data. */ -int mail_cache_reset(struct mail_cache *cache); - -/* Explicitly lock the cache file. Returns -1 if error, 1 if ok, - 0 if we couldn't lock (nonblock = TRUE or index file needs a cache reset) */ +/* Explicitly lock the cache file. Returns -1 if error, 1 if ok, 0 if we + couldn't lock */ int mail_cache_lock(struct mail_cache *cache, int nonblock); int mail_cache_unlock(struct mail_cache *cache); /* Returns TRUE if cache file is locked. */ int mail_cache_is_locked(struct mail_cache *cache); -/* Returns TRUE if index's cache_file_seq doesn't match the latest cache file */ -int mail_cache_need_reset(struct mail_cache *cache, uint32_t *new_file_seq_r); - struct mail_cache_view * mail_cache_view_open(struct mail_cache *cache, struct mail_index_view *iview); void mail_cache_view_close(struct mail_cache_view *view);
--- a/src/lib-index/mail-index-sync.c Sat Jul 03 23:17:37 2004 +0300 +++ b/src/lib-index/mail-index-sync.c Sun Jul 04 14:50:48 2004 +0300 @@ -207,7 +207,7 @@ (index->hdr->log_file_seq == log_file_seq && index->hdr->log_file_offset >= log_file_offset)) { /* already synced */ - return 0; + return mail_cache_need_compress(index->cache); } return 1; @@ -219,7 +219,7 @@ uint32_t log_file_seq, uoff_t log_file_offset) { struct mail_index_sync_ctx *ctx; - uint32_t seq, new_file_seq; + uint32_t seq; uoff_t offset; unsigned int lock_id; @@ -270,17 +270,6 @@ return -1; } - /* check here if cache file's sequence has changed unexpectedly */ - if (mail_cache_need_reset(index->cache, &new_file_seq)) { - uint32_t seq; - uoff_t offset; - struct mail_index_transaction *t; - - t = mail_index_transaction_begin(ctx->view, FALSE); - mail_index_reset_cache(t, new_file_seq); - mail_index_transaction_commit(t, &seq, &offset); - } - *ctx_r = ctx; *view_r = ctx->view; return 1;
--- a/src/lib-storage/index/index-mail.c Sat Jul 03 23:17:37 2004 +0300 +++ b/src/lib-storage/index/index-mail.c Sun Jul 04 14:50:48 2004 +0300 @@ -128,10 +128,17 @@ if (mail->trans->cache_trans != NULL) return TRUE; + if (mail->trans->cache_trans_failed) { + /* don't try more than once */ + return FALSE; + } + if (mail_cache_transaction_begin(mail->trans->cache_view, TRUE, mail->trans->trans, - &mail->trans->cache_trans) <= 0) + &mail->trans->cache_trans) <= 0) { + mail->trans->cache_trans_failed = TRUE; return FALSE; + } mail->data.cached_fields = mail_cache_get_fields(mail->trans->cache_view, mail->data.seq);
--- a/src/lib-storage/index/index-storage.h Sat Jul 03 23:17:37 2004 +0300 +++ b/src/lib-storage/index/index-storage.h Sun Jul 04 14:50:48 2004 +0300 @@ -118,6 +118,7 @@ struct mail_cache_transaction_ctx *cache_trans; struct index_mail fetch_mail; /* for index_storage_fetch() */ + unsigned int cache_trans_failed:1; }; int mail_storage_set_index_error(struct index_mailbox *ibox);