Mercurial > dovecot > core-2.2
changeset 15313:51f2be9aa8ad
lib-index: Added MAIL_INDEX_OPEN_FLAG_SAVEONLY to do only minimal reads from cache file.
author | Timo Sirainen <tss@iki.fi> |
---|---|
date | Thu, 18 Oct 2012 05:22:36 +0300 |
parents | 0d1de37ad9d8 |
children | b7c6215b2b7c |
files | src/lib-index/mail-cache-compress.c src/lib-index/mail-cache-private.h src/lib-index/mail-cache.c src/lib-index/mail-index.h |
diffstat | 4 files changed, 96 insertions(+), 44 deletions(-) [+] |
line wrap: on
line diff
--- a/src/lib-index/mail-cache-compress.c Thu Oct 18 05:16:54 2012 +0300 +++ b/src/lib-index/mail-cache-compress.c Thu Oct 18 05:22:36 2012 +0300 @@ -461,6 +461,9 @@ if (MAIL_INDEX_IS_IN_MEMORY(cache->index) || cache->index->readonly) return 0; + /* compression isn't very efficient with small read()s */ + cache->map_with_read = FALSE; + if (cache->index->lock_method == FILE_LOCK_METHOD_DOTLOCK) { /* we're using dotlocking, cache file creation itself creates the dotlock file we need. */ @@ -495,5 +498,6 @@ bool mail_cache_need_compress(struct mail_cache *cache) { return cache->need_compress_file_seq != 0 && + (cache->index->flags & MAIL_INDEX_OPEN_FLAG_SAVEONLY) == 0 && !cache->index->readonly; }
--- a/src/lib-index/mail-cache-private.h Thu Oct 18 05:16:54 2012 +0300 +++ b/src/lib-index/mail-cache-private.h Thu Oct 18 05:22:36 2012 +0300 @@ -130,9 +130,14 @@ ino_t st_ino; dev_t st_dev; + size_t mmap_length; + /* a) mmaping the whole file */ void *mmap_base; - size_t mmap_length; + /* b) using file cache */ struct file_cache *file_cache; + /* c) using small read() calls with MAIL_INDEX_OPEN_FLAG_SAVEONLY */ + uoff_t read_offset; + buffer_t *read_buf; /* mail_cache_map() increases this always. */ unsigned int remap_counter; @@ -169,6 +174,7 @@ unsigned int hdr_modified:1; unsigned int field_header_write_pending:1; unsigned int compressing:1; + unsigned int map_with_read:1; }; struct mail_cache_loop_track {
--- a/src/lib-index/mail-cache.c Thu Oct 18 05:16:54 2012 +0300 +++ b/src/lib-index/mail-cache.c Thu Oct 18 05:22:36 2012 +0300 @@ -7,6 +7,7 @@ #include "nfs-workarounds.h" #include "file-cache.h" #include "mmap-util.h" +#include "read-full.h" #include "write-full.h" #include "mail-cache-private.h" @@ -264,17 +265,87 @@ return TRUE; } +static int +mail_cache_map_finish(struct mail_cache *cache, uoff_t offset, size_t size, + const void *data, bool copy_hdr) +{ + if (offset == 0) { + const struct mail_cache_header *hdr = data; + + if (!mail_cache_verify_header(cache, hdr)) { + cache->need_compress_file_seq = + !MAIL_CACHE_IS_UNUSABLE(cache) && + cache->hdr->file_seq != 0 ? + cache->hdr->file_seq : 0; + return -1; + } + if (!copy_hdr) + cache->hdr = data; + else { + memcpy(&cache->hdr_ro_copy, data, + sizeof(cache->hdr_ro_copy)); + cache->hdr = &cache->hdr_ro_copy; + } + mail_cache_update_need_compress(cache); + } + + if (offset + size > cache->mmap_length) + return 0; + return 1; +} + +static int +mail_cache_map_with_read(struct mail_cache *cache, size_t offset, size_t size, + const void **data_r) +{ + void *data; + ssize_t ret; + + if (cache->read_buf == NULL) { + cache->read_buf = + buffer_create_dynamic(default_pool, size); + } else if (cache->read_offset <= offset && + cache->read_offset + cache->read_buf->used >= offset+size) { + /* already mapped */ + *data_r = CONST_PTR_OFFSET(cache->mmap_base, + offset - cache->read_offset); + return mail_cache_map_finish(cache, offset, size, *data_r, TRUE); + } else { + buffer_set_used_size(cache->read_buf, 0); + } + data = buffer_append_space_unsafe(cache->read_buf, size); + ret = pread(cache->fd, data, size, offset); + if (ret < 0) { + if (errno != ESTALE) + mail_cache_set_syscall_error(cache, "read()"); + + buffer_set_used_size(cache->read_buf, 0); + cache->hdr = NULL; + cache->mmap_length = 0; + return -1; + } + buffer_set_used_size(cache->read_buf, ret); + + cache->read_offset = offset; + cache->mmap_length = offset + size; + + *data_r = data; + return mail_cache_map_finish(cache, offset, size, data, TRUE); +} + int mail_cache_map(struct mail_cache *cache, size_t offset, size_t size, const void **data_r) { const void *data; ssize_t ret; - cache->remap_counter++; - if (size == 0) size = sizeof(struct mail_cache_header); + cache->remap_counter++; + if (cache->map_with_read) + return mail_cache_map_with_read(cache, offset, size, data_r); + if (cache->file_cache != NULL) { cache->hdr = NULL; @@ -294,28 +365,9 @@ data = file_cache_get_map(cache->file_cache, &cache->mmap_length); - - if (offset == 0) { - if (!mail_cache_verify_header(cache, data)) { - cache->need_compress_file_seq = - !MAIL_CACHE_IS_UNUSABLE(cache) && - cache->hdr->file_seq != 0 ? - cache->hdr->file_seq : 0; - return -1; - } - memcpy(&cache->hdr_ro_copy, data, - sizeof(cache->hdr_ro_copy)); - } - cache->hdr = &cache->hdr_ro_copy; - if (offset == 0) - mail_cache_update_need_compress(cache); - - if (offset > cache->mmap_length) { - *data_r = NULL; - return 0; - } - *data_r = CONST_PTR_OFFSET(data, offset); - return offset + size > cache->mmap_length ? 0 : 1; + *data_r = offset > cache->mmap_length ? NULL : + CONST_PTR_OFFSET(data, offset); + return mail_cache_map_finish(cache, offset, size, *data_r, TRUE); } if (offset < cache->mmap_length && @@ -348,24 +400,9 @@ mail_cache_set_syscall_error(cache, "mmap()"); return -1; } - - if (!mail_cache_verify_header(cache, cache->mmap_base)) { - cache->need_compress_file_seq = - !MAIL_CACHE_IS_UNUSABLE(cache) && - cache->hdr->file_seq != 0 ? - cache->hdr->file_seq : 0; - return -1; - } - - cache->hdr = cache->mmap_base; - if (offset == 0) - mail_cache_update_need_compress(cache); - if (offset > cache->mmap_length) { - *data_r = NULL; - return 0; - } - *data_r = CONST_PTR_OFFSET(cache->mmap_base, offset); - return offset + size > cache->mmap_length ? 0 : 1; + *data_r = offset > cache->mmap_length ? NULL : + CONST_PTR_OFFSET(cache->mmap_base, offset); + return mail_cache_map_finish(cache, offset, size, *data_r, FALSE); } static int mail_cache_try_open(struct mail_cache *cache) @@ -437,6 +474,8 @@ if (!MAIL_INDEX_IS_IN_MEMORY(index) && (index->flags & MAIL_INDEX_OPEN_FLAG_MMAP_DISABLE) != 0) cache->file_cache = file_cache_new(-1); + cache->map_with_read = + (cache->index->flags & MAIL_INDEX_OPEN_FLAG_SAVEONLY) != 0; cache->ext_id = mail_index_ext_register(index, "cache", 0,
--- a/src/lib-index/mail-index.h Thu Oct 18 05:16:54 2012 +0300 +++ b/src/lib-index/mail-index.h Thu Oct 18 05:22:36 2012 +0300 @@ -27,7 +27,10 @@ MAIL_INDEX_OPEN_FLAG_KEEP_BACKUPS = 0x100, /* If we run out of disk space, fail modifications instead of moving indexes to memory. */ - MAIL_INDEX_OPEN_FLAG_NEVER_IN_MEMORY = 0x200 + MAIL_INDEX_OPEN_FLAG_NEVER_IN_MEMORY = 0x200, + /* We're only going to save new messages to the index. + Avoid unnecessary reads. */ + MAIL_INDEX_OPEN_FLAG_SAVEONLY = 0x400 }; enum mail_index_header_compat_flags {