Mercurial > dovecot > original-hg > dovecot-1.2
changeset 4896:3d00a9636393 HEAD
Mailbox list index works now with mmap_disable=yes.
author | Timo Sirainen <tss@iki.fi> |
---|---|
date | Sun, 10 Dec 2006 18:04:46 +0200 |
parents | 95594bcf13ab |
children | 30834ce4362f |
files | src/lib-index/mailbox-list-index-private.h src/lib-index/mailbox-list-index-sync.c src/lib-index/mailbox-list-index.c src/lib-storage/list/index-mailbox-list.c |
diffstat | 4 files changed, 169 insertions(+), 42 deletions(-) [+] |
line wrap: on
line diff
--- a/src/lib-index/mailbox-list-index-private.h Sun Dec 10 16:43:21 2006 +0200 +++ b/src/lib-index/mailbox-list-index-private.h Sun Dec 10 18:04:46 2006 +0200 @@ -51,15 +51,21 @@ char *filepath; char separator; struct mail_index *mail_index; + struct file_cache *file_cache; int fd; void *mmap_base; + const void *const_mmap_base; size_t mmap_size; const struct mailbox_list_index_header *hdr; + + unsigned int mmap_disable:1; }; #define MAILBOX_LIST_RECORDS(dir) \ + ((const struct mailbox_list_record *)(dir + 1)) +#define MAILBOX_LIST_RECORDS_MODIFIABLE(dir) \ ((struct mailbox_list_record *)(dir + 1)) #define MAILBOX_LIST_RECORD_IDX(dir, rec) \ ((rec) - MAILBOX_LIST_RECORDS(dir))
--- a/src/lib-index/mailbox-list-index-sync.c Sun Dec 10 16:43:21 2006 +0200 +++ b/src/lib-index/mailbox-list-index-sync.c Sun Dec 10 18:04:46 2006 +0200 @@ -4,8 +4,10 @@ #include "array.h" #include "bsearch-insert-pos.h" #include "crc32.h" +#include "file-cache.h" #include "file-set-size.h" #include "mmap-util.h" +#include "ostream.h" #include "mail-index-private.h" #include "mailbox-list-index-private.h" @@ -53,6 +55,9 @@ struct mailbox_list_index_header hdr; struct mailbox_list_sync_dir *root, *sync_root; + struct ostream *output; + buffer_t *output_buf; + unsigned int failed:1; unsigned int partial:1; unsigned int seen_sync_root:1; @@ -97,6 +102,9 @@ recs = MAILBOX_LIST_RECORDS(dir); for (i = 0; i < dir->count; i++) { + if (recs[i].deleted) + continue; + sync_rec = array_append_space(&sync_dir->records); sync_rec->name_hash = recs[i].name_hash; sync_rec->uid = recs[i].uid; @@ -104,7 +112,7 @@ mail_index_offset_to_uint32(recs[i].dir_offset); max_len = ctx->index->mmap_size - recs[i].name_offset; - name = CONST_PTR_OFFSET(ctx->index->mmap_base, + name = CONST_PTR_OFFSET(ctx->index->const_mmap_base, recs[i].name_offset); sync_rec->name = p_strndup(ctx->pool, name, max_len); @@ -430,15 +438,30 @@ /* all allocations must be 32bit aligned */ pos = (pos + 3) & ~3; - if (pos + size > ctx->index->mmap_size) { - if (mailbox_list_index_sync_grow(ctx, size + 3) < 0) - return -1; + if (ctx->index->mmap_disable) { + /* write the data into temporary buffer first */ + buffer_set_used_size(ctx->output_buf, 0); + + if (ctx->output->offset != pos) { + i_assert(pos > ctx->output->offset && + pos - ctx->output->offset <= 3); - i_assert(pos + size < ctx->index->mmap_size); + buffer_append_zero(ctx->output_buf, + pos - ctx->output->offset); + } + + *base_r = buffer_append_space_unsafe(ctx->output_buf, size); + } else { + if (pos + size > ctx->index->mmap_size) { + if (mailbox_list_index_sync_grow(ctx, size + 3) < 0) + return -1; + + i_assert(pos + size < ctx->index->mmap_size); + } + *base_r = PTR_OFFSET(ctx->index->mmap_base, pos); } *base_offset_r = pos; - *base_r = PTR_OFFSET(ctx->index->mmap_base, *base_offset_r); ctx->hdr.used_space = pos + size; return 0; } @@ -448,8 +471,11 @@ struct mailbox_list_sync_dir *sync_dir, uint32_t offset_pos, bool partial) { - struct mailbox_list_dir_record *dir, *new_dir; - struct mailbox_list_record *recs, *new_recs; + struct mailbox_list_index *index = ctx->index; + const struct mailbox_list_dir_record *dir; + const struct mailbox_list_record *recs; + struct mailbox_list_dir_record *new_dir; + struct mailbox_list_record *new_recs; struct mailbox_list_sync_record *sync_recs; unsigned int src, dest, orig, count, nondeleted_count; unsigned int name_space_needed, deleted_space; @@ -457,7 +483,7 @@ void *base; i_assert((offset_pos % sizeof(uint32_t)) == 0); - i_assert(offset_pos < ctx->index->mmap_size); + i_assert(offset_pos < index->mmap_size); /* count how much space we need and how much we wasted for deleted records */ @@ -492,15 +518,16 @@ } else { /* the offset should have been verified already to be valid */ i_assert(sync_dir->offset == offset_pos); - i_assert(sync_dir->offset < ctx->index->mmap_size); - dir = PTR_OFFSET(ctx->index->mmap_base, sync_dir->offset); + i_assert(sync_dir->offset < index->mmap_size); + dir = CONST_PTR_OFFSET(index->const_mmap_base, + sync_dir->offset); recs = MAILBOX_LIST_RECORDS(dir); } new_dir = base; new_dir->count = nondeleted_count; - new_recs = MAILBOX_LIST_RECORDS(new_dir); + new_recs = MAILBOX_LIST_RECORDS_MODIFIABLE(new_dir); name_pos = (const char *)(new_recs + nondeleted_count) - (const char *)base; for (src = dest = 0; src < count; src++) { @@ -547,16 +574,42 @@ i_assert(dest == nondeleted_count); i_assert(name_pos == name_space_needed); + if (index->mmap_disable) { + file_cache_write(index->file_cache, ctx->output_buf->data, + ctx->output_buf->used, ctx->output->offset); + o_stream_send(ctx->output, ctx->output_buf->data, + ctx->output_buf->used); + } + if (offset_pos == 0) { /* we're writing the root directory */ - i_assert(base_offset == sizeof(*ctx->index->hdr)); + i_assert(base_offset == sizeof(*index->hdr)); } else { /* add a link to this newly created directory. */ - uint32_t *pos; + uint32_t data = mail_index_uint32_to_offset(base_offset); + + if (!index->mmap_disable) { + uint32_t *pos; + + pos = PTR_OFFSET(index->mmap_base, offset_pos); + i_assert(mail_index_offset_to_uint32(*pos) == 0); + *pos = data; + } else { + uoff_t old_offset = ctx->output->offset; - pos = PTR_OFFSET(ctx->index->mmap_base, offset_pos); - i_assert(mail_index_offset_to_uint32(*pos) == 0); - *pos = mail_index_uint32_to_offset(base_offset); + file_cache_write(index->file_cache, + &data, sizeof(data), offset_pos); + + o_stream_seek(ctx->output, offset_pos); + o_stream_send(ctx->output, &data, sizeof(data)); + o_stream_seek(ctx->output, old_offset); + } + } + + if (index->mmap_disable) { + /* file_cache_write() calls may have moved mmaping */ + index->const_mmap_base = file_cache_get_map(index->file_cache, + &index->mmap_size); } sync_dir->offset = base_offset; @@ -581,11 +634,34 @@ i_assert(dir->count == count); i_assert(sync_dir->seen_records_count < count); - recs = MAILBOX_LIST_RECORDS(dir); + if (!ctx->index->mmap_disable) + recs = MAILBOX_LIST_RECORDS_MODIFIABLE(dir); + else { + /* @UNSAFE: copy the records into a temporary buffer that + we modify and then write back to disk */ + t_push(); + recs = t_new(struct mailbox_list_record, dir->count); + memcpy(recs, MAILBOX_LIST_RECORDS(dir), + sizeof(struct mailbox_list_record) * dir->count); + } for (i = 0; i < dir->count; i++) { if (!sync_recs[i].seen) recs[i].deleted = TRUE; } + if (ctx->index->mmap_disable) { + uoff_t offset, old_offset; + size_t size = sizeof(struct mailbox_list_record) * dir->count; + + offset = sync_dir->offset + + sizeof(struct mailbox_list_dir_record); + file_cache_write(ctx->index->file_cache, recs, size, offset); + + old_offset = ctx->output->offset; + o_stream_seek(ctx->output, offset); + o_stream_send(ctx->output, recs, size); + o_stream_seek(ctx->output, old_offset); + t_pop(); + } return 0; } @@ -643,7 +719,8 @@ continue; /* these may change after each sync_write_dir() call */ - dir = CONST_PTR_OFFSET(ctx->index->mmap_base, sync_dir->offset); + dir = CONST_PTR_OFFSET(ctx->index->const_mmap_base, + sync_dir->offset); recs = MAILBOX_LIST_RECORDS(dir); /* child_offset_pos needs to point to record's dir_offset */ @@ -654,7 +731,7 @@ i_assert(j < dir->count); child_offset_pos = (const char *)&recs[j].dir_offset - - (const char *)ctx->index->mmap_base; + (const char *)ctx->index->const_mmap_base; if (mailbox_list_index_sync_write_dir(ctx, sync_recs[i].dir, child_offset_pos, partial) < 0) @@ -669,6 +746,13 @@ struct mailbox_list_index_header *hdr; bool partial; + if (ctx->index->mmap_disable) { + ctx->output = o_stream_create_file(ctx->index->fd, default_pool, + 0, FALSE); + ctx->output_buf = buffer_create_dynamic(default_pool, 4096); + o_stream_seek(ctx->output, ctx->hdr.used_space); + } + if (ctx->sync_root == ctx->root) { ctx->seen_sync_root = TRUE; partial = (ctx->flags & MAILBOX_LIST_SYNC_FLAG_PARTIAL) != 0; @@ -681,15 +765,25 @@ if (mailbox_list_index_sync_write_dir(ctx, ctx->root, 0, partial) < 0) return -1; - /* update header */ - hdr = ctx->index->mmap_base; - hdr->next_uid = ctx->hdr.next_uid; - hdr->used_space = ctx->hdr.used_space; - hdr->deleted_space = ctx->hdr.deleted_space; + if (!ctx->index->mmap_disable) { + /* update header */ + hdr = ctx->index->mmap_base; + hdr->next_uid = ctx->hdr.next_uid; + hdr->used_space = ctx->hdr.used_space; + hdr->deleted_space = ctx->hdr.deleted_space; - if (msync(ctx->index->mmap_base, hdr->used_space, MS_SYNC) < 0) { - mailbox_list_index_set_syscall_error(ctx->index, "msync()"); - return -1; + if (msync(ctx->index->mmap_base, + hdr->used_space, MS_SYNC) < 0) { + mailbox_list_index_set_syscall_error(ctx->index, + "msync()"); + return -1; + } + } else { + o_stream_seek(ctx->output, 0); + o_stream_send(ctx->output, &ctx->hdr, sizeof(ctx->hdr)); + + o_stream_destroy(&ctx->output); + buffer_free(ctx->output_buf); } return 0; }
--- a/src/lib-index/mailbox-list-index.c Sun Dec 10 16:43:21 2006 +0200 +++ b/src/lib-index/mailbox-list-index.c Sun Dec 10 18:04:46 2006 +0200 @@ -5,6 +5,7 @@ #include "crc32.h" #include "ioloop.h" #include "str.h" +#include "file-cache.h" #include "file-dotlock.h" #include "mmap-util.h" #include "write-full.h" @@ -56,12 +57,16 @@ static void mailbox_list_index_unmap(struct mailbox_list_index *index) { + if (index->file_cache != NULL) + file_cache_invalidate(index->file_cache, 0, (uoff_t)-1); + if (index->mmap_base != NULL) { if (munmap(index->mmap_base, index->mmap_size) < 0) mailbox_list_index_set_syscall_error(index, "munmap()"); index->mmap_base = NULL; - index->mmap_size = 0; } + index->const_mmap_base = NULL; + index->mmap_size = 0; index->hdr = NULL; } @@ -70,6 +75,8 @@ { mailbox_list_index_unmap(index); + if (index->file_cache != NULL) + file_cache_free(&index->file_cache); if (index->fd != -1) { if (close(index->fd) < 0) mailbox_list_index_set_syscall_error(index, "close()"); @@ -123,24 +130,44 @@ int mailbox_list_index_map(struct mailbox_list_index *index) { const struct mailbox_list_index_header *hdr; + ssize_t ret; mailbox_list_index_unmap(index); - // FIXME: handle non-mmaps - index->mmap_base = mmap_rw_file(index->fd, &index->mmap_size); - if (index->mmap_base == MAP_FAILED) { - index->mmap_base = NULL; - return mailbox_list_index_set_syscall_error(index, "mmap()"); + if (!index->mmap_disable) { + index->mmap_base = mmap_rw_file(index->fd, &index->mmap_size); + if (index->mmap_base == MAP_FAILED) { + index->mmap_base = NULL; + mailbox_list_index_set_syscall_error(index, "mmap()"); + return -1; + } + + index->const_mmap_base = index->mmap_base; + } else { + if (index->file_cache == NULL) + index->file_cache = file_cache_new(index->fd); + + ret = file_cache_read(index->file_cache, 0, SSIZE_T_MAX); + if (ret < 0) { + mailbox_list_index_set_syscall_error(index, + "file_cache_read()"); + return -1; + } + index->const_mmap_base = file_cache_get_map(index->file_cache, + &index->mmap_size); } if (index->mmap_size < sizeof(*hdr)) { mailbox_list_index_set_corrupted(index, "File too small"); + index->const_mmap_base = NULL; return 0; } - hdr = index->mmap_base; - if (mailbox_list_index_check_header(index, hdr) < 0) + hdr = index->const_mmap_base; + if (mailbox_list_index_check_header(index, hdr) < 0) { + index->const_mmap_base = NULL; return 0; + } index->hdr = hdr; return 1; @@ -278,6 +305,7 @@ index->separator = separator; index->mail_index = mail_index; index->fd = -1; + index->mmap_disable = mail_index->mmap_disable; return index; } @@ -287,6 +315,7 @@ *_index = NULL; + mailbox_list_index_file_close(index); i_free(index->filepath); i_free(index); } @@ -314,7 +343,7 @@ return -1; } max_len = index->mmap_size - rec->name_offset; - name = CONST_PTR_OFFSET(index->mmap_base, rec->name_offset); + name = CONST_PTR_OFFSET(index->const_mmap_base, rec->name_offset); /* get name length. don't bother checking if it's not NUL-terminated, because practically it always is even if the file is corrupted. just make sure we don't crash if it happens. */ @@ -365,7 +394,7 @@ "dir_offset not 32bit aligned"); } - dir = CONST_PTR_OFFSET(index->mmap_base, cur_offset); + dir = CONST_PTR_OFFSET(index->const_mmap_base, cur_offset); next_offset = mail_index_offset_to_uint32(dir->next_offset); if (next_offset != 0 && next_offset <= cur_offset) { return mailbox_list_index_set_corrupted(index, @@ -374,7 +403,7 @@ cur_offset = next_offset; } while (cur_offset != 0); - cur_offset = (const char *)dir - (const char *)index->mmap_base; + cur_offset = (const char *)dir - (const char *)index->const_mmap_base; if (dir->count > INT_MAX/sizeof(struct mailbox_list_record) || dir->count * sizeof(struct mailbox_list_record) > index->mmap_size - cur_offset) {
--- a/src/lib-storage/list/index-mailbox-list.c Sun Dec 10 16:43:21 2006 +0200 +++ b/src/lib-storage/list/index-mailbox-list.c Sun Dec 10 18:04:46 2006 +0200 @@ -328,10 +328,8 @@ /* FIXME: a bit ugly way to get the flags, but this will do for now.. */ index_flags = MAIL_INDEX_OPEN_FLAG_CREATE; storage_flags = *list->set.mail_storage_flags; - if ((storage_flags & MAIL_STORAGE_FLAG_MMAP_DISABLE) != 0) { + if ((storage_flags & MAIL_STORAGE_FLAG_MMAP_DISABLE) != 0) index_flags |= MAIL_INDEX_OPEN_FLAG_MMAP_DISABLE; - return; /* FIXME: we currently don't support mmap_disable */ - } #ifndef MMAP_CONFLICTS_WRITE if ((storage_flags & MAIL_STORAGE_FLAG_MMAP_NO_WRITE) != 0) #endif