Mercurial > dovecot > original-hg > dovecot-1.2
changeset 4929:791c9c8d3a02 HEAD
Compress list index when its deleted space amount gets too high. Also fixed
several bugs.
author | Timo Sirainen <tss@iki.fi> |
---|---|
date | Wed, 20 Dec 2006 10:16:41 +0200 |
parents | bb4ec18672fe |
children | 8a5985446cce |
files | src/lib-index/mailbox-list-index-private.h src/lib-index/mailbox-list-index-sync.c src/lib-index/mailbox-list-index.c src/lib-storage/list/index-mailbox-list.c |
diffstat | 4 files changed, 171 insertions(+), 36 deletions(-) [+] |
line wrap: on
line diff
--- a/src/lib-index/mailbox-list-index-private.h Wed Dec 20 08:20:25 2006 +0200 +++ b/src/lib-index/mailbox-list-index-private.h Wed Dec 20 10:16:41 2006 +0200 @@ -6,6 +6,9 @@ #define MAILBOX_LIST_INDEX_MAJOR_VERSION 1 #define MAILBOX_LIST_INDEX_MINOR_VERSION 0 +#define MAILBOX_LIST_COMPRESS_PERCENTAGE 10 +#define MAILBOX_LIST_COMPRESS_MIN_SIZE 1024 + struct mailbox_list_index_header { uint8_t major_version; uint8_t minor_version; @@ -82,4 +85,10 @@ const struct mailbox_list_dir_record **dir_r); int mailbox_list_index_map(struct mailbox_list_index *index); +int mailbox_list_index_file_create(struct mailbox_list_index *index, + uint32_t uid_validity); +void mailbox_list_index_file_close(struct mailbox_list_index *index); + +int mailbox_list_index_refresh(struct mailbox_list_index *index); + #endif
--- a/src/lib-index/mailbox-list-index-sync.c Wed Dec 20 08:20:25 2006 +0200 +++ b/src/lib-index/mailbox-list-index-sync.c Wed Dec 20 10:16:41 2006 +0200 @@ -69,6 +69,9 @@ bool *match; }; +static bool mailbox_list_index_need_compress(struct mailbox_list_index *index); +static int mailbox_list_index_compress(struct mailbox_list_index_sync_ctx *ctx); + static struct mailbox_list_sync_dir * mailbox_list_alloc_sync_dir(struct mailbox_list_index_sync_ctx *ctx, unsigned int initial_count) @@ -105,6 +108,11 @@ if (recs[i].deleted) continue; + if (recs[i].uid == 0) { + return mailbox_list_index_set_corrupted(ctx->index, + "Record with UID=0"); + } + sync_rec = array_append_space(&sync_dir->records); sync_rec->name_hash = recs[i].name_hash; sync_rec->uid = recs[i].uid; @@ -317,10 +325,9 @@ &ctx->sync_root, &seq); } -static int sync_init_mail_sync(struct mailbox_list_index_sync_ctx *ctx) +static int sync_mail_sync_init(struct mailbox_list_index_sync_ctx *ctx) { struct mail_index_sync_rec sync_rec; - const struct mail_index_header *hdr; if (mail_index_sync_begin(ctx->index->mail_index, &ctx->mail_sync_ctx, &ctx->view, (uint32_t)-1, 0, @@ -331,6 +338,14 @@ don't need to do anything but write them to the index */ while (mail_index_sync_next(ctx->mail_sync_ctx, &sync_rec) > 0) ; + return 0; +} + +static int sync_mail_sync_init2(struct mailbox_list_index_sync_ctx *ctx) +{ + const struct mail_index_header *hdr; + + ctx->hdr = *ctx->index->hdr; hdr = mail_index_get_header(ctx->view); if (hdr->uid_validity != 0) { @@ -348,7 +363,7 @@ TRUE); } - return mailbox_list_index_get_root(ctx); + return 0; } int mailbox_list_index_sync_init(struct mailbox_list_index *index, @@ -372,11 +387,13 @@ ctx->index = index; ctx->sync_path = p_strdup(pool, path); ctx->flags = flags; - ctx->hdr = *index->hdr; /* mail index syncing acts as the only locking for us */ - if (sync_init_mail_sync(ctx) < 0) { - mailbox_list_index_sync_commit(&ctx); + if (sync_mail_sync_init(ctx) < 0 || + mailbox_list_index_refresh(index) < 0 || + sync_mail_sync_init2(ctx) < 0 || + mailbox_list_index_get_root(ctx) < 0) { + mailbox_list_index_sync_rollback(&ctx); return -1; } @@ -745,6 +762,7 @@ { struct mailbox_list_index_header *hdr; bool partial; + int ret = 0; if (ctx->index->mmap_disable) { ctx->output = o_stream_create_file(ctx->index->fd, default_pool, @@ -763,29 +781,30 @@ } if (mailbox_list_index_sync_write_dir(ctx, ctx->root, 0, partial) < 0) - return -1; + ret = -1; if (!ctx->index->mmap_disable) { /* update header */ hdr = ctx->index->mmap_base; - hdr->next_uid = ctx->hdr.next_uid; - hdr->used_space = ctx->hdr.used_space; - hdr->deleted_space = ctx->hdr.deleted_space; + if (ret == 0) + memcpy(hdr, &ctx->hdr, sizeof(*hdr)); if (msync(ctx->index->mmap_base, hdr->used_space, MS_SYNC) < 0) { mailbox_list_index_set_syscall_error(ctx->index, "msync()"); - return -1; + ret = -1; } } else { - o_stream_seek(ctx->output, 0); - o_stream_send(ctx->output, &ctx->hdr, sizeof(ctx->hdr)); + if (ret == 0) { + o_stream_seek(ctx->output, 0); + o_stream_send(ctx->output, &ctx->hdr, sizeof(ctx->hdr)); + } o_stream_destroy(&ctx->output); buffer_free(ctx->output_buf); } - return 0; + return ret; } int mailbox_list_index_sync_commit(struct mailbox_list_index_sync_ctx **_ctx) @@ -798,6 +817,8 @@ if (!ctx->failed) { /* write all the changes to the index */ ret = mailbox_list_index_sync_write(ctx); + if (ret == 0 && mailbox_list_index_need_compress(ctx->index)) + ret = mailbox_list_index_compress(ctx); } if (ctx->mail_sync_ctx != NULL) { @@ -829,3 +850,67 @@ (*ctx)->failed = TRUE; (void)mailbox_list_index_sync_commit(ctx); } + +static bool mailbox_list_index_need_compress(struct mailbox_list_index *index) +{ + uoff_t max_del_space; + + max_del_space = index->hdr->used_space / 100 * + MAILBOX_LIST_COMPRESS_PERCENTAGE; + if (index->hdr->deleted_space >= max_del_space && + index->hdr->used_space >= MAILBOX_LIST_COMPRESS_MIN_SIZE) + return TRUE; + + return FALSE; +} + +static int mailbox_list_copy_to_mem_all(struct mailbox_list_index_sync_ctx *ctx, + struct mailbox_list_sync_dir *dir) +{ + struct mailbox_list_sync_record *recs; + unsigned int i, count; + + /* mark the directories as new */ + dir->offset = 0; + dir->new_records_count = 1; + + recs = array_get_modifiable(&dir->records, &count); + for (i = 0; i < count; i++) { + recs[i].created = TRUE; + recs[i].seen = TRUE; + + if (recs[i].dir == NULL) { + if (recs[i].dir_offset == 0) + continue; + + if (mailbox_list_copy_sync_dir(ctx, recs[i].dir_offset, + &recs[i].dir) < 0) + return -1; + } + recs[i].dir_offset = 0; + + if (mailbox_list_copy_to_mem_all(ctx, recs[i].dir) < 0) + return -1; + } + return 0; +} + +static int mailbox_list_index_compress(struct mailbox_list_index_sync_ctx *ctx) +{ + /* first read everything to memory */ + if (mailbox_list_copy_to_mem_all(ctx, ctx->root) < 0) + return -1; + + /* truncate the index file */ + mailbox_list_index_file_close(ctx->index); + if (mailbox_list_index_file_create(ctx->index, + ctx->hdr.uid_validity) < 0) + return -1; + + /* reset header */ + ctx->hdr.used_space = sizeof(ctx->hdr); + ctx->hdr.deleted_space = 0; + + /* and write everything back */ + return mailbox_list_index_sync_write(ctx); +}
--- a/src/lib-index/mailbox-list-index.c Wed Dec 20 08:20:25 2006 +0200 +++ b/src/lib-index/mailbox-list-index.c Wed Dec 20 10:16:41 2006 +0200 @@ -71,7 +71,7 @@ index->hdr = NULL; } -static void mailbox_list_index_file_close(struct mailbox_list_index *index) +void mailbox_list_index_file_close(struct mailbox_list_index *index) { mailbox_list_index_unmap(index); @@ -80,6 +80,7 @@ if (index->fd != -1) { if (close(index->fd) < 0) mailbox_list_index_set_syscall_error(index, "close()"); + index->fd = -1; } } @@ -87,7 +88,7 @@ const char *str) { (void)unlink(index->filepath); - // FIXME: reopen or something + mailbox_list_index_file_close(index); i_error("Corrupted mailbox list index file %s: %s", index->filepath, str); @@ -174,7 +175,8 @@ } static void -mailbox_list_index_init_header(struct mailbox_list_index_header *hdr) +mailbox_list_index_init_header(struct mailbox_list_index_header *hdr, + uint32_t uid_validity) { memset(hdr, 0, sizeof(*hdr)); hdr->major_version = MAILBOX_LIST_INDEX_MAJOR_VERSION; @@ -183,7 +185,7 @@ hdr->header_size = sizeof(*hdr); hdr->used_space = hdr->header_size; - hdr->uid_validity = ioloop_time; + hdr->uid_validity = uid_validity; hdr->next_uid = 1; } @@ -191,7 +193,13 @@ { struct stat st1, st2; + if (index->fd == -1) + return 1; + if (stat(index->filepath, &st1) < 0) { + if (errno == ENOENT) + return 1; + mailbox_list_index_set_syscall_error(index, "stat()"); return -1; } @@ -204,8 +212,8 @@ !CMP_DEV_T(st1.st_dev, st2.st_dev); } -static int -mailbox_list_index_file_create(struct mailbox_list_index *index) +int mailbox_list_index_file_create(struct mailbox_list_index *index, + uint32_t uid_validity) { struct mailbox_list_index_header hdr; struct dotlock *dotlock; @@ -228,7 +236,7 @@ } } - mailbox_list_index_init_header(&hdr); + mailbox_list_index_init_header(&hdr, uid_validity); if (write_full(fd, &hdr, sizeof(hdr)) < 0) { mailbox_list_index_set_syscall_error(index, "write_full()"); (void)file_dotlock_delete(&dotlock); @@ -278,7 +286,7 @@ } } - ret = mailbox_list_index_file_create(index); + ret = mailbox_list_index_file_create(index, ioloop_time); if (ret <= 0) mailbox_list_index_file_close(index); return ret; @@ -483,12 +491,18 @@ return mailbox_list_index_lookup_rec(index, dir_offset, p + 1, rec_r); } -static int mailbox_list_index_refresh(struct mailbox_list_index *index) +int mailbox_list_index_refresh(struct mailbox_list_index *index) { int ret; - if ((ret = mailbox_list_index_is_recreated(index)) <= 0) + if ((ret = mailbox_list_index_is_recreated(index)) <= 0) { + if (ret < 0) + return -1; + + if (mailbox_list_index_map(index) < 0) + ret = -1; return ret; + } mailbox_list_index_file_close(index); return mailbox_list_index_open_or_create(index); @@ -529,7 +543,9 @@ (unsigned int)recurse_level; ctx->name_path = str_new(default_pool, 512); - if (*path != '\0') { + if (mailbox_list_index_refresh(index) < 0) + ctx->failed = TRUE; + if (!ctx->failed && *path != '\0') { ret = mailbox_list_index_lookup_rec(index, offset, path, &rec); if (ret < 0) ctx->failed = TRUE;
--- a/src/lib-storage/list/index-mailbox-list.c Wed Dec 20 08:20:25 2006 +0200 +++ b/src/lib-storage/list/index-mailbox-list.c Wed Dec 20 10:16:41 2006 +0200 @@ -22,12 +22,34 @@ static void (*index_next_hook_mailbox_list_created)(struct mailbox_list *list); static int +index_mailbox_view_sync(struct index_mailbox_list_iterate_context *ctx) +{ + struct mail_index_view_sync_ctx *sync_ctx; + struct mail_index_view_sync_rec sync_rec; + int ret; + + if (mail_index_view_sync_begin(ctx->view, MAIL_INDEX_SYNC_MASK_ALL, + &sync_ctx) < 0) { + mailbox_list_set_internal_error(ctx->ctx.list); + return -1; + } + + while ((ret = mail_index_view_sync_next(sync_ctx, &sync_rec)) > 0) ; + + mail_index_view_sync_end(&sync_ctx); + return ret; +} + +static int index_mailbox_list_is_synced(struct index_mailbox_list_iterate_context *ctx) { const struct mail_index_header *hdr; struct stat st; const char *path = ctx->ctx.list->set.root_dir; + if (index_mailbox_view_sync(ctx) < 0) + return -1; + /* FIXME: single sync_stamp works only with maildir++ */ if (stat(path, &st) < 0) { mailbox_list_set_critical(ctx->ctx.list, @@ -117,14 +139,14 @@ } else { /* FIXME: this works nicely with maildir++, but not others */ sync_flags = MAILBOX_LIST_SYNC_FLAG_RECURSIVE; - mask = "*"; - prefix = ""; - if (mailbox_list_index_sync_init(ilist->list_index, prefix, + if (mailbox_list_index_sync_init(ilist->list_index, "", sync_flags, &ctx->sync_ctx) == 0) { - ctx->trans = - mailbox_list_index_sync_get_transaction(ctx->sync_ctx); + mask = "*"; + prefix = ""; + ctx->trans = mailbox_list_index_sync_get_transaction( + ctx->sync_ctx); } ctx->backend_ctx = ilist->super.iter_init(list, mask, flags); @@ -253,11 +275,12 @@ /* if the sync fails, just ignore it. we don't require synced indexes to return valid output. */ if (mailbox_list_index_sync_more(ctx->sync_ctx, info->name, - &seq) < 0) - return info; - - flags = index_mailbox_list_info_flags_translate(info->flags); - mail_index_update_flags(ctx->trans, seq, MODIFY_REPLACE, flags); + &seq) == 0) { + flags = index_mailbox_list_info_flags_translate( + info->flags); + mail_index_update_flags(ctx->trans, seq, MODIFY_REPLACE, + flags); + } } while (imap_match(ctx->glob, info->name) != IMAP_MATCH_YES || !info_flags_match(ctx, info)); @@ -280,7 +303,7 @@ if (ctx->view != NULL) mail_index_view_close(&ctx->view); - if (ctx->backend_ctx != NULL) { + if (ctx->sync_ctx != NULL) { /* FIXME: single sync_stamp works only with maildir++ */ mail_index_update_header(ctx->trans, offsetof(struct mail_index_header, sync_stamp), @@ -293,6 +316,8 @@ fails, we've still returned full output. */ (void)mailbox_list_index_sync_commit(&ctx->sync_ctx); } + } else if (ctx->backend_ctx != NULL) { + ret = ilist->super.iter_deinit(ctx->backend_ctx); } imap_match_deinit(&ctx->glob);