Mercurial > dovecot > original-hg > dovecot-1.2
changeset 4886:ae2114f11a0e HEAD
Memory leak fixes. Also when building a large mailbox flush once in a while
to free memory.
author | Timo Sirainen <tss@iki.fi> |
---|---|
date | Sun, 10 Dec 2006 01:01:13 +0200 |
parents | 902d24d14932 |
children | 463a4ebba685 |
files | src/plugins/fts-squat/squat-trie.c src/plugins/fts-squat/squat-uidlist.c src/plugins/fts-squat/squat-uidlist.h |
diffstat | 3 files changed, 119 insertions(+), 53 deletions(-) [+] |
line wrap: on
line diff
--- a/src/plugins/fts-squat/squat-trie.c Sat Dec 09 23:53:13 2006 +0200 +++ b/src/plugins/fts-squat/squat-trie.c Sun Dec 10 01:01:13 2006 +0200 @@ -58,6 +58,8 @@ char *uidlist_filepath; struct squat_uidlist *uidlist; + + pool_t node_pool; struct trie_node *root; buffer_t *buf; @@ -87,6 +89,7 @@ const char *tmp_path; struct ostream *output; + int fd; struct squat_uidlist_compress_ctx *uidlist_ctx; @@ -158,7 +161,8 @@ struct trie_node *node, unsigned int level); static int trie_write_node(struct squat_trie_build_context *ctx, unsigned int level, struct trie_node *node); -static int squat_trie_build_flush(struct squat_trie_build_context *ctx); +static int +squat_trie_build_flush(struct squat_trie_build_context *ctx, bool finish); static int chr_8bit_cmp(const void *_key, const void *_chr) { @@ -373,7 +377,8 @@ chars16_count * sizeof(struct trie_node *); } - node = i_malloc(sizeof(*node) + chars8_memsize + chars16_memsize); + node = p_malloc(trie->node_pool, + sizeof(*node) + chars8_memsize + chars16_memsize); node->chars_8bit_count = chars8_count; node->chars_16bit_count = chars16_count; node->file_offset = offset; @@ -428,6 +433,9 @@ trie->mmap_size = 0; trie->hdr = NULL; trie->const_mmap_base = NULL; + + p_clear(trie->node_pool); + trie->root = NULL; } static void trie_file_close(struct squat_trie *trie) @@ -557,9 +565,28 @@ return 1; } +static void trie_file_open_fd(struct squat_trie *trie, int fd) +{ + struct stat st; + + if (fstat(fd, &st) < 0) { + /* don't bother adding complexity by trying to handle this + error here. we'll break later anyway in easier error + handling paths. */ + squat_trie_set_syscall_error(trie, "fstat()"); + trie->ino = 0; + } else { + trie->dev = st.st_dev; + trie->ino = st.st_ino; + } + trie->fd = fd; + + if (trie->mmap_disable) + trie->file_cache = file_cache_new(trie->fd); +} + static int trie_file_open(struct squat_trie *trie, bool create) { - struct stat st; int fd; i_assert(trie->fd == -1); @@ -572,18 +599,7 @@ squat_trie_set_syscall_error(trie, "open()"); return -1; } - if (fstat(fd, &st) < 0) { - squat_trie_set_syscall_error(trie, "fstat()"); - (void)close(fd); - return -1; - } - - trie->fd = fd; - trie->dev = st.st_dev; - trie->ino = st.st_ino; - - if (trie->mmap_disable) - trie->file_cache = file_cache_new(trie->fd); + trie_file_open_fd(trie, fd); return 1; } @@ -625,6 +641,7 @@ trie->lock_method = lock_method; trie->mmap_disable = mmap_disable; trie->buf = buffer_create_dynamic(default_pool, 1024); + trie->node_pool = pool_alloconly_create("trie node pool", 1024*64); trie->uidlist_filepath = i_strconcat(path, ".uids", NULL); trie->uidlist = @@ -637,6 +654,7 @@ { buffer_free(trie->buf); squat_uidlist_deinit(trie->uidlist); + pool_unref(trie->node_pool); i_free(trie->uidlist_filepath); i_free(trie->filepath); i_free(trie); @@ -679,6 +697,19 @@ CMP_DEV_T(st.st_dev, trie->dev) ? 0 : 1; } +static int +squat_trie_file_lock(struct squat_trie *trie, int fd, const char *path, + int lock_type, struct file_lock **lock_r) +{ + int ret; + + ret = file_wait_lock(fd, path, lock_type, trie->lock_method, + SQUAT_TRIE_LOCK_TIMEOUT, lock_r); + if (ret == 0) + squat_trie_set_syscall_error(trie, "file_wait_lock()"); + return ret; +} + int squat_trie_lock(struct squat_trie *trie, int lock_type) { bool created = FALSE; @@ -712,16 +743,10 @@ for (;;) { i_assert(trie->file_lock == NULL); - ret = file_wait_lock(trie->fd, trie->filepath, lock_type, - trie->lock_method, SQUAT_TRIE_LOCK_TIMEOUT, - &trie->file_lock); - if (ret <= 0) { - if (ret == 0) { - squat_trie_set_syscall_error(trie, - "file_wait_lock()"); - } + ret = squat_trie_file_lock(trie, trie->fd, trie->filepath, + lock_type, &trie->file_lock); + if (ret <= 0) return ret; - } /* if the trie has been compressed, we need to reopen the file and try to lock again */ @@ -773,7 +798,7 @@ } static struct trie_node * -node_alloc(uint16_t chr, unsigned int level) +node_alloc(struct squat_trie *trie, uint16_t chr, unsigned int level) { struct trie_node *node; unsigned int idx_size, idx_offset = sizeof(*node); @@ -785,7 +810,7 @@ uint8_t *chrp; idx_offset += ALIGN(sizeof(*chrp)); - node = i_malloc(idx_offset + idx_size); + node = p_malloc(trie->node_pool, idx_offset + idx_size); node->chars_8bit_count = 1; chrp = PTR_OFFSET(node, sizeof(*node)); @@ -794,7 +819,7 @@ uint16_t *chrp; idx_offset += ALIGN(sizeof(*chrp)); - node = i_malloc(idx_offset + idx_size); + node = p_malloc(trie->node_pool, idx_offset + idx_size); node->chars_16bit_count = 1; chrp = PTR_OFFSET(node, sizeof(*node)); @@ -807,8 +832,8 @@ } static struct trie_node * -node_realloc(struct trie_node *node, uint32_t char_idx, uint16_t chr, - unsigned int level) +node_realloc(struct squat_trie *trie, struct trie_node *node, + uint32_t char_idx, uint16_t chr, unsigned int level) { struct trie_node *new_node; unsigned int old_size_8bit, old_size_16bit, old_idx_offset; @@ -836,7 +861,7 @@ (node->chars_16bit_count + 1) * idx_size; } - new_node = i_malloc(new_size); + new_node = p_malloc(trie->node_pool, new_size); if (chr < 256) { hole1_pos = sizeof(*node) + char_idx; old_idx_offset = sizeof(*node) + ALIGN(node->chars_8bit_count); @@ -877,7 +902,6 @@ old_size - hole2_pos); new_node->resized = TRUE; - i_free(node); return new_node; } @@ -897,7 +921,7 @@ if (node == NULL) { ctx->node_count++; - node = *parent = node_alloc(*data, level); + node = *parent = node_alloc(trie, *data, level); char_idx = 0; count = 1; modified = TRUE; @@ -911,7 +935,7 @@ chr_8bit_cmp); char_idx = pos - chars; if (char_idx == count || *pos != *data) { - node = node_realloc(node, char_idx, + node = node_realloc(trie, node, char_idx, *data, level); *parent = node; modified = TRUE; @@ -925,7 +949,7 @@ if (node == NULL) { ctx->node_count++; - node = *parent = node_alloc(*data, level); + node = *parent = node_alloc(trie, *data, level); char_idx = 0; count = 1; modified = TRUE; @@ -945,7 +969,7 @@ chr_16bit_cmp); char_idx = pos - chars; if (char_idx == count || *pos != *data) { - node = node_realloc(node, char_idx, + node = node_realloc(trie, node, char_idx, *data, level); *parent = node; modified = TRUE; @@ -1089,7 +1113,7 @@ int ret = ctx->failed ? -1 : 0; if (ret == 0) - ret = squat_trie_build_flush(ctx); + ret = squat_trie_build_flush(ctx, TRUE); if (ctx->locked) squat_trie_unlock(ctx->trie); @@ -1138,8 +1162,16 @@ t_pop(); return 0; } + } else if (squat_uidlist_want_flush(ctx->trie->uidlist)) { + if (squat_trie_build_flush(ctx, FALSE) < 0) { + ctx->failed = TRUE; + t_pop(); + return -1; + } + str = data_normalize(data, size, ctx->trie->buf); } + ctx->prev_uid = uid; for (i = 0; i + BLOCK_SIZE <= size; i++) { if (block_want_add(str+i)) { if (trie_insert_node(ctx, &ctx->trie->root, @@ -1368,7 +1400,8 @@ current_message_count); } -static int squat_trie_build_flush(struct squat_trie_build_context *ctx) +static int +squat_trie_build_flush(struct squat_trie_build_context *ctx, bool finish) { struct squat_trie *trie = ctx->trie; uint32_t uidvalidity; @@ -1378,6 +1411,9 @@ return 0; } + if (trie->corrupted) + return -1; + if (trie_nodes_write(ctx, &uidvalidity) < 0) return -1; if (squat_uidlist_flush(trie->uidlist, uidvalidity) < 0) @@ -1387,8 +1423,8 @@ if (squat_trie_map(trie) <= 0) return -1; - if (squat_trie_need_compress(trie, (unsigned int)-1)) { - if (ctx->locked) { + /*if (squat_trie_need_compress(trie, (unsigned int)-1))*/ { + if (ctx->locked && finish) { squat_trie_unlock(ctx->trie); ctx->locked = FALSE; } @@ -1469,7 +1505,6 @@ children[i] = NULL; need_char_compress = TRUE; } - i_free(child_node); if (ret < 0) return -1; @@ -1574,19 +1609,18 @@ struct squat_trie *trie) { struct squat_trie_header hdr; - int fd; memset(ctx, 0, sizeof(*ctx)); ctx->tmp_path = t_strconcat(trie->filepath, ".tmp", NULL); - fd = open(ctx->tmp_path, O_RDWR | O_CREAT | O_TRUNC, 0600); - if (fd == -1) { + ctx->fd = open(ctx->tmp_path, O_RDWR | O_CREAT | O_TRUNC, 0600); + if (ctx->fd == -1) { i_error("open(%s, O_CREAT) failed: %m", ctx->tmp_path); return -1; } ctx->trie = trie; - ctx->output = o_stream_create_file(fd, default_pool, 0, TRUE); + ctx->output = o_stream_create_file(ctx->fd, default_pool, 0, FALSE); ctx->node_count = trie->hdr->node_count; /* write a dummy header first */ @@ -1617,11 +1651,11 @@ { struct squat_trie_compress_context ctx; struct trie_node *node; + struct file_lock *file_lock = NULL; + unsigned int orig_lock_count; int ret; - /* reopening the file loses locks, so we can't be locked initially */ - i_assert(trie->lock_count == 0); - + orig_lock_count = trie->lock_count; if (squat_trie_lock(trie, F_WRLCK) <= 0) return -1; @@ -1642,7 +1676,13 @@ squat_trie_compress_write_header(&ctx, node); } - i_free(node); + } + + if (ret == 0 && orig_lock_count > 0) { + /* lock the file before renaming so we can keep it locked. */ + if (squat_trie_file_lock(trie, ctx.fd, ctx.tmp_path, F_WRLCK, + &file_lock) <= 0) + ret = -1; } if (ret == 0) { @@ -1652,14 +1692,21 @@ ret = -1; } } + o_stream_destroy(&ctx.output); squat_trie_unlock(trie); - if (ret < 0) + if (ret < 0) { + if (file_lock != NULL) + file_lock_free(&file_lock); + (void)close(ctx.fd); (void)unlink(ctx.tmp_path); - else { + } else { trie_file_close(trie); - if (trie_file_open(trie, FALSE) < 0) + trie_file_open_fd(trie, ctx.fd); + + trie->file_lock = file_lock; + if (squat_trie_map(trie) <= 0) return -1; } return ret;
--- a/src/plugins/fts-squat/squat-uidlist.c Sat Dec 09 23:53:13 2006 +0200 +++ b/src/plugins/fts-squat/squat-uidlist.c Sun Dec 10 01:01:13 2006 +0200 @@ -18,6 +18,7 @@ #define UIDLIST_COMPRESS_PERCENTAGE 30 #define UIDLIST_UID_COMPRESS_PERCENTAGE 20 #define UIDLIST_COMPRESS_MIN_SIZE (1024*8) +#define SQUAT_UIDLIST_FLUSH_THRESHOLD (1024*1024*31) #define UID_NODE_PREV_FLAG_OLD 0x00000001 #define UID_LIST_IDX_FLAG_SINGLE 0x80000000 @@ -67,8 +68,10 @@ ARRAY_DEFINE(lists, struct uid_node); uint32_t first_new_list_idx; + uint32_t current_uid; pool_t node_pool; + size_t node_pool_used; buffer_t *tmp_buf, *list_buf; unsigned int check_expunges:1; @@ -331,7 +334,7 @@ uint32_t uid_list_idx = *_uid_list_idx; struct uid_node *node, *old_node; - i_assert(uid >= uidlist->hdr.uid_max); + i_assert(uid > uidlist->hdr.uid_max || uid == uidlist->current_uid); if (uid_list_idx == 0) { *_uid_list_idx = uid | UID_LIST_IDX_FLAG_SINGLE; @@ -339,6 +342,7 @@ } if (uid > uidlist->hdr.uid_max) { + uidlist->current_uid = uid; uidlist->hdr.uid_max = uid; uidlist->hdr.uid_count++; } @@ -361,6 +365,7 @@ } /* convert single UID to a list */ + uidlist->node_pool_used += sizeof(struct uid_node); old_node = p_new(uidlist->node_pool, struct uid_node, 1); old_node->uid = old_uid; @@ -385,6 +390,7 @@ return 0; } + uidlist->node_pool_used += sizeof(struct uid_node); old_node = p_new(uidlist->node_pool, struct uid_node, 1); *old_node = *node; } @@ -667,8 +673,12 @@ array_clear(&uidlist->lists); p_clear(uidlist->node_pool); + uidlist->node_pool_used = 0; + uidlist->write_failed = FALSE; + uidlist->current_uid = 0; - uidlist->write_failed = FALSE; + if (squat_uidlist_map(uidlist) <= 0) + ret = -1; return ret; } @@ -1051,6 +1061,11 @@ } } +bool squat_uidlist_want_flush(struct squat_uidlist *uidlist) +{ + return uidlist->node_pool_used >= SQUAT_UIDLIST_FLUSH_THRESHOLD; +} + size_t squat_uidlist_mem_used(struct squat_uidlist *uidlist, unsigned int *count_r) {
--- a/src/plugins/fts-squat/squat-uidlist.h Sat Dec 09 23:53:13 2006 +0200 +++ b/src/plugins/fts-squat/squat-uidlist.h Sun Dec 10 01:01:13 2006 +0200 @@ -54,6 +54,10 @@ int squat_uidlist_filter(struct squat_uidlist *uidlist, uint32_t uid_list_idx, ARRAY_TYPE(seq_range) *result); +/* Returns TRUE when uidlist has used so much memory that it'd prefer to + get flushed. */ +bool squat_uidlist_want_flush(struct squat_uidlist *uidlist); + size_t squat_uidlist_mem_used(struct squat_uidlist *uidlist, unsigned int *count_r);