Mercurial > dovecot > original-hg > dovecot-1.2
changeset 6931:97702c9c4111 HEAD
Locking and error handling fixes.
author | Timo Sirainen <tss@iki.fi> |
---|---|
date | Tue, 04 Dec 2007 19:59:57 +0200 |
parents | 48ddc0c4036c |
children | 90b63ce0c6a5 |
files | src/plugins/fts-squat/squat-trie.c src/plugins/fts-squat/squat-uidlist.c src/plugins/fts-squat/squat-uidlist.h |
diffstat | 3 files changed, 439 insertions(+), 314 deletions(-) [+] |
line wrap: on
line diff
--- a/src/plugins/fts-squat/squat-trie.c Tue Dec 04 15:20:01 2007 +0200 +++ b/src/plugins/fts-squat/squat-trie.c Tue Dec 04 19:59:57 2007 +0200 @@ -29,6 +29,9 @@ struct squat_trie_build_context { struct squat_trie *trie; struct ostream *output; + struct squat_uidlist_build_context *uidlist_build_ctx; + + struct file_lock *file_lock; uint32_t first_uid; unsigned int compress_nodes:1; @@ -46,7 +49,7 @@ bool failed; }; -static int squat_trie_map(struct squat_trie *trie); +static int squat_trie_map(struct squat_trie *trie, bool building); void squat_trie_delete(struct squat_trie *trie) { @@ -115,14 +118,6 @@ } } -static void squat_trie_clear(struct squat_trie *trie) -{ - trie->corrupted = FALSE; - node_free(trie, &trie->root); - memset(&trie->root, 0, sizeof(trie->root)); - memset(&trie->hdr, 0, sizeof(trie->hdr)); -} - struct squat_trie * squat_trie_init(const char *path, uint32_t uidvalidity, enum file_lock_method lock_method, bool mmap_disable) @@ -139,19 +134,13 @@ return trie; } -void squat_trie_deinit(struct squat_trie **_trie) -{ - struct squat_trie *trie = *_trie; - - *_trie = NULL; - squat_trie_clear(trie); - squat_uidlist_deinit(trie->uidlist); - i_free(trie->path); - i_free(trie); -} - static void squat_trie_close(struct squat_trie *trie) { + trie->corrupted = FALSE; + node_free(trie, &trie->root); + memset(&trie->root, 0, sizeof(trie->root)); + memset(&trie->hdr, 0, sizeof(trie->hdr)); + if (trie->mmap_size != 0) { if (munmap(trie->mmap_base, trie->mmap_size) < 0) i_error("munmap(%s) failed: %m", trie->path); @@ -162,7 +151,17 @@ trie->fd = -1; } trie->locked_file_size = 0; - squat_uidlist_close(trie->uidlist); +} + +void squat_trie_deinit(struct squat_trie **_trie) +{ + struct squat_trie *trie = *_trie; + + *_trie = NULL; + squat_trie_close(trie); + squat_uidlist_deinit(trie->uidlist); + i_free(trie->path); + i_free(trie); } static void squat_trie_header_init(struct squat_trie *trie) @@ -197,14 +196,10 @@ static int squat_trie_open(struct squat_trie *trie) { squat_trie_close(trie); - squat_trie_clear(trie); if (squat_trie_open_fd(trie) < 0) return -1; - if (squat_trie_map(trie) < 0) - return -1; - - return squat_uidlist_open(trie->uidlist); + return squat_trie_map(trie, FALSE); } static int squat_trie_is_file_stale(struct squat_trie *trie) @@ -522,7 +517,8 @@ } static inline void -node_add_uid(struct squat_trie *trie, uint32_t uid, struct squat_node *node) +node_add_uid(struct squat_trie_build_context *ctx, uint32_t uid, + struct squat_node *node) { if (uid < node->next_uid) { /* duplicate */ @@ -532,11 +528,12 @@ node->next_uid = uid + 1; node->uid_list_idx = - squat_uidlist_build_add_uid(trie->uidlist, + squat_uidlist_build_add_uid(ctx->uidlist_build_ctx, node->uid_list_idx, uid); } -static void node_split_string(struct squat_trie *trie, struct squat_node *node) +static void +node_split_string(struct squat_trie_build_context *ctx, struct squat_node *node) { struct squat_node *child; unsigned char *str; @@ -557,14 +554,14 @@ node->leaf_string_length = 0; /* create a new child node for the rest of the string */ - idx = node_add_child(trie, node, str[0], MAX_FAST_LEVEL); + idx = node_add_child(ctx->trie, node, str[0], MAX_FAST_LEVEL); child = NODE_CHILDREN_NODES(node) + idx; /* update uidlist to contain all of parent's UIDs */ child->next_uid = node->next_uid - node->unused_uids; for (uid = 0; uid < child->next_uid; uid++) { child->uid_list_idx = - squat_uidlist_build_add_uid(trie->uidlist, + squat_uidlist_build_add_uid(ctx->uidlist_build_ctx, child->uid_list_idx, uid); } @@ -585,7 +582,8 @@ } static bool -node_leaf_string_add_or_split(struct squat_trie *trie, struct squat_node *node, +node_leaf_string_add_or_split(struct squat_trie_build_context *ctx, + struct squat_node *node, const unsigned char *data, unsigned int data_len) { const unsigned char *str = NODE_LEAF_STRING(node); @@ -594,23 +592,24 @@ if (data_len != str_len) { /* different lengths, can't match */ - node_split_string(trie, node); + node_split_string(ctx, node); return FALSE; } for (i = 0; i < data_len; i++) { if (data[i] != str[i]) { /* non-match */ - node_split_string(trie, node); + node_split_string(ctx, node); return FALSE; } } return TRUE; } -static int squat_build_add(struct squat_trie *trie, uint32_t uid, +static int squat_build_add(struct squat_trie_build_context *ctx, uint32_t uid, const unsigned char *data, unsigned int size) { + struct squat_trie *trie = ctx->trie; struct squat_node *node = &trie->root; const unsigned char *end = data + size; unsigned char *chars; @@ -626,14 +625,14 @@ if (node->leaf_string_length != 0) { /* the whole string must match or we need to split the node */ - if (node_leaf_string_add_or_split(trie, node, data, + if (node_leaf_string_add_or_split(ctx, node, data, end - data)) { - node_add_uid(trie, uid, node); + node_add_uid(ctx, uid, node); return 0; } } - node_add_uid(trie, uid, node); + node_add_uid(ctx, uid, node); if (unlikely(uid < node->unused_uids)) { squat_trie_set_corrupted(trie); @@ -677,7 +676,7 @@ size - (end - data) + 1); node = NODE_CHILDREN_NODES(node) + idx; - node_add_uid(trie, uid, node); + node_add_uid(ctx, uid, node); uid = 0; if (++data == end) @@ -703,23 +702,24 @@ } static int -squat_build_word_bytes(struct squat_trie *trie, uint32_t uid, +squat_build_word_bytes(struct squat_trie_build_context *ctx, uint32_t uid, const unsigned char *data, unsigned int size) { + struct squat_trie *trie = ctx->trie; unsigned int i; if (trie->hdr.full_len <= trie->hdr.partial_len) i = 0; else { /* the first word is longer than others */ - if (squat_build_add(trie, uid, data, + if (squat_build_add(ctx, uid, data, I_MIN(size, trie->hdr.full_len)) < 0) return -1; i = 1; } for (; i < size; i++) { - if (squat_build_add(trie, uid, data + i, + if (squat_build_add(ctx, uid, data + i, I_MIN(trie->hdr.partial_len, size-i)) < 0) return -1; } @@ -727,15 +727,16 @@ } static int -squat_build_word(struct squat_trie *trie, uint32_t uid, +squat_build_word(struct squat_trie_build_context *ctx, uint32_t uid, const unsigned char *data, const uint8_t *char_lengths, unsigned int size) { + struct squat_trie *trie = ctx->trie; unsigned int i, j, bytelen; if (char_lengths == NULL) { /* optimization path: all characters are bytes */ - return squat_build_word_bytes(trie, uid, data, size); + return squat_build_word_bytes(ctx, uid, data, size); } if (trie->hdr.full_len <= trie->hdr.partial_len) @@ -747,7 +748,7 @@ bytelen += char_lengths[bytelen]; i_assert(bytelen <= size); - if (squat_build_add(trie, uid, data, bytelen) < 0) + if (squat_build_add(ctx, uid, data, bytelen) < 0) return -1; i = char_lengths[0]; } @@ -758,7 +759,7 @@ bytelen += char_lengths[i + bytelen]; i_assert(i + bytelen <= size); - if (squat_build_add(trie, uid, data + i, bytelen) < 0) + if (squat_build_add(ctx, uid, data + i, bytelen) < 0) return -1; } return 0; @@ -803,7 +804,7 @@ while (start < i && data[start] == '\0') start++; if (i != start) { - if (squat_build_word(trie, uid, data + start, + if (squat_build_word(ctx, uid, data + start, !multibyte_chars ? NULL : char_lengths + start, i - start) < 0) { @@ -817,7 +818,7 @@ while (start < i && data[start] == '\0') start++; if (i != start) { - if (squat_build_word(trie, uid, data + start, + if (squat_build_word(ctx, uid, data + start, !multibyte_chars ? NULL : char_lengths + start, i - start) < 0) ret = -1; @@ -1014,31 +1015,37 @@ } } -static int squat_trie_renumber_uidlists(struct squat_trie *trie, bool finish) +static int +squat_trie_renumber_uidlists(struct squat_trie_build_context *ctx, + bool compress) { struct squat_trie_iterate_context *iter; struct squat_node *node; - struct squat_uidlist *uidlist = trie->uidlist; - struct squat_uidlist_rebuild_context *ctx; + struct squat_uidlist_rebuild_context *rebuild_ctx; ARRAY_TYPE(uint32_t) uids; uint32_t new_uid_list_idx, max_count=0; int ret = 0; - if ((ret = squat_uidlist_rebuild_init(uidlist, finish, &ctx)) <= 0) + /* FIXME: update indexid */ + if ((ret = squat_uidlist_rebuild_init(ctx->uidlist_build_ctx, + compress, &rebuild_ctx)) <= 0) return ret; + ctx->trie->hdr.indexid = I_MAX(ioloop_time, ctx->trie->hdr.indexid + 1); + i_array_init(&uids, 1024); - iter = squat_trie_iterate_uidlist_init(trie); + iter = squat_trie_iterate_uidlist_init(ctx->trie); node = squat_trie_iterate_uidlist_first(iter); new_uid_list_idx = 0x100; while (node != NULL) { array_clear(&uids); - if (squat_uidlist_get(uidlist, node->uid_list_idx, &uids) < 0) { + if (squat_uidlist_get(ctx->trie->uidlist, node->uid_list_idx, + &uids) < 0) { ret = -1; break; } max_count = I_MAX(max_count, array_count(&uids)); - squat_uidlist_rebuild_next(ctx, &uids); + squat_uidlist_rebuild_next(rebuild_ctx, &uids); node->uid_list_idx = new_uid_list_idx << 1; new_uid_list_idx++; @@ -1047,7 +1054,10 @@ squat_trie_iterate_uidlist_deinit(iter); array_free(&uids); - return squat_uidlist_rebuild_finish(ctx, ret < 0); + /* lock the trie before we rename uidlist */ + if (squat_trie_lock(ctx->trie, F_WRLCK, &ctx->file_lock) <= 0) + ret = -1; + return squat_uidlist_rebuild_finish(rebuild_ctx, ret < 0); } static bool squat_trie_check_header(struct squat_trie *trie) @@ -1103,7 +1113,7 @@ return 0; } -static int squat_trie_map(struct squat_trie *trie) +static int squat_trie_map(struct squat_trie *trie, bool building) { struct file_lock *file_lock = NULL; bool changed; @@ -1134,6 +1144,11 @@ } } + if (ret == 0 && !building) { + /* do this while we're still locked */ + ret = squat_uidlist_refresh(trie->uidlist); + } + if (file_lock != NULL) file_unlock(&file_lock); if (ret < 0) @@ -1147,6 +1162,7 @@ struct squat_trie_build_context **ctx_r) { struct squat_trie_build_context *ctx; + struct squat_uidlist_build_context *uidlist_build_ctx; if (trie->fd == -1) { trie->fd = open(trie->path, O_RDWR | O_CREAT, 0600); @@ -1157,14 +1173,17 @@ } /* uidlist locks building */ - if (squat_uidlist_build_init(trie->uidlist) < 0) + if (squat_uidlist_build_init(trie->uidlist, &uidlist_build_ctx) < 0) return -1; - if (squat_trie_map(trie) < 0) + if (squat_trie_map(trie, TRUE) < 0) { + squat_uidlist_build_deinit(&uidlist_build_ctx); return -1; + } ctx = i_new(struct squat_trie_build_context, 1); ctx->trie = trie; + ctx->uidlist_build_ctx = uidlist_build_ctx; ctx->first_uid = trie->root.next_uid; *last_uid_r = I_MAX((trie->root.next_uid+1)/2, 1) - 1; @@ -1172,10 +1191,19 @@ return 0; } +static int squat_trie_write_lock(struct squat_trie_build_context *ctx) +{ + if (ctx->file_lock != NULL) + return 0; + + if (squat_trie_lock(ctx->trie, F_WRLCK, &ctx->file_lock) <= 0) + return -1; + return 0; +} + static int squat_trie_write(struct squat_trie_build_context *ctx) { struct squat_trie *trie = ctx->trie; - struct file_lock *file_lock; struct ostream *output; const char *path; int fd = -1, ret = 0; @@ -1191,25 +1219,35 @@ i_error("creat(%s) failed: %m", path); return -1; } + ret = file_wait_lock(fd, path, F_WRLCK, trie->lock_method, + SQUAT_TRIE_LOCK_TIMEOUT, &ctx->file_lock); + if (ret <= 0) { + if (ret == 0) + i_error("file_wait_lock(%s) failed: %m", path); + (void)close(fd); + return -1; + } + output = o_stream_create_fd(fd, 0, FALSE); o_stream_cork(output); o_stream_send(output, &trie->hdr, sizeof(trie->hdr)); - file_lock = NULL; } else { - /* we need to lock only the header update */ - if (squat_trie_lock(trie, F_WRLCK, &file_lock) <= 0) - return -1; - + /* we need to lock only while header is being written */ path = trie->path; ctx->compress_nodes = trie->hdr.used_file_size == sizeof(trie->hdr); output = o_stream_create_fd(trie->fd, 0, FALSE); o_stream_cork(output); - if (trie->hdr.used_file_size == 0) + if (trie->hdr.used_file_size != 0) + o_stream_seek(output, trie->hdr.used_file_size); + else { + if (squat_trie_write_lock(ctx) < 0) { + o_stream_unref(&output); + return -1; + } o_stream_send(output, &trie->hdr, sizeof(trie->hdr)); - else - o_stream_seek(output, trie->hdr.used_file_size); + } } ctx->output = output; @@ -1218,38 +1256,44 @@ if (trie->corrupted) ret = -1; + if (ret == 0) + ret = squat_trie_write_lock(ctx); if (ret == 0) { trie->hdr.used_file_size = output->offset; o_stream_seek(output, 0); o_stream_send(output, &trie->hdr, sizeof(trie->hdr)); - o_stream_flush(output); } - if (file_lock != NULL) - file_unlock(&file_lock); - if (output->last_failed_errno != 0) { errno = output->last_failed_errno; i_error("write() to %s failed: %m", path); ret = -1; } - o_stream_unref(&output); - if (fd != -1) { - if (ret < 0) { - if (close(fd) < 0) - i_error("close(%s) failed: %m", path); - } else if (rename(path, trie->path) < 0) { - i_error("rename(%s, %s) failed: %m", path, trie->path); - ret = -1; - } + o_stream_destroy(&output); + + if (fd == -1) { + /* appended to the existing file */ + return ret; + } - if (ret < 0) { - if (unlink(path) < 0 && errno != ENOENT) - i_error("unlink(%s) failed: %m", path); - } else { + /* recreating the trie file */ + if (ret < 0) { + if (close(fd) < 0) + i_error("close(%s) failed: %m", path); + fd = -1; + } else if (rename(path, trie->path) < 0) { + i_error("rename(%s, %s) failed: %m", path, trie->path); + ret = -1; + } + + if (ret < 0) { + if (unlink(path) < 0 && errno != ENOENT) + i_error("unlink(%s) failed: %m", path); + } else { + if (trie->fd != -1) { if (close(trie->fd) < 0) i_error("close(%s) failed: %m", trie->path); - trie->fd = fd; } + trie->fd = fd; } return ret; } @@ -1264,12 +1308,20 @@ compress = (ctx->trie->root.next_uid - ctx->first_uid) > 10; - ret = squat_uidlist_build_deinit(ctx->trie->uidlist); - if (squat_trie_renumber_uidlists(ctx->trie, compress) < 0) - ret = -1; - else + /* keep trie locked while header is being written and when files are + being renamed, so that while trie is read locked, uidlist can't + change under. */ + squat_uidlist_build_flush(ctx->uidlist_build_ctx); + ret = squat_trie_renumber_uidlists(ctx, compress); + if (ret == 0) ret = squat_trie_write(ctx); + if (ret == 0) + ret = squat_uidlist_build_finish(ctx->uidlist_build_ctx); + if (ctx->file_lock != NULL) + file_unlock(&ctx->file_lock); + squat_uidlist_build_deinit(&ctx->uidlist_build_ctx); + i_free(ctx); return ret; }
--- a/src/plugins/fts-squat/squat-uidlist.c Tue Dec 04 15:20:01 2007 +0200 +++ b/src/plugins/fts-squat/squat-uidlist.c Tue Dec 04 19:59:57 2007 +0200 @@ -33,21 +33,14 @@ struct squat_trie *trie; char *path; - struct ostream *output; int fd; struct file_lock *file_lock; uoff_t locked_file_size; - ARRAY_DEFINE(lists, struct uidlist_list); - ARRAY_TYPE(uint32_t) block_offsets; - ARRAY_TYPE(uint32_t) block_end_indexes; - uint32_t list_start_idx; - void *mmap_base; size_t mmap_size; struct squat_uidlist_file_header hdr; - struct squat_uidlist_file_header build_hdr; unsigned int cur_block_count; const uint32_t *cur_block_offsets; @@ -55,10 +48,27 @@ size_t max_size; unsigned int corrupted:1; + unsigned int building:1; +}; + +struct squat_uidlist_build_context { + struct squat_uidlist *uidlist; + struct ostream *output; + + ARRAY_TYPE(uint32_t) block_offsets; + ARRAY_TYPE(uint32_t) block_end_indexes; + + ARRAY_DEFINE(lists, struct uidlist_list); + uint32_t list_start_idx; + + struct squat_uidlist_file_header build_hdr; + unsigned int need_reopen:1; }; struct squat_uidlist_rebuild_context { struct squat_uidlist *uidlist; + struct squat_uidlist_build_context *build_ctx; + int fd; struct ostream *output; @@ -70,6 +80,8 @@ unsigned int new_count; }; +static void squat_uidlist_close(struct squat_uidlist *uidlist); + void squat_uidlist_delete(struct squat_uidlist *uidlist) { if (unlink(uidlist->path) < 0 && errno != ENOENT) @@ -87,10 +99,10 @@ squat_trie_delete(uidlist->trie); } -static uint32_t +static int uidlist_write_array(struct ostream *output, const uint32_t *uid_list, unsigned int uid_count, uint32_t packed_flags, - uint32_t offset, bool write_size) + uint32_t offset, bool write_size, uint32_t *size_r) { uint8_t *uidbuf, *bufp, sizebuf[SQUAT_PACK_MAX_SIZE], *sizebufp; uint8_t listbuf[SQUAT_PACK_MAX_SIZE], *listbufp = listbuf; @@ -175,7 +187,8 @@ prev = 0; for (i = 0; i < uid_count; i++) { uid = uid_list[i]; - i_assert((uid & ~UID_LIST_MASK_RANGE) >= prev); + if (unlikely((uid & ~UID_LIST_MASK_RANGE) < prev)) + return -1; if ((uid & UID_LIST_MASK_RANGE) == 0) { squat_pack_num(&bufp, (uid - prev) << 1); prev = uid + 1; @@ -208,12 +221,13 @@ i_free(uidbuf); t_pop(); - return size_value; + *size_r = size_value; + return 0; } -static uint32_t +static int uidlist_write(struct ostream *output, const struct uidlist_list *list, - bool write_size) + bool write_size, uint32_t *size_r) { const uint32_t *uid_list = list->uid_list; uint8_t buf[SQUAT_PACK_MAX_SIZE], *bufp; @@ -230,7 +244,8 @@ bufp = buf; squat_pack_num(&bufp, offset); o_stream_send(output, buf, bufp - buf); - return (bufp - buf) << 2 | packed_flags; + *size_r = (bufp - buf) << 2 | packed_flags; + return 0; } } else { i_assert(list->uid_count > 1); @@ -242,7 +257,7 @@ } return uidlist_write_array(output, uid_list, uid_count, - packed_flags, offset, write_size); + packed_flags, offset, write_size, size_r); } static int node_uidlist_map_blocks(struct squat_uidlist *uidlist) @@ -286,26 +301,44 @@ return 0; } -static int squat_uidlist_map(struct squat_uidlist *uidlist, uoff_t offset) +static int squat_uidlist_map_header(struct squat_uidlist *uidlist) +{ + if (uidlist->hdr.indexid == 0) { + /* still being built */ + return 0; + } + if (uidlist->hdr.indexid != uidlist->trie->hdr.indexid) { + /* see if trie was recreated */ + squat_trie_refresh(uidlist->trie); + } + if (uidlist->hdr.indexid != uidlist->trie->hdr.indexid) { + squat_uidlist_set_corrupted(uidlist, "wrong indexid"); + return -1; + } + if (uidlist->hdr.used_file_size < sizeof(uidlist->hdr) || + uidlist->hdr.used_file_size > uidlist->mmap_size) { + squat_uidlist_set_corrupted(uidlist, "broken used_file_size"); + return -1; + } + if (node_uidlist_map_blocks(uidlist) < 0) + return -1; + return 0; +} + +static int squat_uidlist_mmap(struct squat_uidlist *uidlist) { struct stat st; - if (uidlist->mmap_size > offset) - return 0; - if (fstat(uidlist->fd, &st) < 0) { i_error("fstat(%s) failed: %m", uidlist->path); return -1; } if (st.st_size < (off_t)sizeof(uidlist->hdr)) { + if (st.st_size == 0) + return 0; squat_uidlist_set_corrupted(uidlist, "File too small"); return -1; } - if (offset >= (uoff_t)st.st_size && offset != (uoff_t)-1) { - squat_uidlist_set_corrupted(uidlist, - "Offset points outside file"); - return -1; - } if (uidlist->mmap_size != 0) { if (munmap(uidlist->mmap_base, uidlist->mmap_size) < 0) @@ -321,20 +354,29 @@ i_error("mmap(%s) failed: %m", uidlist->path); return -1; } - memcpy(&uidlist->hdr, uidlist->mmap_base, sizeof(uidlist->hdr)); + return 0; +} - if (uidlist->hdr.indexid != uidlist->trie->hdr.indexid) { - squat_uidlist_set_corrupted(uidlist, "wrong indexid"); - return -1; +static int squat_uidlist_map(struct squat_uidlist *uidlist) +{ + const struct squat_uidlist_file_header *mmap_hdr = uidlist->mmap_base; + + if (mmap_hdr != NULL && !uidlist->building && + uidlist->hdr.block_list_offset == mmap_hdr->block_list_offset) { + /* file hasn't changed */ + return 0; } - if (uidlist->hdr.used_file_size < sizeof(uidlist->hdr) || - uidlist->hdr.used_file_size > uidlist->mmap_size) { - squat_uidlist_set_corrupted(uidlist, "broken used_file_size"); - return -1; + + if (mmap_hdr == NULL || uidlist->building || + uidlist->mmap_size < mmap_hdr->used_file_size) { + if (squat_uidlist_mmap(uidlist) < 0) + return -1; } - if (node_uidlist_map_blocks(uidlist) < 0) - return -1; - return 0; + + if (!uidlist->building) + memcpy(&uidlist->hdr, uidlist->mmap_base, sizeof(uidlist->hdr)); + + return squat_uidlist_map_header(uidlist); } struct squat_uidlist *squat_uidlist_init(struct squat_trie *trie) @@ -353,17 +395,11 @@ { squat_uidlist_close(uidlist); - if (array_is_created(&uidlist->block_offsets)) - array_free(&uidlist->block_offsets); - if (array_is_created(&uidlist->block_end_indexes)) - array_free(&uidlist->block_end_indexes); - if (array_is_created(&uidlist->lists)) - array_free(&uidlist->lists); i_free(uidlist->path); i_free(uidlist); } -int squat_uidlist_open(struct squat_uidlist *uidlist) +static int squat_uidlist_open(struct squat_uidlist *uidlist) { squat_uidlist_close(uidlist); @@ -376,13 +412,50 @@ i_error("open(%s) failed: %m", uidlist->path); return -1; } - return squat_uidlist_map(uidlist, 0); + return squat_uidlist_map(uidlist); +} + +static void squat_uidlist_close(struct squat_uidlist *uidlist) +{ + i_assert(!uidlist->building); + + if (uidlist->file_lock != NULL) + file_lock_free(&uidlist->file_lock); + if (uidlist->mmap_size != 0) { + if (munmap(uidlist->mmap_base, uidlist->mmap_size) < 0) + i_error("munmap(%s) failed: %m", uidlist->path); + uidlist->mmap_base = NULL; + uidlist->mmap_size = 0; + } + if (uidlist->fd != -1) { + if (close(uidlist->fd) < 0) + i_error("close(%s) failed: %m", uidlist->path); + uidlist->fd = -1; + } + uidlist->cur_block_end_indexes = NULL; + uidlist->cur_block_offsets = NULL; + uidlist->corrupted = FALSE; +} + +int squat_uidlist_refresh(struct squat_uidlist *uidlist) +{ + if (uidlist->fd == -1 || + uidlist->hdr.indexid != uidlist->trie->hdr.indexid) { + if (squat_uidlist_open(uidlist) < 0) + return -1; + } else { + if (squat_uidlist_map(uidlist) < 0) + return -1; + } + return 0; } static int squat_uidlist_is_file_stale(struct squat_uidlist *uidlist) { struct stat st, st2; + i_assert(uidlist->fd != -1); + if (stat(uidlist->path, &st) < 0) { if (errno == ENOENT) return 1; @@ -405,6 +478,7 @@ int ret; for (;;) { + i_assert(uidlist->fd != -1); i_assert(uidlist->file_lock == NULL); ret = file_wait_lock(uidlist->fd, uidlist->path, F_WRLCK, @@ -449,7 +523,7 @@ return -1; if (uidlist->locked_file_size != 0) { - if (squat_uidlist_map(uidlist, 0) < 0) { + if (squat_uidlist_map(uidlist) < 0) { /* broken file, truncate */ if (ftruncate(uidlist->fd, 0) < 0) { i_error("ftruncate(%s) failed: %m", @@ -461,8 +535,7 @@ } if (uidlist->locked_file_size == 0) { /* write using 0 until we're finished */ - uidlist->hdr.indexid = 0; - uidlist->hdr.used_file_size = sizeof(uidlist->hdr); + memset(&uidlist->hdr, 0, sizeof(uidlist->hdr)); if (write_full(uidlist->fd, &uidlist->hdr, sizeof(uidlist->hdr)) < 0) { i_error("write(%s) failed: %m", uidlist->path); @@ -472,27 +545,13 @@ return 0; } -void squat_uidlist_close(struct squat_uidlist *uidlist) +int squat_uidlist_build_init(struct squat_uidlist *uidlist, + struct squat_uidlist_build_context **ctx_r) { - if (uidlist->file_lock != NULL) - file_lock_free(&uidlist->file_lock); - if (uidlist->mmap_size != 0) { - if (munmap(uidlist->mmap_base, uidlist->mmap_size) < 0) - i_error("munmap(%s) failed: %m", uidlist->path); - uidlist->mmap_size = 0; - } - if (uidlist->output != NULL) - o_stream_unref(&uidlist->output); - if (uidlist->fd != -1) { - if (close(uidlist->fd) < 0) - i_error("close(%s) failed: %m", uidlist->path); - uidlist->fd = -1; - } - uidlist->corrupted = FALSE; -} + struct squat_uidlist_build_context *ctx; -int squat_uidlist_build_init(struct squat_uidlist *uidlist) -{ + i_assert(!uidlist->building); + if (squat_uidlist_open_or_create(uidlist) < 0) { if (uidlist->file_lock != NULL) file_unlock(&uidlist->file_lock); @@ -505,29 +564,35 @@ return -1; } - uidlist->output = o_stream_create_fd(uidlist->fd, 0, FALSE); - if (uidlist->output->offset == 0) { + ctx = i_new(struct squat_uidlist_build_context, 1); + ctx->uidlist = uidlist; + ctx->output = o_stream_create_fd(uidlist->fd, 0, FALSE); + if (ctx->output->offset == 0) { struct squat_uidlist_file_header hdr; memset(&hdr, 0, sizeof(hdr)); - o_stream_send(uidlist->output, &hdr, sizeof(hdr)); + o_stream_send(ctx->output, &hdr, sizeof(hdr)); } - o_stream_cork(uidlist->output); - i_array_init(&uidlist->lists, 10240); - i_array_init(&uidlist->block_offsets, 128); - i_array_init(&uidlist->block_end_indexes, 128); - uidlist->list_start_idx = uidlist->hdr.count; - uidlist->build_hdr = uidlist->hdr; + o_stream_cork(ctx->output); + i_array_init(&ctx->lists, 10240); + i_array_init(&ctx->block_offsets, 128); + i_array_init(&ctx->block_end_indexes, 128); + ctx->list_start_idx = uidlist->hdr.count; + ctx->build_hdr = uidlist->hdr; + + uidlist->building = TRUE; + *ctx_r = ctx; return 0; } -static int -uidlist_write_block_list_and_header(struct squat_uidlist *uidlist, +static void +uidlist_write_block_list_and_header(struct squat_uidlist_build_context *ctx, struct ostream *output, ARRAY_TYPE(uint32_t) *block_offsets, ARRAY_TYPE(uint32_t) *block_end_indexes, bool write_old_blocks) { + struct squat_uidlist *uidlist = ctx->uidlist; unsigned int align, old_block_count, new_block_count; uint32_t block_offset_count; uoff_t block_list_offset; @@ -555,21 +620,17 @@ old_block_count * sizeof(uint32_t)); o_stream_send(output, array_idx(block_offsets, 0), new_block_count * sizeof(uint32_t)); + o_stream_flush(output); - /* write header */ - uidlist->build_hdr.indexid = uidlist->trie->hdr.indexid; - uidlist->build_hdr.block_list_offset = block_list_offset; - uidlist->build_hdr.used_file_size = output->offset; - uidlist->hdr = uidlist->build_hdr; - - o_stream_seek(output, 0); - o_stream_send(output, &uidlist->build_hdr, sizeof(uidlist->build_hdr)); - o_stream_seek(output, uidlist->build_hdr.used_file_size); - o_stream_flush(output); - return 0; + /* update header - it's written later when trie is locked */ + i_assert(uidlist->trie->hdr.indexid != 0); + ctx->build_hdr.indexid = uidlist->trie->hdr.indexid; + ctx->build_hdr.block_list_offset = block_list_offset; + ctx->build_hdr.used_file_size = output->offset; + uidlist->hdr = ctx->build_hdr; } -static int squat_uidlist_build_flush(struct squat_uidlist *uidlist) +void squat_uidlist_build_flush(struct squat_uidlist_build_context *ctx) { struct uidlist_list *lists; uint8_t buf[SQUAT_PACK_MAX_SIZE], *bufp; @@ -578,77 +639,105 @@ uint32_t list_sizes[UIDLIST_BLOCK_LIST_COUNT]; size_t mem_size; - if (uidlist->corrupted) - return -1; + if (ctx->uidlist->corrupted) + return; - lists = array_get_modifiable(&uidlist->lists, &count); + lists = array_get_modifiable(&ctx->lists, &count); if (count == 0) - return 0; + return; /* write the lists and save the written sizes to uid_list[0] */ for (i = 0; i < count; i += UIDLIST_BLOCK_LIST_COUNT) { - start_offset = uidlist->output->offset; + start_offset = ctx->output->offset; max = I_MIN(count - i, UIDLIST_BLOCK_LIST_COUNT); for (j = 0; j < max; j++) { - list_sizes[j] = uidlist_write(uidlist->output, - &lists[i+j], FALSE); + if (uidlist_write(ctx->output, &lists[i+j], + FALSE, &list_sizes[j]) < 0) { + squat_uidlist_set_corrupted(ctx->uidlist, + "Broken uidlists"); + return; + } } - block_offset = uidlist->output->offset; - block_end_idx = uidlist->list_start_idx + i + max; - array_append(&uidlist->block_offsets, &block_offset, 1); - array_append(&uidlist->block_end_indexes, &block_end_idx, 1); + block_offset = ctx->output->offset; + block_end_idx = ctx->list_start_idx + i + max; + array_append(&ctx->block_offsets, &block_offset, 1); + array_append(&ctx->block_end_indexes, &block_end_idx, 1); /* write the full size of the uidlists */ bufp = buf; squat_pack_num(&bufp, block_offset - start_offset); - o_stream_send(uidlist->output, buf, bufp - buf); + o_stream_send(ctx->output, buf, bufp - buf); /* write the sizes/flags */ for (j = 0; j < max; j++) { bufp = buf; squat_pack_num(&bufp, list_sizes[j]); - o_stream_send(uidlist->output, buf, bufp - buf); + o_stream_send(ctx->output, buf, bufp - buf); } } - mem_size = uidlist->lists.arr.buffer->used + - uidlist->block_offsets.arr.buffer->used + - uidlist->block_end_indexes.arr.buffer->used; - if (uidlist->max_size < mem_size) - uidlist->max_size = mem_size; + mem_size = ctx->lists.arr.buffer->used + + ctx->block_offsets.arr.buffer->used + + ctx->block_end_indexes.arr.buffer->used; + if (ctx->uidlist->max_size < mem_size) + ctx->uidlist->max_size = mem_size; - uidlist->list_start_idx += count; - array_clear(&uidlist->lists); + ctx->list_start_idx += count; + array_clear(&ctx->lists); + + uidlist_write_block_list_and_header(ctx, ctx->output, + &ctx->block_offsets, + &ctx->block_end_indexes, TRUE); + + (void)squat_uidlist_map(ctx->uidlist); - if (uidlist_write_block_list_and_header(uidlist, uidlist->output, - &uidlist->block_offsets, - &uidlist->block_end_indexes, - TRUE) < 0) + array_clear(&ctx->block_offsets); + array_clear(&ctx->block_end_indexes); +} + +int squat_uidlist_build_finish(struct squat_uidlist_build_context *ctx) +{ + if (ctx->uidlist->corrupted) return -1; - if (uidlist->output->last_failed_errno != 0) { - errno = uidlist->output->last_failed_errno; - i_error("write() to %s failed: %m", uidlist->path); + + o_stream_seek(ctx->output, 0); + o_stream_send(ctx->output, &ctx->build_hdr, sizeof(ctx->build_hdr)); + o_stream_seek(ctx->output, ctx->build_hdr.used_file_size); + o_stream_flush(ctx->output); + + if (ctx->output->last_failed_errno != 0) { + errno = ctx->output->last_failed_errno; + i_error("write() to %s failed: %m", ctx->uidlist->path); return -1; } - - (void)squat_uidlist_map(uidlist, (uoff_t)-1); - - array_clear(&uidlist->block_offsets); - array_clear(&uidlist->block_end_indexes); return 0; } -int squat_uidlist_build_deinit(struct squat_uidlist *uidlist) +void squat_uidlist_build_deinit(struct squat_uidlist_build_context **_ctx) { - int ret; + struct squat_uidlist_build_context *ctx = *_ctx; + + *_ctx = NULL; + + i_assert(array_count(&ctx->lists) == 0 || ctx->uidlist->corrupted); + i_assert(ctx->uidlist->building); + ctx->uidlist->building = FALSE; - ret = squat_uidlist_build_flush(uidlist); - file_unlock(&uidlist->file_lock); - return ret; + file_unlock(&ctx->uidlist->file_lock); + + if (ctx->need_reopen) + (void)squat_uidlist_open(ctx->uidlist); + + array_free(&ctx->block_offsets); + array_free(&ctx->block_end_indexes); + array_free(&ctx->lists); + o_stream_unref(&ctx->output); + i_free(ctx); } -int squat_uidlist_rebuild_init(struct squat_uidlist *uidlist, bool finish, +int squat_uidlist_rebuild_init(struct squat_uidlist_build_context *build_ctx, + bool compress, struct squat_uidlist_rebuild_context **ctx_r) { struct squat_uidlist_rebuild_context *ctx; @@ -656,15 +745,16 @@ const char *temp_path; int fd; - if (uidlist->hdr.link_count == 0) + if (build_ctx->build_hdr.link_count == 0) return 0; - if (!finish) { - if (uidlist->hdr.link_count < uidlist->hdr.count*2/3) + if (!compress) { + if (build_ctx->build_hdr.link_count < + build_ctx->build_hdr.count*2/3) return 0; } - temp_path = t_strconcat(uidlist->path, ".tmp", NULL); + temp_path = t_strconcat(build_ctx->uidlist->path, ".tmp", NULL); fd = open(temp_path, O_RDWR | O_TRUNC | O_CREAT, 0600); if (fd < 0) { i_error("open(%s) failed: %m", temp_path); @@ -672,7 +762,8 @@ } ctx = i_new(struct squat_uidlist_rebuild_context, 1); - ctx->uidlist = uidlist; + ctx->uidlist = build_ctx->uidlist; + ctx->build_ctx = build_ctx; ctx->fd = fd; ctx->output = o_stream_create_fd(ctx->fd, 0, FALSE); o_stream_cork(ctx->output); @@ -682,9 +773,9 @@ ctx->cur_block_start_offset = ctx->output->offset; i_array_init(&ctx->new_block_offsets, - uidlist->build_hdr.count / UIDLIST_BLOCK_LIST_COUNT); + build_ctx->build_hdr.count / UIDLIST_BLOCK_LIST_COUNT); i_array_init(&ctx->new_block_end_indexes, - uidlist->build_hdr.count / UIDLIST_BLOCK_LIST_COUNT); + build_ctx->build_hdr.count / UIDLIST_BLOCK_LIST_COUNT); *ctx_r = ctx; return 1; } @@ -721,9 +812,11 @@ void squat_uidlist_rebuild_next(struct squat_uidlist_rebuild_context *ctx, const ARRAY_TYPE(uint32_t) *uids) { - ctx->list_sizes[ctx->list_idx] = - uidlist_write_array(ctx->output, array_idx(uids, 0), - array_count(uids), 0, 0, FALSE); + if (uidlist_write_array(ctx->output, array_idx(uids, 0), + array_count(uids), 0, 0, FALSE, + &ctx->list_sizes[ctx->list_idx]) < 0) + squat_uidlist_set_corrupted(ctx->uidlist, "Broken uidlists"); + if (++ctx->list_idx == UIDLIST_BLOCK_LIST_COUNT) { uidlist_rebuild_flush_block(ctx); ctx->list_idx = 0; @@ -733,53 +826,53 @@ int squat_uidlist_rebuild_finish(struct squat_uidlist_rebuild_context *ctx, bool cancel) { - struct squat_uidlist *uidlist = ctx->uidlist; const char *temp_path; int ret = 1; if (ctx->list_idx != 0) uidlist_rebuild_flush_block(ctx); - if (array_count(&ctx->new_block_end_indexes) == 0 || cancel) + if (array_count(&ctx->new_block_end_indexes) == 0 || + cancel || ctx->uidlist->corrupted) ret = 0; temp_path = t_strconcat(ctx->uidlist->path, ".tmp", NULL); - squat_uidlist_close(ctx->uidlist); - if (ret > 0) { - uidlist->build_hdr.count = ctx->new_count; - uidlist->build_hdr.link_count = 0; - uidlist_write_block_list_and_header(uidlist, ctx->output, + ctx->build_ctx->build_hdr.indexid = + ctx->uidlist->trie->hdr.indexid; + ctx->build_ctx->build_hdr.count = ctx->new_count; + ctx->build_ctx->build_hdr.link_count = 0; + uidlist_write_block_list_and_header(ctx->build_ctx, ctx->output, &ctx->new_block_offsets, &ctx->new_block_end_indexes, FALSE); - if (ctx->output->last_failed_errno != 0) { + o_stream_seek(ctx->output, 0); + o_stream_send(ctx->output, &ctx->build_ctx->build_hdr, + sizeof(ctx->build_ctx->build_hdr)); + o_stream_seek(ctx->output, + ctx->build_ctx->build_hdr.used_file_size); + o_stream_flush(ctx->output); + + if (ctx->uidlist->corrupted) + ret = -1; + else if (ctx->output->last_failed_errno != 0) { errno = ctx->output->last_failed_errno; i_error("write() to %s failed: %m", temp_path); ret = -1; - } else if (rename(temp_path, uidlist->path) < 0) { + } else if (rename(temp_path, ctx->uidlist->path) < 0) { i_error("rename(%s, %s) failed: %m", - temp_path, uidlist->path); + temp_path, ctx->uidlist->path); ret = -1; } + ctx->build_ctx->need_reopen = TRUE; } + o_stream_unref(&ctx->output); + if (close(ctx->fd) < 0) + i_error("close(%s) failed: %m", temp_path); + if (ret <= 0) { - o_stream_unref(&ctx->output); - if (close(ctx->fd) < 0) - i_error("close(%s) failed: %m", temp_path); if (unlink(temp_path) < 0) i_error("unlink(%s) failed: %m", temp_path); - } else { - array_clear(&uidlist->block_offsets); - array_clear(&uidlist->block_end_indexes); - uidlist->fd = ctx->fd; - uidlist->output = ctx->output; - uidlist->list_start_idx = ctx->new_count; - - i_assert(array_count(&uidlist->lists) == 0); - i_assert(uidlist->mmap_size == 0); - - (void)squat_uidlist_map(uidlist, (uoff_t)-1); } array_free(&ctx->new_block_offsets); array_free(&ctx->new_block_end_indexes); @@ -787,41 +880,15 @@ return ret < 0 ? -1 : 0; } -static int uidlist_rebuild(struct squat_uidlist *uidlist) +static void +uidlist_flush(struct squat_uidlist_build_context *ctx, + struct uidlist_list *list, uint32_t uid) { - struct squat_uidlist_rebuild_context *ctx; - unsigned int i; - ARRAY_TYPE(uint32_t) uids; - int ret = 0; - - if (uidlist->hdr.link_count == 0) - return 0; - - if (squat_uidlist_rebuild_init(uidlist, TRUE, &ctx) < 0) - return -1; + uint32_t size, offset = ctx->output->offset; - i_array_init(&uids, 1024); - for (i = 0; i < uidlist->hdr.count; i++) { - array_clear(&uids); - if (squat_uidlist_get(uidlist, (i + 0x100) << 1, &uids) < 0) { - ret = -1; - break; - } - squat_uidlist_rebuild_next(ctx, &uids); - } - array_free(&uids); - - return squat_uidlist_rebuild_finish(ctx, ret < 0); -} - -static void -uidlist_flush(struct squat_uidlist *uidlist, struct uidlist_list *list, - uint32_t uid) -{ - uint32_t offset = uidlist->output->offset; - - uidlist->build_hdr.link_count++; - uidlist_write(uidlist->output, list, TRUE); + ctx->build_hdr.link_count++; + if (uidlist_write(ctx->output, list, TRUE, &size) < 0) + squat_uidlist_set_corrupted(ctx->uidlist, "Broken uidlists"); list->uid_count = 2; list->uid_begins_with_pointer = TRUE; @@ -831,22 +898,22 @@ } static struct uidlist_list * -uidlist_add_new(struct squat_uidlist *uidlist, unsigned int count, +uidlist_add_new(struct squat_uidlist_build_context *ctx, unsigned int count, uint32_t *uid_list_idx_r) { struct uidlist_list *list; - i_assert(array_count(&uidlist->lists) + - uidlist->list_start_idx == uidlist->build_hdr.count); - *uid_list_idx_r = (uidlist->build_hdr.count + 0x100) << 1; - list = array_append_space(&uidlist->lists); - uidlist->build_hdr.count++; + i_assert(array_count(&ctx->lists) + + ctx->list_start_idx == ctx->build_hdr.count); + *uid_list_idx_r = (ctx->build_hdr.count + 0x100) << 1; + list = array_append_space(&ctx->lists); + ctx->build_hdr.count++; list->uid_count = count; return list; } -uint32_t squat_uidlist_build_add_uid(struct squat_uidlist *uidlist, +uint32_t squat_uidlist_build_add_uid(struct squat_uidlist_build_context *ctx, uint32_t uid_list_idx, uint32_t uid) { struct uidlist_list *list; @@ -858,7 +925,7 @@ uint32_t prev_uid = uid_list_idx >> 1; i_assert(prev_uid != uid); - list = uidlist_add_new(uidlist, 2, &uid_list_idx); + list = uidlist_add_new(ctx, 2, &uid_list_idx); list->uid_list[0] = prev_uid; if (prev_uid + 1 == uid) list->uid_list[0] |= UID_LIST_MASK_RANGE; @@ -882,7 +949,7 @@ /* create a new list */ old_list_idx = uid_list_idx >> 1; - list = uidlist_add_new(uidlist, 1, &uid_list_idx); + list = uidlist_add_new(ctx, 1, &uid_list_idx); /* add the first UID ourself */ idx = 0; i_assert((old_list_idx & 0xff) != 0); @@ -895,7 +962,7 @@ } for (; mask <= 128; mask <<= 1, idx++) { if ((old_list_idx & mask) != 0) { - squat_uidlist_build_add_uid(uidlist, + squat_uidlist_build_add_uid(ctx, uid_list_idx, idx); } } @@ -903,25 +970,26 @@ /* add to existing list */ idx = (uid_list_idx >> 1) - 0x100; - if (idx < uidlist->list_start_idx) { - list = uidlist_add_new(uidlist, 2, &uid_list_idx); + if (idx < ctx->list_start_idx) { + list = uidlist_add_new(ctx, 2, &uid_list_idx); list->uid_list[0] = UID_LIST_POINTER_MASK_LIST_IDX | idx; list->uid_list[1] = uid; list->uid_begins_with_pointer = TRUE; - uidlist->build_hdr.link_count++; + ctx->build_hdr.link_count++; return uid_list_idx; } - idx -= uidlist->list_start_idx; - if (idx >= array_count(&uidlist->lists)) { - squat_uidlist_set_corrupted(uidlist, "missing/broken uidlist"); + idx -= ctx->list_start_idx; + if (idx >= array_count(&ctx->lists)) { + squat_uidlist_set_corrupted(ctx->uidlist, + "missing/broken uidlist"); return 0; } - list = array_idx_modifiable(&uidlist->lists, idx); + list = array_idx_modifiable(&ctx->lists, idx); i_assert(list->uid_count > 0); p = &list->uid_list[list->uid_count-1]; - i_assert(uid != *p || uidlist->corrupted || + i_assert(uid != *p || ctx->uidlist->corrupted || (list->uid_count == 1 && list->uid_begins_with_pointer)); if (uid == *p + 1 && (list->uid_count > 1 || !list->uid_begins_with_pointer)) { @@ -934,7 +1002,7 @@ } if (list->uid_count == UIDLIST_LIST_SIZE) { - uidlist_flush(uidlist, list, uid); + uidlist_flush(ctx, list, uid); return uid_list_idx; } /* create a new range */ @@ -942,7 +1010,7 @@ } if (list->uid_count == UIDLIST_LIST_SIZE) { - uidlist_flush(uidlist, list, uid); + uidlist_flush(ctx, list, uid); return uid_list_idx; } @@ -1011,8 +1079,6 @@ uint32_t size, base_uid; unsigned int i, j, extra = 0; - if (squat_uidlist_map(uidlist, offset) < 0) - return -1; p = CONST_PTR_OFFSET(uidlist->mmap_base, offset); end = CONST_PTR_OFFSET(uidlist->mmap_base, uidlist->mmap_size); @@ -1093,6 +1159,11 @@ unsigned int idx; uint32_t num, skip_bytes, uidlists_offset; + if (uidlist->fd == -1) { + squat_uidlist_set_corrupted(uidlist, "no uidlists"); + return -1; + } + if (bsearch_insert_pos(&uid_list_idx, uidlist->cur_block_end_indexes, uidlist->cur_block_count, sizeof(uint32_t), uint32_cmp, &idx)) @@ -1188,10 +1259,8 @@ unsigned int i, count; t_array_init(&tmp_uid_arr, 128); - if (squat_uidlist_get(uidlist, uid_list_idx, &tmp_uid_arr) < 0) { - t_pop(); + if (squat_uidlist_get(uidlist, uid_list_idx, &tmp_uid_arr) < 0) return -1; - } tmp_uids = array_get(&tmp_uid_arr, &count); for (i = 0; i < count; i++) {
--- a/src/plugins/fts-squat/squat-uidlist.h Tue Dec 04 15:20:01 2007 +0200 +++ b/src/plugins/fts-squat/squat-uidlist.h Tue Dec 04 19:59:57 2007 +0200 @@ -2,6 +2,7 @@ #define SQUAT_UIDLIST_H struct squat_trie; +struct squat_uidlist_build_context; struct squat_uidlist_rebuild_context; struct squat_uidlist_file_header { @@ -33,15 +34,18 @@ struct squat_uidlist *squat_uidlist_init(struct squat_trie *trie); void squat_uidlist_deinit(struct squat_uidlist *uidlist); -int squat_uidlist_open(struct squat_uidlist *uidlist); -void squat_uidlist_close(struct squat_uidlist *uidlist); +int squat_uidlist_refresh(struct squat_uidlist *uidlist); -int squat_uidlist_build_init(struct squat_uidlist *uidlist); -uint32_t squat_uidlist_build_add_uid(struct squat_uidlist *uidlist, +int squat_uidlist_build_init(struct squat_uidlist *uidlist, + struct squat_uidlist_build_context **ctx_r); +uint32_t squat_uidlist_build_add_uid(struct squat_uidlist_build_context *ctx, uint32_t uid_list_idx, uint32_t uid); -int squat_uidlist_build_deinit(struct squat_uidlist *uidlist); +void squat_uidlist_build_flush(struct squat_uidlist_build_context *ctx); +int squat_uidlist_build_finish(struct squat_uidlist_build_context *ctx); +void squat_uidlist_build_deinit(struct squat_uidlist_build_context **ctx); -int squat_uidlist_rebuild_init(struct squat_uidlist *uidlist, bool finish, +int squat_uidlist_rebuild_init(struct squat_uidlist_build_context *build_ctx, + bool compress, struct squat_uidlist_rebuild_context **ctx_r); void squat_uidlist_rebuild_next(struct squat_uidlist_rebuild_context *ctx, const ARRAY_TYPE(uint32_t) *uids);