Mercurial > dovecot > original-hg > dovecot-1.2
changeset 4883:d8adbe93c969 HEAD
Added support for mmap_disable=yes and several other fixes.
author | Timo Sirainen <tss@iki.fi> |
---|---|
date | Sat, 09 Dec 2006 23:08:56 +0200 |
parents | 969cf7a88c25 |
children | 8a09d03e6628 |
files | src/plugins/fts-squat/fts-backend-squat.c src/plugins/fts-squat/squat-trie.c src/plugins/fts-squat/squat-uidlist.c src/plugins/fts-squat/squat-uidlist.h |
diffstat | 4 files changed, 460 insertions(+), 157 deletions(-) [+] |
line wrap: on
line diff
--- a/src/plugins/fts-squat/fts-backend-squat.c Sat Dec 09 23:08:27 2006 +0200 +++ b/src/plugins/fts-squat/fts-backend-squat.c Sat Dec 09 23:08:56 2006 +0200 @@ -38,7 +38,9 @@ if (mailbox_get_status(box, STATUS_UIDVALIDITY, &status) < 0) return NULL; - mmap_disable = (storage->flags & MAIL_STORAGE_FLAG_MMAP_DISABLE) != 0; + mmap_disable = (storage->flags & + (MAIL_STORAGE_FLAG_MMAP_DISABLE | + MAIL_STORAGE_FLAG_MMAP_NO_WRITE)) != 0; backend = i_new(struct squat_fts_backend, 1); backend->backend = fts_backend_squat;
--- a/src/plugins/fts-squat/squat-trie.c Sat Dec 09 23:08:27 2006 +0200 +++ b/src/plugins/fts-squat/squat-trie.c Sat Dec 09 23:08:56 2006 +0200 @@ -3,9 +3,11 @@ #include "lib.h" #include "array.h" #include "bsearch-insert-pos.h" +#include "file-cache.h" #include "file-lock.h" #include "istream.h" #include "ostream.h" +#include "read-full.h" #include "write-full.h" #include "mmap-util.h" #include "squat-uidlist.h" @@ -43,9 +45,12 @@ struct file_lock *file_lock; int lock_count; int lock_type; /* F_RDLCK / F_WRLCK */ - bool mmap_disable; - void *mmap_base; + struct file_cache *file_cache; + uint32_t file_cache_modify_counter; + + void *mmap_base; /* NULL with mmap_disable=yes */ + const uint8_t *const_mmap_base; size_t mmap_size; const struct squat_trie_header *hdr; @@ -57,6 +62,7 @@ buffer_t *buf; unsigned int corrupted:1; + unsigned int mmap_disable:1; }; struct squat_trie_build_context { @@ -71,6 +77,7 @@ unsigned int node_count; unsigned int deleted_space; + unsigned int modified:1; unsigned int failed:1; unsigned int locked:1; }; @@ -94,6 +101,7 @@ uint32_t used_file_size; uint32_t deleted_space; uint32_t node_count; + uint32_t modify_counter; uint32_t root_offset; }; @@ -273,6 +281,24 @@ #endif } +static int trie_map_area(struct squat_trie *trie, uoff_t offset, size_t len) +{ + ssize_t ret; + + if (trie->file_cache == NULL) + return 0; + + ret = file_cache_read(trie->file_cache, offset, len); + if (ret < 0) { + squat_trie_set_syscall_error(trie, "file_cache_read()"); + return -1; + } + trie->const_mmap_base = + file_cache_get_map(trie->file_cache, &trie->mmap_size); + trie->hdr = (const void *)trie->const_mmap_base; + return 0; +} + static int trie_map_node(struct squat_trie *trie, uint32_t offset, unsigned int level, struct trie_node **node_r) @@ -280,52 +306,74 @@ struct trie_node *node; const uint8_t *p, *end, *chars8_src, *chars16_src; uint32_t num, chars8_count, chars16_count; - unsigned int chars8_size, chars16_size; + unsigned int chars8_offset, chars8_size, chars8_memsize; + unsigned int chars16_offset, chars16_size, chars16_memsize; i_assert(trie->fd != -1); + if (trie_map_area(trie, offset, 2+256) < 0) + return -1; + if (offset >= trie->mmap_size) { squat_trie_set_corrupted(trie, "trie offset too large"); return -1; } - p = CONST_PTR_OFFSET(trie->mmap_base, offset); - end = CONST_PTR_OFFSET(trie->mmap_base, trie->mmap_size); + p = trie->const_mmap_base + offset; + end = trie->const_mmap_base + trie->mmap_size; /* get 8bit char count and check that it's valid */ num = _squat_trie_unpack_num(&p, end); chars8_count = num >> 1; - if (chars8_count > 256 || p + chars8_count >= end) { + chars8_offset = p - trie->const_mmap_base; + chars8_size = chars8_count * (sizeof(uint8_t) + sizeof(uint32_t)); + + if (chars8_count > 256 || + chars8_offset + chars8_size > trie->mmap_size) { squat_trie_set_corrupted(trie, "trie offset broken"); return -1; } - chars8_src = p; - chars8_size = ALIGN(chars8_count * sizeof(uint8_t)) + + chars8_memsize = ALIGN(chars8_count * sizeof(uint8_t)) + chars8_count * sizeof(struct trie_node *); + + if (trie_map_area(trie, chars8_offset, chars8_size + 8) < 0) + return -1; + if ((num & 1) == 0) { /* no 16bit chars */ chars16_count = 0; - chars16_size = 0; - chars16_src = NULL; + chars16_memsize = 0; + chars16_offset = 0; } else { - /* get the 16bit char count and check that it's valid */ - p = CONST_PTR_OFFSET(p, chars8_count * - (sizeof(uint8_t) + sizeof(uint32_t))); + /* get the 16bit char count */ + p = trie->const_mmap_base + chars8_offset + chars8_size; + end = trie->const_mmap_base + trie->mmap_size; + chars16_count = _squat_trie_unpack_num(&p, end); - if (chars16_count > 65536 || - p + chars16_count*sizeof(uint16_t) >= end) { + if (chars16_count > 65536) { + squat_trie_set_corrupted(trie, "trie offset broken"); + return -1; + } + chars16_offset = p - trie->const_mmap_base; + + /* map the required area size and make sure it exists */ + chars16_size = chars16_count * + (sizeof(uint16_t) + sizeof(uint32_t)); + if (trie_map_area(trie, chars16_offset, chars16_size) < 0) + return -1; + + if (chars16_offset + chars16_size > trie->mmap_size) { squat_trie_set_corrupted(trie, "trie offset broken"); return -1; } - chars16_src = p; - chars16_size = ALIGN(chars16_count * sizeof(uint16_t)) + + chars16_memsize = ALIGN(chars16_count * sizeof(uint16_t)) + chars16_count * sizeof(struct trie_node *); } - node = i_malloc(sizeof(*node) + chars8_size + chars16_size); + node = i_malloc(sizeof(*node) + chars8_memsize + chars16_memsize); node->chars_8bit_count = chars8_count; node->chars_16bit_count = chars16_count; node->file_offset = offset; @@ -338,6 +386,9 @@ const uint32_t *src_idx; const void *end_offset; + chars8_src = trie->const_mmap_base + chars8_offset; + chars16_src = trie->const_mmap_base + chars16_offset; + memcpy(chars8, chars8_src, sizeof(uint8_t) * chars8_count); memcpy(chars16, chars16_src, sizeof(uint16_t) * chars16_count); @@ -355,8 +406,8 @@ end_offset = &src_idx[chars16_count]; } - node->orig_size = ((const char *)end_offset - - (const char *)trie->mmap_base) - offset; + node->orig_size = ((const uint8_t *)end_offset - + trie->const_mmap_base) - offset; } *node_r = node; @@ -365,6 +416,9 @@ static void squat_trie_unmap(struct squat_trie *trie) { + if (trie->file_cache != NULL) + file_cache_invalidate(trie->file_cache, 0, (uoff_t)-1); + if (trie->mmap_base != NULL) { if (munmap(trie->mmap_base, trie->mmap_size) < 0) squat_trie_set_syscall_error(trie, "munmap()"); @@ -373,10 +427,13 @@ trie->mmap_size = 0; trie->hdr = NULL; + trie->const_mmap_base = NULL; } static void trie_file_close(struct squat_trie *trie) { + if (trie->file_cache != NULL) + file_cache_free(&trie->file_cache); if (trie->file_lock != NULL) file_lock_free(&trie->file_lock); @@ -391,18 +448,19 @@ trie->corrupted = FALSE; } -static int trie_map_check_header(struct squat_trie *trie, - const struct squat_trie_header *hdr) +static int +trie_map_check_header(struct squat_trie *trie, + const struct squat_trie_header *hdr, uoff_t file_size) { if (hdr->version != SQUAT_TRIE_VERSION) return -1; - if (hdr->used_file_size > trie->mmap_size) { + if (hdr->used_file_size > file_size) { squat_trie_set_corrupted(trie, "used_file_size too large"); return -1; } if (hdr->root_offset != 0 && - (hdr->root_offset > trie->mmap_size || + (hdr->root_offset > file_size || hdr->root_offset < sizeof(*hdr))) { squat_trie_set_corrupted(trie, "invalid root_offset"); return -1; @@ -415,14 +473,42 @@ return 0; } +static int squat_trie_file_was_modified(struct squat_trie *trie) +{ + struct squat_trie_header hdr; + int ret; + + ret = pread_full(trie->fd, &hdr.modify_counter, + sizeof(hdr.modify_counter), + offsetof(struct squat_trie_header, modify_counter)); + if (ret < 0) { + squat_trie_set_syscall_error(trie, "pread_full()"); + return -1; + } + if (ret == 0) { + /* broken file, treat as modified */ + return 1; + } + return hdr.modify_counter == trie->file_cache_modify_counter ? 0 : 1; +} + static int squat_trie_map(struct squat_trie *trie) { + const struct squat_trie_header *hdr; struct stat st; + ssize_t ret; - if (trie->hdr != NULL && - trie->hdr->used_file_size <= trie->mmap_size) { - /* everything is already mapped */ - return 1; + if (trie->hdr != NULL) { + if (!trie->mmap_disable) { + if (trie->hdr->used_file_size <= trie->mmap_size) { + /* everything is already mapped */ + return 1; + } + } else { + ret = squat_trie_file_was_modified(trie); + if (ret <= 0) + return ret < 0 ? -1 : 1; + } } if (fstat(trie->fd, &st) < 0) { @@ -434,58 +520,71 @@ squat_trie_unmap(trie); - trie->mmap_size = st.st_size; - trie->mmap_base = mmap(NULL, trie->mmap_size, PROT_READ | PROT_WRITE, - MAP_SHARED, trie->fd, 0); - if (trie->mmap_base == MAP_FAILED) { - trie->mmap_size = 0; - trie->mmap_base = NULL; - squat_trie_set_syscall_error(trie, "mmap()"); - return -1; + if (!trie->mmap_disable) { + trie->mmap_size = st.st_size; + trie->mmap_base = mmap(NULL, trie->mmap_size, + PROT_READ | PROT_WRITE, + MAP_SHARED, trie->fd, 0); + if (trie->mmap_base == MAP_FAILED) { + trie->mmap_size = 0; + trie->mmap_base = NULL; + squat_trie_set_syscall_error(trie, "mmap()"); + return -1; + } + trie->const_mmap_base = trie->mmap_base; + } else { + ret = file_cache_read(trie->file_cache, 0, sizeof(*trie->hdr)); + if (ret < 0) { + squat_trie_set_syscall_error(trie, "file_cache_read()"); + return -1; + } + if ((size_t)ret < sizeof(*trie->hdr)) { + squat_trie_set_corrupted(trie, "file too small"); + return -1; + } + trie->const_mmap_base = + file_cache_get_map(trie->file_cache, &trie->mmap_size); } - if (trie_map_check_header(trie, trie->mmap_base) < 0) + hdr = (const void *)trie->const_mmap_base; + if (trie_map_check_header(trie, hdr, st.st_size) < 0) return -1; - trie->hdr = trie->mmap_base; + trie->hdr = hdr; + trie->file_cache_modify_counter = trie->hdr->modify_counter; - if (trie_map_node(trie, trie->hdr->root_offset, 1, &trie->root) < 0) { - trie_file_close(trie); + if (trie_map_node(trie, trie->hdr->root_offset, 1, &trie->root) < 0) return 0; - } return 1; } -static int trie_file_open(struct squat_trie *trie) +static int trie_file_open(struct squat_trie *trie, bool create) { - trie->fd = open(trie->filepath, O_RDWR); - if (trie->fd == -1) { + struct stat st; + int fd; + + i_assert(trie->fd == -1); + + fd = open(trie->filepath, O_RDWR | (create ? O_CREAT : 0), 0660); + if (fd == -1) { if (errno == ENOENT) return 0; squat_trie_set_syscall_error(trie, "open()"); return -1; } - - return 1; -} - -static int trie_file_create(struct squat_trie *trie) -{ - struct stat st; - - trie->fd = open(trie->filepath, O_RDWR | O_CREAT, 0660); - if (trie->fd == -1) { - squat_trie_set_syscall_error(trie, "open()"); + if (fstat(fd, &st) < 0) { + squat_trie_set_syscall_error(trie, "fstat()"); + (void)close(fd); return -1; } - if (fstat(trie->fd, &st) < 0) { - squat_trie_set_syscall_error(trie, "fstat()"); - return -1; - } + trie->fd = fd; trie->dev = st.st_dev; trie->ino = st.st_ino; - return 0; + + if (trie->mmap_disable) + trie->file_cache = file_cache_new(trie->fd); + return 1; } static int trie_file_create_finish(struct squat_trie *trie) @@ -513,15 +612,6 @@ return 0; } -static int squat_trie_reopen(struct squat_trie *trie) -{ - trie_file_close(trie); - if (trie_file_open(trie) < 0) - return -1; - - return 0; -} - struct squat_trie * squat_trie_open(const char *path, uint32_t uidvalidity, enum file_lock_method lock_method, bool mmap_disable) @@ -538,7 +628,8 @@ trie->uidlist_filepath = i_strconcat(path, ".uids", NULL); trie->uidlist = - squat_uidlist_init(trie, trie->uidlist_filepath, uidvalidity); + squat_uidlist_init(trie, trie->uidlist_filepath, + uidvalidity, mmap_disable); return trie; } @@ -553,7 +644,23 @@ int squat_trie_get_last_uid(struct squat_trie *trie, uint32_t *uid_r) { - return squat_uidlist_get_last_uid(trie->uidlist, uid_r); + int ret; + + if (trie->fd == -1) { + if ((ret = trie_file_open(trie, FALSE)) < 0) + return ret; + if (ret == 0) { + *uid_r = 0; + return 0; + } + } + + if (squat_trie_lock(trie, F_RDLCK) <= 0) + return -1; + + ret = squat_uidlist_get_last_uid(trie->uidlist, uid_r); + squat_trie_unlock(trie); + return ret; } static int squat_trie_is_file_stale(struct squat_trie *trie) @@ -561,6 +668,9 @@ struct stat st; if (stat(trie->filepath, &st) < 0) { + if (errno == ENOENT) + return 1; + squat_trie_set_syscall_error(trie, "stat()"); return -1; } @@ -587,15 +697,15 @@ if (trie->fd == -1 || trie->corrupted) { trie_file_close(trie); if (lock_type == F_WRLCK) { - if ((ret = trie_file_open(trie)) < 0) + if ((ret = trie_file_open(trie, FALSE)) < 0) return -1; if (ret == 0) { - if (trie_file_create(trie) < 0) + if (trie_file_open(trie, TRUE) < 0) return -1; created = TRUE; } } else { - if (trie_file_open(trie) <= 0) + if (trie_file_open(trie, FALSE) <= 0) return -1; } } @@ -605,8 +715,13 @@ ret = file_wait_lock(trie->fd, trie->filepath, lock_type, trie->lock_method, SQUAT_TRIE_LOCK_TIMEOUT, &trie->file_lock); - if (ret <= 0) + if (ret <= 0) { + if (ret == 0) { + squat_trie_set_syscall_error(trie, + "file_wait_lock()"); + } return ret; + } /* if the trie has been compressed, we need to reopen the file and try to lock again */ @@ -618,7 +733,8 @@ if (ret < 0) return -1; - if (squat_trie_reopen(trie) < 0) + trie_file_close(trie); + if (trie_file_open(trie, FALSE) <= 0) return -1; } @@ -636,6 +752,10 @@ file_unlock(&trie->file_lock); return -1; } + if (squat_uidlist_refresh(trie->uidlist) < 0) { + file_unlock(&trie->file_lock); + return -1; + } trie->lock_count++; trie->lock_type = lock_type; @@ -681,6 +801,7 @@ *chrp = chr; } + node->modified = TRUE; node->resized = TRUE; return node; } @@ -1141,12 +1262,22 @@ struct trie_node **children8 = NODE_CHILDREN8(node); struct trie_node **children16 = NODE_CHILDREN16(node); - trie_write_node_children(ctx, level + 1, - children8, node->chars_8bit_count); - trie_write_node_children(ctx, level + 1, - children16, node->chars_16bit_count); + if (trie_write_node_children(ctx, level + 1, + children8, + node->chars_8bit_count) < 0) + return -1; + if (trie_write_node_children(ctx, level + 1, + children16, + node->chars_16bit_count) < 0) + return -1; + } + + if (!node->modified) + return 0; + + if (level < BLOCK_SIZE) node_pack(trie->buf, node); - } else { + else { if (node_leaf_finish(trie, node) < 0) return -1; @@ -1165,7 +1296,7 @@ o_stream_send(ctx->output, trie->buf->data, trie->buf->used); ctx->deleted_space += node->orig_size; - } else if (node->modified) { + } else { /* overwrite node's contents */ i_assert(node->file_offset != 0); i_assert(trie->buf->used <= node->orig_size); @@ -1177,6 +1308,8 @@ ctx->deleted_space += trie->buf->used - node->orig_size; } + + ctx->modified = TRUE; return 0; } @@ -1193,20 +1326,25 @@ } ctx->output = o_stream_create_file(trie->fd, default_pool, 0, FALSE); - if (hdr.used_file_size == 0) + if (hdr.used_file_size == 0) { o_stream_send(ctx->output, &hdr, sizeof(hdr)); + ctx->modified = TRUE; + } ctx->deleted_space = 0; if (trie_write_node(ctx, 1, trie->root) < 0) return -1; - /* update the header */ - hdr.root_offset = trie->root->file_offset; - hdr.used_file_size = ctx->output->offset; - hdr.deleted_space += ctx->deleted_space; - hdr.node_count = ctx->node_count; - o_stream_seek(ctx->output, 0); - o_stream_send(ctx->output, &hdr, sizeof(hdr)); + if (ctx->modified) { + /* update the header */ + hdr.root_offset = trie->root->file_offset; + hdr.used_file_size = ctx->output->offset; + hdr.deleted_space += ctx->deleted_space; + hdr.node_count = ctx->node_count; + hdr.modify_counter++; + o_stream_seek(ctx->output, 0); + o_stream_send(ctx->output, &hdr, sizeof(hdr)); + } o_stream_destroy(&ctx->output); *uidvalidity_r = hdr.uidvalidity; @@ -1250,6 +1388,11 @@ return -1; if (squat_trie_need_compress(trie, (unsigned int)-1)) { + if (ctx->locked) { + squat_trie_unlock(ctx->trie); + ctx->locked = FALSE; + } + if (squat_trie_compress(trie, NULL) < 0) return -1; } @@ -1433,6 +1576,8 @@ struct squat_trie_header hdr; int fd; + memset(ctx, 0, sizeof(*ctx)); + ctx->tmp_path = t_strconcat(trie->filepath, ".tmp", NULL); fd = open(ctx->tmp_path, O_RDWR | O_CREAT | O_TRUNC, 0600); if (fd == -1) { @@ -1440,7 +1585,6 @@ return -1; } - memset(ctx, 0, sizeof(*ctx)); ctx->trie = trie; ctx->output = o_stream_create_file(fd, default_pool, 0, TRUE); ctx->node_count = trie->hdr->node_count; @@ -1475,6 +1619,9 @@ struct trie_node *node; int ret; + /* reopening the file loses locks, so we can't be locked initially */ + i_assert(trie->lock_count == 0); + if (squat_trie_lock(trie, F_WRLCK) <= 0) return -1; @@ -1511,7 +1658,8 @@ if (ret < 0) (void)unlink(ctx.tmp_path); else { - if (squat_trie_reopen(trie) < 0) + trie_file_close(trie); + if (trie_file_open(trie, FALSE) < 0) return -1; } return ret; @@ -1524,7 +1672,11 @@ bool compress; int ret; + if ((ret = squat_trie_lock(trie, F_RDLCK)) <= 0) + return ret; compress = squat_trie_need_compress(trie, current_message_count); + squat_trie_unlock(trie); + ret = squat_uidlist_mark_having_expunges(trie->uidlist, compress); if (compress)
--- a/src/plugins/fts-squat/squat-uidlist.c Sat Dec 09 23:08:27 2006 +0200 +++ b/src/plugins/fts-squat/squat-uidlist.c Sat Dec 09 23:08:56 2006 +0200 @@ -3,7 +3,9 @@ #include "lib.h" #include "array.h" #include "ostream.h" +#include "file-cache.h" #include "mmap-util.h" +#include "read-full.h" #include "write-full.h" #include "squat-trie.h" #include "squat-uidlist.h" @@ -53,8 +55,14 @@ int fd; struct ostream *output; + dev_t dev; + ino_t ino; + void *mmap_base; + const uint8_t *const_mmap_base; size_t mmap_size; + + struct file_cache *file_cache; struct squat_uidlist_header hdr; ARRAY_DEFINE(lists, struct uid_node); @@ -65,6 +73,7 @@ unsigned int check_expunges:1; unsigned int write_failed:1; + unsigned int mmap_disable:1; }; struct squat_uidlist_compress_ctx { @@ -84,6 +93,8 @@ unsigned int failed:1; }; +static void squat_uidlist_close(struct squat_uidlist *uidlist); + static void squat_uidlist_set_syscall_error(struct squat_uidlist *uidlist, const char *function) @@ -93,14 +104,15 @@ } static int squat_uidlist_check_header(struct squat_uidlist *uidlist, - const struct squat_uidlist_header *hdr) + const struct squat_uidlist_header *hdr, + uoff_t file_size) { if (hdr->uidvalidity != uidlist->uidvalidity) { squat_trie_set_corrupted(uidlist->trie, "uidlist: uidvalidity changed"); return -1; } - if (hdr->used_file_size > uidlist->mmap_size) { + if (hdr->used_file_size > file_size) { squat_trie_set_corrupted(uidlist->trie, "uidlist: used_file_size too large"); return -1; @@ -109,68 +121,135 @@ return 0; } +static int squat_uidlist_read_header(struct squat_uidlist *uidlist) +{ + int ret; + + ret = pread_full(uidlist->fd, &uidlist->hdr, sizeof(uidlist->hdr), 0); + if (ret < 0) + squat_uidlist_set_syscall_error(uidlist, "pread_full()"); + return ret; +} + static int squat_uidlist_map(struct squat_uidlist *uidlist) { struct stat st; + int ret; + + if (!uidlist->mmap_disable) { + const struct squat_uidlist_header *hdr = uidlist->mmap_base; + + if (hdr != NULL && hdr->used_file_size <= uidlist->mmap_size) { + /* everything is already mapped */ + uidlist->hdr = *hdr; + return 1; + } + } else { + if ((ret = squat_uidlist_read_header(uidlist)) < 0) + return -1; + if (ret == 0) + return 0; + } if (fstat(uidlist->fd, &st) < 0) { squat_uidlist_set_syscall_error(uidlist, "fstat()"); return -1; } + uidlist->dev = st.st_dev; + uidlist->ino = st.st_ino; - if (st.st_size <= sizeof(uidlist->hdr)) { - memset(&uidlist->hdr, 0, sizeof(uidlist->hdr)); - uidlist->hdr.used_file_size = sizeof(uidlist->hdr); + if (st.st_size <= sizeof(uidlist->hdr)) return 0; - } if (uidlist->mmap_base != NULL) { if (munmap(uidlist->mmap_base, uidlist->mmap_size) < 0) squat_uidlist_set_syscall_error(uidlist, "munmap()"); } - uidlist->mmap_size = st.st_size; + uidlist->const_mmap_base = NULL; - uidlist->mmap_base = - mmap(NULL, uidlist->mmap_size, PROT_READ | PROT_WRITE, - MAP_SHARED, uidlist->fd, 0); - if (uidlist->mmap_base == MAP_FAILED) { - uidlist->mmap_size = 0; - uidlist->mmap_base = NULL; - squat_uidlist_set_syscall_error(uidlist, "mmap()"); - return -1; + if (!uidlist->mmap_disable) { + uidlist->mmap_size = st.st_size; + uidlist->mmap_base = + mmap(NULL, uidlist->mmap_size, PROT_READ | PROT_WRITE, + MAP_SHARED, uidlist->fd, 0); + if (uidlist->mmap_base == MAP_FAILED) { + uidlist->mmap_size = 0; + uidlist->mmap_base = NULL; + squat_uidlist_set_syscall_error(uidlist, "mmap()"); + return -1; + } + uidlist->const_mmap_base = uidlist->mmap_base; + memcpy(&uidlist->hdr, uidlist->mmap_base, sizeof(uidlist->hdr)); + } else { + /* the header is always read separately. everything between it + and the used_file_size doesn't change */ + file_cache_invalidate(uidlist->file_cache, + uidlist->hdr.used_file_size, (uoff_t)-1); } - memcpy(&uidlist->hdr, uidlist->mmap_base, sizeof(uidlist->hdr)); - if (squat_uidlist_check_header(uidlist, &uidlist->hdr) < 0) - return -1; + if (squat_uidlist_check_header(uidlist, &uidlist->hdr, st.st_size) < 0) + return 0; if (uidlist->hdr.uids_expunged) uidlist->check_expunges = TRUE; - uidlist->first_new_list_idx = uidlist->mmap_size; + uidlist->first_new_list_idx = uidlist->hdr.used_file_size; return 1; } static int squat_uidlist_open(struct squat_uidlist *uidlist) { + int ret; + i_assert(uidlist->fd == -1); - uidlist->fd = open(uidlist->filepath, O_RDWR | O_CREAT, 0600); + uidlist->fd = open(uidlist->filepath, O_RDWR, 0600); + if (uidlist->fd == -1) { + if (errno == ENOENT) + return 0; + + squat_uidlist_set_syscall_error(uidlist, "open()"); + return -1; + } + + if (uidlist->mmap_disable) + uidlist->file_cache = file_cache_new(uidlist->fd); + + if ((ret = squat_uidlist_map(uidlist)) == 0) { + /* broken */ + if (unlink(uidlist->filepath) < 0) + squat_uidlist_set_syscall_error(uidlist, "unlink()"); + squat_uidlist_close(uidlist); + } + return ret; +} + +static int squat_uidlist_create(struct squat_uidlist *uidlist) +{ + i_assert(uidlist->fd == -1); + + /* we should get here only if normal file opening failed */ + uidlist->fd = open(uidlist->filepath, O_RDWR | O_CREAT | O_TRUNC, 0600); if (uidlist->fd == -1) { squat_uidlist_set_syscall_error(uidlist, "open()"); return -1; } - return squat_uidlist_map(uidlist); + if (uidlist->mmap_disable) + uidlist->file_cache = file_cache_new(uidlist->fd); + return 0; } static void squat_uidlist_close(struct squat_uidlist *uidlist) { + if (uidlist->file_cache != NULL) + file_cache_free(&uidlist->file_cache); if (uidlist->mmap_base != NULL) { if (munmap(uidlist->mmap_base, uidlist->mmap_size) < 0) squat_uidlist_set_syscall_error(uidlist, "munmap()"); uidlist->mmap_base = NULL; } + uidlist->const_mmap_base = NULL; uidlist->mmap_size = 0; if (uidlist->fd != -1) { @@ -182,7 +261,7 @@ struct squat_uidlist * squat_uidlist_init(struct squat_trie *trie, const char *path, - uint32_t uidvalidity) + uint32_t uidvalidity, bool mmap_disable) { struct squat_uidlist *uidlist; @@ -192,12 +271,12 @@ uidlist->uidvalidity = uidvalidity; uidlist->fd = -1; uidlist->first_new_list_idx = 1; + uidlist->mmap_disable = mmap_disable; i_array_init(&uidlist->lists, 65536); uidlist->node_pool = pool_alloconly_create("squat uidlist node pool", 65536); uidlist->tmp_buf = buffer_create_dynamic(default_pool, 16); uidlist->list_buf = buffer_create_dynamic(default_pool, 256); - (void)squat_uidlist_open(uidlist); return uidlist; } @@ -212,6 +291,34 @@ i_free(uidlist); } +int squat_uidlist_refresh(struct squat_uidlist *uidlist) +{ + struct stat st; + int ret; + + if (uidlist->fd != -1) { + if (stat(uidlist->filepath, &st) < 0) { + if (errno == ENOENT) + return 0; + + squat_uidlist_set_syscall_error(uidlist, "stat()"); + return -1; + } + if (st.st_ino == uidlist->ino && + CMP_DEV_T(st.st_dev, uidlist->dev)) { + /* no need to reopen, just remap */ + if ((ret = squat_uidlist_map(uidlist)) != 0) + return ret < 0 ? -1 : 0; + /* broken file */ + } + squat_uidlist_close(uidlist); + } + + if (squat_uidlist_open(uidlist) < 0) + return -1; + return 0; +} + int squat_uidlist_get_last_uid(struct squat_uidlist *uidlist, uint32_t *uid_r) { *uid_r = uidlist->hdr.uid_max; @@ -289,20 +396,61 @@ } static int -squat_uidlist_copy_existing(struct squat_uidlist *uidlist, size_t offset, +squat_uidlist_map_area(struct squat_uidlist *uidlist, + size_t offset, size_t size) +{ + ssize_t ret; + + if (uidlist->file_cache == NULL) + return 0; + + ret = file_cache_read(uidlist->file_cache, offset, size); + if (ret < 0) { + squat_uidlist_set_syscall_error(uidlist, "file_cache_read()"); + return -1; + } + uidlist->const_mmap_base = + file_cache_get_map(uidlist->file_cache, &uidlist->mmap_size); + return 0; +} + +static int +squat_uidlist_map_list(struct squat_uidlist *uidlist, size_t offset, + const uint8_t **data_r, uint32_t *size_r) +{ + const uint8_t *data, *end; + size_t data_offset; + uint32_t size; + + if (squat_uidlist_map_area(uidlist, offset, 128) < 0) + return -1; + if (offset >= uidlist->mmap_size) + return -1; + + data = uidlist->const_mmap_base + offset; + end = uidlist->const_mmap_base + uidlist->mmap_size; + + size = _squat_trie_unpack_num(&data, end); + data_offset = data - uidlist->const_mmap_base; + + if (squat_uidlist_map_area(uidlist, data_offset, size) < 0) + return -1; + if (data_offset + size > uidlist->mmap_size) + return -1; + + *data_r = uidlist->const_mmap_base + data_offset; + *size_r = size; + return 0; +} + +static int +squat_uidlist_copy_existing(struct squat_uidlist *uidlist, size_t offset, uint32_t *prev_uid_r, uint32_t *written_uid_r) { const uint8_t *data, *data_start, *end, *p = NULL; uint32_t size, num, prev_uid, next_uid; - if (offset >= uidlist->mmap_size) - return -1; - - data = CONST_PTR_OFFSET(uidlist->mmap_base, offset); - end = CONST_PTR_OFFSET(uidlist->mmap_base, uidlist->mmap_size); - - size = _squat_trie_unpack_num(&data, end); - if (data + size > end) + if (squat_uidlist_map_list(uidlist, offset, &data, &size) < 0) return -1; data_start = data; @@ -337,7 +485,7 @@ *prev_uid_r = next_uid; uidlist->hdr.deleted_space += - (end - (const uint8_t *)uidlist->mmap_base) - offset; + (end - (const uint8_t *)uidlist->const_mmap_base) - offset; buffer_append(uidlist->list_buf, data_start, p - data_start); return 0; @@ -410,10 +558,15 @@ return 0; } -static void squat_uidlist_write_init(struct squat_uidlist *uidlist) +static int squat_uidlist_write_init(struct squat_uidlist *uidlist) { i_assert(uidlist->output == NULL); + if (uidlist->fd == -1) { + if (squat_uidlist_create(uidlist) < 0) + return -1; + } + uidlist->output = o_stream_create_file(uidlist->fd, default_pool, 0, FALSE); if (uidlist->hdr.used_file_size < sizeof(uidlist->hdr)) { @@ -425,6 +578,7 @@ o_stream_seek(uidlist->output, uidlist->hdr.used_file_size); } + return 0; } static int squat_uidlist_write_listbuf(struct squat_uidlist *uidlist, @@ -476,8 +630,12 @@ return -1; } - if (uidlist->output == NULL) - squat_uidlist_write_init(uidlist); + if (uidlist->output == NULL) { + if (squat_uidlist_write_init(uidlist) < 0) { + uidlist->write_failed = TRUE; + return -1; + } + } /* new uidlist index is the offset in uidlist file */ *_uid_list_idx = uidlist->output->offset; @@ -511,8 +669,6 @@ p_clear(uidlist->node_pool); uidlist->write_failed = FALSE; - - (void)squat_uidlist_map(uidlist); return ret; } @@ -588,6 +744,9 @@ ctx->output = o_stream_create_file(fd, default_pool, 0, TRUE); o_stream_send(ctx->output, &ctx->hdr, sizeof(ctx->hdr)); } + + if (squat_uidlist_refresh(uidlist) < 0) + ctx->failed = TRUE; return ctx; } @@ -683,8 +842,8 @@ uint32_t *uid_list_idx) { struct squat_uidlist *uidlist = ctx->uidlist; - const uint8_t *data, *p, *end; - uint32_t size; + const uint8_t *data, *data_start; + uint32_t size, old_offset; int ret; if ((*uid_list_idx & UID_LIST_IDX_FLAG_SINGLE) != 0) { @@ -700,22 +859,13 @@ if (ctx->output == NULL) return -1; - if (*uid_list_idx >= uidlist->mmap_size) { - squat_trie_set_corrupted(uidlist->trie, - "uidlist index points outside file (compressing)"); - return -1; - } - - data = p = CONST_PTR_OFFSET(uidlist->mmap_base, *uid_list_idx); - end = CONST_PTR_OFFSET(uidlist->mmap_base, uidlist->mmap_size); - - size = _squat_trie_unpack_num(&p, end); - if (data + size > end) { + if (squat_uidlist_map_list(uidlist, *uid_list_idx, &data, &size) < 0) { squat_trie_set_corrupted(uidlist->trie, "corrupted uidlist index (compressing)"); return -1; } + old_offset = *uid_list_idx; *uid_list_idx = ctx->output->offset; if (!ctx->uidlist->check_expunges) @@ -723,7 +873,7 @@ else { bool all_expunged; - ret = squat_uidlist_remove_expunged(ctx, p, size, + ret = squat_uidlist_remove_expunged(ctx, data, size, &all_expunged); if (ret < 0) { ctx->failed = TRUE; @@ -734,7 +884,11 @@ } if (ret == 0) { - if (o_stream_send(ctx->output, data, p - data + size) < 0) { + data_start = data = uidlist->const_mmap_base + old_offset; + (void)_squat_trie_unpack_num(&data, NULL); + + if (o_stream_send(ctx->output, data_start, + data - data_start + size) < 0) { ctx->failed = TRUE; return -1; } @@ -826,17 +980,8 @@ const uint8_t *data, *end; uint32_t size, num, prev_uid, next_uid; - if (offset >= ctx->uidlist->mmap_size) + if (squat_uidlist_map_list(ctx->uidlist, offset, &data, &size) < 0) return -1; - - data = CONST_PTR_OFFSET(ctx->uidlist->mmap_base, offset); - end = CONST_PTR_OFFSET(ctx->uidlist->mmap_base, - ctx->uidlist->mmap_size); - - size = _squat_trie_unpack_num(&data, end); - if (data + size > end) - return -1; - end = data + size; prev_uid = _squat_trie_unpack_num(&data, end);
--- a/src/plugins/fts-squat/squat-uidlist.h Sat Dec 09 23:08:27 2006 +0200 +++ b/src/plugins/fts-squat/squat-uidlist.h Sat Dec 09 23:08:56 2006 +0200 @@ -8,9 +8,13 @@ struct squat_uidlist * squat_uidlist_init(struct squat_trie *trie, const char *path, - uint32_t uidvalidity); + uint32_t uidvalidity, bool mmap_disable); void squat_uidlist_deinit(struct squat_uidlist *uidlist); +/* Make sure that we've the latest uidlist file fully mapped. */ +int squat_uidlist_refresh(struct squat_uidlist *uidlist); + +/* Get the last UID added to the file. */ int squat_uidlist_get_last_uid(struct squat_uidlist *uidlist, uint32_t *uid_r); /* Add new UID to given UID list. The uid_list_idx is updated to contain the