Mercurial > dovecot > core-2.2
changeset 13101:7b9978eb6f91
fts: Redesigned/enhanced FTS API and how virtual plugin works with it.
The changes include:
- Only indexer process and "doveadm index" do FTS indexing now. Other
processes connect to the indexer process via UNIX socket and request
indexing a specific mailbox.
- FTS backends can now index/search any search key. Current backends
implement indexing for some specific header fields (Subject, From, To, Cc,
Bcc), but it would also be possible to add indexing for e.g. message size
or date.
- CLucene support is fixed and fully functional.
- Solr is split to "solr" and "solr_old" backends. The new "solr" backend
supports the new header fields. It also uses mailbox GUIDs as mailbox
identifiers, so that renaming a mailbox doesn't require reindexing.
The "solr_old" uses the old Solr schema and doesn't support any new features.
line wrap: on
line diff
--- a/configure.in Fri Jul 22 13:13:29 2011 +0300 +++ b/configure.in Fri Jul 22 13:21:59 2011 +0300 @@ -146,9 +146,6 @@ AS_HELP_STRING([--with-lucene], [Build with CLucene full text search support]), TEST_WITH(lucene, $withval), want_lucene=no) -if test "$want_lucene" = "yes"; then - AC_ERROR([CLucene support isn't working very well. Use Solr instead.]) -fi AM_CONDITIONAL(BUILD_LUCENE, test "$want_lucene" = "yes") AC_ARG_WITH(solr,
--- a/doc/solr-schema.xml Fri Jul 22 13:13:29 2011 +0300 +++ b/doc/solr-schema.xml Fri Jul 22 13:21:59 2011 +0300 @@ -6,7 +6,7 @@ This is the Solr schema file, place it into solr/conf/schema.xml. You may want to modify the tokenizers and filters. --> -<schema name="dovecot" version="1.1"> +<schema name="dovecot" version="2.1"> <types> <!-- IMAP has 32bit unsigned ints but java ints are signed, so use longs --> <fieldType name="string" class="solr.StrField" omitNorms="true"/> @@ -38,15 +38,19 @@ <fields> - <field name="id" type="string" indexed="true" stored="true" required="true" /> - <field name="uid" type="slong" indexed="true" stored="true" required="true" /> - <field name="uidv" type="long" indexed="true" stored="true" required="true" /> - <field name="box" type="string" indexed="true" stored="true" required="true" /> - <field name="user" type="string" indexed="true" stored="true" required="true" /> - <field name="ns" type="string" indexed="true" stored="true" required="false" /> - <field name="last_uid" type="boolean" indexed="true" stored="false" /> - <field name="hdr" type="text" indexed="true" stored="false" /> - <field name="body" type="text" indexed="true" stored="false" /> + <field name="id" type="string" indexed="true" stored="true" required="true" /> + <field name="uid" type="slong" indexed="true" stored="true" required="true" /> + <field name="box" type="string" indexed="true" stored="true" required="true" /> + <field name="user" type="string" indexed="true" stored="true" required="true" /> + + <field name="hdr" type="text" indexed="true" stored="false" /> + <field name="body" type="text" indexed="true" stored="false" /> + + <field name="from" type="text" indexed="true" stored="false" /> + <field name="to" type="text" indexed="true" stored="false" /> + <field name="cc" type="text" indexed="true" stored="false" /> + <field name="bcc" type="text" indexed="true" stored="false" /> + <field name="subject" type="text" indexed="true" stored="false" /> </fields> <uniqueKey>id</uniqueKey>
--- a/src/plugins/fts-lucene/fts-backend-lucene.c Fri Jul 22 13:13:29 2011 +0300 +++ b/src/plugins/fts-lucene/fts-backend-lucene.c Fri Jul 22 13:21:59 2011 +0300 @@ -2,212 +2,334 @@ #include "lib.h" #include "array.h" -#include "mkdir-parents.h" +#include "hash.h" +#include "hex-binary.h" +#include "file-dotlock.h" +#include "mail-namespace.h" #include "mail-storage-private.h" #include "lucene-wrapper.h" #include "fts-lucene-plugin.h" -#define LUCENE_INDEX_DIR_NAME "lucene-indexes" -#define LUCENE_LOCK_SUBDIR_NAME "locks" - -#define LUCENE_CONTEXT(obj) \ - MODULE_CONTEXT(obj, fts_lucene_storage_module) +#include <wchar.h> -struct lucene_mail_storage { - union mail_storage_module_context module_ctx; - struct lucene_index *index; - struct mailbox *selected_box; - int refcount; -}; +#define LUCENE_INDEX_DIR_NAME "lucene-indexes" +#define LUCENE_EXPUNGE_FILENAME "pending-expunges" +#define LUCENE_OPTIMIZE_BATCH_MSGS_COUNT 100 struct lucene_fts_backend { struct fts_backend backend; - struct lucene_mail_storage *lstorage; - struct mailbox *box; -}; + char *dir_path; -struct lucene_fts_backend_build_context { - struct fts_backend_build_context ctx; + struct lucene_index *index; + struct mailbox *selected_box; - uint32_t uid; - bool hdr; + unsigned int dir_created:1; + unsigned int updating:1; }; -static MODULE_CONTEXT_DEFINE_INIT(fts_lucene_storage_module, - &mail_storage_module_register); +struct lucene_fts_backend_update_context { + struct fts_backend_update_context ctx; + + struct mailbox *box; + uint32_t last_uid; -static void fts_backend_select(struct lucene_fts_backend *backend) + uint32_t uid; + char *hdr_name; + + unsigned int added_msgs, expunges; +}; + +static int fts_backend_lucene_mkdir(struct lucene_fts_backend *backend) { - if (backend->lstorage->selected_box != backend->box) { - lucene_index_select_mailbox(backend->lstorage->index, - mailbox_get_name(backend->box)); - backend->lstorage->selected_box = backend->box; - } + if (backend->dir_created) + return 0; + + backend->dir_created = TRUE; + return mailbox_list_mkdir_root(backend->backend.ns->list, + backend->dir_path, + MAILBOX_LIST_PATH_TYPE_INDEX); } -static struct fts_backend *fts_backend_lucene_init(struct mailbox *box) +static int +fts_backend_select(struct lucene_fts_backend *backend, struct mailbox *box) { - struct lucene_mail_storage *lstorage; - struct lucene_fts_backend *backend; - const char *path, *lock_path; + struct mailbox_metadata metadata; + buffer_t buf; + unsigned char guid_hex[MAILBOX_GUID_HEX_LENGTH]; + wchar_t wguid_hex[MAILBOX_GUID_HEX_LENGTH]; + unsigned int i; - lstorage = LUCENE_CONTEXT(box->storage); - if (lstorage == NULL) { - path = mailbox_list_get_path(box->list, "INBOX", - MAILBOX_LIST_PATH_TYPE_INDEX); - if (path == NULL) { - /* in-memory indexes */ - if (box->storage->set->mail_debug) - i_debug("fts squat: Disabled with in-memory indexes"); - return NULL; + if (backend->selected_box == box) + return 0; + + if (fts_backend_lucene_mkdir(backend) < 0) + return -1; + + if (box != NULL) { + if (mailbox_get_metadata(box, MAILBOX_METADATA_GUID, + &metadata) < 0) { + i_error("fts-lucene: Couldn't get mailbox %s GUID: %s", + box->vname, mailbox_get_last_error(box, NULL)); + return -1; } - path = t_strconcat(path, "/"LUCENE_INDEX_DIR_NAME, NULL); - lock_path = t_strdup_printf("%s/"LUCENE_LOCK_SUBDIR_NAME, path); - if (mkdir_parents(lock_path, 0700) < 0 && errno != EEXIST) { - i_error("mkdir_parents(%s) failed: %m", lock_path); - return NULL; - } + buffer_create_data(&buf, guid_hex, sizeof(guid_hex)); + binary_to_hex_append(&buf, metadata.guid, MAIL_GUID_128_SIZE); + for (i = 0; i < N_ELEMENTS(wguid_hex); i++) + wguid_hex[i] = guid_hex[i]; - lstorage = i_new(struct lucene_mail_storage, 1); - lstorage->index = lucene_index_init(path, lock_path); - MODULE_CONTEXT_SET(box->storage, fts_lucene_storage_module, - lstorage); + lucene_index_select_mailbox(backend->index, wguid_hex); + } else { + lucene_index_unselect_mailbox(backend->index); } - lstorage->refcount++; + backend->selected_box = box; + return 0; +} + +static struct fts_backend *fts_backend_lucene_alloc(void) +{ + struct lucene_fts_backend *backend; backend = i_new(struct lucene_fts_backend, 1); backend->backend = fts_backend_lucene; - backend->lstorage = lstorage; - backend->box = box; return &backend->backend; } +static int +fts_backend_lucene_init(struct fts_backend *_backend, + const char **error_r ATTR_UNUSED) +{ + struct lucene_fts_backend *backend = + (struct lucene_fts_backend *)_backend; + struct mailbox_list *list = _backend->ns->list; + const char *path; + + path = mailbox_list_get_path(list, NULL, + MAILBOX_LIST_PATH_TYPE_INDEX); + i_assert(path != NULL); /* fts already checked this */ + + backend->dir_path = i_strconcat(path, "/"LUCENE_INDEX_DIR_NAME, NULL); + backend->index = lucene_index_init(backend->dir_path); + return 0; +} + static void fts_backend_lucene_deinit(struct fts_backend *_backend) { struct lucene_fts_backend *backend = (struct lucene_fts_backend *)_backend; - if (--backend->lstorage->refcount == 0) { - MODULE_CONTEXT_UNSET(backend->box->storage, - fts_lucene_storage_module); - lucene_index_deinit(backend->lstorage->index); - i_free(backend->lstorage); - } + lucene_index_deinit(backend->index); + i_free(backend->dir_path); i_free(backend); } static int fts_backend_lucene_get_last_uid(struct fts_backend *_backend, - uint32_t *last_uid_r) + struct mailbox *box, uint32_t *last_uid_r) { struct lucene_fts_backend *backend = (struct lucene_fts_backend *)_backend; - fts_backend_select(backend); - return lucene_index_get_last_uid(backend->lstorage->index, last_uid_r); + if (fts_index_get_last_uid(box, last_uid_r)) + return 0; + + /* either nothing has been indexed, or the index was corrupted. + do it the slow way. */ + if (fts_backend_select(backend, box) < 0) + return -1; + if (lucene_index_get_last_uid(backend->index, last_uid_r) < 0) + return -1; + + (void)fts_index_set_last_uid(box, *last_uid_r); + return 0; } -static int -fts_backend_lucene_build_init(struct fts_backend *_backend, - uint32_t *last_uid_r, - struct fts_backend_build_context **ctx_r) +static struct fts_backend_update_context * +fts_backend_lucene_update_init(struct fts_backend *_backend) { struct lucene_fts_backend *backend = (struct lucene_fts_backend *)_backend; - struct lucene_fts_backend_build_context *ctx; - uint32_t last_uid; + struct lucene_fts_backend_update_context *ctx; - fts_backend_select(backend); - if (lucene_index_build_init(backend->lstorage->index, - &last_uid) < 0) - return -1; - - ctx = i_new(struct lucene_fts_backend_build_context, 1); - ctx->ctx.backend = _backend; - ctx->uid = last_uid + 1; + i_assert(!backend->updating); - *last_uid_r = last_uid; - *ctx_r = &ctx->ctx; - return 0; -} + ctx = i_new(struct lucene_fts_backend_update_context, 1); + ctx->ctx.backend = _backend; + backend->updating = TRUE; -static void -fts_backend_lucene_build_hdr(struct fts_backend_build_context *_ctx, - uint32_t uid) -{ - struct lucene_fts_backend_build_context *ctx = - (struct lucene_fts_backend_build_context *)_ctx; - - i_assert(uid >= ctx->uid); - - ctx->uid = uid; - ctx->hdr = TRUE; + if (fts_backend_lucene_mkdir(backend) < 0) + ctx->ctx.failed = TRUE; + if (lucene_index_build_init(backend->index) < 0) + ctx->ctx.failed = TRUE; + return &ctx->ctx; } static bool -fts_backend_lucene_build_body_begin(struct fts_backend_build_context *_ctx, - uint32_t uid, const char *content_type, - const char *content_disposition ATTR_UNUSED) +fts_backend_lucene_need_optimize(struct lucene_fts_backend_update_context *ctx) { - struct lucene_fts_backend_build_context *ctx = - (struct lucene_fts_backend_build_context *)_ctx; + struct lucene_fts_backend *backend = + (struct lucene_fts_backend *)ctx->ctx.backend; + const struct dotlock_settings dotlock_set = { + .timeout = 1, + .stale_timeout = 30, + .use_excl_lock = TRUE + }; + struct dotlock *dotlock; + const char *path; + char buf[MAX_INT_STRLEN]; + unsigned int expunges = 0; + uint32_t numdocs; + int fdw, fdr; - i_assert(uid >= ctx->uid); + if (ctx->added_msgs >= LUCENE_OPTIMIZE_BATCH_MSGS_COUNT) + return TRUE; - if (!fts_backend_default_can_index(content_type)) + if (ctx->expunges == 0) + return FALSE; + + if (lucene_index_get_doc_count(backend->index, &numdocs) < 0) return FALSE; - ctx->uid = uid; - ctx->hdr = FALSE; - return TRUE; + /* update pending expunges count */ + path = t_strconcat(backend->dir_path, "/"LUCENE_EXPUNGE_FILENAME, NULL); + fdw = file_dotlock_open(&dotlock_set, path, 0, &dotlock); + if (fdw == -1) + return FALSE; + + fdr = open(path, O_RDONLY); + if (fdr == -1) { + if (errno != ENOENT) + i_error("open(%s) failed: %m", path); + } else { + if (read(fdr, buf, sizeof(buf)) < 0) + i_error("read(%s) failed: %m", path); + if (str_to_uint(buf, &expunges) < 0) + i_error("%s is corrupted", path); + if (close(fdr) < 0) + i_error("close(%s) failed: %m", path); + } + expunges += ctx->expunges; + + i_snprintf(buf, sizeof(buf), "%u", expunges); + if (write(fdw, buf, strlen(buf)) < 0) + i_error("write(%s) failed: %m", path); + if (close(fdw) < 0) + i_error("close(%s) failed: %m", path); + return numdocs / expunges <= 50; /* >2% of index has been expunged */ } static int -fts_backend_lucene_build_more(struct fts_backend_build_context *_ctx, - const unsigned char *data, size_t size) +fts_backend_lucene_update_deinit(struct fts_backend_update_context *_ctx) { - struct lucene_fts_backend_build_context *ctx = - (struct lucene_fts_backend_build_context *)_ctx; + struct lucene_fts_backend_update_context *ctx = + (struct lucene_fts_backend_update_context *)_ctx; struct lucene_fts_backend *backend = (struct lucene_fts_backend *)_ctx->backend; - - if (_ctx->failed) - return -1; + int ret = _ctx->failed ? -1 : 0; - i_assert(backend->lstorage->selected_box == backend->box); - return lucene_index_build_more(backend->lstorage->index, - ctx->uid, data, size, ctx->hdr); -} + i_assert(backend->updating); -static int -fts_backend_lucene_build_deinit(struct fts_backend_build_context *ctx) -{ - struct lucene_fts_backend *backend = - (struct lucene_fts_backend *)ctx->backend; - int ret = ctx->failed ? -1 : 0; + backend->updating = FALSE; + lucene_index_build_deinit(backend->index); - i_assert(backend->lstorage->selected_box == backend->box); - lucene_index_build_deinit(backend->lstorage->index); + if (fts_backend_lucene_need_optimize(ctx)) + (void)fts_backend_optimize(_ctx->backend); + i_free(ctx); return ret; } static void -fts_backend_lucene_expunge(struct fts_backend *_backend, struct mail *mail) +fts_backend_lucene_update_set_mailbox(struct fts_backend_update_context *_ctx, + struct mailbox *box) { + struct lucene_fts_backend_update_context *ctx = + (struct lucene_fts_backend_update_context *)_ctx; struct lucene_fts_backend *backend = - (struct lucene_fts_backend *)_backend; + (struct lucene_fts_backend *)_ctx->backend; - fts_backend_select(backend); - (void)lucene_index_expunge(backend->lstorage->index, mail->uid); + if (ctx->last_uid != 0) { + (void)fts_index_set_last_uid(ctx->box, ctx->last_uid); + ctx->last_uid = 0; + } + ctx->box = box; + + if (fts_backend_select(backend, box) < 0) + _ctx->failed = TRUE; } static void -fts_backend_lucene_expunge_finish(struct fts_backend *_backend ATTR_UNUSED, - struct mailbox *box ATTR_UNUSED, - bool committed ATTR_UNUSED) +fts_backend_lucene_update_expunge(struct fts_backend_update_context *_ctx, + uint32_t uid ATTR_UNUSED) +{ + struct lucene_fts_backend_update_context *ctx = + (struct lucene_fts_backend_update_context *)_ctx; + + ctx->expunges++; +} + +static bool +fts_backend_lucene_update_set_build_key(struct fts_backend_update_context *_ctx, + const struct fts_backend_build_key *key) { + struct lucene_fts_backend_update_context *ctx = + (struct lucene_fts_backend_update_context *)_ctx; + + switch (key->type) { + case FTS_BACKEND_BUILD_KEY_HDR: + case FTS_BACKEND_BUILD_KEY_MIME_HDR: + i_assert(key->hdr_name != NULL); + + i_free(ctx->hdr_name); + ctx->hdr_name = i_strdup(key->hdr_name); + break; + case FTS_BACKEND_BUILD_KEY_BODY_PART: + i_free_and_null(ctx->hdr_name); + break; + case FTS_BACKEND_BUILD_KEY_BODY_PART_BINARY: + i_unreached(); + } + + if (key->uid != ctx->last_uid) { + i_assert(key->uid >= ctx->last_uid); + ctx->last_uid = key->uid; + ctx->added_msgs++; + } + + ctx->uid = key->uid; + return TRUE; +} + +static void +fts_backend_lucene_update_unset_build_key(struct fts_backend_update_context *_ctx) +{ + struct lucene_fts_backend_update_context *ctx = + (struct lucene_fts_backend_update_context *)_ctx; + + ctx->uid = 0; + i_free_and_null(ctx->hdr_name); +} + +static int +fts_backend_lucene_update_build_more(struct fts_backend_update_context *_ctx, + const unsigned char *data, size_t size) +{ + struct lucene_fts_backend_update_context *ctx = + (struct lucene_fts_backend_update_context *)_ctx; + struct lucene_fts_backend *backend = + (struct lucene_fts_backend *)_ctx->backend; + int ret; + + i_assert(ctx->uid != 0); + + if (_ctx->failed) + return -1; + + T_BEGIN { + ret = lucene_index_build_more(backend->index, ctx->uid, + data, size, ctx->hdr_name); + } T_END; + return ret; } static int @@ -217,20 +339,186 @@ } static int -fts_backend_lucene_lookup(struct fts_backend *_backend, - const char *key, enum fts_lookup_flags flags, - ARRAY_TYPE(seq_range) *definite_uids, - ARRAY_TYPE(seq_range) *maybe_uids) +optimize_mailbox_get_uids(struct mailbox *box, ARRAY_TYPE(seq_range) *uids) +{ + struct mailbox_status status; + + if (mailbox_get_status(box, STATUS_MESSAGES, &status) < 0) + return -1; + + if (status.messages > 0) T_BEGIN { + ARRAY_TYPE(seq_range) seqs; + + t_array_init(&seqs, 2); + seq_range_array_add_range(&seqs, 1, status.messages); + mailbox_get_uid_range(box, &seqs, uids); + } T_END; + return 0; +} + +static int +optimize_mailbox_update_last_uid(struct mailbox *box, + const ARRAY_TYPE(seq_range) *missing_uids) +{ + struct mailbox_status status; + const struct seq_range *range; + unsigned int count; + uint32_t last_uid; + + /* FIXME: missing_uids from the middle of mailbox could probably + be handled better. they now create duplicates on next indexing? */ + + range = array_get(missing_uids, &count); + if (count > 0) + last_uid = range[0].seq1 - 1; + else { + mailbox_get_open_status(box, STATUS_UIDNEXT, &status); + last_uid = status.uidnext - 1; + } + return fts_index_set_last_uid(box, last_uid); +} + +static int fts_backend_lucene_optimize(struct fts_backend *_backend) { struct lucene_fts_backend *backend = (struct lucene_fts_backend *)_backend; + struct mailbox_list_iterate_context *iter; + const struct mailbox_info *info; + struct mailbox *box; + ARRAY_TYPE(seq_range) uids, missing_uids; + int ret = 0; - i_assert((flags & FTS_LOOKUP_FLAG_INVERT) == 0); + i_array_init(&uids, 128); + i_array_init(&missing_uids, 32); + iter = mailbox_list_iter_init(_backend->ns->list, "*", + MAILBOX_LIST_ITER_NO_AUTO_BOXES | + MAILBOX_LIST_ITER_RETURN_NO_FLAGS); + while ((info = mailbox_list_iter_next(iter)) != NULL) { + if ((info->flags & + (MAILBOX_NOSELECT | MAILBOX_NONEXISTENT)) != 0) + continue; + + box = mailbox_alloc(info->ns->list, info->name, + MAILBOX_FLAG_KEEP_RECENT); + if (fts_backend_select(backend, box) < 0 || + optimize_mailbox_get_uids(box, &uids) < 0 || + lucene_index_optimize_scan(backend->index, &uids, + &missing_uids) < 0 || + optimize_mailbox_update_last_uid(box, &missing_uids)) + ret = -1; + + fts_backend_select(backend, NULL); + mailbox_free(&box); + array_clear(&uids); + array_clear(&missing_uids); + } + if (mailbox_list_iter_deinit(&iter) < 0) + ret = -1; + + /* FIXME: optionally go through mailboxes in index and delete those + that don't exist anymre */ + if (lucene_index_optimize_finish(backend->index) < 0) + ret = -1; + + array_free(&uids); + array_free(&missing_uids); + return ret; +} + +static int +fts_backend_lucene_lookup(struct fts_backend *_backend, struct mailbox *box, + struct mail_search_arg *args, bool and_args, + struct fts_result *result) +{ + struct lucene_fts_backend *backend = + (struct lucene_fts_backend *)_backend; + int ret; + + if (fts_backend_select(backend, box) < 0) + return -1; + T_BEGIN { + ret = lucene_index_lookup(backend->index, args, and_args, + result); + } T_END; + return ret; +} + +/* a char* hash function from ASU -- from glib */ +static unsigned int wstr_hash(const void *p) +{ + const wchar_t *s = p; + unsigned int g, h = 0; - array_clear(maybe_uids); - fts_backend_select(backend); - return lucene_index_lookup(backend->lstorage->index, - flags, key, definite_uids); + while (*s != '\0') { + h = (h << 4) + *s; + if ((g = h & 0xf0000000UL)) { + h = h ^ (g >> 24); + h = h ^ g; + } + s++; + } + + return h; +} + +static int +mailboxes_get_guids(struct mailbox *const boxes[], + struct hash_table *guids, struct fts_multi_result *result) +{ + ARRAY_DEFINE(box_results, struct fts_result); + struct fts_result *box_result; + const char *guid; + wchar_t *guid_dup; + unsigned int i, j; + + p_array_init(&box_results, result->pool, 32); + for (i = 0; boxes[i] != NULL; i++) { + if (fts_mailbox_get_guid(boxes[i], &guid) < 0) + return -1; + + i_assert(strlen(guid) == MAILBOX_GUID_HEX_LENGTH); + guid_dup = t_new(wchar_t, MAILBOX_GUID_HEX_LENGTH + 1); + for (j = 0; j < MAILBOX_GUID_HEX_LENGTH; j++) + guid_dup[j] = guid[j]; + + box_result = array_append_space(&box_results); + box_result->box = boxes[i]; + hash_table_insert(guids, guid_dup, box_result); + } + + (void)array_append_space(&box_results); + result->box_results = array_idx_modifiable(&box_results, 0); + return 0; +} + +static int +fts_backend_lucene_lookup_multi(struct fts_backend *_backend, + struct mailbox *const boxes[], + struct mail_search_arg *args, bool and_args, + struct fts_multi_result *result) +{ + struct lucene_fts_backend *backend = + (struct lucene_fts_backend *)_backend; + int ret; + + if (fts_backend_lucene_mkdir(backend) < 0) + return -1; + + T_BEGIN { + struct hash_table *guids; + + guids = hash_table_create(default_pool, default_pool, 0, + wstr_hash, + (hash_cmp_callback_t *)wcscmp); + ret = mailboxes_get_guids(boxes, guids, result); + if (ret == 0) { + ret = lucene_index_lookup_multi(backend->index, + guids, args, and_args, + result); + } + hash_table_destroy(&guids); + } T_END; + return ret; } struct fts_backend fts_backend_lucene = { @@ -238,21 +526,21 @@ .flags = 0, { + fts_backend_lucene_alloc, fts_backend_lucene_init, fts_backend_lucene_deinit, fts_backend_lucene_get_last_uid, - NULL, - fts_backend_lucene_build_init, - fts_backend_lucene_build_hdr, - fts_backend_lucene_build_body_begin, - NULL, - fts_backend_lucene_build_more, - fts_backend_lucene_build_deinit, - fts_backend_lucene_expunge, - fts_backend_lucene_expunge_finish, + fts_backend_lucene_update_init, + fts_backend_lucene_update_deinit, + fts_backend_lucene_update_set_mailbox, + fts_backend_lucene_update_expunge, + fts_backend_lucene_update_set_build_key, + fts_backend_lucene_update_unset_build_key, + fts_backend_lucene_update_build_more, fts_backend_lucene_refresh, + fts_backend_lucene_optimize, + fts_backend_default_can_lookup, fts_backend_lucene_lookup, - NULL, - NULL + fts_backend_lucene_lookup_multi } };
--- a/src/plugins/fts-lucene/lucene-wrapper.cc Fri Jul 22 13:13:29 2011 +0300 +++ b/src/plugins/fts-lucene/lucene-wrapper.cc Fri Jul 22 13:21:59 2011 +0300 @@ -5,38 +5,33 @@ #include "array.h" #include "env-util.h" #include "unichar.h" +#include "hash.h" #include "str.h" -#include "str-sanitize.h" +#include "strescape.h" +#include "mail-search.h" #include "lucene-wrapper.h" #include <dirent.h> #include <sys/stat.h> }; #include <CLucene.h> +#include <CLucene/util/CLStreams.h> +#include <CLucene/search/MultiPhraseQuery.h> /* Lucene's default is 10000. Use it here also.. */ #define MAX_TERMS_PER_DOCUMENT 10000 -/* If all the files in the lucene index directory are older than this many - seconds, assume we can delete stale locks */ -#define STALE_INDEX_SECS 60 -/* When index is determined to be stale, delete all locks older than this */ -#define STALE_LOCK_SECS 60 -/* Minimum interval between staleness checks */ -#define STALENESS_CHECK_INTERVAL 10 - using namespace lucene::document; using namespace lucene::index; using namespace lucene::search; using namespace lucene::queryParser; using namespace lucene::analysis; +using namespace lucene::analysis; +using namespace lucene::util; struct lucene_index { - char *path, *lock_path; - wchar_t *mailbox_name; - - time_t last_stale_check; - bool lock_error; + char *path; + wchar_t mailbox_guid[MAILBOX_GUID_HEX_LENGTH + 1]; IndexReader *reader; IndexWriter *writer; @@ -44,99 +39,16 @@ Analyzer *analyzer; Document *doc; - uint32_t prev_uid, last_uid; + uint32_t prev_uid; }; -static bool lucene_dir_scan(const char *dir, const char *skip_path, - time_t stale_stamp, bool unlink_staled) -{ - DIR *d; - struct dirent *dp; - struct stat st; - string_t *path; - unsigned int dir_len; - bool found_nonstale = FALSE; - - d = opendir(dir); - if (d == NULL) { - i_error("opendir(%s) failed: %m", dir); - return TRUE; - } - - t_push(); - path = t_str_new(256); - str_append(path, dir); - str_append_c(path, '/'); - dir_len = str_len(path); - - while ((dp = readdir(d)) != NULL) { - if (*dp->d_name == '.') { - if (dp->d_name[1] == '\0') - continue; - if (dp->d_name[1] == '.' && dp->d_name[2] == '\0') - continue; - } - - str_truncate(path, dir_len); - str_append(path, dp->d_name); - - if (skip_path != NULL && - strcmp(str_c(path), skip_path) == 0) - continue; - - if (stat(str_c(path), &st) < 0) { - if (errno != ENOENT) - i_error("stat(%s) failed: %m", str_c(path)); - found_nonstale = TRUE; - } else if (st.st_ctime <= stale_stamp && - st.st_mtime <= stale_stamp) { - if (unlink_staled) { - if (unlink(str_c(path)) < 0 && - errno != ENOENT) { - i_error("unlink(%s) failed: %m", - str_c(path)); - } - } - } else { - found_nonstale = TRUE; - } - } - if (closedir(d) < 0) - i_error("closedir(%s) failed: %m", dir); - t_pop(); - return found_nonstale; -} - -static void lucene_delete_stale_locks(struct lucene_index *index) -{ - time_t now; - - now = time(NULL); - if (index->last_stale_check + STALENESS_CHECK_INTERVAL > now) - return; - index->last_stale_check = now; - - if (lucene_dir_scan(index->path, index->lock_path, - now - STALE_INDEX_SECS, FALSE)) { - /* the index is probably being updated */ - return; - } - (void)lucene_dir_scan(index->lock_path, NULL, - now - STALE_LOCK_SECS, TRUE); -} - -struct lucene_index *lucene_index_init(const char *path, const char *lock_path) +struct lucene_index *lucene_index_init(const char *path) { struct lucene_index *index; - env_put(t_strconcat(LUCENE_LOCK_DIR_ENV_1"=", lock_path, NULL)); - index = i_new(struct lucene_index, 1); index->path = i_strdup(path); - index->lock_path = i_strdup(lock_path); index->analyzer = _CLNEW standard::StandardAnalyzer(); - - lucene_delete_stale_locks(index); return index; } @@ -151,34 +63,16 @@ { lucene_index_close(index); _CLDELETE(index->analyzer); - i_free(index->mailbox_name); i_free(index->path); - i_free(index->lock_path); i_free(index); } static void -lucene_utf8_to_tchar(const char *src, wchar_t *dest, size_t destsize) -{ - ARRAY_TYPE(unichars) dest_arr; - buffer_t buf = { 0, 0 }; - - i_assert(sizeof(wchar_t) == sizeof(unichar_t)); - - buffer_create_data(&buf, dest, sizeof(wchar_t) * destsize); - array_create_from_buffer(&dest_arr, &buf, sizeof(wchar_t)); - if (uni_utf8_to_ucs4(src, &dest_arr) < 0) - i_unreached(); - i_assert(array_count(&dest_arr)+1 == destsize); - dest[destsize-1] = 0; -} - -static void lucene_utf8_n_to_tchar(const unsigned char *src, size_t srcsize, wchar_t *dest, size_t destsize) { ARRAY_TYPE(unichars) dest_arr; - buffer_t buf = { 0, 0 }; + buffer_t buf = { 0, 0, { 0, 0, 0, 0, 0 } }; i_assert(sizeof(wchar_t) == sizeof(unichar_t)); @@ -190,16 +84,32 @@ dest[destsize-1] = 0; } -void lucene_index_select_mailbox(struct lucene_index *index, - const char *mailbox_name) +static const wchar_t *t_lucene_utf8_to_tchar(const char *str) { - size_t size; + ARRAY_TYPE(unichars) dest_arr; + const unichar_t *ret; + + i_assert(sizeof(wchar_t) == sizeof(unichar_t)); - i_free(index->mailbox_name); + t_array_init(&dest_arr, strlen(str) + 1); + if (uni_utf8_to_ucs4(str, &dest_arr) < 0) + i_unreached(); + (void)array_append_space(&dest_arr); + ret = array_idx(&dest_arr, 0); + return (const wchar_t *)ret; +} - size = uni_utf8_strlen_n(mailbox_name, (size_t)-1) + 1; - index->mailbox_name = i_new(wchar_t, size); - lucene_utf8_to_tchar(mailbox_name, index->mailbox_name, size); +void lucene_index_select_mailbox(struct lucene_index *index, + const wchar_t guid[MAILBOX_GUID_HEX_LENGTH]) +{ + memcpy(index->mailbox_guid, guid, + MAILBOX_GUID_HEX_LENGTH * sizeof(wchar_t)); + index->mailbox_guid[MAILBOX_GUID_HEX_LENGTH] = '\0'; +} + +void lucene_index_unselect_mailbox(struct lucene_index *index) +{ + memset(index->mailbox_guid, 0, sizeof(index->mailbox_guid)); } static void lucene_handle_error(struct lucene_index *index, CLuceneError &err, @@ -207,15 +117,6 @@ { const char *what = err.what(); - if (err.number() == CL_ERR_IO && strncasecmp(what, "Lock", 4) == 0) { - /* "Lock obtain timed out". delete any stale locks. */ - lucene_delete_stale_locks(index); - if (index->lock_error) { - /* we've already complained about this */ - return; - } - index->lock_error = TRUE; - } i_error("lucene index %s: %s failed: %s", index->path, msg, what); } @@ -271,120 +172,72 @@ return 0; } -static int -lucene_index_get_last_uid_int(struct lucene_index *index, bool delete_old) +int lucene_index_get_last_uid(struct lucene_index *index, uint32_t *last_uid_r) { - ARRAY_TYPE(uint32_t) delete_doc_ids; - uint32_t del_id; int ret = 0; - bool deleted = false; - index->last_uid = 0; + *last_uid_r = 0; if ((ret = lucene_index_open_search(index)) <= 0) return ret; - /* find all the existing last_uids for selected mailbox. - if there are more than one, delete the smaller ones. this is normal - behavior because we can't update/delete documents in writer, so - we'll do it only in here.. */ - Term mailbox_term(_T("box"), index->mailbox_name); - Term last_uid_term(_T("last_uid"), _T("*")); - TermQuery mailbox_query(&mailbox_term); - WildcardQuery last_uid_query(&last_uid_term); + Term mailbox_term(_T("box"), index->mailbox_guid); + TermQuery query(&mailbox_term); - BooleanQuery query; - query.add(&mailbox_query, true, false); - query.add(&last_uid_query, true, false); - - t_push(); - t_array_init(&delete_doc_ids, 10); - int32_t last_doc_id = -1; + uint32_t last_uid = 0; try { Hits *hits = index->searcher->search(&query); - for (int32_t i = 0; i < hits->length(); i++) { + for (size_t i = 0; i < hits->length(); i++) { uint32_t uid; if (lucene_doc_get_uid(index, &hits->doc(i), - _T("last_uid"), &uid) < 0) { + _T("uid"), &uid) < 0) { ret = -1; break; } - if (uid > index->last_uid) { - if (last_doc_id >= 0) { - del_id = last_doc_id; - array_append_i(&delete_doc_ids.arr, - (void *)&del_id, 1); - } - index->last_uid = uid; - last_doc_id = hits->id(i); - } else { - del_id = hits->id(i); - array_append_i(&delete_doc_ids.arr, - (void *)&del_id, 1); - } + if (uid > last_uid) + last_uid = uid; } - if (delete_old && array_count(&delete_doc_ids) > 0) { - const uint32_t *ids; - unsigned int i, count; - - ids = array_get(&delete_doc_ids, &count); - for (i = 0; i < count; i++) - index->reader->deleteDocument(ids[i]); - deleted = true; - } - index->lock_error = FALSE; _CLDELETE(hits); } catch (CLuceneError &err) { lucene_handle_error(index, err, "last_uid search"); ret = -1; } - - if (deleted) { - /* the index was modified. we'll need to release the locks - before opening a writer */ - lucene_index_close(index); - } - t_pop(); + *last_uid_r = last_uid; return ret; } -int lucene_index_get_last_uid(struct lucene_index *index, uint32_t *last_uid_r) +int lucene_index_get_doc_count(struct lucene_index *index, uint32_t *count_r) { - /* delete the old last_uids in here, since we've not write-locked - the index yet */ - if (lucene_index_get_last_uid_int(index, true) < 0) - return -1; + int ret; - *last_uid_r = index->last_uid; - return 0; + if (index->reader == NULL) { + lucene_index_close(index); + if ((ret = lucene_index_open(index)) < 0) + return -1; + if (ret == 0) { + *count_r = 0; + return 0; + } + } + return index->reader->numDocs(); } -int lucene_index_build_init(struct lucene_index *index, uint32_t *last_uid_r) +int lucene_index_build_init(struct lucene_index *index) { - i_assert(index->mailbox_name != NULL); - - /* set this even if we fail so fts-storage won't crash */ - *last_uid_r = index->last_uid; - lucene_index_close(index); bool exists = IndexReader::indexExists(index->path); try { index->writer = _CLNEW IndexWriter(index->path, index->analyzer, !exists); - index->lock_error = FALSE; } catch (CLuceneError &err) { lucene_handle_error(index, err, "IndexWriter()"); return -1; } index->writer->setMaxFieldLength(MAX_TERMS_PER_DOCUMENT); - - if (lucene_index_get_last_uid_int(index, false) < 0) - return -1; - *last_uid_r = index->last_uid; return 0; } @@ -409,53 +262,41 @@ int lucene_index_build_more(struct lucene_index *index, uint32_t uid, const unsigned char *data, size_t size, - bool headers) + const char *hdr_name) { - size_t destsize; - - i_assert(uid > index->last_uid); - i_assert(size > 0); - - destsize = uni_utf8_strlen_n(data, size) + 1; - wchar_t dest[destsize]; - lucene_utf8_n_to_tchar(data, size, dest, destsize); + wchar_t id[MAX_INT_STRLEN]; + size_t namesize, datasize; if (uid != index->prev_uid) { - wchar_t id[MAX_INT_STRLEN]; - if (lucene_index_build_flush(index) < 0) return -1; index->prev_uid = uid; index->doc = _CLNEW Document(); swprintf(id, N_ELEMENTS(id), L"%u", uid); - index->doc->add(*new Field(_T("uid"), id, Field::STORE_YES | Field::INDEX_UNTOKENIZED)); - index->doc->add(*new Field(_T("box"), index->mailbox_name, Field::STORE_YES | Field::INDEX_UNTOKENIZED)); + index->doc->add(*_CLNEW Field(_T("uid"), id, Field::STORE_YES | Field::INDEX_UNTOKENIZED)); + index->doc->add(*_CLNEW Field(_T("box"), index->mailbox_guid, Field::STORE_YES | Field::INDEX_UNTOKENIZED)); } - if (headers) - index->doc->add(*new Field(_T("headers"), dest, Field::STORE_NO | Field::INDEX_TOKENIZED)); - else - index->doc->add(*new Field(_T("body"), dest, Field::STORE_NO | Field::INDEX_TOKENIZED)); - return 0; -} - -static int lucene_index_update_last_uid(struct lucene_index *index) -{ - Document doc; - wchar_t id[MAX_INT_STRLEN]; + datasize = uni_utf8_strlen_n(data, size) + 1; + wchar_t dest[datasize]; + lucene_utf8_n_to_tchar(data, size, dest, datasize); - swprintf(id, N_ELEMENTS(id), L"%u", index->last_uid); - doc.add(*new Field(_T("last_uid"), id, Field::STORE_YES | Field::INDEX_UNTOKENIZED)); - doc.add(*new Field(_T("box"), index->mailbox_name, Field::STORE_YES | Field::INDEX_UNTOKENIZED)); + if (hdr_name != NULL) { + /* hdr_name should be ASCII, but don't break in case it isn't */ + namesize = uni_utf8_strlen(hdr_name) + 1; + wchar_t wname[namesize]; + lucene_utf8_n_to_tchar((const unsigned char *)hdr_name, + strlen(hdr_name), wname, namesize); + index->doc->add(*_CLNEW Field(_T("hdr"), wname, Field::STORE_NO | Field::INDEX_UNTOKENIZED)); + index->doc->add(*_CLNEW Field(_T("hdr"), dest, Field::STORE_NO | Field::INDEX_TOKENIZED)); - try { - index->writer->addDocument(&doc); - return 0; - } catch (CLuceneError &err) { - lucene_handle_error(index, err, "IndexWriter::addDocument()"); - return -1; + if (fts_header_want_indexed(hdr_name)) + index->doc->add(*_CLNEW Field(wname, dest, Field::STORE_NO | Field::INDEX_TOKENIZED)); + } else if (size > 0) { + index->doc->add(*_CLNEW Field(_T("body"), dest, Field::STORE_NO | Field::INDEX_TOKENIZED)); } + return 0; } int lucene_index_build_deinit(struct lucene_index *index) @@ -466,9 +307,6 @@ /* no changes. */ return 0; } - - if (index->prev_uid > index->last_uid) - index->last_uid = index->prev_uid; index->prev_uid = 0; if (index->writer == NULL) { @@ -478,8 +316,6 @@ if (lucene_index_build_flush(index) < 0) ret = -1; - if (lucene_index_update_last_uid(index) < 0) - ret = -1; try { index->writer->optimize(); @@ -498,102 +334,319 @@ return ret; } -int lucene_index_expunge(struct lucene_index *index, uint32_t uid) -{ - wchar_t id[MAX_INT_STRLEN]; - int ret; - - if ((ret = lucene_index_open_search(index)) <= 0) - return ret; +struct uid_id_map { + uint32_t imap_uid; + int32_t lucene_id; +}; +ARRAY_DEFINE_TYPE(uid_id_map, struct uid_id_map); - swprintf(id, N_ELEMENTS(id), L"%u", uid); +static int uid_id_map_cmp(const struct uid_id_map *u1, + const struct uid_id_map *u2) +{ + if (u1->imap_uid < u2->imap_uid) + return -1; + if (u1->imap_uid > u2->imap_uid) + return 1; + return 0; +} - Term mailbox_term(_T("box"), index->mailbox_name); - Term uid_term(_T("uid"), id); - TermQuery mailbox_query(&mailbox_term); - TermQuery uid_query(&uid_term); +static int get_mailbox_uid_id_map(struct lucene_index *index, + ARRAY_TYPE(uid_id_map) *uid_id_map) +{ + int ret = 0; - BooleanQuery query; - query.add(&mailbox_query, true, false); - query.add(&uid_query, true, false); + /* get a sorted map of imap uid -> lucene id */ + Term mailbox_term(_T("box"), index->mailbox_guid); + TermQuery query(&mailbox_term); try { Hits *hits = index->searcher->search(&query); - for (int32_t i = 0; i < hits->length(); i++) - index->reader->deleteDocument(hits->id(i)); - index->lock_error = FALSE; + for (size_t i = 0; i < hits->length(); i++) { + uint32_t uid; + + if (lucene_doc_get_uid(index, &hits->doc(i), + _T("uid"), &uid) < 0) { + ret = -1; + break; + } + struct uid_id_map *ui = array_append_space(uid_id_map); + ui->imap_uid = uid; + ui->lucene_id = hits->id(i); + } _CLDELETE(hits); } catch (CLuceneError &err) { lucene_handle_error(index, err, "expunge search"); ret = -1; } + array_sort(uid_id_map, uid_id_map_cmp); + return ret; +} + +int lucene_index_optimize_scan(struct lucene_index *index, + const ARRAY_TYPE(seq_range) *existing_uids, + ARRAY_TYPE(seq_range) *missing_uids_r) +{ + ARRAY_TYPE(uid_id_map) uid_id_map_arr; + const struct uid_id_map *uid_id_map; + struct seq_range_iter iter; + unsigned int n, i, count; + uint32_t uid; + int ret; + + if ((ret = lucene_index_open_search(index)) <= 0) + return ret; + + i_array_init(&uid_id_map_arr, 128); + if (get_mailbox_uid_id_map(index, &uid_id_map_arr) < 0) + return -1; + uid_id_map = array_get(&uid_id_map_arr, &count); + + seq_range_array_iter_init(&iter, existing_uids); n = i = 0; + while (seq_range_array_iter_nth(&iter, n++, &uid)) { + while (i < count && uid_id_map[i].imap_uid < uid) { + /* expunged message */ + index->reader->deleteDocument(uid_id_map[i].lucene_id); + i++; + } + + if (i == count || uid_id_map[i].imap_uid > uid) { + /* uid is missing from index */ + seq_range_array_add(missing_uids_r, 0, uid); + } else { + i++; + } + } + for (; i < count; i++) + index->reader->deleteDocument(uid_id_map[i].lucene_id); + + array_free(&uid_id_map_arr); + return ret; +} + +int lucene_index_optimize_finish(struct lucene_index *index) +{ + int ret = 0; + + if (IndexReader::isLocked(index->path)) + IndexReader::unlock(index->path); + + IndexWriter *writer = + _CLNEW IndexWriter(index->path, index->analyzer, false); try { - index->reader->close(); + writer->optimize(); } catch (CLuceneError &err) { - lucene_handle_error(index, err, "IndexReader::close()"); + lucene_handle_error(index, err, "IndexWriter::optimize()"); ret = -1; } - - lucene_index_close(index); + _CLDELETE(writer); return ret; } -int lucene_index_lookup(struct lucene_index *index, enum fts_lookup_flags flags, - const char *key, ARRAY_TYPE(seq_range) *result) +// Mostly copy&pasted from CLucene's QueryParser +static Query* getFieldQuery(Analyzer *analyzer, const TCHAR* _field, const TCHAR* queryText) { + // Use the analyzer to get all the tokens, and then build a TermQuery, + // PhraseQuery, or nothing based on the term count + + StringReader reader(queryText); + TokenStream* source = analyzer->tokenStream(_field, &reader); + + CLVector<CL_NS(analysis)::Token*, Deletor::Object<CL_NS(analysis)::Token> > v; + CL_NS(analysis)::Token* t = NULL; + int32_t positionCount = 0; + bool severalTokensAtSamePosition = false; + + while (true) { + t = _CLNEW Token(); + try { + Token* _t = source->next(t); + if (_t == NULL) _CLDELETE(t); + }_CLCATCH_ERR(CL_ERR_IO, _CLLDELETE(source);_CLLDELETE(t);,{ + t = NULL; + }); + if (t == NULL) + break; + v.push_back(t); + if (t->getPositionIncrement() != 0) + positionCount += t->getPositionIncrement(); + else + severalTokensAtSamePosition = true; + } + try { + source->close(); + } + _CLCATCH_ERR_CLEANUP(CL_ERR_IO, {_CLLDELETE(source);_CLLDELETE(t);} ); /* cleanup */ + _CLLDELETE(source); + + if (v.size() == 0) + return NULL; + else if (v.size() == 1) { + Term* tm = _CLNEW Term(_field, v.at(0)->termBuffer()); + Query* ret = _CLNEW PrefixQuery( tm ); + _CLDECDELETE(tm); + return ret; + } else { + if (severalTokensAtSamePosition) { + if (positionCount == 1) { + // no phrase query: + BooleanQuery* q = _CLNEW BooleanQuery(true); + for(size_t i=0; i<v.size(); i++ ){ + Term* tm = _CLNEW Term(_field, v.at(i)->termBuffer()); + q->add(_CLNEW TermQuery(tm), true, BooleanClause::SHOULD); + _CLDECDELETE(tm); + } + return q; + }else { + MultiPhraseQuery* mpq = _CLNEW MultiPhraseQuery(); + CLArrayList<Term*> multiTerms; + int32_t position = -1; + for (size_t i = 0; i < v.size(); i++) { + t = v.at(i); + if (t->getPositionIncrement() > 0 && multiTerms.size() > 0) { + ValueArray<Term*> termsArray(multiTerms.size()); + multiTerms.toArray(termsArray.values); + mpq->add(&termsArray,position); + multiTerms.clear(); + } + position += t->getPositionIncrement(); + multiTerms.push_back(_CLNEW Term(_field, t->termBuffer())); + } + ValueArray<Term*> termsArray(multiTerms.size()); + multiTerms.toArray(termsArray.values); + mpq->add(&termsArray,position); + return mpq; + } + }else { + PhraseQuery* pq = _CLNEW PhraseQuery(); + int32_t position = -1; + + for (size_t i = 0; i < v.size(); i++) { + t = v.at(i); + Term* tm = _CLNEW Term(_field, t->termBuffer()); + position += t->getPositionIncrement(); + pq->add(tm,position); + _CLDECDELETE(tm); + } + return pq; + } + } +} + +static Query * +lucene_get_query(struct lucene_index *index, + const TCHAR *key, const char *value) { - const char *quoted_key; + const TCHAR *wvalue = t_lucene_utf8_to_tchar(value); + return getFieldQuery(index->analyzer, key, wvalue); +} + +static bool +lucene_add_definite_query(struct lucene_index *index, BooleanQuery &query, + struct mail_search_arg *arg, bool and_args) +{ + Query *q; + + if (arg->match_not && !and_args) { + /* FIXME: we could handle this by doing multiple queries.. */ + return false; + } + + switch (arg->type) { + case SEARCH_TEXT: { + BooleanQuery *bq = _CLNEW BooleanQuery(); + Query *q1 = lucene_get_query(index, _T("hdr"), arg->value.str); + Query *q2 = lucene_get_query(index, _T("body"), arg->value.str); + + bq->add(q1, true, BooleanClause::SHOULD); + bq->add(q2, true, BooleanClause::SHOULD); + q = bq; + break; + } + case SEARCH_BODY: + q = lucene_get_query(index, _T("body"), arg->value.str); + break; + case SEARCH_HEADER: + case SEARCH_HEADER_ADDRESS: + case SEARCH_HEADER_COMPRESS_LWSP: + if (!fts_header_want_indexed(arg->hdr_field_name)) + return false; + + q = lucene_get_query(index, + t_lucene_utf8_to_tchar(arg->hdr_field_name), + arg->value.str); + break; + default: + return false; + } + + if (!and_args) + query.add(q, true, BooleanClause::SHOULD); + else if (!arg->match_not) + query.add(q, true, BooleanClause::MUST); + else + query.add(q, true, BooleanClause::MUST_NOT); + return true; +} + +static bool +lucene_add_maybe_query(struct lucene_index *index, BooleanQuery &query, + struct mail_search_arg *arg, bool and_args) +{ + Query *q; + + if (arg->match_not && !and_args) { + /* FIXME: we could handle this by doing multiple queries.. */ + return false; + } + + switch (arg->type) { + case SEARCH_HEADER: + case SEARCH_HEADER_ADDRESS: + case SEARCH_HEADER_COMPRESS_LWSP: + if (fts_header_want_indexed(arg->hdr_field_name)) + return false; + + /* we can check if the search key exists in some header and + filter out the messages that have no chance of matching */ + q = lucene_get_query(index, _T("hdr"), arg->value.str); + break; + default: + return false; + } + + if (!and_args) + query.add(q, true, BooleanClause::SHOULD); + else if (!arg->match_not) + query.add(q, true, BooleanClause::MUST); + else + query.add(q, true, BooleanClause::MUST_NOT); + return true; +} + +static int +lucene_index_search(struct lucene_index *index, + Query &search_query, struct fts_result *result, + ARRAY_TYPE(seq_range) *uids_r) +{ + struct fts_score_map *score; int ret = 0; - i_assert((flags & (FTS_LOOKUP_FLAG_HEADER|FTS_LOOKUP_FLAG_BODY)) != 0); - - if (lucene_index_open_search(index) <= 0) - return -1; - - t_push(); - quoted_key = strchr(key, ' ') == NULL ? - t_strdup_printf("%s*", key) : - t_strdup_printf("\"%s\"", key); - unsigned int keysize = uni_utf8_strlen_n(quoted_key, (size_t)-1) + 1; - wchar_t wkey[keysize]; - lucene_utf8_to_tchar(quoted_key, wkey, keysize); - t_pop(); + BooleanQuery query; + query.add(&search_query, BooleanClause::MUST); - BooleanQuery lookup_query; - Query *content_query1 = NULL, *content_query2 = NULL; - try { - if ((flags & FTS_LOOKUP_FLAG_HEADER) != 0) { - content_query1 = QueryParser::parse(wkey, _T("headers"), - index->analyzer); - lookup_query.add(content_query1, false, false); - } - if ((flags & FTS_LOOKUP_FLAG_BODY) != 0) { - content_query2 = QueryParser::parse(wkey, _T("body"), - index->analyzer); - lookup_query.add(content_query2, false, false); - } - } catch (CLuceneError &err) { - if (getenv("DEBUG") != NULL) { - i_info("lucene: QueryParser::parse(%s) failed: %s", - str_sanitize(key, 40), err.what()); - } - if (content_query1 != NULL) - _CLDELETE(content_query1); - lucene_index_close(index); - return -1; - } - - BooleanQuery query; - Term mailbox_term(_T("box"), index->mailbox_name); + Term mailbox_term(_T("box"), index->mailbox_guid); TermQuery mailbox_query(&mailbox_term); - query.add(&lookup_query, true, false); - query.add(&mailbox_query, true, false); + query.add(&mailbox_query, BooleanClause::MUST); try { Hits *hits = index->searcher->search(&query); - for (int32_t i = 0; i < hits->length(); i++) { + uint32_t last_uid = 0; + if (result != NULL) + result->scores_sorted = true; + + for (size_t i = 0; i < hits->length(); i++) { uint32_t uid; if (lucene_doc_get_uid(index, &hits->doc(i), @@ -602,18 +655,157 @@ break; } - seq_range_array_add(result, 0, uid); + if (result != NULL) { + if (uid < last_uid) + result->scores_sorted = false; + last_uid = uid; + + seq_range_array_add(uids_r, 0, uid); + score = array_append_space(&result->scores); + score->uid = uid; + score->score = hits->score(i); + } } - index->lock_error = FALSE; _CLDELETE(hits); + return ret; } catch (CLuceneError &err) { lucene_handle_error(index, err, "search"); - ret = -1; + return -1; + } +} + +int lucene_index_lookup(struct lucene_index *index, + struct mail_search_arg *args, bool and_args, + struct fts_result *result) +{ + struct mail_search_arg *arg; + + if (lucene_index_open_search(index) <= 0) + return -1; + + BooleanQuery def_query; + bool have_definites = false; + + for (arg = args; arg != NULL; arg = arg->next) { + if (lucene_add_definite_query(index, def_query, arg, and_args)) { + arg->match_always = true; + have_definites = true; + } + } + + if (have_definites) { + if (lucene_index_search(index, def_query, result, + &result->definite_uids) < 0) + return -1; + } + + BooleanQuery maybe_query; + bool have_maybies = false; + + for (arg = args; arg != NULL; arg = arg->next) { + if (lucene_add_maybe_query(index, maybe_query, arg, and_args)) { + arg->match_always = true; + have_maybies = true; + } } - if (content_query1 != NULL) - _CLDELETE(content_query1); - if (content_query2 != NULL) - _CLDELETE(content_query2); - return ret; + if (have_maybies) { + if (lucene_index_search(index, maybe_query, NULL, + &result->maybe_uids) < 0) + return -1; + } + return 0; } + +static int +lucene_index_search_multi(struct lucene_index *index, struct hash_table *guids, + Query &search_query, struct fts_multi_result *result) +{ + struct fts_score_map *score; + int ret = 0; + + BooleanQuery query; + query.add(&search_query, BooleanClause::MUST); + + BooleanQuery mailbox_query; + struct hash_iterate_context *iter; + void *key, *value; + iter = hash_table_iterate_init(guids); + while (hash_table_iterate(iter, &key, &value)) { + Term *term = _CLNEW Term(_T("box"), (wchar_t *)key); + TermQuery *q = _CLNEW TermQuery(term); + mailbox_query.add(q, true, BooleanClause::SHOULD); + } + hash_table_iterate_deinit(&iter); + + query.add(&mailbox_query, BooleanClause::MUST); + try { + Hits *hits = index->searcher->search(&query); + + for (size_t i = 0; i < hits->length(); i++) { + uint32_t uid; + + Field *field = hits->doc(i).getField(_T("box")); + const TCHAR *box_guid = field == NULL ? NULL : field->stringValue(); + if (box_guid == NULL) { + i_error("lucene: Corrupted FTS index %s: No mailbox for document", + index->path); + ret = -1; + break; + } + struct fts_result *br = (struct fts_result *) + hash_table_lookup(guids, (const void *)box_guid); + if (br == NULL) { + i_warning("lucene: Returned unexpected mailbox with GUID %ls", box_guid); + continue; + } + + if (lucene_doc_get_uid(index, &hits->doc(i), + _T("uid"), &uid) < 0) { + ret = -1; + break; + } + + if (!array_is_created(&br->definite_uids)) { + p_array_init(&br->definite_uids, result->pool, 32); + p_array_init(&br->scores, result->pool, 32); + } + seq_range_array_add(&br->definite_uids, 0, uid); + score = array_append_space(&br->scores); + score->uid = uid; + score->score = hits->score(i); + } + _CLDELETE(hits); + return ret; + } catch (CLuceneError &err) { + lucene_handle_error(index, err, "multi search"); + return -1; + } +} +int lucene_index_lookup_multi(struct lucene_index *index, + struct hash_table *guids, + struct mail_search_arg *args, bool and_args, + struct fts_multi_result *result) +{ + struct mail_search_arg *arg; + + if (lucene_index_open_search(index) <= 0) + return -1; + + BooleanQuery def_query; + bool have_definites = false; + + for (arg = args; arg != NULL; arg = arg->next) { + if (lucene_add_definite_query(index, def_query, arg, and_args)) { + arg->match_always = true; + have_definites = true; + } + } + + if (have_definites) { + if (lucene_index_search_multi(index, guids, + def_query, result) < 0) + return -1; + } + return 0; +}
--- a/src/plugins/fts-lucene/lucene-wrapper.h Fri Jul 22 13:13:29 2011 +0300 +++ b/src/plugins/fts-lucene/lucene-wrapper.h Fri Jul 22 13:21:59 2011 +0300 @@ -3,22 +3,35 @@ #include "fts-api-private.h" -struct lucene_index *lucene_index_init(const char *path, const char *lock_path); +#define MAILBOX_GUID_HEX_LENGTH (MAIL_GUID_128_SIZE*2) + +struct lucene_index *lucene_index_init(const char *path); void lucene_index_deinit(struct lucene_index *index); void lucene_index_select_mailbox(struct lucene_index *index, - const char *mailbox_name); + const wchar_t guid[MAILBOX_GUID_HEX_LENGTH]); +void lucene_index_unselect_mailbox(struct lucene_index *index); int lucene_index_get_last_uid(struct lucene_index *index, uint32_t *last_uid_r); +int lucene_index_get_doc_count(struct lucene_index *index, uint32_t *count_r); -int lucene_index_build_init(struct lucene_index *index, uint32_t *last_uid_r); +int lucene_index_build_init(struct lucene_index *index); int lucene_index_build_more(struct lucene_index *index, uint32_t uid, const unsigned char *data, size_t size, - bool headers); + const char *hdr_name); int lucene_index_build_deinit(struct lucene_index *index); -int lucene_index_expunge(struct lucene_index *index, uint32_t uid); +int lucene_index_optimize_scan(struct lucene_index *index, + const ARRAY_TYPE(seq_range) *existing_uids, + ARRAY_TYPE(seq_range) *missing_uids_r); +int lucene_index_optimize_finish(struct lucene_index *index); -int lucene_index_lookup(struct lucene_index *index, enum fts_lookup_flags flags, - const char *key, ARRAY_TYPE(seq_range) *result); +int lucene_index_lookup(struct lucene_index *index, + struct mail_search_arg *args, bool and_args, + struct fts_result *result); + +int lucene_index_lookup_multi(struct lucene_index *index, + struct hash_table *guids, + struct mail_search_arg *args, bool and_args, + struct fts_multi_result *result); #endif
--- a/src/plugins/fts-solr/Makefile.am Fri Jul 22 13:13:29 2011 +0300 +++ b/src/plugins/fts-solr/Makefile.am Fri Jul 22 13:21:59 2011 +0300 @@ -22,6 +22,7 @@ lib21_fts_solr_plugin_la_SOURCES = \ fts-backend-solr.c \ + fts-backend-solr-old.c \ fts-solr-plugin.c \ solr-connection.c
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/plugins/fts-solr/fts-backend-solr-old.c Fri Jul 22 13:21:59 2011 +0300 @@ -0,0 +1,811 @@ +/* Copyright (c) 2006-2011 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "str.h" +#include "hash.h" +#include "strescape.h" +#include "unichar.h" +#include "mail-storage-private.h" +#include "mailbox-list-private.h" +#include "mail-search.h" +#include "fts-api.h" +#include "solr-connection.h" +#include "fts-solr-plugin.h" + +#include <ctype.h> + +#define SOLR_CMDBUF_SIZE (1024*64) +#define SOLR_MAX_MULTI_ROWS 100000 + +struct solr_fts_backend { + struct fts_backend backend; + char *id_username, *id_namespace; + struct mail_namespace *default_ns; +}; + +struct solr_fts_backend_update_context { + struct fts_backend_update_context ctx; + + struct mailbox *cur_box; + char *id_box_name; + + struct solr_connection_post *post; + uint32_t prev_uid, uid_validity; + string_t *cmd, *hdr; + + bool headers_open; + bool documents_added; +}; + +static struct solr_connection *solr_conn = NULL; + +static bool is_valid_xml_char(unichar_t chr) +{ + /* Valid characters in XML: + + #x9 | #xA | #xD | [#x20-#xD7FF] | [#xE000-#xFFFD] | + [#x10000-#x10FFFF] + + This function gets called only for #x80 and higher */ + if (chr > 0xd7ff && chr < 0xe000) + return FALSE; + if (chr > 0xfffd && chr < 0x10000) + return FALSE; + return chr < 0x10ffff; +} + +static void +xml_encode_data(string_t *dest, const unsigned char *data, unsigned int len) +{ + unichar_t chr; + unsigned int i; + + for (i = 0; i < len; i++) { + switch (data[i]) { + case '&': + str_append(dest, "&"); + break; + case '<': + str_append(dest, "<"); + break; + case '>': + str_append(dest, ">"); + break; + case '\t': + case '\n': + case '\r': + /* exceptions to the following control char check */ + str_append_c(dest, data[i]); + break; + default: + if (data[i] < 32) { + /* SOLR doesn't like control characters. + replace them with spaces. */ + str_append_c(dest, ' '); + } else if (data[i] >= 0x80) { + /* make sure the character is valid for XML + so we don't get XML parser errors */ + unsigned int char_len = + uni_utf8_char_bytes(data[i]); + if (i + char_len <= len && + uni_utf8_get_char_n(data + i, char_len, &chr) == 1 && + is_valid_xml_char(chr)) + str_append_n(dest, data + i, char_len); + else { + str_append_n(dest, utf8_replacement_char, + UTF8_REPLACEMENT_CHAR_LEN); + } + i += char_len - 1; + } else { + str_append_c(dest, data[i]); + } + break; + } + } +} + +static void xml_encode(string_t *dest, const char *str) +{ + xml_encode_data(dest, (const unsigned char *)str, strlen(str)); +} + +static const char *solr_escape_id_str(const char *str) +{ + string_t *tmp; + const char *p; + + for (p = str; *p != '\0'; p++) { + if (*p == '/' || *p == '!') + break; + } + if (*p == '\0') + return str; + + tmp = t_str_new(64); + for (p = str; *p != '\0'; p++) { + switch (*p) { + case '/': + str_append(tmp, "!\\"); + break; + case '!': + str_append(tmp, "!!"); + break; + default: + str_append_c(tmp, *p); + break; + } + } + return str_c(tmp); +} + +static void solr_quote(string_t *dest, const char *str) +{ + str_append_c(dest, '"'); + str_append(dest, str_escape(str)); + str_append_c(dest, '"'); +} + +static void solr_quote_http(string_t *dest, const char *str) +{ + str_append(dest, "%22"); + solr_connection_http_escape(solr_conn, dest, str); + str_append(dest, "%22"); +} + +static void fts_solr_set_default_ns(struct solr_fts_backend *backend) +{ + struct mail_namespace *ns = backend->backend.ns; + struct fts_solr_user *fuser = FTS_SOLR_USER_CONTEXT(ns->user); + const struct fts_solr_settings *set = &fuser->set; + const char *str; + + if (backend->default_ns != NULL) + return; + + if (set->default_ns_prefix != NULL) { + backend->default_ns = + mail_namespace_find_prefix(ns->user->namespaces, + set->default_ns_prefix); + if (backend->default_ns == NULL) { + i_error("fts_solr: default_ns setting points to " + "nonexistent namespace"); + } + } + if (backend->default_ns == NULL) { + backend->default_ns = + mail_namespace_find_inbox(ns->user->namespaces); + } + while (backend->default_ns->alias_for != NULL) + backend->default_ns = backend->default_ns->alias_for; + + if (ns != backend->default_ns) { + str = solr_escape_id_str(ns->prefix); + backend->id_namespace = i_strdup(str); + } +} + +static void fts_box_name_get_root(struct mail_namespace **ns, const char **name) +{ + struct mail_namespace *orig_ns = *ns; + + while ((*ns)->alias_for != NULL) + *ns = (*ns)->alias_for; + + if (**name == '\0' && *ns != orig_ns && + ((*ns)->flags & NAMESPACE_FLAG_INBOX_USER) != 0) { + /* ugly workaround to allow selecting INBOX from a Maildir/ + when it's not in the inbox=yes namespace. */ + *name = "INBOX"; + } +} + +static const char * +fts_box_get_root(struct mailbox *box, struct mail_namespace **ns_r) +{ + struct mail_namespace *ns = mailbox_get_namespace(box); + const char *name = box->name; + + fts_box_name_get_root(&ns, &name); + *ns_r = ns; + return name; +} + +static struct fts_backend *fts_backend_solr_alloc(void) +{ + struct solr_fts_backend *backend; + + backend = i_new(struct solr_fts_backend, 1); + backend->backend = fts_backend_solr_old; + return &backend->backend; +} + +static int +fts_backend_solr_init(struct fts_backend *_backend, + const char **error_r ATTR_UNUSED) +{ + struct solr_fts_backend *backend = (struct solr_fts_backend *)_backend; + struct fts_solr_user *fuser = FTS_SOLR_USER_CONTEXT(_backend->ns->user); + const struct fts_solr_settings *set = &fuser->set; + const char *str; + + if (solr_conn == NULL) + solr_conn = solr_connection_init(set->url, set->debug); + + str = solr_escape_id_str(_backend->ns->user->username); + backend->id_username = i_strdup(str); + return 0; +} + +static void fts_backend_solr_deinit(struct fts_backend *_backend) +{ + struct solr_fts_backend *backend = (struct solr_fts_backend *)_backend; + + i_free(backend->id_namespace); + i_free(backend->id_username); + i_free(backend); +} + +static void +solr_add_ns_query(string_t *str, struct solr_fts_backend *backend, + struct mail_namespace *ns, bool neg) +{ + while (ns->alias_for != NULL) + ns = ns->alias_for; + + if (ns == backend->default_ns || *ns->prefix == '\0') { + if (!neg) + str_append(str, " -ns:[* TO *]"); + else + str_append(str, " +ns:[* TO *]"); + } else { + if (!neg) + str_append(str, " +ns:"); + else + str_append(str, " -ns:"); + solr_quote(str, ns->prefix); + } +} + +static void +solr_add_ns_query_http(string_t *str, struct solr_fts_backend *backend, + struct mail_namespace *ns) +{ + string_t *tmp; + + tmp = t_str_new(64); + solr_add_ns_query(tmp, backend, ns, FALSE); + solr_connection_http_escape(solr_conn, str, str_c(tmp)); +} + +static int +fts_backend_solr_get_last_uid_fallback(struct solr_fts_backend *backend, + struct mailbox *box, + uint32_t *last_uid_r) +{ + struct mail_namespace *ns; + struct mailbox_status status; + struct solr_result **results; + const struct seq_range *uidvals; + const char *box_name; + unsigned int count; + string_t *str; + pool_t pool; + int ret = 0; + + str = t_str_new(256); + str_append(str, "fl=uid&rows=1&sort=uid+desc&q="); + + box_name = fts_box_get_root(box, &ns); + + mailbox_get_open_status(box, STATUS_UIDVALIDITY, &status); + str_printfa(str, "uidv:%u+box:", status.uidvalidity); + solr_quote_http(str, box_name); + solr_add_ns_query_http(str, backend, ns); + str_append(str, "+user:"); + solr_quote_http(str, ns->user->username); + + pool = pool_alloconly_create("solr last uid lookup", 1024); + if (solr_connection_select(solr_conn, str_c(str), + pool, &results) < 0) + ret = -1; + else if (results[0] == NULL) { + /* no UIDs */ + *last_uid_r = 0; + } else { + uidvals = array_get(&results[0]->uids, &count); + i_assert(count > 0); + if (count == 1 && uidvals[0].seq1 == uidvals[0].seq2) { + *last_uid_r = uidvals[0].seq1; + } else { + i_error("fts_solr: Last UID lookup returned multiple rows"); + ret = -1; + } + } + pool_unref(&pool); + return ret; +} + +static int +fts_backend_solr_get_last_uid(struct fts_backend *_backend, + struct mailbox *box, uint32_t *last_uid_r) +{ + struct solr_fts_backend *backend = + (struct solr_fts_backend *)_backend; + + if (fts_index_get_last_uid(box, last_uid_r)) + return 0; + + /* either nothing has been indexed, or the index was corrupted. + do it the slow way. */ + if (fts_backend_solr_get_last_uid_fallback(backend, box, last_uid_r) < 0) + return -1; + + (void)fts_index_set_last_uid(box, *last_uid_r); + return 0; +} + +static struct fts_backend_update_context * +fts_backend_solr_update_init(struct fts_backend *_backend) +{ + struct solr_fts_backend *backend = + (struct solr_fts_backend *)_backend; + struct solr_fts_backend_update_context *ctx; + + ctx = i_new(struct solr_fts_backend_update_context, 1); + ctx->ctx.backend = _backend; + ctx->cmd = str_new(default_pool, SOLR_CMDBUF_SIZE); + ctx->hdr = str_new(default_pool, 4096); + fts_solr_set_default_ns(backend); + return &ctx->ctx; +} + +static void xml_encode_id(struct solr_fts_backend_update_context *ctx, + string_t *str, uint32_t uid) +{ + struct solr_fts_backend *backend = + (struct solr_fts_backend *)ctx->ctx.backend; + + if (uid != 0) + str_printfa(str, "%u/", uid); + else + str_append(str, "L/"); + + if (backend->id_namespace != NULL) { + xml_encode(str, backend->id_namespace); + str_append_c(str, '/'); + } + str_printfa(str, "%u/", ctx->uid_validity); + xml_encode(str, backend->id_username); + str_append_c(str, '/'); + xml_encode(str, ctx->id_box_name); +} + +static void +fts_backend_solr_add_doc_prefix(struct solr_fts_backend_update_context *ctx, + uint32_t uid) +{ + struct solr_fts_backend *backend = + (struct solr_fts_backend *)ctx->ctx.backend; + struct mailbox *box = ctx->cur_box; + struct mail_namespace *ns; + const char *box_name; + + ctx->documents_added = TRUE; + + str_printfa(ctx->cmd, "<doc>" + "<field name=\"uid\">%u</field>" + "<field name=\"uidv\">%u</field>", + uid, ctx->uid_validity); + + box_name = fts_box_get_root(box, &ns); + + if (ns != backend->default_ns) { + str_append(ctx->cmd, "<field name=\"ns\">"); + xml_encode(ctx->cmd, ns->prefix); + str_append(ctx->cmd, "</field>"); + } + str_append(ctx->cmd, "<field name=\"box\">"); + xml_encode(ctx->cmd, box_name); + str_append(ctx->cmd, "</field><field name=\"user\">"); + xml_encode(ctx->cmd, ns->user->username); + str_append(ctx->cmd, "</field>"); +} + +static int +fts_backed_solr_build_commit(struct solr_fts_backend_update_context *ctx) +{ + if (ctx->post == NULL) + return 0; + + str_append(ctx->cmd, "</doc></add>"); + + solr_connection_post_more(ctx->post, str_data(ctx->cmd), + str_len(ctx->cmd)); + return solr_connection_post_end(ctx->post); +} + +static int +fts_backend_solr_update_deinit(struct fts_backend_update_context *_ctx) +{ + struct solr_fts_backend_update_context *ctx = + (struct solr_fts_backend_update_context *)_ctx; + const char *str; + int ret; + + ret = fts_backed_solr_build_commit(ctx); + + /* commit and wait until the documents we just indexed are + visible to the following search */ + str = t_strdup_printf("<commit waitFlush=\"false\" " + "waitSearcher=\"%s\"/>", + ctx->documents_added ? "true" : "false"); + if (solr_connection_post(solr_conn, str) < 0) + ret = -1; + + str_free(&ctx->cmd); + str_free(&ctx->hdr); + i_free(ctx->id_box_name); + i_free(ctx); + return ret; +} + +static void +fts_backend_solr_update_set_mailbox(struct fts_backend_update_context *_ctx, + struct mailbox *box) +{ + struct solr_fts_backend_update_context *ctx = + (struct solr_fts_backend_update_context *)_ctx; + struct mailbox_status status; + struct mail_namespace *ns; + + ctx->cur_box = box; + ctx->uid_validity = 0; + i_free_and_null(ctx->id_box_name); + + if (box != NULL) { + ctx->id_box_name = i_strdup(fts_box_get_root(box, &ns)); + + mailbox_get_open_status(box, STATUS_UIDVALIDITY, &status); + ctx->uid_validity = status.uidvalidity; + } +} + +static void +fts_backend_solr_update_expunge(struct fts_backend_update_context *_ctx, + uint32_t uid) +{ + struct solr_fts_backend_update_context *ctx = + (struct solr_fts_backend_update_context *)_ctx; + + T_BEGIN { + string_t *cmd; + + cmd = t_str_new(256); + str_append(cmd, "<delete><id>"); + xml_encode_id(ctx, cmd, uid); + str_append(cmd, "</id></delete>"); + + (void)solr_connection_post(solr_conn, str_c(cmd)); + } T_END; +} + +static void +fts_backend_solr_uid_changed(struct solr_fts_backend_update_context *ctx, + uint32_t uid) +{ + if (ctx->post == NULL) { + i_assert(ctx->prev_uid == 0); + + ctx->post = solr_connection_post_begin(solr_conn); + str_append(ctx->cmd, "<add>"); + } else { + ctx->headers_open = FALSE; + str_append(ctx->cmd, "<field name=\"hdr\">"); + str_append_str(ctx->cmd, ctx->hdr); + str_append(ctx->cmd, "</field>"); + str_truncate(ctx->hdr, 0); + + str_append(ctx->cmd, "</doc>"); + } + ctx->prev_uid = uid; + + fts_backend_solr_add_doc_prefix(ctx, uid); + str_printfa(ctx->cmd, "<field name=\"id\">"); + xml_encode_id(ctx, ctx->cmd, uid); + str_append(ctx->cmd, "</field>"); +} + +static bool +fts_backend_solr_update_set_build_key(struct fts_backend_update_context *_ctx, + const struct fts_backend_build_key *key) +{ + struct solr_fts_backend_update_context *ctx = + (struct solr_fts_backend_update_context *)_ctx; + + if (key->uid != ctx->prev_uid) + fts_backend_solr_uid_changed(ctx, key->uid); + + switch (key->type) { + case FTS_BACKEND_BUILD_KEY_HDR: + case FTS_BACKEND_BUILD_KEY_MIME_HDR: + xml_encode(ctx->hdr, key->hdr_name); + str_append(ctx->hdr, ": "); + ctx->headers_open = TRUE; + break; + case FTS_BACKEND_BUILD_KEY_BODY_PART: + ctx->headers_open = FALSE; + str_append(ctx->cmd, "<field name=\"body\">"); + break; + case FTS_BACKEND_BUILD_KEY_BODY_PART_BINARY: + i_unreached(); + } + return TRUE; +} + +static void +fts_backend_solr_update_unset_build_key(struct fts_backend_update_context *_ctx) +{ + struct solr_fts_backend_update_context *ctx = + (struct solr_fts_backend_update_context *)_ctx; + + if (!ctx->headers_open) + str_append(ctx->cmd, "</field>"); + else + str_append_c(ctx->hdr, '\n'); +} + +static int +fts_backend_solr_update_build_more(struct fts_backend_update_context *_ctx, + const unsigned char *data, size_t size) +{ + struct solr_fts_backend_update_context *ctx = + (struct solr_fts_backend_update_context *)_ctx; + + xml_encode_data(ctx->cmd, data, size); + if (str_len(ctx->cmd) > SOLR_CMDBUF_SIZE-128) { + solr_connection_post_more(ctx->post, str_data(ctx->cmd), + str_len(ctx->cmd)); + str_truncate(ctx->cmd, 0); + } + return 0; +} + +static int fts_backend_solr_refresh(struct fts_backend *backend ATTR_UNUSED) +{ + return 0; +} + +static int fts_backend_solr_optimize(struct fts_backend *backend ATTR_UNUSED) +{ + return 0; +} + +static bool +solr_add_definite_query(string_t *str, struct mail_search_arg *arg) +{ + switch (arg->type) { + case SEARCH_TEXT: { + if (arg->match_not) + str_append_c(str, '-'); + str_append(str, "(hdr:"); + solr_quote_http(str, arg->value.str); + str_append(str, "+OR+body:"); + solr_quote_http(str, arg->value.str); + str_append(str, ")"); + break; + } + case SEARCH_BODY: + if (arg->match_not) + str_append_c(str, '-'); + str_append(str, "body:"); + solr_quote_http(str, arg->value.str); + break; + default: + return FALSE; + } + return TRUE; +} + +static bool +solr_add_definite_query_args(string_t *str, struct mail_search_arg *arg, + bool and_args) +{ + unsigned int last_len; + + last_len = str_len(str); + for (; arg != NULL; arg = arg->next) { + if (solr_add_definite_query(str, arg)) { + arg->match_always = TRUE; + last_len = str_len(str); + if (and_args) + str_append(str, "+AND+"); + else + str_append(str, "+OR+"); + } + } + if (str_len(str) == last_len) + return FALSE; + + str_truncate(str, last_len); + return TRUE; +} + +static int +fts_backend_solr_lookup(struct fts_backend *_backend, struct mailbox *box, + struct mail_search_arg *args, bool and_args, + struct fts_result *result) +{ + struct solr_fts_backend *backend = + (struct solr_fts_backend *)_backend; + struct mail_namespace *ns; + struct mailbox_status status; + string_t *str; + const char *box_name; + pool_t pool; + struct solr_result **results; + int ret; + + fts_solr_set_default_ns(backend); + mailbox_get_open_status(box, STATUS_UIDVALIDITY | STATUS_UIDNEXT, + &status); + + str = t_str_new(256); + str_printfa(str, "fl=uid,score&rows=%u&sort=uid+asc&q=", + status.uidnext); + + if (!solr_add_definite_query_args(str, args, and_args)) { + /* can't search this query */ + return 0; + } + + /* use a separate filter query for selecting the mailbox. it shouldn't + affect the score and there could be some caching benefits too. */ + str_append(str, "&fq=%2Buser:"); + solr_quote_http(str, box->storage->user->username); + box_name = fts_box_get_root(box, &ns); + str_printfa(str, "+%%2Buidv:%u+%%2Bbox:", status.uidvalidity); + solr_quote_http(str, box_name); + solr_add_ns_query_http(str, backend, ns); + + pool = pool_alloconly_create("fts solr search", 1024); + ret = solr_connection_select(solr_conn, str_c(str), pool, &results); + if (ret == 0 && results[0] != NULL) { + array_append_array(&result->definite_uids, &results[0]->uids); + array_append_array(&result->scores, &results[0]->scores); + } + result->scores_sorted = TRUE; + pool_unref(&pool); + return ret; +} + +static char * +mailbox_get_id(struct solr_fts_backend *backend, struct mail_namespace *ns, + const char *mailbox, uint32_t uidvalidity) +{ + string_t *str = t_str_new(64); + + str_printfa(str, "%u\001", uidvalidity); + str_append(str, mailbox); + if (ns != backend->default_ns) + str_printfa(str, "\001%s", ns->prefix); + return str_c_modifiable(str); +} + +static int +solr_search_multi(struct solr_fts_backend *backend, string_t *str, + struct mailbox *const boxes[], + struct fts_multi_result *result) +{ + struct solr_result **solr_results; + struct fts_result *fts_result; + ARRAY_DEFINE(fts_results, struct fts_result); + struct mail_namespace *ns; + struct mailbox_status status; + struct hash_table *mailboxes; + struct mailbox *box; + const char *box_name; + char *box_id; + unsigned int i, len; + + /* use a separate filter query for selecting the mailbox. it shouldn't + affect the score and there could be some caching benefits too. */ + str_append(str, "&fq=%2Buser:"); + if (backend->backend.ns->owner != NULL) + solr_quote_http(str, backend->backend.ns->owner->username); + else + str_append(str, "%22%22"); + + mailboxes = hash_table_create(default_pool, default_pool, 0, + str_hash, (hash_cmp_callback_t *)strcmp); + str_append(str, "%2B("); + len = str_len(str); + for (i = 0; boxes[i] != NULL; i++) { + if (str_len(str) != len) + str_append(str, "+OR+"); + + box_name = fts_box_get_root(boxes[i], &ns); + mailbox_get_open_status(boxes[i], STATUS_UIDVALIDITY, &status); + str_printfa(str, "%%2B(%%2Buidv:%u+%%2Bbox:", status.uidvalidity); + solr_quote_http(str, box_name); + solr_add_ns_query_http(str, backend, ns); + str_append_c(str, ')'); + + box_id = mailbox_get_id(backend, ns, box_name, status.uidvalidity); + hash_table_insert(mailboxes, box_id, boxes[i]); + } + str_append_c(str, ')'); + + if (solr_connection_select(solr_conn, str_c(str), + result->pool, &solr_results) < 0) { + hash_table_destroy(&mailboxes); + return -1; + } + + p_array_init(&fts_results, result->pool, 32); + for (i = 0; solr_results[i] != NULL; i++) { + box = hash_table_lookup(mailboxes, solr_results[i]->box_id); + if (box == NULL) { + i_warning("fts_solr: Lookup returned unexpected mailbox " + "with id=%s", solr_results[i]->box_id); + continue; + } + fts_result = array_append_space(&fts_results); + fts_result->box = box; + fts_result->definite_uids = solr_results[i]->uids; + fts_result->scores = solr_results[i]->scores; + fts_result->scores_sorted = TRUE; + } + (void)array_append_space(&fts_results); + result->box_results = array_idx_modifiable(&fts_results, 0); + hash_table_destroy(&mailboxes); + return 0; +} + +static int +fts_backend_solr_lookup_multi(struct fts_backend *_backend, + struct mailbox *const boxes[], + struct mail_search_arg *args, bool and_args, + struct fts_multi_result *result) +{ + struct solr_fts_backend *backend = + (struct solr_fts_backend *)_backend; + string_t *str; + + fts_solr_set_default_ns(backend); + + str = t_str_new(256); + str_printfa(str, "fl=ns,box,uidv,uid,score&rows=%u&sort=box+asc,uid+asc&q=", + SOLR_MAX_MULTI_ROWS); + + if (solr_add_definite_query_args(str, args, and_args)) { + if (solr_search_multi(backend, str, boxes, result) < 0) + return -1; + } + /* FIXME: maybe_uids could be handled also with some more work.. */ + return 0; +} + +struct fts_backend fts_backend_solr_old = { + .name = "solr_old", + .flags = 0, + + { + fts_backend_solr_alloc, + fts_backend_solr_init, + fts_backend_solr_deinit, + fts_backend_solr_get_last_uid, + fts_backend_solr_update_init, + fts_backend_solr_update_deinit, + fts_backend_solr_update_set_mailbox, + fts_backend_solr_update_expunge, + fts_backend_solr_update_set_build_key, + fts_backend_solr_update_unset_build_key, + fts_backend_solr_update_build_more, + fts_backend_solr_refresh, + fts_backend_solr_optimize, + fts_backend_default_can_lookup, + fts_backend_solr_lookup, + fts_backend_solr_lookup_multi + } +};
--- a/src/plugins/fts-solr/fts-backend-solr.c Fri Jul 22 13:13:29 2011 +0300 +++ b/src/plugins/fts-solr/fts-backend-solr.c Fri Jul 22 13:21:59 2011 +0300 @@ -3,78 +3,42 @@ #include "lib.h" #include "array.h" #include "str.h" +#include "hash.h" #include "strescape.h" #include "unichar.h" #include "mail-storage-private.h" #include "mailbox-list-private.h" -#include "fts-mailbox.h" +#include "mail-search.h" +#include "fts-api.h" #include "solr-connection.h" #include "fts-solr-plugin.h" #include <ctype.h> #define SOLR_CMDBUF_SIZE (1024*64) -#define SOLR_MAX_ROWS 100000 -#define FTS_SOLR_MAX_BOX_INC_PATTERNS 5 -#define FTS_SOLR_MAX_BOX_EXC_PATTERNS 5 +#define SOLR_MAX_MULTI_ROWS 100000 struct solr_fts_backend { struct fts_backend backend; - char *id_username, *id_namespace, *id_box_name; - struct mail_namespace *default_ns; }; -struct solr_fts_backend_build_context { - struct fts_backend_build_context ctx; +struct solr_fts_backend_update_context { + struct fts_backend_update_context ctx; + + struct mailbox *cur_box; + char box_guid[MAILBOX_GUID_HEX_LENGTH+1]; struct solr_connection_post *post; - uint32_t prev_uid, uid_validity; - string_t *cmd; - bool headers; - bool field_open; -}; + uint32_t prev_uid; + string_t *cmd, *hdr, *hdr_fields; -struct solr_virtual_uid_map_context { - struct fts_backend *backend; - struct mailbox *box; -}; - -struct fts_backend_solr_get_last_uids_context { - struct fts_backend *backend; - pool_t pool; - ARRAY_TYPE(fts_backend_uid_map) *last_uids; - - struct mailbox *box; + bool headers_open; + bool cur_header_index; + bool documents_added; }; static struct solr_connection *solr_conn = NULL; -static void fts_box_name_get_root(struct mail_namespace **ns, const char **name) -{ - struct mail_namespace *orig_ns = *ns; - - while ((*ns)->alias_for != NULL) - *ns = (*ns)->alias_for; - - if (**name == '\0' && *ns != orig_ns && - ((*ns)->flags & NAMESPACE_FLAG_INBOX_USER) != 0) { - /* ugly workaround to allow selecting INBOX from a Maildir/ - when it's not in the inbox=yes namespace. */ - *name = "INBOX"; - } -} - -static const char * -fts_box_get_root(struct mailbox *box, struct mail_namespace **ns_r) -{ - struct mail_namespace *ns = mailbox_get_namespace(box); - const char *name = box->name; - - fts_box_name_get_root(&ns, &name); - *ns_r = ns; - return name; -} - static bool is_valid_xml_char(unichar_t chr) { /* Valid characters in XML: @@ -145,42 +109,6 @@ xml_encode_data(dest, (const unsigned char *)str, strlen(str)); } -static const char *solr_escape_id_str(const char *str) -{ - string_t *tmp; - const char *p; - - for (p = str; *p != '\0'; p++) { - if (*p == '/' || *p == '!') - break; - } - if (*p == '\0') - return str; - - tmp = t_str_new(64); - for (p = str; *p != '\0'; p++) { - switch (*p) { - case '/': - str_append(tmp, "!\\"); - break; - case '!': - str_append(tmp, "!!"); - break; - default: - str_append_c(tmp, *p); - break; - } - } - return str_c(tmp); -} - -static void solr_quote(string_t *dest, const char *str) -{ - str_append_c(dest, '"'); - str_append(dest, str_escape(str)); - str_append_c(dest, '"'); -} - static void solr_quote_http(string_t *dest, const char *str) { str_append(dest, "%22"); @@ -188,484 +116,323 @@ str_append(dest, "%22"); } -static struct fts_backend * -fts_backend_solr_init(struct mailbox *box) +static struct fts_backend *fts_backend_solr_alloc(void) { - struct fts_solr_user *fuser = - FTS_SOLR_USER_CONTEXT(box->storage->user); - const struct fts_solr_settings *set = &fuser->set; struct solr_fts_backend *backend; - struct mail_namespace *ns; - const char *str, *box_name; + + backend = i_new(struct solr_fts_backend, 1); + backend->backend = fts_backend_solr; + return &backend->backend; +} - - box_name = fts_box_get_root(box, &ns); - i_assert(*box_name != '\0'); +static int +fts_backend_solr_init(struct fts_backend *_backend, + const char **error_r ATTR_UNUSED) +{ + struct fts_solr_user *fuser = FTS_SOLR_USER_CONTEXT(_backend->ns->user); + const struct fts_solr_settings *set = &fuser->set; if (solr_conn == NULL) solr_conn = solr_connection_init(set->url, set->debug); - - backend = i_new(struct solr_fts_backend, 1); - if (set->default_ns_prefix != NULL) { - backend->default_ns = - mail_namespace_find_prefix(ns->user->namespaces, - set->default_ns_prefix); - if (backend->default_ns == NULL) { - i_fatal("fts_solr: default_ns setting points to " - "nonexistent namespace"); - } - } else { - backend->default_ns = - mail_namespace_find_inbox(ns->user->namespaces); - } - while (backend->default_ns->alias_for != NULL) - backend->default_ns = backend->default_ns->alias_for; - - str = solr_escape_id_str(ns->user->username); - backend->id_username = i_strdup(str); - if (ns != backend->default_ns) { - str = solr_escape_id_str(ns->prefix); - backend->id_namespace = i_strdup(str); - } - backend->id_box_name = i_strdup(box_name); - backend->backend = fts_backend_solr; - - return &backend->backend; + return 0; } static void fts_backend_solr_deinit(struct fts_backend *_backend) { struct solr_fts_backend *backend = (struct solr_fts_backend *)_backend; - i_free(backend->id_box_name); - i_free(backend->id_namespace); - i_free(backend->id_username); i_free(backend); } -static void -solr_add_ns_query(string_t *str, struct fts_backend *_backend, - struct mail_namespace *ns, bool neg) +static int +get_last_uid_fallback(struct fts_backend *_backend, struct mailbox *box, + uint32_t *last_uid_r) { - struct solr_fts_backend *backend = (struct solr_fts_backend *)_backend; - - while (ns->alias_for != NULL) - ns = ns->alias_for; - - if (ns == backend->default_ns || *ns->prefix == '\0') { - if (!neg) - str_append(str, " -ns:[* TO *]"); - else - str_append(str, " +ns:[* TO *]"); - } else { - if (!neg) - str_append(str, " +ns:"); - else - str_append(str, " -ns:"); - solr_quote(str, ns->prefix); - } -} - -static void -solr_add_ns_query_http(string_t *str, struct fts_backend *backend, - struct mail_namespace *ns) -{ - string_t *tmp; - - tmp = t_str_new(64); - solr_add_ns_query(tmp, backend, ns, FALSE); - solr_connection_http_escape(solr_conn, str, str_c(tmp)); -} - -static int fts_backend_solr_get_last_uid_fallback(struct fts_backend *backend, - uint32_t *last_uid_r) -{ - struct mailbox *box = backend->box; - struct mail_namespace *ns; - struct mailbox_status status; - ARRAY_TYPE(seq_range) uids; const struct seq_range *uidvals; - const char *box_name; + const char *box_guid; unsigned int count; + struct solr_result **results; string_t *str; + pool_t pool; + int ret = 0; str = t_str_new(256); str_append(str, "fl=uid&rows=1&sort=uid+desc&q="); - box_name = fts_box_get_root(box, &ns); - - mailbox_get_open_status(box, STATUS_UIDVALIDITY, &status); - str_printfa(str, "uidv:%u+box:", status.uidvalidity); - solr_quote_http(str, box_name); - solr_add_ns_query_http(str, backend, ns); - str_append(str, "+user:"); - solr_quote_http(str, ns->user->username); - - t_array_init(&uids, 1); - if (solr_connection_select(solr_conn, str_c(str), - NULL, NULL, &uids, NULL) < 0) + if (fts_mailbox_get_guid(box, &box_guid) < 0) return -1; - uidvals = array_get(&uids, &count); - if (count == 0) { - /* nothing indexed yet for this mailbox */ + str_printfa(str, "box:%s+user:", box_guid); + if (_backend->ns->owner != NULL) + solr_quote_http(str, _backend->ns->owner->username); + else + str_append(str, "%22%22"); + + pool = pool_alloconly_create("solr last uid lookup", 1024); + if (solr_connection_select(solr_conn, str_c(str), + pool, &results) < 0) + ret = -1; + else if (results[0] == NULL) { + /* no UIDs */ *last_uid_r = 0; - } else if (count == 1 && uidvals[0].seq1 == uidvals[0].seq2) { - *last_uid_r = uidvals[0].seq1; } else { - i_error("fts_solr: Last UID lookup returned multiple rows"); + uidvals = array_get(&results[0]->uids, &count); + i_assert(count > 0); + if (count == 1 && uidvals[0].seq1 == uidvals[0].seq2) { + *last_uid_r = uidvals[0].seq1; + } else { + i_error("fts_solr: Last UID lookup returned multiple rows"); + ret = -1; + } + } + pool_unref(&pool); + return ret; +} + +static int +fts_backend_solr_get_last_uid(struct fts_backend *_backend, + struct mailbox *box, uint32_t *last_uid_r) +{ + if (fts_index_get_last_uid(box, last_uid_r)) + return 0; + + /* either nothing has been indexed, or the index was corrupted. + do it the slow way. */ + if (get_last_uid_fallback(_backend, box, last_uid_r) < 0) return -1; - } + + (void)fts_index_set_last_uid(box, *last_uid_r); return 0; } -static int fts_backend_solr_get_last_uid(struct fts_backend *backend, - uint32_t *last_uid_r) +static struct fts_backend_update_context * +fts_backend_solr_update_init(struct fts_backend *_backend) { - struct mailbox *box = backend->box; - struct mail_namespace *ns; - struct mailbox_status status; - ARRAY_TYPE(seq_range) uids; - const struct seq_range *uidvals; - const char *box_name; - unsigned int count; - string_t *str; - - str = t_str_new(256); - str_append(str, "fl=uid&rows=1&q=last_uid:TRUE+"); - - box_name = fts_box_get_root(box, &ns); + struct solr_fts_backend_update_context *ctx; - mailbox_get_open_status(box, STATUS_UIDVALIDITY, &status); - str_printfa(str, "uidv:%u+box:", status.uidvalidity); - solr_quote_http(str, box_name); - solr_add_ns_query_http(str, backend, ns); - str_append(str, "+user:"); - solr_quote_http(str, ns->user->username); - - t_array_init(&uids, 1); - if (solr_connection_select(solr_conn, str_c(str), - NULL, NULL, &uids, NULL) < 0) - return -1; - - uidvals = array_get(&uids, &count); - if (count == 0) { - /* either nothing is indexed or we're converting from an - older database format without the last_uid fields */ - return fts_backend_solr_get_last_uid_fallback(backend, - last_uid_r); - } else if (count == 1 && uidvals[0].seq1 == uidvals[0].seq2) { - *last_uid_r = uidvals[0].seq1; - } else { - i_error("fts_solr: Last UID lookup returned multiple rows"); - return -1; - } - return 0; + ctx = i_new(struct solr_fts_backend_update_context, 1); + ctx->ctx.backend = _backend; + ctx->cmd = str_new(default_pool, SOLR_CMDBUF_SIZE); + ctx->hdr = str_new(default_pool, 4096); + ctx->hdr_fields = str_new(default_pool, 1024); + return &ctx->ctx; } -static struct mail_namespace * -solr_get_namespaces(struct fts_backend *_backend, - struct mailbox *box, const char *ns_prefix) -{ - struct solr_fts_backend *backend = (struct solr_fts_backend *)_backend; - struct mail_namespace *namespaces = box->storage->user->namespaces; - - if (ns_prefix == NULL) - return backend->default_ns; - else - return mail_namespace_find_prefix(namespaces, ns_prefix); -} - -static bool -solr_virtual_get_last_uids(const char *ns_prefix, const char *mailbox, - uint32_t uidvalidity, uint32_t *uid, void *context) +static void xml_encode_id(struct solr_fts_backend_update_context *ctx, + string_t *str, uint32_t uid) { - struct fts_backend_solr_get_last_uids_context *ctx = context; - struct fts_backend_uid_map *map; - struct mail_namespace *ns; - - ns = solr_get_namespaces(ctx->backend, ctx->box, ns_prefix); - for (; ns != NULL; ns = ns->alias_chain_next) T_BEGIN { - const char *vname = mailbox_list_get_vname(ns->list, mailbox); - map = array_append_space(ctx->last_uids); - map->mailbox = p_strdup(ctx->pool, vname); - map->uidvalidity = uidvalidity; - map->uid = *uid; - } T_END; - return FALSE; -} - -static void -solr_add_pattern(string_t *str, const struct mailbox_virtual_pattern *pattern) -{ - struct mail_namespace *ns = pattern->ns; - const char *name, *p; - - name = pattern->pattern; - fts_box_name_get_root(&ns, &name); - - if (strcmp(name, "*") == 0) { - str_append(str, "[* TO *]"); - return; - } - - /* first check if there are any wildcards in the pattern */ - for (p = name; *p != '\0'; p++) { - if (*p == '%' || *p == '*') - break; - } - if (*p == '\0') { - /* full mailbox name */ - solr_quote(str, name); - return; - } - - /* there are at least some wildcards. */ - for (p = name; *p != '\0'; p++) { - if (*p == '%' || *p == '*') { - if (p == name || (p[-1] != '%' && p[-1] != '*')) - str_append_c(str, '*'); - } else { - if (!i_isalnum(*p)) - str_append_c(str, '\\'); - str_append_c(str, *p); - } + str_printfa(str, "%u/%s", uid, ctx->box_guid); + if (ctx->ctx.backend->ns->owner != NULL) { + str_append_c(str, '/'); + xml_encode(str, ctx->ctx.backend->ns->owner->username); } } static void -fts_backend_solr_filter_mailboxes(struct fts_backend *_backend, - string_t *str, struct mailbox *box) +fts_backend_solr_doc_open(struct solr_fts_backend_update_context *ctx, + uint32_t uid) +{ + ctx->documents_added = TRUE; + + str_printfa(ctx->cmd, "<doc>" + "<field name=\"uid\">%u</field>" + "<field name=\"box\">%s</field>", + uid, ctx->box_guid); + str_append(ctx->cmd, "<field name=\"user\">"); + if (ctx->ctx.backend->ns->owner != NULL) + xml_encode(ctx->cmd, ctx->ctx.backend->ns->owner->username); + str_append(ctx->cmd, "</field>"); + + str_printfa(ctx->cmd, "<field name=\"id\">"); + xml_encode_id(ctx, ctx->cmd, uid); + str_append(ctx->cmd, "</field>"); +} + +static void +fts_backend_solr_doc_close(struct solr_fts_backend_update_context *ctx) +{ + ctx->headers_open = FALSE; + if (str_len(ctx->hdr) > 0) { + str_append(ctx->cmd, "<field name=\"hdr\">"); + str_append_str(ctx->cmd, ctx->hdr); + str_append(ctx->cmd, "</field>"); + str_truncate(ctx->hdr, 0); + } + if (str_len(ctx->hdr_fields) > 0) { + str_append_str(ctx->cmd, ctx->hdr_fields); + str_truncate(ctx->hdr_fields, 0); + } + str_append(ctx->cmd, "</doc>"); +} + +static int +fts_backed_solr_build_commit(struct solr_fts_backend_update_context *ctx) { - ARRAY_TYPE(mailbox_virtual_patterns) includes_arr, excludes_arr; - struct mail_namespace *ns; - const struct mailbox_virtual_pattern *includes, *excludes; - unsigned int i, inc_count, exc_count; - string_t *fq; + if (ctx->post == NULL) + return 0; + + fts_backend_solr_doc_close(ctx); + str_append(ctx->cmd, "</add>"); + + solr_connection_post_more(ctx->post, str_data(ctx->cmd), + str_len(ctx->cmd)); + return solr_connection_post_end(ctx->post); +} + +static int +fts_backend_solr_update_deinit(struct fts_backend_update_context *_ctx) +{ + struct solr_fts_backend_update_context *ctx = + (struct solr_fts_backend_update_context *)_ctx; + const char *str; + int ret = _ctx->failed ? -1 : 0; + + if (fts_backed_solr_build_commit(ctx) < 0) + ret = -1; + + /* commit and wait until the documents we just indexed are + visible to the following search */ + str = t_strdup_printf("<commit waitFlush=\"false\" " + "waitSearcher=\"%s\"/>", + ctx->documents_added ? "true" : "false"); + if (solr_connection_post(solr_conn, str) < 0) + ret = -1; + + str_free(&ctx->cmd); + str_free(&ctx->hdr); + str_free(&ctx->hdr_fields); + i_free(ctx); + return ret; +} + +static void +fts_backend_solr_update_set_mailbox(struct fts_backend_update_context *_ctx, + struct mailbox *box) +{ + struct solr_fts_backend_update_context *ctx = + (struct solr_fts_backend_update_context *)_ctx; + const char *box_guid; + + ctx->cur_box = box; - t_array_init(&includes_arr, 16); - t_array_init(&excludes_arr, 16); - fts_mailbox_get_virtual_box_patterns(box, &includes_arr, &excludes_arr); - includes = array_get(&includes_arr, &inc_count); - excludes = array_get(&excludes_arr, &exc_count); - i_assert(inc_count > 0); + if (box != NULL) { + if (fts_mailbox_get_guid(box, &box_guid) < 0) + _ctx->failed = TRUE; + + i_assert(strlen(box_guid) == sizeof(ctx->box_guid)-1); + memcpy(ctx->box_guid, box_guid, sizeof(ctx->box_guid)-1); + } else { + memset(ctx->box_guid, 0, sizeof(ctx->box_guid)); + } +} + +static void +fts_backend_solr_update_expunge(struct fts_backend_update_context *_ctx, + uint32_t uid) +{ + struct solr_fts_backend_update_context *ctx = + (struct solr_fts_backend_update_context *)_ctx; + + T_BEGIN { + string_t *cmd; + + cmd = t_str_new(256); + str_append(cmd, "<delete><id>"); + xml_encode_id(ctx, cmd, uid); + str_append(cmd, "</id></delete>"); + + (void)solr_connection_post(solr_conn, str_c(cmd)); + } T_END; +} + +static void +fts_backend_solr_uid_changed(struct solr_fts_backend_update_context *ctx, + uint32_t uid) +{ + if (ctx->post == NULL) { + i_assert(ctx->prev_uid == 0); + + ctx->post = solr_connection_post_begin(solr_conn); + str_append(ctx->cmd, "<add>"); + } else { + fts_backend_solr_doc_close(ctx); + } + ctx->prev_uid = uid; + fts_backend_solr_doc_open(ctx, uid); +} - /* First see if there are any patterns that begin with a wildcard. - Solr doesn't allow them, so in that case we'll need to return - all mailboxes. */ - for (i = 0; i < inc_count; i++) { - if (*includes[i].pattern == '*' || - *includes[i].pattern == '%') - break; +static bool +fts_backend_solr_update_set_build_key(struct fts_backend_update_context *_ctx, + const struct fts_backend_build_key *key) +{ + struct solr_fts_backend_update_context *ctx = + (struct solr_fts_backend_update_context *)_ctx; + + if (key->uid != ctx->prev_uid) + fts_backend_solr_uid_changed(ctx, key->uid); + + switch (key->type) { + case FTS_BACKEND_BUILD_KEY_HDR: + if (fts_header_want_indexed(key->hdr_name)) { + ctx->cur_header_index = TRUE; + str_printfa(ctx->hdr_fields, "<field name=\"%s\">", + t_str_lcase(key->hdr_name)); + } + /* fall through */ + case FTS_BACKEND_BUILD_KEY_MIME_HDR: + xml_encode(ctx->hdr, key->hdr_name); + str_append(ctx->hdr, ": "); + ctx->headers_open = TRUE; + break; + case FTS_BACKEND_BUILD_KEY_BODY_PART: + ctx->headers_open = FALSE; + str_append(ctx->cmd, "<field name=\"body\">"); + break; + case FTS_BACKEND_BUILD_KEY_BODY_PART_BINARY: + i_unreached(); + } + return TRUE; +} + +static void +fts_backend_solr_update_unset_build_key(struct fts_backend_update_context *_ctx) +{ + struct solr_fts_backend_update_context *ctx = + (struct solr_fts_backend_update_context *)_ctx; + + if (!ctx->headers_open) + str_append(ctx->cmd, "</field>"); + else { + /* this is called individually for each header line. + headers are finished only when key changes to body */ + str_append_c(ctx->hdr, '\n'); } - fq = t_str_new(128); - if (i == inc_count && inc_count <= FTS_SOLR_MAX_BOX_INC_PATTERNS) { - /* we can filter what mailboxes we want returned */ - str_append_c(fq, '('); - for (i = 0; i < inc_count; i++) { - if (i != 0) - str_append(fq, " OR +"); - str_append_c(fq, '('); - str_append(fq, "+box:"); - solr_add_pattern(fq, &includes[i]); - solr_add_ns_query(fq, _backend, includes[i].ns, FALSE); - str_append_c(fq, ')'); - } - str_append_c(fq, ')'); - } - exc_count = I_MIN(FTS_SOLR_MAX_BOX_EXC_PATTERNS, exc_count); - for (i = 0; i < exc_count; i++) { - if (str_len(fq) > 0) - str_append_c(fq, ' '); - str_append(fq, "NOT ("); - str_append(fq, "box:"); - solr_add_pattern(fq, &excludes[i]); - - for (ns = excludes[i].ns; ns->alias_for != NULL; ) - ns = ns->alias_for; - solr_add_ns_query(fq, _backend, ns, FALSE); - str_append_c(fq, ')'); - } - if (str_len(fq) > 0) { - str_append(str, "&fq="); - solr_connection_http_escape(solr_conn, str, str_c(fq)); + if (ctx->cur_header_index) { + str_append(ctx->hdr_fields, "</field>"); + ctx->cur_header_index = FALSE; } } static int -fts_backend_solr_get_all_last_uids(struct fts_backend *backend, pool_t pool, - ARRAY_TYPE(fts_backend_uid_map) *last_uids) -{ - struct fts_backend_solr_get_last_uids_context ctx; - string_t *str; - - memset(&ctx, 0, sizeof(ctx)); - ctx.backend = backend; - ctx.pool = pool; - ctx.last_uids = last_uids; - ctx.box = backend->box; - - str = t_str_new(256); - str_printfa(str, "fl=uid,box,uidv,ns&rows=%u&q=last_uid:TRUE+user:", - SOLR_MAX_ROWS); - solr_quote_http(str, backend->box->storage->user->username); - fts_backend_solr_filter_mailboxes(backend, str, backend->box); - - return solr_connection_select(solr_conn, str_c(str), - solr_virtual_get_last_uids, &ctx, - NULL, NULL); -} - -static int -fts_backend_solr_build_init(struct fts_backend *backend, uint32_t *last_uid_r, - struct fts_backend_build_context **ctx_r) +fts_backend_solr_update_build_more(struct fts_backend_update_context *_ctx, + const unsigned char *data, size_t size) { - struct solr_fts_backend_build_context *ctx; - struct mailbox_status status; - - *last_uid_r = (uint32_t)-1; - - ctx = i_new(struct solr_fts_backend_build_context, 1); - ctx->ctx.backend = backend; - ctx->cmd = str_new(default_pool, SOLR_CMDBUF_SIZE); - - mailbox_get_open_status(backend->box, STATUS_UIDVALIDITY, &status); - ctx->uid_validity = status.uidvalidity; - - *ctx_r = &ctx->ctx; - return 0; -} - -static void -fts_backend_solr_add_doc_prefix(struct solr_fts_backend_build_context *ctx, - uint32_t uid) -{ - struct solr_fts_backend *backend = - (struct solr_fts_backend *)ctx->ctx.backend; - struct mailbox *box = ctx->ctx.backend->box; - struct mail_namespace *ns; - const char *box_name; - - str_printfa(ctx->cmd, "<doc>" - "<field name=\"uid\">%u</field>" - "<field name=\"uidv\">%u</field>", - uid, ctx->uid_validity); - - box_name = fts_box_get_root(box, &ns); + struct solr_fts_backend_update_context *ctx = + (struct solr_fts_backend_update_context *)_ctx; - if (ns != backend->default_ns) { - str_append(ctx->cmd, "<field name=\"ns\">"); - xml_encode(ctx->cmd, ns->prefix); - str_append(ctx->cmd, "</field>"); - } - str_append(ctx->cmd, "<field name=\"box\">"); - xml_encode(ctx->cmd, box_name); - str_append(ctx->cmd, "</field><field name=\"user\">"); - xml_encode(ctx->cmd, ns->user->username); - str_append(ctx->cmd, "</field>"); -} - -static void xml_encode_id(string_t *str, struct fts_backend *_backend, - uint32_t uid, uint32_t uid_validity) -{ - struct solr_fts_backend *backend = (struct solr_fts_backend *)_backend; - - if (uid != 0) - str_printfa(str, "%u/", uid); - else - str_append(str, "L/"); - if (backend->id_namespace != NULL) { - xml_encode(str, backend->id_namespace); - str_append_c(str, '/'); - } - str_printfa(str, "%u/", uid_validity); - xml_encode(str, backend->id_username); - str_append_c(str, '/'); - xml_encode(str, backend->id_box_name); -} + if (_ctx->failed) + return -1; -static void -fts_backend_solr_uid_changed(struct solr_fts_backend_build_context *ctx, - uint32_t uid) -{ - if (ctx->post == NULL) { - ctx->post = solr_connection_post_begin(solr_conn); - str_append(ctx->cmd, "<add>"); + if (ctx->headers_open) { + if (ctx->cur_header_index) + xml_encode_data(ctx->hdr_fields, data, size); + xml_encode_data(ctx->hdr, data, size); } else { - if (ctx->field_open) { - str_append(ctx->cmd, "</field>"); - ctx->field_open = FALSE; - } - str_append(ctx->cmd, "</doc>"); - } - ctx->prev_uid = uid; - ctx->headers = FALSE; - - fts_backend_solr_add_doc_prefix(ctx, uid); - str_printfa(ctx->cmd, "<field name=\"id\">"); - xml_encode_id(ctx->cmd, ctx->ctx.backend, uid, ctx->uid_validity); - str_append(ctx->cmd, "</field>"); -} - -static void -fts_backend_solr_build_hdr(struct fts_backend_build_context *_ctx, - uint32_t uid) -{ - struct solr_fts_backend_build_context *ctx = - (struct solr_fts_backend_build_context *)_ctx; - - if (uid != ctx->prev_uid) - fts_backend_solr_uid_changed(ctx, uid); - else { - i_assert(!ctx->headers); - - if (ctx->field_open) { - str_append(ctx->cmd, "</field>"); - ctx->field_open = FALSE; - } + i_assert(!ctx->cur_header_index); + xml_encode_data(ctx->cmd, data, size); } - i_assert(!ctx->field_open); - ctx->field_open = TRUE; - ctx->headers = TRUE; - str_append(ctx->cmd, "<field name=\"hdr\">"); -} - -static bool -fts_backend_solr_build_body_begin(struct fts_backend_build_context *_ctx, - uint32_t uid, const char *content_type, - const char *content_disposition ATTR_UNUSED) -{ - struct solr_fts_backend_build_context *ctx = - (struct solr_fts_backend_build_context *)_ctx; - - if (!fts_backend_default_can_index(content_type)) - return FALSE; - - if (uid != ctx->prev_uid) - fts_backend_solr_uid_changed(ctx, uid); - else { - /* body comes first, then headers */ - i_assert(!ctx->headers); - } - - if (!ctx->field_open) { - ctx->field_open = TRUE; - ctx->headers = FALSE; - str_append(ctx->cmd, "<field name=\"body\">"); - } - return TRUE; -} - -static int -fts_backend_solr_build_more(struct fts_backend_build_context *_ctx, - const unsigned char *data, size_t size) -{ - struct solr_fts_backend_build_context *ctx = - (struct solr_fts_backend_build_context *)_ctx; - - xml_encode_data(ctx->cmd, data, size); if (str_len(ctx->cmd) > SOLR_CMDBUF_SIZE-128) { solr_connection_post_more(ctx->post, str_data(ctx->cmd), str_len(ctx->cmd)); @@ -674,217 +441,290 @@ return 0; } -static int -fts_backed_solr_build_commit(struct solr_fts_backend_build_context *ctx) -{ - int ret; - - if (ctx->post == NULL) - return 0; - - if (ctx->field_open) { - str_append(ctx->cmd, "</field>"); - ctx->field_open = FALSE; - } - str_append(ctx->cmd, "</doc>"); - - /* Update the mailbox's last_uid field, replacing the existing - document. Note that since there is no locking, it's possible that - if another session is indexing at the same time, the last_uid value - may shrink. This doesn't really matter, we'll simply do more work - in future by reindexing some messages. */ - fts_backend_solr_add_doc_prefix(ctx, ctx->prev_uid); - str_printfa(ctx->cmd, "<field name=\"last_uid\">TRUE</field>" - "<field name=\"id\">"); - xml_encode_id(ctx->cmd, ctx->ctx.backend, 0, ctx->uid_validity); - str_append(ctx->cmd, "</field></doc></add>"); - - solr_connection_post_more(ctx->post, str_data(ctx->cmd), - str_len(ctx->cmd)); - ret = solr_connection_post_end(ctx->post); - /* commit and wait until the documents we just indexed are - visible to the following search */ - if (solr_connection_post(solr_conn, "<commit waitFlush=\"false\" " - "waitSearcher=\"true\"/>") < 0) - ret = -1; - return ret; -} - -static int -fts_backend_solr_build_deinit(struct fts_backend_build_context *_ctx) -{ - struct solr_fts_backend_build_context *ctx = - (struct solr_fts_backend_build_context *)_ctx; - int ret; - - ret = fts_backed_solr_build_commit(ctx); - str_free(&ctx->cmd); - i_free(ctx); - return ret; -} - -static void -fts_backend_solr_expunge(struct fts_backend *backend, struct mail *mail) -{ - struct mailbox_status status; - - mailbox_get_open_status(mail->box, STATUS_UIDVALIDITY, &status); - - T_BEGIN { - string_t *cmd; - - cmd = t_str_new(256); - str_append(cmd, "<delete><id>"); - xml_encode_id(cmd, backend, mail->uid, status.uidvalidity); - str_append(cmd, "</id></delete>"); - - (void)solr_connection_post(solr_conn, str_c(cmd)); - } T_END; -} - -static void -fts_backend_solr_expunge_finish(struct fts_backend *backend ATTR_UNUSED, - struct mailbox *box ATTR_UNUSED, - bool committed ATTR_UNUSED) -{ - solr_connection_post(solr_conn, - "<commit waitFlush=\"false\" waitSearcher=\"false\"/>"); -} - static int fts_backend_solr_refresh(struct fts_backend *backend ATTR_UNUSED) { return 0; } -static bool solr_virtual_uid_map(const char *ns_prefix, const char *mailbox, - uint32_t uidvalidity, uint32_t *uid, - void *context) +static int fts_backend_solr_optimize(struct fts_backend *backend ATTR_UNUSED) +{ + return 0; +} + +static bool +solr_add_definite_query(string_t *str, struct mail_search_arg *arg) { - struct solr_virtual_uid_map_context *ctx = context; - struct mail_namespace *ns; - bool convert_inbox, ret; + switch (arg->type) { + case SEARCH_TEXT: { + if (arg->match_not) + str_append_c(str, '-'); + str_append(str, "(hdr:"); + solr_quote_http(str, arg->value.str); + str_append(str, "+OR+body:"); + solr_quote_http(str, arg->value.str); + str_append(str, ")"); + break; + } + case SEARCH_BODY: + if (arg->match_not) + str_append_c(str, '-'); + str_append(str, "body:"); + solr_quote_http(str, arg->value.str); + break; + case SEARCH_HEADER: + case SEARCH_HEADER_ADDRESS: + case SEARCH_HEADER_COMPRESS_LWSP: + if (!fts_header_want_indexed(arg->hdr_field_name)) + return FALSE; - ns = solr_get_namespaces(ctx->backend, ctx->box, ns_prefix); - convert_inbox = (ns->flags & NAMESPACE_FLAG_INBOX_USER) != 0 && - strcmp(mailbox, "INBOX") == 0; - for (; ns != NULL; ns = ns->alias_chain_next) { - T_BEGIN { - const char *vname = convert_inbox ? ns->prefix : - mailbox_list_get_vname(ns->list, mailbox); - ret = fts_mailbox_get_virtual_uid(ctx->box, vname, - uidvalidity, - *uid, uid); - } T_END; - if (ret) - return TRUE; + if (arg->match_not) + str_append_c(str, '-'); + str_append(str, arg->hdr_field_name); + str_append_c(str, ':'); + solr_quote_http(str, arg->value.str); + break; + default: + return FALSE; } - return FALSE; + return TRUE; +} + +static bool +solr_add_definite_query_args(string_t *str, struct mail_search_arg *arg, + bool and_args) +{ + unsigned int last_len; + + last_len = str_len(str); + for (; arg != NULL; arg = arg->next) { + if (solr_add_definite_query(str, arg)) { + arg->match_always = TRUE; + last_len = str_len(str); + if (and_args) + str_append(str, "+AND+"); + else + str_append(str, "+OR+"); + } + } + if (str_len(str) == last_len) + return FALSE; + + str_truncate(str, last_len); + return TRUE; } -static int fts_backend_solr_lookup(struct fts_backend_lookup_context *ctx, - ARRAY_TYPE(seq_range) *definite_uids, - ARRAY_TYPE(seq_range) *maybe_uids, - ARRAY_TYPE(fts_score_map) *scores) +static bool +solr_add_maybe_query(string_t *str, struct mail_search_arg *arg) +{ + switch (arg->type) { + case SEARCH_HEADER: + case SEARCH_HEADER_ADDRESS: + case SEARCH_HEADER_COMPRESS_LWSP: + if (fts_header_want_indexed(arg->hdr_field_name)) + return FALSE; + if (arg->match_not) { + /* all matches would be definite, but all non-matches + would be maybies. too much trouble to optimize. */ + return FALSE; + } + + /* we can check if the search key exists in some header and + filter out the messages that have no chance of matching */ + str_append(str, "hdr:"); + solr_quote_http(str, arg->value.str); + break; + default: + return FALSE; + } + return TRUE; +} + +static bool +solr_add_maybe_query_args(string_t *str, struct mail_search_arg *arg, + bool and_args) { - struct mailbox *box = ctx->backend->box; - struct mail_namespace *ns; - struct solr_virtual_uid_map_context uid_map_ctx; - const struct fts_backend_lookup_field *fields; - const char *box_name; - unsigned int i, count; + unsigned int last_len; + + last_len = str_len(str); + for (; arg != NULL; arg = arg->next) { + if (solr_add_maybe_query(str, arg)) { + arg->match_always = TRUE; + last_len = str_len(str); + if (and_args) + str_append(str, "+AND+"); + else + str_append(str, "+OR+"); + } + } + if (str_len(str) == last_len) + return FALSE; + + str_truncate(str, last_len); + return TRUE; +} + +static int solr_search(struct fts_backend *_backend, string_t *str, + const char *box_guid, ARRAY_TYPE(seq_range) *uids_r, + ARRAY_TYPE(fts_score_map) *scores_r) +{ + pool_t pool = pool_alloconly_create("fts solr search", 1024); + struct solr_result **results; + int ret; + + /* use a separate filter query for selecting the mailbox. it shouldn't + affect the score and there could be some caching benefits too. */ + str_printfa(str, "&fq=%%2Bbox:%s+%%2Buser:", box_guid); + if (_backend->ns->owner != NULL) + solr_quote_http(str, _backend->ns->owner->username); + else + str_append(str, "%22%22"); + + ret = solr_connection_select(solr_conn, str_c(str), pool, &results); + if (ret == 0 && results[0] != NULL) { + array_append_array(uids_r, &results[0]->uids); + array_append_array(scores_r, &results[0]->scores); + } + pool_unref(&pool); + return ret; +} + +static int +fts_backend_solr_lookup(struct fts_backend *_backend, struct mailbox *box, + struct mail_search_arg *args, bool and_args, + struct fts_result *result) +{ struct mailbox_status status; string_t *str; - bool virtual; + const char *box_guid; + unsigned int prefix_len; - virtual = strcmp(box->storage->name, "virtual") == 0; - mailbox_get_open_status(box, STATUS_UIDVALIDITY, &status); + if (fts_mailbox_get_guid(box, &box_guid) < 0) + return -1; + mailbox_get_open_status(box, STATUS_UIDNEXT, &status); str = t_str_new(256); - if (!virtual) { - str_printfa(str, "fl=uid,score&rows=%u&sort=uid+asc&q=", - status.uidnext); - } else { - str_printfa(str, "fl=uid,score,box,uidv,ns&rows=%u" - "&sort=box+asc,uid+asc&q=", - SOLR_MAX_ROWS); - } - - /* build a lucene search query from the fields */ - fields = array_get(&ctx->fields, &count); - for (i = 0; i < count; i++) { - if (i > 0) - str_append_c(str, '+'); - - if ((fields[i].flags & FTS_LOOKUP_FLAG_INVERT) != 0) - str_append_c(str, '-'); + str_printfa(str, "fl=uid,score&rows=%u&sort=uid+asc&q=", + status.uidnext); + prefix_len = str_len(str); - if ((fields[i].flags & FTS_LOOKUP_FLAG_HEADER) == 0) { - /* body only */ - i_assert((fields[i].flags & FTS_LOOKUP_FLAG_BODY) != 0); - str_append(str, "body:"); - solr_quote_http(str, fields[i].key); - } else if ((fields[i].flags & FTS_LOOKUP_FLAG_BODY) == 0) { - /* header only */ - str_append(str, "hdr:"); - solr_quote_http(str, fields[i].key); - } else { - /* both */ - str_append(str, "(body:"); - solr_quote_http(str, fields[i].key); - str_append(str, "+OR+hdr:"); - solr_quote_http(str, fields[i].key); - str_append_c(str, ')'); - } + if (solr_add_definite_query_args(str, args, and_args)) { + if (solr_search(_backend, str, box_guid, + &result->definite_uids, &result->scores) < 0) + return -1; } + str_truncate(str, prefix_len); + if (solr_add_maybe_query_args(str, args, and_args)) { + if (solr_search(_backend, str, box_guid, + &result->maybe_uids, &result->scores) < 0) + return -1; + } + result->scores_sorted = TRUE; + return 0; +} + +static int +solr_search_multi(struct fts_backend *_backend, string_t *str, + struct mailbox *const boxes[], + struct fts_multi_result *result) +{ + struct solr_result **solr_results; + struct fts_result *fts_result; + ARRAY_DEFINE(fts_results, struct fts_result); + struct hash_table *mailboxes; + struct mailbox *box; + const char *box_guid; + unsigned int i, len; /* use a separate filter query for selecting the mailbox. it shouldn't affect the score and there could be some caching benefits too. */ str_append(str, "&fq=%2Buser:"); - solr_quote_http(str, box->storage->user->username); - if (virtual) - fts_backend_solr_filter_mailboxes(ctx->backend, str, box); - else { - box_name = fts_box_get_root(box, &ns); - str_printfa(str, "+%%2Buidv:%u+%%2Bbox:", status.uidvalidity); - solr_quote_http(str, box_name); - solr_add_ns_query_http(str, ctx->backend, ns); + if (_backend->ns->owner != NULL) + solr_quote_http(str, _backend->ns->owner->username); + else + str_append(str, "%22%22"); + + mailboxes = hash_table_create(default_pool, default_pool, 0, + str_hash, (hash_cmp_callback_t *)strcmp); + str_append(str, "%2B("); + len = str_len(str); + for (i = 0; boxes[i] != NULL; i++) { + if (fts_mailbox_get_guid(boxes[i], &box_guid) < 0) + continue; + + if (str_len(str) != len) + str_append(str, "+OR+"); + str_printfa(str, "box:%s", box_guid); + hash_table_insert(mailboxes, t_strdup_noconst(box_guid), + boxes[i]); + } + str_append_c(str, ')'); + + if (solr_connection_select(solr_conn, str_c(str), + result->pool, &solr_results) < 0) { + hash_table_destroy(&mailboxes); + return -1; } - array_clear(maybe_uids); - if (!virtual) { - return solr_connection_select(solr_conn, str_c(str), NULL, NULL, - definite_uids, scores); - } else { - memset(&uid_map_ctx, 0, sizeof(uid_map_ctx)); - uid_map_ctx.backend = ctx->backend; - uid_map_ctx.box = box; - return solr_connection_select(solr_conn, str_c(str), - solr_virtual_uid_map, - &uid_map_ctx, - definite_uids, scores); + p_array_init(&fts_results, result->pool, 32); + for (i = 0; solr_results[i] != NULL; i++) { + box = hash_table_lookup(mailboxes, solr_results[i]->box_id); + if (box == NULL) { + i_warning("fts_solr: Lookup returned unexpected mailbox " + "with guid=%s", solr_results[i]->box_id); + continue; + } + fts_result = array_append_space(&fts_results); + fts_result->box = box; + fts_result->definite_uids = solr_results[i]->uids; + fts_result->scores = solr_results[i]->scores; + fts_result->scores_sorted = TRUE; } + (void)array_append_space(&fts_results); + result->box_results = array_idx_modifiable(&fts_results, 0); + hash_table_destroy(&mailboxes); + return 0; +} + +static int +fts_backend_solr_lookup_multi(struct fts_backend *backend, + struct mailbox *const boxes[], + struct mail_search_arg *args, bool and_args, + struct fts_multi_result *result) +{ + string_t *str; + + str = t_str_new(256); + str_printfa(str, "fl=box,uid,score&rows=%u&sort=box+asc,uid+asc&q=", + SOLR_MAX_MULTI_ROWS); + + if (solr_add_definite_query_args(str, args, and_args)) { + if (solr_search_multi(backend, str, boxes, result) < 0) + return -1; + } + /* FIXME: maybe_uids could be handled also with some more work.. */ + return 0; } struct fts_backend fts_backend_solr = { .name = "solr", - .flags = FTS_BACKEND_FLAG_VIRTUAL_LOOKUPS, + .flags = 0, { + fts_backend_solr_alloc, fts_backend_solr_init, fts_backend_solr_deinit, fts_backend_solr_get_last_uid, - fts_backend_solr_get_all_last_uids, - fts_backend_solr_build_init, - fts_backend_solr_build_hdr, - fts_backend_solr_build_body_begin, - NULL, - fts_backend_solr_build_more, - fts_backend_solr_build_deinit, - fts_backend_solr_expunge, - fts_backend_solr_expunge_finish, + fts_backend_solr_update_init, + fts_backend_solr_update_deinit, + fts_backend_solr_update_set_mailbox, + fts_backend_solr_update_expunge, + fts_backend_solr_update_set_build_key, + fts_backend_solr_update_unset_build_key, + fts_backend_solr_update_build_more, fts_backend_solr_refresh, - NULL, - NULL, - fts_backend_solr_lookup + fts_backend_solr_optimize, + fts_backend_default_can_lookup, + fts_backend_solr_lookup, + fts_backend_solr_lookup_multi } };
--- a/src/plugins/fts-solr/fts-solr-plugin.c Fri Jul 22 13:13:29 2011 +0300 +++ b/src/plugins/fts-solr/fts-solr-plugin.c Fri Jul 22 13:21:59 2011 +0300 @@ -73,12 +73,14 @@ void fts_solr_plugin_init(struct module *module) { fts_backend_register(&fts_backend_solr); + fts_backend_register(&fts_backend_solr_old); mail_storage_hooks_add(module, &fts_solr_mail_storage_hooks); } void fts_solr_plugin_deinit(void) { fts_backend_unregister(fts_backend_solr.name); + fts_backend_unregister(fts_backend_solr_old.name); mail_storage_hooks_remove(&fts_solr_mail_storage_hooks); }
--- a/src/plugins/fts-solr/fts-solr-plugin.h Fri Jul 22 13:13:29 2011 +0300 +++ b/src/plugins/fts-solr/fts-solr-plugin.h Fri Jul 22 13:21:59 2011 +0300 @@ -19,6 +19,7 @@ extern const char *fts_solr_plugin_dependencies[]; extern struct fts_backend fts_backend_solr; +extern struct fts_backend fts_backend_solr_old; extern MODULE_CONTEXT_DEFINE(fts_solr_user_module, &mail_user_module_register); void fts_solr_plugin_init(struct module *module);
--- a/src/plugins/fts-solr/solr-connection.c Fri Jul 22 13:13:29 2011 +0300 +++ b/src/plugins/fts-solr/solr-connection.c Fri Jul 22 13:21:59 2011 +0300 @@ -4,6 +4,7 @@ #include "lib.h" #include "array.h" +#include "hash.h" #include "str.h" #include "strescape.h" #include "solr-connection.h" @@ -37,11 +38,10 @@ float score; char *mailbox, *ns; - solr_uid_map_callback_t *callback; - void *context; - - ARRAY_TYPE(seq_range) *uids; - ARRAY_TYPE(fts_score_map) *scores; + pool_t result_pool; + /* box_id -> solr_result */ + struct hash_table *mailboxes; + ARRAY_DEFINE(results, struct solr_result *); }; struct solr_connection_post { @@ -269,28 +269,57 @@ } } +static struct solr_result * +solr_result_get(struct solr_lookup_xml_context *ctx, const char *box_id) +{ + struct solr_result *result; + char *box_id_dup; + + result = hash_table_lookup(ctx->mailboxes, box_id); + if (result != NULL) + return result; + + box_id_dup = p_strdup(ctx->result_pool, box_id); + result = p_new(ctx->result_pool, struct solr_result, 1); + result->box_id = box_id_dup; + p_array_init(&result->uids, ctx->result_pool, 32); + p_array_init(&result->scores, ctx->result_pool, 32); + hash_table_insert(ctx->mailboxes, box_id_dup, result); + array_append(&ctx->results, &result, 1); + return result; +} + static void solr_lookup_add_doc(struct solr_lookup_xml_context *ctx) { struct fts_score_map *score; + struct solr_result *result; + const char *box_id; if (ctx->uid == 0) { i_error("fts_solr: Query didn't return uid"); return; } - if (ctx->callback != NULL) { - if (ctx->mailbox == NULL) { - i_error("fts_solr: Query didn't return mailbox"); - return; - } - if (!ctx->callback(ctx->ns, ctx->mailbox, ctx->uidvalidity, - &ctx->uid, ctx->context)) - return; + if (ctx->mailbox == NULL) { + /* looking up from a single mailbox only */ + box_id = ""; + } else if (ctx->uidvalidity != 0) { + /* old style lookup */ + string_t *str = t_str_new(64); + str_printfa(str, "%u\001", ctx->uidvalidity); + str_append(str, ctx->mailbox); + if (ctx->ns != NULL) + str_printfa(str, "\001%s", ctx->ns); + box_id = str_c(str); + } else { + /* new style lookup */ + box_id = ctx->mailbox; } + result = solr_result_get(ctx, box_id); - seq_range_array_add(ctx->uids, 0, ctx->uid); - if (ctx->scores != NULL && ctx->score != 0) { - score = array_append_space(ctx->scores); + seq_range_array_add(&result->uids, 0, ctx->uid); + if (ctx->score != 0) { + score = array_append_space(&result->scores); score->uid = ctx->uid; score->score = ctx->score; } @@ -302,9 +331,19 @@ i_assert(ctx->depth >= (int)ctx->state); + if (ctx->state == SOLR_XML_RESPONSE_STATE_CONTENT && + ctx->content_state == SOLR_XML_CONTENT_STATE_MAILBOX && + ctx->mailbox == NULL) { + /* mailbox is namespace prefix */ + ctx->mailbox = i_strdup(""); + } + if (ctx->depth == (int)ctx->state) { - if (ctx->state == SOLR_XML_RESPONSE_STATE_DOC) - solr_lookup_add_doc(ctx); + if (ctx->state == SOLR_XML_RESPONSE_STATE_DOC) { + T_BEGIN { + solr_lookup_add_doc(ctx); + } T_END; + } ctx->state--; ctx->content_state = SOLR_XML_CONTENT_STATE_NONE; } @@ -367,21 +406,21 @@ } int solr_connection_select(struct solr_connection *conn, const char *query, - solr_uid_map_callback_t *callback, void *context, - ARRAY_TYPE(seq_range) *uids, - ARRAY_TYPE(fts_score_map) *scores) + pool_t pool, struct solr_result ***box_results_r) { struct solr_lookup_xml_context solr_lookup_context; CURLcode ret; long httpret; + int parse_ret; i_assert(!conn->posting); memset(&solr_lookup_context, 0, sizeof(solr_lookup_context)); - solr_lookup_context.uids = uids; - solr_lookup_context.scores = scores; - solr_lookup_context.callback = callback; - solr_lookup_context.context = context; + solr_lookup_context.result_pool = pool; + solr_lookup_context.mailboxes = + hash_table_create(default_pool, default_pool, 0, + str_hash, (hash_cmp_callback_t *)strcmp); + p_array_init(&solr_lookup_context.results, pool, 32); i_free_and_null(conn->http_failure); conn->xml_failed = FALSE; @@ -407,7 +446,12 @@ i_error("fts_solr: Lookup failed: %s", conn->http_failure); return -1; } - return solr_xml_parse(conn, NULL, 0, TRUE); + parse_ret = solr_xml_parse(conn, NULL, 0, TRUE); + hash_table_destroy(&solr_lookup_context.mailboxes); + + (void)array_append_space(&solr_lookup_context.results); + *box_results_r = array_idx_modifiable(&solr_lookup_context.results, 0); + return parse_ret; } struct solr_connection_post *
--- a/src/plugins/fts-solr/solr-connection.h Fri Jul 22 13:13:29 2011 +0300 +++ b/src/plugins/fts-solr/solr-connection.h Fri Jul 22 13:21:59 2011 +0300 @@ -4,10 +4,12 @@ #include "seq-range-array.h" #include "fts-api.h" -/* Returns TRUE if UID conversion was done, FALSE if uid should be skipped. */ -typedef bool solr_uid_map_callback_t(const char *ns_prefix, const char *mailbox, - uint32_t uidvalidity, uint32_t *uid, - void *context); +struct solr_result { + const char *box_id; + + ARRAY_TYPE(seq_range) uids; + ARRAY_TYPE(fts_score_map) scores; +}; struct solr_connection *solr_connection_init(const char *url, bool debug); void solr_connection_deinit(struct solr_connection *conn); @@ -16,9 +18,7 @@ const char *str); int solr_connection_select(struct solr_connection *conn, const char *query, - solr_uid_map_callback_t *callback, void *context, - ARRAY_TYPE(seq_range) *uids, - ARRAY_TYPE(fts_score_map) *scores); + pool_t pool, struct solr_result ***box_results_r); int solr_connection_post(struct solr_connection *conn, const char *cmd); struct solr_connection_post *
--- a/src/plugins/fts-squat/fts-backend-squat.c Fri Jul 22 13:13:29 2011 +0300 +++ b/src/plugins/fts-squat/fts-backend-squat.c Fri Jul 22 13:21:59 2011 +0300 @@ -2,6 +2,7 @@ #include "lib.h" #include "array.h" +#include "str.h" #include "mail-user.h" #include "mail-namespace.h" #include "mail-storage-private.h" @@ -15,59 +16,104 @@ struct squat_fts_backend { struct fts_backend backend; + + struct mailbox *box; struct squat_trie *trie; + + unsigned int partial_len, full_len; + bool refresh; }; -struct squat_fts_backend_build_context { - struct fts_backend_build_context ctx; +struct squat_fts_backend_update_context { + struct fts_backend_update_context ctx; struct squat_trie_build_context *build_ctx; + enum squat_index_type squat_type; uint32_t uid; + string_t *hdr; + + bool failed; }; -static void -fts_backend_squat_set(struct squat_fts_backend *backend, const char *str) +static struct fts_backend *fts_backend_squat_alloc(void) { - const char *const *tmp; + struct squat_fts_backend *backend; + + backend = i_new(struct squat_fts_backend, 1); + backend->backend = fts_backend_squat; + return &backend->backend; +} + +static int +fts_backend_squat_init(struct fts_backend *_backend, const char **error_r) +{ + struct squat_fts_backend *backend = + (struct squat_fts_backend *)_backend; + const char *const *tmp, *env; unsigned int len; - for (tmp = t_strsplit_spaces(str, " "); *tmp != NULL; tmp++) { + env = mail_user_plugin_getenv(_backend->ns->user, "fts_squat"); + if (env == NULL) + return 0; + + for (tmp = t_strsplit_spaces(env, " "); *tmp != NULL; tmp++) { if (strncmp(*tmp, "partial=", 8) == 0) { if (str_to_uint(*tmp + 8, &len) < 0 || len == 0) { - i_fatal("fts_squat: Invalid partial len: %s", - *tmp + 8); + *error_r = t_strdup_printf( + "Invalid partial length: %s", *tmp + 8); + return -1; } - squat_trie_set_partial_len(backend->trie, len); + backend->partial_len = len; } else if (strncmp(*tmp, "full=", 5) == 0) { if (str_to_uint(*tmp + 5, &len) < 0 || len == 0) { - i_fatal("fts_squat: Invalid full len: %s", - *tmp + 5); + *error_r = t_strdup_printf( + "Invalid full length: %s", *tmp + 5); + return -1; } - squat_trie_set_full_len(backend->trie, len); + backend->full_len = len; } else { - i_fatal("fts_squat: Invalid setting: %s", *tmp); + *error_r = t_strdup_printf("Invalid setting: %s", *tmp); + return -1; } } + return 0; +} + +static void +fts_backend_squat_unset_box(struct squat_fts_backend *backend) +{ + if (backend->trie != NULL) + squat_trie_deinit(&backend->trie); + backend->box = NULL; } -static struct fts_backend *fts_backend_squat_init(struct mailbox *box) +static void fts_backend_squat_deinit(struct fts_backend *_backend) +{ + struct squat_fts_backend *backend = + (struct squat_fts_backend *)_backend; + + fts_backend_squat_unset_box(backend); + i_free(backend); +} + +static void +fts_backend_squat_set_box(struct squat_fts_backend *backend, + struct mailbox *box) { const struct mailbox_permissions *perm = mailbox_get_permissions(box); - struct squat_fts_backend *backend; struct mail_storage *storage; struct mailbox_status status; - const char *path, *env; + const char *path; enum squat_index_flags flags = 0; + if (backend->box == box) + return; + fts_backend_squat_unset_box(backend); + storage = mailbox_get_storage(box); path = mailbox_list_get_path(box->list, box->name, MAILBOX_LIST_PATH_TYPE_INDEX); - if (*path == '\0') { - /* in-memory indexes */ - if (storage->set->mail_debug) - i_debug("fts squat: Disabled with in-memory indexes"); - return NULL; - } + i_assert(*path != '\0'); /* fts already checked this */ mailbox_get_open_status(box, STATUS_UIDVALIDITY, &status); if (storage->set->mmap_disable) @@ -77,8 +123,6 @@ if (storage->set->dotlock_use_excl) flags |= SQUAT_INDEX_FLAG_DOTLOCK_USE_EXCL; - backend = i_new(struct squat_fts_backend, 1); - backend->backend = fts_backend_squat; backend->trie = squat_trie_init(t_strconcat(path, "/"SQUAT_FILE_PREFIX, NULL), status.uidvalidity, @@ -86,86 +130,33 @@ flags, perm->file_create_mode, perm->file_create_gid); - env = mail_user_plugin_getenv(box->storage->user, "fts_squat"); - if (env != NULL) - fts_backend_squat_set(backend, env); - return &backend->backend; + if (backend->partial_len != 0) + squat_trie_set_partial_len(backend->trie, backend->partial_len); + if (backend->full_len != 0) + squat_trie_set_full_len(backend->trie, backend->full_len); + backend->box = box; } -static void fts_backend_squat_deinit(struct fts_backend *_backend) -{ - struct squat_fts_backend *backend = - (struct squat_fts_backend *)_backend; - - squat_trie_deinit(&backend->trie); - i_free(backend); -} - -static int fts_backend_squat_get_last_uid(struct fts_backend *_backend, - uint32_t *last_uid_r) +static int +fts_backend_squat_get_last_uid(struct fts_backend *_backend, + struct mailbox *box, uint32_t *last_uid_r) { struct squat_fts_backend *backend = (struct squat_fts_backend *)_backend; + fts_backend_squat_set_box(backend, box); return squat_trie_get_last_uid(backend->trie, last_uid_r); } -static int -fts_backend_squat_build_init(struct fts_backend *_backend, uint32_t *last_uid_r, - struct fts_backend_build_context **ctx_r) +static struct fts_backend_update_context * +fts_backend_squat_update_init(struct fts_backend *_backend) { - struct squat_fts_backend *backend = - (struct squat_fts_backend *)_backend; - struct squat_fts_backend_build_context *ctx; - struct squat_trie_build_context *build_ctx; - - if (squat_trie_build_init(backend->trie, last_uid_r, &build_ctx) < 0) - return -1; - - ctx = i_new(struct squat_fts_backend_build_context, 1); - ctx->ctx.backend = _backend; - ctx->build_ctx = build_ctx; - - *ctx_r = &ctx->ctx; - return 0; -} - -static void -fts_backend_squat_build_hdr(struct fts_backend_build_context *_ctx, - uint32_t uid) -{ - struct squat_fts_backend_build_context *ctx = - (struct squat_fts_backend_build_context *)_ctx; + struct squat_fts_backend_update_context *ctx; - ctx->squat_type = SQUAT_INDEX_TYPE_HEADER; - ctx->uid = uid; -} - -static bool -fts_backend_squat_build_body_begin(struct fts_backend_build_context *_ctx, - uint32_t uid, const char *content_type, - const char *content_disposition ATTR_UNUSED) -{ - struct squat_fts_backend_build_context *ctx = - (struct squat_fts_backend_build_context *)_ctx; - - if (!fts_backend_default_can_index(content_type)) - return FALSE; - - ctx->squat_type = SQUAT_INDEX_TYPE_BODY; - ctx->uid = uid; - return TRUE; -} - -static int -fts_backend_squat_build_more(struct fts_backend_build_context *_ctx, - const unsigned char *data, size_t size) -{ - struct squat_fts_backend_build_context *ctx = - (struct squat_fts_backend_build_context *)_ctx; - - return squat_trie_build_more(ctx->build_ctx, ctx->uid, ctx->squat_type, - data, size); + ctx = i_new(struct squat_fts_backend_update_context, 1); + ctx->ctx.backend = _backend; + ctx->hdr = str_new(default_pool, 1024*32); + return &ctx->ctx; } static int get_all_msg_uids(struct mailbox *box, ARRAY_TYPE(seq_range) *uids) @@ -194,67 +185,265 @@ } static int -fts_backend_squat_build_deinit(struct fts_backend_build_context *_ctx) +fts_backend_squat_update_uid_changed(struct squat_fts_backend_update_context *ctx) { - struct squat_fts_backend_build_context *ctx = - (struct squat_fts_backend_build_context *)_ctx; + int ret = 0; + + if (ctx->uid == 0) + return 0; + + if (squat_trie_build_more(ctx->build_ctx, ctx->uid, + SQUAT_INDEX_TYPE_HEADER, + str_data(ctx->hdr), str_len(ctx->hdr)) < 0) + ret = -1; + str_truncate(ctx->hdr, 0); + return ret; +} + +static int +fts_backend_squat_build_deinit(struct squat_fts_backend_update_context *ctx) +{ + struct squat_fts_backend *backend = + (struct squat_fts_backend *)ctx->ctx.backend; ARRAY_TYPE(seq_range) uids; - int ret; + int ret = 0; + + if (ctx->build_ctx == NULL) + return 0; + + if (fts_backend_squat_update_uid_changed(ctx) < 0) + ret = -1; i_array_init(&uids, 1024); - if (get_all_msg_uids(ctx->ctx.backend->box, &uids) < 0) - ret = squat_trie_build_deinit(&ctx->build_ctx, NULL); - else { + if (get_all_msg_uids(backend->box, &uids) < 0) { + (void)squat_trie_build_deinit(&ctx->build_ctx, NULL); + ret = -1; + } else { seq_range_array_invert(&uids, 2, (uint32_t)-2); - ret = squat_trie_build_deinit(&ctx->build_ctx, &uids); + if (squat_trie_build_deinit(&ctx->build_ctx, &uids) < 0) + ret = -1; } array_free(&uids); + return ret; +} + +static int +fts_backend_squat_update_deinit(struct fts_backend_update_context *_ctx) +{ + struct squat_fts_backend_update_context *ctx = + (struct squat_fts_backend_update_context *)_ctx; + int ret = ctx->failed ? -1 : 0; + + if (fts_backend_squat_build_deinit(ctx) < 0) + ret = -1; + str_free(&ctx->hdr); i_free(ctx); return ret; } static void -fts_backend_squat_expunge(struct fts_backend *_backend ATTR_UNUSED, - struct mail *mail ATTR_UNUSED) +fts_backend_squat_update_set_mailbox(struct fts_backend_update_context *_ctx, + struct mailbox *box) { + struct squat_fts_backend_update_context *ctx = + (struct squat_fts_backend_update_context *)_ctx; + struct squat_fts_backend *backend = + (struct squat_fts_backend *)ctx->ctx.backend; + + if (fts_backend_squat_build_deinit(ctx) < 0) + ctx->failed = TRUE; + fts_backend_squat_set_box(backend, box); + + if (squat_trie_build_init(backend->trie, &ctx->build_ctx) < 0) + ctx->failed = TRUE; } static void -fts_backend_squat_expunge_finish(struct fts_backend *_backend ATTR_UNUSED, - struct mailbox *box ATTR_UNUSED, - bool committed ATTR_UNUSED) +fts_backend_squat_update_expunge(struct fts_backend_update_context *_ctx ATTR_UNUSED, + uint32_t last_uid ATTR_UNUSED) { /* FIXME */ } +static bool +fts_backend_squat_update_set_build_key(struct fts_backend_update_context *_ctx, + const struct fts_backend_build_key *key) +{ + struct squat_fts_backend_update_context *ctx = + (struct squat_fts_backend_update_context *)_ctx; + + if (ctx->failed) + return FALSE; + + if (key->uid != ctx->uid) { + if (fts_backend_squat_update_uid_changed(ctx) < 0) + ctx->failed = TRUE; + } + + switch (key->type) { + case FTS_BACKEND_BUILD_KEY_HDR: + case FTS_BACKEND_BUILD_KEY_MIME_HDR: + str_printfa(ctx->hdr, "%s: ", key->hdr_name); + ctx->squat_type = SQUAT_INDEX_TYPE_HEADER; + break; + case FTS_BACKEND_BUILD_KEY_BODY_PART: + ctx->squat_type = SQUAT_INDEX_TYPE_BODY; + break; + case FTS_BACKEND_BUILD_KEY_BODY_PART_BINARY: + i_unreached(); + } + ctx->uid = key->uid; + return TRUE; +} + +static void +fts_backend_squat_update_unset_build_key(struct fts_backend_update_context *_ctx) +{ + struct squat_fts_backend_update_context *ctx = + (struct squat_fts_backend_update_context *)_ctx; + + if (ctx->squat_type == SQUAT_INDEX_TYPE_HEADER) + str_append_c(ctx->hdr, '\n'); +} + +static int +fts_backend_squat_update_build_more(struct fts_backend_update_context *_ctx, + const unsigned char *data, size_t size) +{ + struct squat_fts_backend_update_context *ctx = + (struct squat_fts_backend_update_context *)_ctx; + + if (ctx->squat_type == SQUAT_INDEX_TYPE_HEADER) { + str_append_n(ctx->hdr, data, size); + return 0; + } + return squat_trie_build_more(ctx->build_ctx, ctx->uid, ctx->squat_type, + data, size); +} + static int fts_backend_squat_refresh(struct fts_backend *_backend) { struct squat_fts_backend *backend = (struct squat_fts_backend *)_backend; - return squat_trie_refresh(backend->trie); + backend->refresh = TRUE; + return 0; +} + +static int fts_backend_squat_optimize(struct fts_backend *_backend ATTR_UNUSED) +{ + /* FIXME: drop expunged messages */ + return 0; +} + +static int squat_lookup_arg(struct squat_fts_backend *backend, + const struct mail_search_arg *arg, bool and_args, + ARRAY_TYPE(seq_range) *definite_uids, + ARRAY_TYPE(seq_range) *maybe_uids) +{ + enum squat_index_type squat_type; + ARRAY_TYPE(seq_range) tmp_definite_uids, tmp_maybe_uids; + uint32_t last_uid; + int ret; + + switch (arg->type) { + case SEARCH_TEXT: + squat_type = SQUAT_INDEX_TYPE_HEADER | + SQUAT_INDEX_TYPE_BODY; + break; + case SEARCH_BODY: + squat_type = SQUAT_INDEX_TYPE_BODY; + break; + case SEARCH_HEADER: + case SEARCH_HEADER_ADDRESS: + case SEARCH_HEADER_COMPRESS_LWSP: + squat_type = SQUAT_INDEX_TYPE_HEADER; + break; + default: + return 0; + } + + i_array_init(&tmp_definite_uids, 128); + i_array_init(&tmp_maybe_uids, 128); + + ret = squat_trie_lookup(backend->trie, arg->value.str, squat_type, + &tmp_definite_uids, &tmp_maybe_uids); + if (arg->match_not) { + /* definite -> non-match + maybe -> maybe + non-match -> maybe */ + array_clear(&tmp_maybe_uids); + + if (squat_trie_get_last_uid(backend->trie, &last_uid) < 0) + i_unreached(); + seq_range_array_add_range(&tmp_maybe_uids, 1, last_uid); + seq_range_array_remove_seq_range(&tmp_maybe_uids, + &tmp_definite_uids); + array_clear(&tmp_definite_uids); + } + + if (and_args) { + /* AND: + definite && definite -> definite + definite && maybe -> maybe + maybe && maybe -> maybe */ + + /* put definites among maybies, so they can be intersected */ + seq_range_array_merge(maybe_uids, definite_uids); + seq_range_array_merge(&tmp_maybe_uids, &tmp_definite_uids); + + seq_range_array_intersect(maybe_uids, &tmp_maybe_uids); + seq_range_array_intersect(definite_uids, &tmp_definite_uids); + /* remove duplicate maybies that are also definites */ + seq_range_array_remove_seq_range(maybe_uids, definite_uids); + } else { + /* OR: + definite || definite -> definite + definite || maybe -> definite + maybe || maybe -> maybe */ + + /* remove maybies that are now definites */ + seq_range_array_remove_seq_range(&tmp_maybe_uids, + definite_uids); + seq_range_array_remove_seq_range(maybe_uids, + &tmp_definite_uids); + + seq_range_array_merge(definite_uids, &tmp_definite_uids); + seq_range_array_merge(maybe_uids, &tmp_maybe_uids); + } + + array_free(&tmp_definite_uids); + array_free(&tmp_maybe_uids); + return ret < 0 ? -1 : 1; } static int -fts_backend_squat_lookup(struct fts_backend *_backend, const char *key, - enum fts_lookup_flags flags, - ARRAY_TYPE(seq_range) *definite_uids, - ARRAY_TYPE(seq_range) *maybe_uids) +fts_backend_squat_lookup(struct fts_backend *_backend, struct mailbox *box, + struct mail_search_arg *args, bool and_args, + struct fts_result *result) { struct squat_fts_backend *backend = (struct squat_fts_backend *)_backend; - enum squat_index_type squat_type = 0; + int ret; - i_assert((flags & FTS_LOOKUP_FLAG_INVERT) == 0); + fts_backend_squat_set_box(backend, box); + if (backend->refresh) { + if (squat_trie_refresh(backend->trie) < 0) + return -1; + backend->refresh = FALSE; + } - if ((flags & FTS_LOOKUP_FLAG_HEADER) != 0) - squat_type |= SQUAT_INDEX_TYPE_HEADER; - if ((flags & FTS_LOOKUP_FLAG_BODY) != 0) - squat_type |= SQUAT_INDEX_TYPE_BODY; - i_assert(squat_type != 0); + for (; args != NULL; args = args->next) { + ret = squat_lookup_arg(backend, args, and_args, + &result->definite_uids, + &result->maybe_uids); + if (ret < 0) + return -1; + if (ret > 0) + args->match_always = TRUE; - return squat_trie_lookup(backend->trie, key, squat_type, - definite_uids, maybe_uids); + } + return 0; } struct fts_backend fts_backend_squat = { @@ -262,21 +451,21 @@ .flags = 0, { + fts_backend_squat_alloc, fts_backend_squat_init, fts_backend_squat_deinit, fts_backend_squat_get_last_uid, - NULL, - fts_backend_squat_build_init, - fts_backend_squat_build_hdr, - fts_backend_squat_build_body_begin, - NULL, - fts_backend_squat_build_more, - fts_backend_squat_build_deinit, - fts_backend_squat_expunge, - fts_backend_squat_expunge_finish, + fts_backend_squat_update_init, + fts_backend_squat_update_deinit, + fts_backend_squat_update_set_mailbox, + fts_backend_squat_update_expunge, + fts_backend_squat_update_set_build_key, + fts_backend_squat_update_unset_build_key, + fts_backend_squat_update_build_more, fts_backend_squat_refresh, + fts_backend_squat_optimize, + fts_backend_default_can_lookup, fts_backend_squat_lookup, - NULL, NULL } };
--- a/src/plugins/fts-squat/squat-test.c Fri Jul 22 13:13:29 2011 +0300 +++ b/src/plugins/fts-squat/squat-test.c Fri Jul 22 13:21:59 2011 +0300 @@ -47,7 +47,6 @@ enum squat_index_type index_type; bool data_header = TRUE, first = TRUE, skip_body = FALSE; bool mime_header = TRUE; - uint32_t last_uid; size_t trie_mem, uidlist_mem; clock_t clock_start, clock_end; struct timeval tv_start, tv_end; @@ -66,7 +65,7 @@ if (fd == -1) return 1; - if (squat_trie_build_init(trie, &last_uid, &build_ctx) < 0) + if (squat_trie_build_init(trie, &build_ctx) < 0) return 1; valid = buffer_create_dynamic(default_pool, 4096);
--- a/src/plugins/fts-squat/squat-trie.c Fri Jul 22 13:13:29 2011 +0300 +++ b/src/plugins/fts-squat/squat-trie.c Fri Jul 22 13:21:59 2011 +0300 @@ -1561,7 +1561,7 @@ return fd; } -int squat_trie_build_init(struct squat_trie *trie, uint32_t *last_uid_r, +int squat_trie_build_init(struct squat_trie *trie, struct squat_trie_build_context **ctx_r) { struct squat_trie_build_context *ctx; @@ -1591,7 +1591,6 @@ ctx->uidlist_build_ctx = uidlist_build_ctx; ctx->first_uid = trie->root.next_uid; - *last_uid_r = I_MAX((trie->root.next_uid+1)/2, 1) - 1; *ctx_r = ctx; return 0; }
--- a/src/plugins/fts-squat/squat-trie.h Fri Jul 22 13:13:29 2011 +0300 +++ b/src/plugins/fts-squat/squat-trie.h Fri Jul 22 13:21:59 2011 +0300 @@ -28,7 +28,7 @@ int squat_trie_refresh(struct squat_trie *trie); -int squat_trie_build_init(struct squat_trie *trie, uint32_t *last_uid_r, +int squat_trie_build_init(struct squat_trie *trie, struct squat_trie_build_context **ctx_r); /* bodies must be added before headers */ int squat_trie_build_more(struct squat_trie_build_context *ctx,
--- a/src/plugins/fts/Makefile.am Fri Jul 22 13:13:29 2011 +0300 +++ b/src/plugins/fts/Makefile.am Fri Jul 22 13:21:59 2011 +0300 @@ -13,14 +13,20 @@ lib20_fts_plugin_la_SOURCES = \ fts-api.c \ - fts-mailbox.c \ + fts-build.c \ + fts-build-indexer.c \ + fts-build-mailbox.c \ + fts-build-virtual.c \ fts-plugin.c \ fts-search.c \ + fts-search-serialize.c \ fts-storage.c noinst_HEADERS = \ fts-api.h \ fts-api-private.h \ - fts-mailbox.h \ + fts-build.h \ + fts-build-private.h \ fts-plugin.h \ + fts-search-serialize.h \ fts-storage.h
--- a/src/plugins/fts/fts-api-private.h Fri Jul 22 13:13:29 2011 +0300 +++ b/src/plugins/fts/fts-api-private.h Fri Jul 22 13:21:59 2011 +0300 @@ -3,51 +3,51 @@ #include "fts-api.h" +#define MAILBOX_GUID_HEX_LENGTH (MAIL_GUID_128_SIZE*2) + struct fts_backend_vfuncs { - struct fts_backend *(*init)(struct mailbox *box); + struct fts_backend *(*alloc)(void); + int (*init)(struct fts_backend *backend, const char **error_r); void (*deinit)(struct fts_backend *backend); - int (*get_last_uid)(struct fts_backend *backend, uint32_t *last_uid_r); - int (*get_all_last_uids)(struct fts_backend *backend, pool_t pool, - ARRAY_TYPE(fts_backend_uid_map) *last_uids); + int (*get_last_uid)(struct fts_backend *backend, struct mailbox *box, + uint32_t *last_uid_r); + + struct fts_backend_update_context * + (*update_init)(struct fts_backend *backend); + int (*update_deinit)(struct fts_backend_update_context *ctx); - int (*build_init)(struct fts_backend *backend, uint32_t *last_uid_r, - struct fts_backend_build_context **ctx_r); - void (*build_hdr)(struct fts_backend_build_context *ctx, uint32_t uid); - bool (*build_body_begin)(struct fts_backend_build_context *ctx, - uint32_t uid, const char *content_type, - const char *content_disposition); - void (*build_body_end)(struct fts_backend_build_context *ctx); - int (*build_more)(struct fts_backend_build_context *ctx, - const unsigned char *data, size_t size); - int (*build_deinit)(struct fts_backend_build_context *ctx); + void (*update_set_mailbox)(struct fts_backend_update_context *ctx, + struct mailbox *box); + void (*update_expunge)(struct fts_backend_update_context *ctx, + uint32_t uid); - void (*expunge)(struct fts_backend *backend, struct mail *mail); - void (*expunge_finish)(struct fts_backend *backend, - struct mailbox *box, bool committed); + /* Start a build for specified key */ + bool (*update_set_build_key)(struct fts_backend_update_context *ctx, + const struct fts_backend_build_key *key); + /* Finish a build for specified key - guaranteed to be called */ + void (*update_unset_build_key)(struct fts_backend_update_context *ctx); + /* Add data for current build key */ + int (*update_build_more)(struct fts_backend_update_context *ctx, + const unsigned char *data, size_t size); int (*refresh)(struct fts_backend *backend); + int (*optimize)(struct fts_backend *backend); - int (*lookup)(struct fts_backend *backend, const char *key, - enum fts_lookup_flags flags, - ARRAY_TYPE(seq_range) *definite_uids, - ARRAY_TYPE(seq_range) *maybe_uids); - int (*filter)(struct fts_backend *backend, const char *key, - enum fts_lookup_flags flags, - ARRAY_TYPE(seq_range) *definite_uids, - ARRAY_TYPE(seq_range) *maybe_uids); - - int (*lookup2)(struct fts_backend_lookup_context *ctx, - ARRAY_TYPE(seq_range) *definite_uids, - ARRAY_TYPE(seq_range) *maybe_uids, - ARRAY_TYPE(fts_score_map) *scores); + bool (*can_lookup)(struct fts_backend *backend, + const struct mail_search_arg *args); + int (*lookup)(struct fts_backend *backend, struct mailbox *box, + struct mail_search_arg *args, bool and_args, + struct fts_result *result); + int (*lookup_multi)(struct fts_backend *backend, + struct mailbox *const boxes[], + struct mail_search_arg *args, bool and_args, + struct fts_multi_result *result); }; enum fts_backend_flags { - /* Backend supports virtual mailbox lookups. */ - FTS_BACKEND_FLAG_VIRTUAL_LOOKUPS = 0x02, /* Backend supports indexing binary MIME parts */ - FTS_BACKEND_FLAG_BINARY_MIME_PARTS = 0x04 + FTS_BACKEND_FLAG_BINARY_MIME_PARTS = 0x01 }; struct fts_backend { @@ -55,37 +55,48 @@ enum fts_backend_flags flags; struct fts_backend_vfuncs v; - struct mailbox *box; + struct mail_namespace *ns; - unsigned int building:1; + unsigned int updating:1; }; -struct fts_backend_build_context { +struct fts_backend_update_context { struct fts_backend *backend; + struct mailbox *cur_box, *backend_box; + + unsigned int build_key_open:1; unsigned int failed:1; }; -struct fts_backend_lookup_field { - const char *key; - enum fts_lookup_flags flags; -}; +struct fts_index_header { + uint32_t last_indexed_uid; -struct fts_backend_lookup_context { - struct fts_backend *backend; - pool_t pool; - - ARRAY_DEFINE(fields, struct fts_backend_lookup_field); + /* highest UID and number of messages when optimization was last + time done */ + uint32_t last_optimize_uid; + uint32_t last_optimize_msgcount; }; void fts_backend_register(const struct fts_backend *backend); void fts_backend_unregister(const char *name); -bool fts_backend_default_can_index(const char *content_type); +bool fts_backend_default_can_lookup(struct fts_backend *backend, + const struct mail_search_arg *args); void fts_filter_uids(ARRAY_TYPE(seq_range) *definite_dest, const ARRAY_TYPE(seq_range) *definite_filter, ARRAY_TYPE(seq_range) *maybe_dest, const ARRAY_TYPE(seq_range) *maybe_filter); +/* Returns TRUE if ok, FALSE if no fts header */ +bool fts_index_get_last_uid(struct mailbox *box, uint32_t *last_uid_r); +int fts_index_set_last_uid(struct mailbox *box, uint32_t last_uid); + +/* Returns TRUE if FTS backend should index the header for optimizing + separate lookups */ +bool fts_header_want_indexed(const char *hdr_name); + +int fts_mailbox_get_guid(struct mailbox *box, const char **guid_r); + #endif
--- a/src/plugins/fts/fts-api.c Fri Jul 22 13:13:29 2011 +0300 +++ b/src/plugins/fts/fts-api.c Fri Jul 22 13:21:59 2011 +0300 @@ -2,6 +2,11 @@ #include "lib.h" #include "array.h" +#include "hex-binary.h" +#include "mail-index.h" +#include "mail-storage-private.h" +#include "mail-search.h" +#include "../virtual/virtual-storage.h" #include "fts-api-private.h" static ARRAY_DEFINE(backends, const struct fts_backend *); @@ -48,23 +53,26 @@ return NULL; } -struct fts_backend * -fts_backend_init(const char *backend_name, struct mailbox *box) +int fts_backend_init(const char *backend_name, struct mail_namespace *ns, + const char **error_r, struct fts_backend **backend_r) { const struct fts_backend *be; struct fts_backend *backend; be = fts_backend_class_lookup(backend_name); if (be == NULL) { - i_error("Unknown FTS backend: %s", backend_name); - return NULL; + *error_r = "Unknown backend"; + return -1; } - backend = be->v.init(box); - if (backend == NULL) - return NULL; - backend->box = box; - return backend; + backend = be->v.alloc(); + backend->ns = ns; + if (backend->v.init(backend, error_r) < 0) { + i_free(backend); + return -1; + } + *backend_r = backend; + return 0; } void fts_backend_deinit(struct fts_backend **_backend) @@ -75,78 +83,98 @@ backend->v.deinit(backend); } -int fts_backend_get_last_uid(struct fts_backend *backend, uint32_t *last_uid_r) +int fts_backend_get_last_uid(struct fts_backend *backend, struct mailbox *box, + uint32_t *last_uid_r) { - return backend->v.get_last_uid(backend, last_uid_r); -} + if (strcmp(box->storage->name, VIRTUAL_STORAGE_NAME) == 0) { + /* virtual mailboxes themselves don't have any indexes, + so catch this call here */ + if (!fts_index_get_last_uid(box, last_uid_r)) + *last_uid_r = 0; + return 0; + } -int fts_backend_get_all_last_uids(struct fts_backend *backend, pool_t pool, - ARRAY_TYPE(fts_backend_uid_map) *last_uids) -{ - return backend->v.get_all_last_uids(backend, pool, last_uids); + return backend->v.get_last_uid(backend, box, last_uid_r); } -int fts_backend_build_init(struct fts_backend *backend, uint32_t *last_uid_r, - struct fts_backend_build_context **ctx_r) +bool fts_backend_is_updating(struct fts_backend *backend) { - int ret; - - i_assert(!backend->building); - - ret = backend->v.build_init(backend, last_uid_r, ctx_r); - if (ret == 0) - backend->building = TRUE; - return ret; + return backend->updating; } -void fts_backend_build_hdr(struct fts_backend_build_context *ctx, uint32_t uid) +struct fts_backend_update_context * +fts_backend_update_init(struct fts_backend *backend) { - ctx->backend->v.build_hdr(ctx, uid); + i_assert(!backend->updating); + + backend->updating = TRUE; + return backend->v.update_init(backend); +} + +static void fts_backend_set_cur_mailbox(struct fts_backend_update_context *ctx) +{ + fts_backend_update_unset_build_key(ctx); + if (ctx->backend_box != ctx->cur_box) { + ctx->backend->v.update_set_mailbox(ctx, ctx->cur_box); + ctx->backend_box = ctx->cur_box; + } } -bool fts_backend_build_body_begin(struct fts_backend_build_context *ctx, - uint32_t uid, const char *content_type, - const char *content_disposition) +int fts_backend_update_deinit(struct fts_backend_update_context **_ctx) { - return ctx->backend->v.build_body_begin(ctx, uid, content_type, - content_disposition); + struct fts_backend_update_context *ctx = *_ctx; + + *_ctx = NULL; + + ctx->cur_box = NULL; + fts_backend_set_cur_mailbox(ctx); + + return ctx->backend->v.update_deinit(ctx); } -void fts_backend_build_body_end(struct fts_backend_build_context *ctx) +void fts_backend_update_set_mailbox(struct fts_backend_update_context *ctx, + struct mailbox *box) { - if (ctx->backend->v.build_body_end != NULL) - ctx->backend->v.build_body_end(ctx); -} - -int fts_backend_build_more(struct fts_backend_build_context *ctx, - const unsigned char *data, size_t size) -{ - return ctx->backend->v.build_more(ctx, data, size); + if (ctx->backend_box != NULL && box != ctx->backend_box) { + /* make sure we don't reference the backend box anymore */ + ctx->backend->v.update_set_mailbox(ctx, NULL); + ctx->backend_box = NULL; + } + ctx->cur_box = box; } -int fts_backend_build_deinit(struct fts_backend_build_context **_ctx) +void fts_backend_update_expunge(struct fts_backend_update_context *ctx, + uint32_t uid) { - struct fts_backend_build_context *ctx = *_ctx; - - *_ctx = NULL; - ctx->backend->building = FALSE; - return ctx->backend->v.build_deinit(ctx); + fts_backend_set_cur_mailbox(ctx); + ctx->backend->v.update_expunge(ctx, uid); } -bool fts_backend_is_building(struct fts_backend *backend) +bool fts_backend_update_set_build_key(struct fts_backend_update_context *ctx, + const struct fts_backend_build_key *key) { - return backend->building; + fts_backend_set_cur_mailbox(ctx); + + if (!ctx->backend->v.update_set_build_key(ctx, key)) + return FALSE; + ctx->build_key_open = TRUE; + return TRUE; } -void fts_backend_expunge(struct fts_backend *backend, struct mail *mail) +void fts_backend_update_unset_build_key(struct fts_backend_update_context *ctx) { - backend->v.expunge(backend, mail); + if (ctx->build_key_open) { + ctx->backend->v.update_unset_build_key(ctx); + ctx->build_key_open = FALSE; + } } -void fts_backend_expunge_finish(struct fts_backend *backend, - struct mailbox *box, bool committed) +int fts_backend_update_build_more(struct fts_backend_update_context *ctx, + const unsigned char *data, size_t size) { - backend->v.expunge_finish(backend, box, committed); + i_assert(ctx->build_key_open); + + return ctx->backend->v.update_build_more(ctx, data, size); } int fts_backend_refresh(struct fts_backend *backend) @@ -154,6 +182,12 @@ return backend->v.refresh(backend); } +int fts_backend_optimize(struct fts_backend *backend) +{ + return backend->v.optimize == NULL ? 0 : + backend->v.optimize(backend); +} + static void fts_merge_maybies(ARRAY_TYPE(seq_range) *dest_maybe, const ARRAY_TYPE(seq_range) *dest_definite, @@ -203,131 +237,149 @@ seq_range_array_intersect(definite_dest, definite_filter); } -static void fts_lookup_invert(ARRAY_TYPE(seq_range) *definite_uids, - const ARRAY_TYPE(seq_range) *maybe_uids) +bool fts_backend_default_can_lookup(struct fts_backend *backend, + const struct mail_search_arg *args) { - /* we'll begin by inverting definite UIDs */ - seq_range_array_invert(definite_uids, 1, (uint32_t)-1); - - /* from that list remove UIDs in the maybe list. - the maybe list itself isn't touched. */ - (void)seq_range_array_remove_seq_range(definite_uids, maybe_uids); + for (; args != NULL; args = args->next) { + switch (args->type) { + case SEARCH_OR: + case SEARCH_SUB: + case SEARCH_INTHREAD: + if (fts_backend_default_can_lookup(backend, + args->value.subargs)) + return TRUE; + break; + case SEARCH_HEADER: + case SEARCH_HEADER_ADDRESS: + case SEARCH_HEADER_COMPRESS_LWSP: + case SEARCH_BODY: + case SEARCH_TEXT: + return TRUE; + default: + break; + } + } + return FALSE; } -static int fts_backend_lookup(struct fts_backend *backend, const char *key, - enum fts_lookup_flags flags, - ARRAY_TYPE(seq_range) *definite_uids, - ARRAY_TYPE(seq_range) *maybe_uids) +bool fts_backend_can_lookup(struct fts_backend *backend, + const struct mail_search_arg *args) { - int ret; + return backend->v.can_lookup(backend, args); +} - ret = backend->v.lookup(backend, key, flags & ~FTS_LOOKUP_FLAG_INVERT, - definite_uids, maybe_uids); - if (unlikely(ret < 0)) +static int fts_score_map_sort(const struct fts_score_map *m1, + const struct fts_score_map *m2) +{ + if (m1->uid < m2->uid) return -1; - if ((flags & FTS_LOOKUP_FLAG_INVERT) != 0) - fts_lookup_invert(definite_uids, maybe_uids); + if (m1->uid > m2->uid) + return 1; return 0; } -static int fts_backend_filter(struct fts_backend *backend, const char *key, - enum fts_lookup_flags flags, - ARRAY_TYPE(seq_range) *definite_uids, - ARRAY_TYPE(seq_range) *maybe_uids) +int fts_backend_lookup(struct fts_backend *backend, struct mailbox *box, + struct mail_search_arg *args, bool and_args, + struct fts_result *result) { - ARRAY_TYPE(seq_range) tmp_definite, tmp_maybe; - int ret; - - if (backend->v.filter != NULL) { - return backend->v.filter(backend, key, flags, - definite_uids, maybe_uids); - } - - /* do this ourself */ - i_array_init(&tmp_definite, 64); - i_array_init(&tmp_maybe, 64); - ret = fts_backend_lookup(backend, key, flags, - &tmp_definite, &tmp_maybe); - if (ret < 0) { - array_clear(definite_uids); - array_clear(maybe_uids); - } else { - fts_filter_uids(definite_uids, &tmp_definite, - maybe_uids, &tmp_maybe); - } - array_free(&tmp_maybe); - array_free(&tmp_definite); - return ret; -} - -struct fts_backend_lookup_context * -fts_backend_lookup_init(struct fts_backend *backend) -{ - struct fts_backend_lookup_context *ctx; - pool_t pool; + array_clear(&result->definite_uids); + array_clear(&result->maybe_uids); + array_clear(&result->scores); - pool = pool_alloconly_create("fts backend lookup", 256); - ctx = p_new(pool, struct fts_backend_lookup_context, 1); - ctx->pool = pool; - ctx->backend = backend; - p_array_init(&ctx->fields, pool, 8); - return ctx; -} - -void fts_backend_lookup_add(struct fts_backend_lookup_context *ctx, - const char *key, enum fts_lookup_flags flags) -{ - struct fts_backend_lookup_field *field; - - field = array_append_space(&ctx->fields); - field->key = p_strdup(ctx->pool, key); - field->flags = flags; -} + if (backend->v.lookup(backend, box, args, and_args, result) < 0) + return -1; -static int fts_backend_lookup_old(struct fts_backend_lookup_context *ctx, - ARRAY_TYPE(seq_range) *definite_uids, - ARRAY_TYPE(seq_range) *maybe_uids) -{ - const struct fts_backend_lookup_field *fields; - unsigned int i, count; - - fields = array_get(&ctx->fields, &count); - i_assert(count > 0); - - if (fts_backend_lookup(ctx->backend, fields[0].key, fields[0].flags, - definite_uids, maybe_uids) < 0) - return -1; - for (i = 1; i < count; i++) { - if (fts_backend_filter(ctx->backend, - fields[i].key, fields[i].flags, - definite_uids, maybe_uids) < 0) - return -1; + if (!result->scores_sorted && array_is_created(&result->scores)) { + array_sort(&result->scores, fts_score_map_sort); + result->scores_sorted = TRUE; } return 0; } -int fts_backend_lookup_deinit(struct fts_backend_lookup_context **_ctx, - ARRAY_TYPE(seq_range) *definite_uids, - ARRAY_TYPE(seq_range) *maybe_uids, - ARRAY_TYPE(fts_score_map) *scores) +int fts_backend_lookup_multi(struct fts_backend *backend, + struct mailbox *const boxes[], + struct mail_search_arg *args, bool and_args, + struct fts_multi_result *result) { - struct fts_backend_lookup_context *ctx = *_ctx; - int ret; + i_assert(boxes[0] != NULL); + + return backend->v.lookup_multi(backend, boxes, args, and_args, result); +} + +static bool +fts_index_get_header(struct mailbox *box, struct fts_index_header *hdr_r, + uint32_t *ext_id_r) +{ + const void *data; + size_t data_size; - *_ctx = NULL; - if (ctx->backend->v.lookup2 != NULL) { - ret = ctx->backend->v.lookup2(ctx, definite_uids, maybe_uids, - scores); - } else { - array_clear(scores); - ret = fts_backend_lookup_old(ctx, definite_uids, maybe_uids); + *ext_id_r = mail_index_ext_register(box->index, "fts", + sizeof(struct fts_index_header), + 0, 0); + mail_index_get_header_ext(box->view, *ext_id_r, &data, &data_size); + if (data_size < sizeof(*hdr_r)) { + memset(hdr_r, 0, sizeof(*hdr_r)); + return FALSE; } - pool_unref(&ctx->pool); - return ret; + + memcpy(hdr_r, data, data_size); + return TRUE; +} + +bool fts_index_get_last_uid(struct mailbox *box, uint32_t *last_uid_r) +{ + struct fts_index_header hdr; + uint32_t ext_id; + + if (!fts_index_get_header(box, &hdr, &ext_id)) { + *last_uid_r = 0; + return FALSE; + } + + *last_uid_r = hdr.last_indexed_uid; + return TRUE; } -bool fts_backend_default_can_index(const char *content_type) +int fts_index_set_last_uid(struct mailbox *box, uint32_t last_uid) +{ + struct mail_index_transaction *trans; + struct fts_index_header hdr; + uint32_t ext_id; + + (void)fts_index_get_header(box, &hdr, &ext_id); + + hdr.last_indexed_uid = last_uid; + trans = mail_index_transaction_begin(box->view, 0); + mail_index_update_header_ext(trans, ext_id, 0, &hdr, sizeof(hdr)); + return mail_index_transaction_commit(&trans); +} + +static const char *indexed_headers[] = { + "From", "To", "Cc", "Bcc", "Subject" +}; + +bool fts_header_want_indexed(const char *hdr_name) { - return strncasecmp(content_type, "text/", 5) == 0 || - strcasecmp(content_type, "message/rfc822") == 0; + unsigned int i; + + for (i = 0; i < N_ELEMENTS(indexed_headers); i++) { + if (strcasecmp(hdr_name, indexed_headers[i]) == 0) + return TRUE; + } + return FALSE; } + +int fts_mailbox_get_guid(struct mailbox *box, const char **guid_r) +{ + struct mailbox_metadata metadata; + buffer_t buf; + unsigned char guid_hex[MAILBOX_GUID_HEX_LENGTH]; + + if (mailbox_get_metadata(box, MAILBOX_METADATA_GUID, &metadata) < 0) + return -1; + + buffer_create_data(&buf, guid_hex, sizeof(guid_hex)); + binary_to_hex_append(&buf, metadata.guid, MAIL_GUID_128_SIZE); + *guid_r = t_strndup(guid_hex, sizeof(guid_hex)); + return 0; +}
--- a/src/plugins/fts/fts-api.h Fri Jul 22 13:13:29 2011 +0300 +++ b/src/plugins/fts/fts-api.h Fri Jul 22 13:21:59 2011 +0300 @@ -3,26 +3,40 @@ struct mail; struct mailbox; -struct fts_backend_build_context; +struct mail_namespace; +struct mail_search_arg; + +struct fts_backend; #include "seq-range-array.h" -enum fts_lookup_flags { - /* Search within header and/or body. - At least one of these must be set. */ - FTS_LOOKUP_FLAG_HEADER = 0x01, - FTS_LOOKUP_FLAG_BODY = 0x02, - - /* The key must NOT be found */ - FTS_LOOKUP_FLAG_INVERT = 0x04 +enum fts_backend_build_key_type { + /* Header */ + FTS_BACKEND_BUILD_KEY_HDR, + /* MIME part header */ + FTS_BACKEND_BUILD_KEY_MIME_HDR, + /* MIME body part */ + FTS_BACKEND_BUILD_KEY_BODY_PART, + /* Binary MIME body part, if backend supports binary data */ + FTS_BACKEND_BUILD_KEY_BODY_PART_BINARY }; -struct fts_backend_uid_map { - const char *mailbox; - uint32_t uidvalidity; +struct fts_backend_build_key { uint32_t uid; + enum fts_backend_build_key_type type; + + /* for _KEY_HDR: */ + const char *hdr_name; + + /* for _KEY_BODY_PART and _KEY_BODY_PART_BINARY: */ + + /* Contains a valid parsed "type/subtype" string. For messages without + (valid) Content-Type: header, it's set to "text/plain". */ + const char *body_content_type; + /* Content-Disposition: header without parsing/validation if it exists, + otherwise NULL. */ + const char *body_content_disposition; }; -ARRAY_DEFINE_TYPE(fts_backend_uid_map, struct fts_backend_uid_map); struct fts_score_map { uint32_t uid; @@ -30,80 +44,94 @@ }; ARRAY_DEFINE_TYPE(fts_score_map, struct fts_score_map); -struct fts_backend * -fts_backend_init(const char *backend_name, struct mailbox *box); +struct fts_result { + struct mailbox *box; + + ARRAY_TYPE(seq_range) definite_uids; + /* The maybe_uids is useful with backends that can only filter out + messages, but can't definitively say if the search matched a + message. */ + ARRAY_TYPE(seq_range) maybe_uids; + ARRAY_TYPE(fts_score_map) scores; + bool scores_sorted; +}; + +struct fts_multi_result { + pool_t pool; + /* box=NULL-terminated array of mailboxes and matching UIDs, + all allocated from the given pool. */ + struct fts_result *box_results; +}; + +int fts_backend_init(const char *backend_name, struct mail_namespace *ns, + const char **error_r, struct fts_backend **backend_r); void fts_backend_deinit(struct fts_backend **backend); /* Get the last_uid for the mailbox. */ -int fts_backend_get_last_uid(struct fts_backend *backend, uint32_t *last_uid_r); -/* Get last_uids for all mailboxes that might be backend mailboxes for a - virtual mailbox. The backend can use mailbox_get_virtual_backend_boxes() or - mailbox_get_virtual_box_patterns() functions to get the list of mailboxes. +int fts_backend_get_last_uid(struct fts_backend *backend, struct mailbox *box, + uint32_t *last_uid_r); - Depending on virtual mailbox configuration, this function may also return - mailboxes that don't even match the virtual mailbox patterns. The caller - needs to be able to ignore the unnecessary ones. */ -int fts_backend_get_all_last_uids(struct fts_backend *backend, pool_t pool, - ARRAY_TYPE(fts_backend_uid_map) *last_uids); +/* Returns TRUE if there exists an update context. */ +bool fts_backend_is_updating(struct fts_backend *backend); -/* Initialize adding new data to the index. last_uid_r is set to the last - indexed message's IMAP UID */ -int fts_backend_build_init(struct fts_backend *backend, uint32_t *last_uid_r, - struct fts_backend_build_context **ctx_r); -/* Switch to building index for mail's headers or MIME part headers. */ -void fts_backend_build_hdr(struct fts_backend_build_context *ctx, uint32_t uid); -/* Switch to building index for the next body part. If backend doesn't want - to index this body part (based on content type/disposition check), it can - return FALSE and caller will skip to next part. The backend must return - TRUE for all text/xxx and message/rfc822 content types. +/* Start an index update. */ +struct fts_backend_update_context * +fts_backend_update_init(struct fts_backend *backend); +/* Finish an index update. Returns 0 if ok, -1 if some updates failed. + If updates failed, the index is in unspecified state. */ +int fts_backend_update_deinit(struct fts_backend_update_context **ctx); - The content_type contains a valid parsed "type/subtype" string. For messages - without (valid) Content-Type header, the content_type is set to "text/plain". - The content_disposition is passed without parsing/validation if it exists, - otherwise it's NULL. */ -bool fts_backend_build_body_begin(struct fts_backend_build_context *ctx, - uint32_t uid, const char *content_type, - const char *content_disposition); -/* Called once when the whole body part has been sent. */ -void fts_backend_build_body_end(struct fts_backend_build_context *ctx); -/* Add more content to the index for the currently selected header/body part. - The data must contain only full valid UTF-8 characters, but it doesn't need - to be NUL-terminated. size contains the data size in bytes, not characters. - This function may be called many times and the data block sizes may be - small. Backend returns 0 if ok, -1 if build should be aborted. */ -int fts_backend_build_more(struct fts_backend_build_context *ctx, - const unsigned char *data, size_t size); -/* Finish adding new data to the index. */ -int fts_backend_build_deinit(struct fts_backend_build_context **ctx); +/* Switch to updating the specified mailbox. box may also be set to NULL to + make sure the previous mailbox won't tried to be accessed anymore. */ +void fts_backend_update_set_mailbox(struct fts_backend_update_context *ctx, + struct mailbox *box); +/* Expunge the specified mail. */ +void fts_backend_update_expunge(struct fts_backend_update_context *ctx, + uint32_t uid); -/* Returns TRUE if there exists a build context. */ -bool fts_backend_is_building(struct fts_backend *backend); +/* Switch to building index for specified key. If backend doesn't want to + index this key, it can return FALSE and caller will skip to next key. */ +bool fts_backend_update_set_build_key(struct fts_backend_update_context *ctx, + const struct fts_backend_build_key *key); +/* Make sure that if _build_more() is called, we'll assert-crash. */ +void fts_backend_update_unset_build_key(struct fts_backend_update_context *ctx); +/* Add more content to the index for the currently specified build key. + Non-BODY_PART_BINARY data must contain only full valid UTF-8 characters, + but it doesn't need to be NUL-terminated. size contains the data size in + bytes, not characters. This function may be called many times and the data + block sizes may be small. Backend returns 0 if ok, -1 if build should be + aborted. */ +int fts_backend_update_build_more(struct fts_backend_update_context *ctx, + const unsigned char *data, size_t size); -/* Expunge given mail from the backend. Note that the transaction may still - fail later, so backend shouldn't do anything irreversible. */ -void fts_backend_expunge(struct fts_backend *backend, struct mail *mail); -/* Called after transaction has been committed or rollbacked. */ -void fts_backend_expunge_finish(struct fts_backend *backend, - struct mailbox *box, bool committed); - -/* Refresh database to make sure we see latest changes from lookups. +/* Refresh index to make sure we see latest changes from lookups. Returns 0 if ok, -1 if error. */ int fts_backend_refresh(struct fts_backend *backend); +/* Optimize the index. This can be a somewhat heavy operation. */ +int fts_backend_optimize(struct fts_backend *backend); -/* Start building a FTS lookup. */ -struct fts_backend_lookup_context * -fts_backend_lookup_init(struct fts_backend *backend); -/* Add a new search key to the lookup. The keys are ANDed together. */ -void fts_backend_lookup_add(struct fts_backend_lookup_context *ctx, - const char *key, enum fts_lookup_flags flags); -/* Finish the lookup and return found UIDs. The definite_uids are returned - to client directly, while for maybe_uids Dovecot first verifies (by - opening and reading the mail) that they really do contain the searched - keys. The maybe_uids is useful with backends that can only filter out - messages, but can't definitively say if the search matched a message. */ -int fts_backend_lookup_deinit(struct fts_backend_lookup_context **ctx, - ARRAY_TYPE(seq_range) *definite_uids, - ARRAY_TYPE(seq_range) *maybe_uids, - ARRAY_TYPE(fts_score_map) *scores); +/* Returns TRUE if fts_backend_lookup() should even be tried for the + given args. */ +bool fts_backend_can_lookup(struct fts_backend *backend, + const struct mail_search_arg *args); +/* Do a FTS lookup for the given search args. Backends can support different + kinds of search arguments, so match_always=TRUE must be set to all search + args that were actually used to produce the search results. The other args + are handled by the regular search code. The backends MUST ignore all args + that have subargs (SEARCH_OR, SEARCH_SUB), since they are looked up + separately. + + and_args specifies if the args should be ANDed or ORed together. + + The arrays in result must be initialized by caller. */ +int fts_backend_lookup(struct fts_backend *backend, struct mailbox *box, + struct mail_search_arg *args, bool and_args, + struct fts_result *result); + +/* Search from multiple mailboxes. result->pool must be initialized. */ +int fts_backend_lookup_multi(struct fts_backend *backend, + struct mailbox *const boxes[], + struct mail_search_arg *args, bool and_args, + struct fts_multi_result *result); #endif
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/plugins/fts/fts-build-indexer.c Fri Jul 22 13:21:59 2011 +0300 @@ -0,0 +1,160 @@ +/* Copyright (c) 2006-2011 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "ioloop.h" +#include "network.h" +#include "istream.h" +#include "write-full.h" +#include "strescape.h" +#include "mail-user.h" +#include "mail-storage-private.h" +#include "fts-api-private.h" +#include "fts-build-private.h" + +#define INDEXER_SOCKET_NAME "indexer" +#define INDEXER_WAIT_MSECS 250 +#define INDEXER_HANDSHAKE "VERSION\tindexer\t1\t0\n" + +struct indexer_fts_storage_build_context { + struct fts_storage_build_context ctx; + + char *path; + int fd; + struct istream *input; +}; + +static int +fts_build_indexer_init(struct fts_backend *backend, struct mailbox *box, + struct fts_storage_build_context **build_ctx_r) +{ + struct indexer_fts_storage_build_context *ctx; + struct mailbox_status status; + uint32_t last_uid, seq1, seq2; + const char *path, *cmd; + int fd; + + if (fts_backend_get_last_uid(backend, box, &last_uid) < 0) + return -1; + + mailbox_get_open_status(box, STATUS_UIDNEXT, &status); + if (status.uidnext == last_uid+1) { + /* everything is already indexed */ + return 0; + } + + mailbox_get_seq_range(box, last_uid+1, (uint32_t)-1, &seq1, &seq2); + if (seq1 == 0) { + /* no new messages (last messages in mailbox were expunged) */ + return 0; + } + + path = t_strconcat(box->storage->user->set->base_dir, + "/"INDEXER_SOCKET_NAME, NULL); + fd = net_connect_unix_with_retries(path, 1000); + if (fd == -1) { + i_error("net_connect_unix(%s) failed: %m", path); + return -1; + } + + cmd = t_strdup_printf(INDEXER_HANDSHAKE"PREPEND\t1\t%s\t%s\n", + str_tabescape(box->storage->user->username), + str_tabescape(box->vname)); + if (write_full(fd, cmd, strlen(cmd)) < 0) { + i_error("write(%s) failed: %m", path); + (void)close(fd); + return -1; + } + + /* connect to indexer and request immediate indexing of the mailbox */ + ctx = i_new(struct indexer_fts_storage_build_context, 1); + ctx->ctx.mail_count = 100; + ctx->path = i_strdup(path); + ctx->fd = fd; + ctx->input = i_stream_create_fd(fd, 128, FALSE); + + *build_ctx_r = &ctx->ctx; + return 1; +} + +static int +fts_build_indexer_deinit(struct fts_storage_build_context *_ctx) +{ + struct indexer_fts_storage_build_context *ctx = + (struct indexer_fts_storage_build_context *)_ctx; + + i_stream_destroy(&ctx->input); + if (close(ctx->fd) < 0) + i_error("close(%s) failed: %m", ctx->path); + i_free(ctx->path); + return 0; +} + +static int +fts_build_indexer_input(struct indexer_fts_storage_build_context *ctx) +{ + const char *line; + int percentage; + + while ((line = i_stream_read_next_line(ctx->input)) != NULL) { + /* initial reply: <tag> \t OK + following: <tag> \t <percentage> */ + if (strncmp(line, "1\t", 2) != 0) { + i_error("indexer sent invalid reply: %s", line); + return -1; + } + line += 2; + if (strcmp(line, "OK") == 0) + continue; + if (str_to_int(line, &percentage) < 0 || percentage > 100) { + i_error("indexer sent invalid percentage: %s", line); + return -1; + } + if (percentage < 0) { + /* indexing failed */ + i_error("indexer failed to index mailbox %s", + ctx->ctx.box->vname); + return -1; + } + ctx->ctx.mail_idx = percentage; + if (percentage == 100) { + /* finished */ + return 1; + } + } + if (ctx->input->eof || ctx->input->stream_errno != 0) { + i_error("indexer disconnected unexpectedly"); + return -1; + } + return 0; +} + +static int fts_build_indexer_more(struct fts_storage_build_context *_ctx) +{ + struct indexer_fts_storage_build_context *ctx = + (struct indexer_fts_storage_build_context *)_ctx; + struct ioloop *ioloop; + struct io *io; + struct timeout *to; + int ret; + + if ((ret = fts_build_indexer_input(ctx)) != 0) + return ret; + + /* wait for a while for the reply. FIXME: once search API supports + asynchronous waits, get rid of this wait and use the mail IO loop */ + ioloop = io_loop_create(); + io = io_add(ctx->fd, IO_READ, io_loop_stop, ioloop); + to = timeout_add(INDEXER_WAIT_MSECS, io_loop_stop, ioloop); + io_loop_run(ioloop); + io_remove(&io); + timeout_remove(&to); + io_loop_destroy(&ioloop); + + return fts_build_indexer_input(ctx); +} + +const struct fts_storage_build_vfuncs fts_storage_build_indexer_vfuncs = { + fts_build_indexer_init, + fts_build_indexer_deinit, + fts_build_indexer_more +};
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/plugins/fts/fts-build-mailbox.c Fri Jul 22 13:21:59 2011 +0300 @@ -0,0 +1,91 @@ +/* Copyright (c) 2006-2011 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "ioloop.h" +#include "mail-storage-private.h" +#include "mail-search-build.h" +#include "fts-api-private.h" +#include "fts-build-private.h" + +#define FTS_SEARCH_NONBLOCK_COUNT 50 + +static int +fts_build_mailbox_init(struct fts_backend *backend, struct mailbox *box, + struct fts_storage_build_context **build_ctx_r) +{ + struct fts_storage_build_context *ctx; + struct mail_search_args *search_args; + struct fts_backend_update_context *update_ctx; + struct mailbox_status status; + uint32_t last_uid, seq1, seq2; + + if (fts_backend_get_last_uid(backend, box, &last_uid) < 0) + return -1; + + mailbox_get_open_status(box, STATUS_UIDNEXT, &status); + if (status.uidnext == last_uid+1) { + /* everything is already indexed */ + return 0; + } + + mailbox_get_seq_range(box, last_uid+1, (uint32_t)-1, &seq1, &seq2); + if (seq1 == 0) { + /* no new messages (last messages in mailbox were expunged) */ + return 0; + } + + update_ctx = fts_backend_update_init(backend); + fts_backend_update_set_mailbox(update_ctx, box); + + search_args = mail_search_build_init(); + mail_search_build_add_seqset(search_args, seq1, seq2); + + ctx = i_new(struct fts_storage_build_context, 1); + ctx->update_ctx = update_ctx; + ctx->mail_count = seq2 - seq1 + 1; + + ctx->trans = mailbox_transaction_begin(box, 0); + ctx->search_ctx = mailbox_search_init(ctx->trans, search_args, + NULL, 0, NULL); + ctx->search_ctx->progress_hidden = TRUE; + mail_search_args_unref(&search_args); + + *build_ctx_r = ctx; + return 1; +} + +static int fts_build_mailbox_deinit(struct fts_storage_build_context *ctx) +{ + int ret; + + ret = mailbox_search_deinit(&ctx->search_ctx); + (void)mailbox_transaction_commit(&ctx->trans); + return ret; +} + +static int fts_build_mailbox_more(struct fts_storage_build_context *ctx) +{ + struct mail *mail = NULL; + unsigned int count = 0; + int ret; + + while (mailbox_search_next(ctx->search_ctx, &mail)) { + T_BEGIN { + ret = fts_build_mail(ctx, mail); + } T_END; + + if (ret < 0) + return -1; + + ctx->mail_idx++; + if (++count == FTS_SEARCH_NONBLOCK_COUNT) + return 0; + } + return 1; +} + +const struct fts_storage_build_vfuncs fts_storage_build_mailbox_vfuncs = { + fts_build_mailbox_init, + fts_build_mailbox_deinit, + fts_build_mailbox_more +};
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/plugins/fts/fts-build-private.h Fri Jul 22 13:21:59 2011 +0300 @@ -0,0 +1,42 @@ +#ifndef FTS_BUILD_PRIVATE_H +#define FTS_BUILD_PRIVATE_H + +#include "fts-build.h" + +struct fts_storage_build_context; + +#define FTS_SEARCH_NONBLOCK_COUNT 50 + +struct fts_storage_build_vfuncs { + int (*init)(struct fts_backend *backend, struct mailbox *box, + struct fts_storage_build_context **build_ctx_r); + int (*deinit)(struct fts_storage_build_context *ctx); + int (*more)(struct fts_storage_build_context *ctx); +}; + +struct fts_storage_build_context { + struct mailbox *box; + struct fts_backend_update_context *update_ctx; + struct fts_storage_build_vfuncs v; + + struct timeval search_start_time, last_notify; + unsigned int mail_idx, mail_count; + + struct mailbox_transaction_context *trans; + struct mail_search_context *search_ctx; + + uint32_t uid; + char *content_type, *content_disposition; + + unsigned int binary_mime_parts:1; + unsigned int notified:1; + unsigned int failed:1; +}; + +extern const struct fts_storage_build_vfuncs fts_storage_build_mailbox_vfuncs; +extern const struct fts_storage_build_vfuncs fts_storage_build_virtual_vfuncs; +extern const struct fts_storage_build_vfuncs fts_storage_build_indexer_vfuncs; + +int fts_build_mail(struct fts_storage_build_context *ctx, struct mail *mail); + +#endif
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/plugins/fts/fts-build-virtual.c Fri Jul 22 13:21:59 2011 +0300 @@ -0,0 +1,203 @@ +/* Copyright (c) 2006-2011 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "ioloop.h" +#include "mail-storage-private.h" +#include "mail-search-build.h" +#include "../virtual/virtual-storage.h" +#include "fts-api-private.h" +#include "fts-storage.h" +#include "fts-build-private.h" + +#define FTS_SEARCH_NONBLOCK_COUNT 50 + +struct virtual_fts_storage_build_context { + struct fts_storage_build_context ctx; + + struct fts_backend *update_backend; + uint32_t virtual_last_uid; + + ARRAY_TYPE(mailboxes) mailboxes; + unsigned int mailbox_idx; +}; + +static int +fts_mailbox_get_seqs(struct mailbox *box, uint32_t *seq1_r, uint32_t *seq2_r) +{ + struct mailbox_status status; + uint32_t last_uid; + + if (fts_backend_get_last_uid(fts_mailbox_backend(box), + box, &last_uid) < 0) + return -1; + + mailbox_get_open_status(box, STATUS_UIDNEXT, &status); + if (status.uidnext <= last_uid+1) + *seq1_r = *seq2_r = 0; + else { + mailbox_get_seq_range(box, last_uid+1, (uint32_t)-1, + seq1_r, seq2_r); + } + return 0; +} + +static int +fts_build_virtual_mailboxes_get(struct virtual_fts_storage_build_context *ctx) +{ + struct virtual_mailbox *vbox = + (struct virtual_mailbox *)ctx->ctx.box; + struct mailbox *const *boxp; + ARRAY_TYPE(mailboxes) all_mailboxes; + uint32_t seq1, seq2; + + t_array_init(&all_mailboxes, 64); + i_array_init(&ctx->mailboxes, 64); + vbox->vfuncs.get_virtual_backend_boxes(ctx->ctx.box, + &all_mailboxes, TRUE); + + array_foreach(&all_mailboxes, boxp) { + if (fts_mailbox_get_seqs(*boxp, &seq1, &seq2) < 0) { + array_free(&ctx->mailboxes); + return -1; + } + if (seq1 != 0) { + ctx->ctx.mail_count += seq2 - seq1 + 1; + array_append(&ctx->mailboxes, boxp, 1); + } + } + return 0; +} + +static void +fts_build_virtual_mailbox_close(struct virtual_fts_storage_build_context *ctx) +{ + if (mailbox_search_deinit(&ctx->ctx.search_ctx) < 0) + ctx->ctx.failed = TRUE; + (void)mailbox_transaction_commit(&ctx->ctx.trans); +} + +static bool +fts_build_virtual_mailbox_next(struct virtual_fts_storage_build_context *ctx) +{ + struct mail_search_args *search_args; + struct mailbox *const *boxes, *box; + struct fts_backend *backend; + unsigned int count; + uint32_t seq1, seq2; + + boxes = array_get(&ctx->mailboxes, &count); + if (ctx->mailbox_idx == count) + return FALSE; + box = boxes[ctx->mailbox_idx++]; + + if (ctx->ctx.trans != NULL) + fts_build_virtual_mailbox_close(ctx); + + if (fts_mailbox_get_seqs(box, &seq1, &seq2) < 0) { + ctx->ctx.failed = TRUE; + return fts_build_virtual_mailbox_next(ctx); + } + + backend = fts_mailbox_backend(box); + if (ctx->update_backend != backend) { + if (ctx->ctx.update_ctx != NULL) { + if (fts_backend_update_deinit(&ctx->ctx.update_ctx) < 0) + ctx->ctx.failed = TRUE; + } + ctx->update_backend = backend; + ctx->ctx.update_ctx = fts_backend_update_init(backend); + } + + + fts_backend_update_set_mailbox(ctx->ctx.update_ctx, box); + search_args = mail_search_build_init(); + mail_search_build_add_seqset(search_args, seq1, seq2); + + ctx->ctx.trans = mailbox_transaction_begin(box, 0); + ctx->ctx.search_ctx = mailbox_search_init(ctx->ctx.trans, search_args, + NULL, 0, NULL); + ctx->ctx.search_ctx->progress_hidden = TRUE; + mail_search_args_unref(&search_args); + return TRUE; +} + +static int +fts_build_virtual_init(struct fts_backend *backend, struct mailbox *box, + struct fts_storage_build_context **build_ctx_r) +{ + struct virtual_fts_storage_build_context *ctx; + struct mailbox_status status; + uint32_t last_uid; + + /* first do a quick check: is the virtual mailbox's last indexed + UID up to date? */ + if (fts_backend_get_last_uid(backend, box, &last_uid) < 0) + return -1; + + mailbox_get_open_status(box, STATUS_UIDNEXT, &status); + if (status.uidnext == last_uid+1) + return 0; + + /* nope. we'll need to go through its mailboxes and check their + indexes. FIXME: we could optimize by going through only those + mailboxes that exist in >last_uid mails */ + ctx = i_new(struct virtual_fts_storage_build_context, 1); + ctx->ctx.box = box; + ctx->virtual_last_uid = status.uidnext - 1; + + if (fts_build_virtual_mailboxes_get(ctx) < 0) { + i_free(ctx); + return -1; + } + fts_build_virtual_mailbox_next(ctx); + + *build_ctx_r = &ctx->ctx; + return 1; +} + +static int fts_build_virtual_deinit(struct fts_storage_build_context *_ctx) +{ + struct virtual_fts_storage_build_context *ctx = + (struct virtual_fts_storage_build_context *)_ctx; + + if (!_ctx->failed) { + (void)fts_index_set_last_uid(ctx->ctx.box, + ctx->virtual_last_uid); + } + + fts_build_virtual_mailbox_close(ctx); + array_free(&ctx->mailboxes); + return 0; +} + +static int fts_build_virtual_more(struct fts_storage_build_context *_ctx) +{ + struct virtual_fts_storage_build_context *ctx = + (struct virtual_fts_storage_build_context *)_ctx; + struct mail *mail; + unsigned int count = 0; + int ret; + + while (mailbox_search_next(_ctx->search_ctx, &mail)) { + T_BEGIN { + ret = fts_build_mail(_ctx, mail); + } T_END; + + if (ret < 0) + return -1; + + _ctx->mail_idx++; + if (++count == FTS_SEARCH_NONBLOCK_COUNT) + return 0; + } + + if (fts_build_virtual_mailbox_next(ctx)) + return fts_build_virtual_more(_ctx); + return 1; +} + +const struct fts_storage_build_vfuncs fts_storage_build_virtual_vfuncs = { + fts_build_virtual_init, + fts_build_virtual_deinit, + fts_build_virtual_more +};
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/plugins/fts/fts-build.c Fri Jul 22 13:21:59 2011 +0300 @@ -0,0 +1,319 @@ +/* Copyright (c) 2006-2011 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "ioloop.h" +#include "istream.h" +#include "str.h" +#include "time-util.h" +#include "rfc822-parser.h" +#include "message-address.h" +#include "message-parser.h" +#include "message-decoder.h" +#include "../virtual/virtual-storage.h" +#include "fts-api-private.h" +#include "fts-build-private.h" + +#define FTS_BUILD_NOTIFY_INTERVAL_SECS 10 + +static void fts_build_parse_content_type(struct fts_storage_build_context *ctx, + const struct message_header_line *hdr) +{ + struct rfc822_parser_context parser; + string_t *content_type; + + rfc822_parser_init(&parser, hdr->full_value, hdr->full_value_len, NULL); + (void)rfc822_skip_lwsp(&parser); + + T_BEGIN { + content_type = t_str_new(64); + if (rfc822_parse_content_type(&parser, content_type) >= 0) { + i_free(ctx->content_type); + ctx->content_type = i_strdup(str_c(content_type)); + } + } T_END; +} + +static void +fts_build_parse_content_disposition(struct fts_storage_build_context *ctx, + const struct message_header_line *hdr) +{ + /* just pass it as-is to backend. */ + i_free(ctx->content_disposition); + ctx->content_disposition = + i_strndup(hdr->full_value, hdr->full_value_len); +} + +static void fts_parse_mail_header(struct fts_storage_build_context *ctx, + const struct message_block *raw_block) +{ + const struct message_header_line *hdr = raw_block->hdr; + + if (strcasecmp(hdr->name, "Content-Type") == 0) + fts_build_parse_content_type(ctx, hdr); + else if (strcasecmp(hdr->name, "Content-Disposition") == 0) + fts_build_parse_content_disposition(ctx, hdr); +} + +static void fts_build_mail_header(struct fts_storage_build_context *ctx, + const struct message_block *block) +{ + const struct message_header_line *hdr = block->hdr; + struct fts_backend_build_key key; + + if (hdr->eoh) + return; + + /* hdr->full_value is always set because we get the block from + message_decoder */ + memset(&key, 0, sizeof(key)); + key.uid = ctx->uid; + key.type = block->part->physical_pos == 0 ? + FTS_BACKEND_BUILD_KEY_HDR : FTS_BACKEND_BUILD_KEY_MIME_HDR; + key.hdr_name = hdr->name; + + if (!fts_backend_update_set_build_key(ctx->update_ctx, &key)) + return; + + if (!message_header_is_address(hdr->name)) { + /* regular unstructured header */ + (void)fts_backend_update_build_more(ctx->update_ctx, + hdr->full_value, + hdr->full_value_len); + } else T_BEGIN { + /* message address. normalize it to give better + search results. */ + struct message_address *addr; + string_t *str; + + addr = message_address_parse(pool_datastack_create(), + hdr->full_value, + hdr->full_value_len, + -1U, TRUE); + str = t_str_new(hdr->full_value_len); + message_address_write(str, addr); + + (void)fts_backend_update_build_more(ctx->update_ctx, + str_data(str), + str_len(str)); + } T_END; +} + +static bool fts_build_body_begin(struct fts_storage_build_context *ctx) +{ + const char *content_type; + struct fts_backend_build_key key; + + memset(&key, 0, sizeof(key)); + key.uid = ctx->uid; + + content_type = ctx->content_type != NULL ? + ctx->content_type : "text/plain"; + if (strncmp(content_type, "text/", 5) == 0 || + strncmp(content_type, "message/", 8) == 0) { + /* text body parts */ + key.type = FTS_BACKEND_BUILD_KEY_BODY_PART; + } else { + /* possibly binary */ + if (!ctx->binary_mime_parts) + return FALSE; + key.type = FTS_BACKEND_BUILD_KEY_BODY_PART_BINARY; + } + key.body_content_type = content_type; + key.body_content_disposition = ctx->content_disposition; + return fts_backend_update_set_build_key(ctx->update_ctx, &key); +} + +int fts_build_mail(struct fts_storage_build_context *ctx, struct mail *mail) +{ + enum message_decoder_flags decoder_flags = MESSAGE_DECODER_FLAG_DTCASE; + struct istream *input; + struct message_parser_ctx *parser; + struct message_decoder_context *decoder; + struct message_block raw_block, block; + struct message_part *prev_part, *parts; + bool skip_body = FALSE, body_part = FALSE; + int ret; + + ctx->uid = mail->uid; + + if (mail_get_stream(mail, NULL, NULL, &input) < 0) + return -1; + + prev_part = NULL; + parser = message_parser_init(pool_datastack_create(), input, + MESSAGE_HEADER_PARSER_FLAG_CLEAN_ONELINE, + 0); + + + if (ctx->binary_mime_parts) + decoder_flags |= MESSAGE_DECODER_FLAG_RETURN_BINARY; + decoder = message_decoder_init(decoder_flags); + for (;;) { + ret = message_parser_parse_next_block(parser, &raw_block); + i_assert(ret != 0); + if (ret < 0) { + if (input->stream_errno == 0) + ret = 0; + break; + } + + if (raw_block.part != prev_part) { + /* body part changed. we're now parsing the end of + boundary, possibly followed by message epilogue */ + fts_backend_update_unset_build_key(ctx->update_ctx); + prev_part = raw_block.part; + i_free_and_null(ctx->content_type); + i_free_and_null(ctx->content_disposition); + + if (raw_block.size != 0) { + /* multipart. skip until beginning of next + part's headers */ + skip_body = TRUE; + } + } + + if (raw_block.hdr != NULL) { + /* always handle headers */ + } else if (raw_block.size == 0) { + /* end of headers */ + skip_body = !fts_build_body_begin(ctx); + body_part = TRUE; + } else { + if (skip_body) + continue; + } + + if (!message_decoder_decode_next_block(decoder, &raw_block, + &block)) + continue; + + if (block.hdr != NULL) { + fts_parse_mail_header(ctx, &raw_block); + fts_build_mail_header(ctx, &block); + } else if (block.size == 0) { + /* end of headers */ + } else { + i_assert(body_part); + if (fts_backend_update_build_more(ctx->update_ctx, + block.data, + block.size) < 0) { + ret = -1; + break; + } + } + } + if (message_parser_deinit(&parser, &parts) < 0) + mail_set_cache_corrupted(mail, MAIL_FETCH_MESSAGE_PARTS); + message_decoder_deinit(&decoder); + return ret; +} + +static void fts_build_notify(struct fts_storage_build_context *ctx) +{ + double completed_frac; + unsigned int eta_secs; + + if (ioloop_time - ctx->last_notify.tv_sec < FTS_BUILD_NOTIFY_INTERVAL_SECS) + return; + ctx->last_notify = ioloop_timeval; + + if (ctx->box->storage->callbacks.notify_ok == NULL || + ctx->mail_idx == 0) + return; + + /* mail_count is counted before indexing actually begins. + by the time the mailbox is actually indexed it may already + have more (or less) mails. so mail_idx can be higher than + mail_count. */ + completed_frac = ctx->mail_idx >= ctx->mail_count ? 1 : + (double)ctx->mail_idx / ctx->mail_count; + + if (completed_frac >= 0.000001) { + unsigned int elapsed_msecs, est_total_msecs; + + elapsed_msecs = timeval_diff_msecs(&ioloop_timeval, + &ctx->search_start_time); + est_total_msecs = elapsed_msecs / completed_frac; + eta_secs = (est_total_msecs - elapsed_msecs) / 1000; + } else { + eta_secs = 0; + } + + T_BEGIN { + const char *text; + + text = t_strdup_printf("Indexed %d%% of the mailbox, " + "ETA %d:%02d", (int)(completed_frac * 100.0), + eta_secs/60, eta_secs%60); + ctx->box->storage->callbacks. + notify_ok(ctx->box, text, + ctx->box->storage->callback_context); + ctx->notified = TRUE; + } T_END; +} + +int fts_build_init(struct fts_backend *backend, struct mailbox *box, + bool precache, + struct fts_storage_build_context **build_ctx_r) +{ + const struct fts_storage_build_vfuncs *v; + int ret; + + *build_ctx_r = NULL; + + /* unless we're precaching (i.e. indexer service, doveadm index) + use the indexer service */ + if (!precache) + v = &fts_storage_build_indexer_vfuncs; + else if (strcmp(box->storage->name, VIRTUAL_STORAGE_NAME) == 0) + v = &fts_storage_build_virtual_vfuncs; + else + v = &fts_storage_build_mailbox_vfuncs; + + if ((ret = v->init(backend, box, build_ctx_r)) <= 0) + return ret; + + (*build_ctx_r)->box = box; + (*build_ctx_r)->v = *v; + (*build_ctx_r)->binary_mime_parts = + (backend->flags & FTS_BACKEND_FLAG_BINARY_MIME_PARTS) != 0; + return 1; +} + +int fts_build_deinit(struct fts_storage_build_context **_ctx) +{ + struct fts_storage_build_context *ctx = *_ctx; + int ret = ctx->failed ? -1 : 0; + + *_ctx = NULL; + + if (ctx->v.deinit(ctx) < 0) + ret = -1; + if (ctx->update_ctx != NULL) { + if (fts_backend_update_deinit(&ctx->update_ctx) < 0) + ret = -1; + } + if (ctx->notified) { + /* we notified at least once */ + ctx->box->storage->callbacks. + notify_ok(ctx->box, "Mailbox indexing finished", + ctx->box->storage->callback_context); + } + i_free(ctx->content_type); + i_free(ctx->content_disposition); + i_free(ctx); + return ret; +} + +int fts_build_more(struct fts_storage_build_context *ctx) +{ + int ret; + + if ((ret = ctx->v.more(ctx)) < 0) { + ctx->failed = TRUE; + return -1; + } + + fts_build_notify(ctx); + return ret; +}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/plugins/fts/fts-build.h Fri Jul 22 13:21:59 2011 +0300 @@ -0,0 +1,18 @@ +#ifndef FTS_BUILD_H +#define FTS_BUILD_H + +struct fts_storage_build_context; + +/* Initialize building. Returns 1 if we need to build (build_ctx set), + 0 if not (build_ctx NULL) or -1 if error. */ +int fts_build_init(struct fts_backend *backend, struct mailbox *box, + bool precache, + struct fts_storage_build_context **build_ctx_r); +/* Returns 0 if ok, -1 if error. */ +int fts_build_deinit(struct fts_storage_build_context **ctx); + +/* Build more. Returns 1 if finished, 0 if this function needs to be called + again, -1 if error. */ +int fts_build_more(struct fts_storage_build_context *ctx); + +#endif
--- a/src/plugins/fts/fts-mailbox.c Fri Jul 22 13:13:29 2011 +0300 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,56 +0,0 @@ -/* Copyright (c) 2006-2011 Dovecot authors, see the included COPYING file */ - -#include "lib.h" -#include "mail-storage.h" -#include "fts-mailbox.h" -#include "../virtual/virtual-storage.h" - -bool fts_mailbox_get_virtual_uid(struct mailbox *box, - const char *backend_mailbox, - uint32_t backend_uidvalidity, - uint32_t backend_uid, uint32_t *uid_r) -{ - struct virtual_mailbox *vbox; - - if (strcmp(box->storage->name, VIRTUAL_STORAGE_NAME) != 0) - return FALSE; - - vbox = (struct virtual_mailbox *)box; - return vbox->vfuncs.get_virtual_uid(box, backend_mailbox, - backend_uidvalidity, - backend_uid, uid_r); -} - -void fts_mailbox_get_virtual_backend_boxes(struct mailbox *box, - ARRAY_TYPE(mailboxes) *mailboxes, - bool only_with_msgs) -{ - struct virtual_mailbox *vbox; - - if (strcmp(box->storage->name, VIRTUAL_STORAGE_NAME) != 0) - array_append(mailboxes, &box, 1); - else { - vbox = (struct virtual_mailbox *)box; - vbox->vfuncs.get_virtual_backend_boxes(box, mailboxes, - only_with_msgs); - } -} - -void fts_mailbox_get_virtual_box_patterns(struct mailbox *box, - ARRAY_TYPE(mailbox_virtual_patterns) *includes, - ARRAY_TYPE(mailbox_virtual_patterns) *excludes) -{ - struct virtual_mailbox *vbox; - - if (strcmp(box->storage->name, VIRTUAL_STORAGE_NAME) != 0) { - struct mailbox_virtual_pattern pat; - - memset(&pat, 0, sizeof(pat)); - pat.ns = mailbox_list_get_namespace(box->list); - pat.pattern = box->name; - array_append(includes, &pat, 1); - } else { - vbox = (struct virtual_mailbox *)box; - vbox->vfuncs.get_virtual_box_patterns(box, includes, excludes); - } -}
--- a/src/plugins/fts/fts-mailbox.h Fri Jul 22 13:13:29 2011 +0300 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,22 +0,0 @@ -#ifndef FTS_MAILBOX_H -#define FTS_MAILBOX_H - -/* If box is a virtual mailbox, look up UID for the given backend message. - Returns TRUE if found, FALSE if not. */ -bool fts_mailbox_get_virtual_uid(struct mailbox *box, - const char *backend_mailbox, - uint32_t backend_uidvalidity, - uint32_t backend_uid, uint32_t *uid_r); -/* If box is a virtual mailbox, return all backend mailboxes. If - only_with_msgs=TRUE, return only those mailboxes that have at least one - message existing in the virtual mailbox. */ -void fts_mailbox_get_virtual_backend_boxes(struct mailbox *box, - ARRAY_TYPE(mailboxes) *mailboxes, - bool only_with_msgs); -/* If mailbox is a virtual mailbox, return all mailbox list patterns that - are used to figure out which mailboxes belong to the virtual mailbox. */ -void fts_mailbox_get_virtual_box_patterns(struct mailbox *box, - ARRAY_TYPE(mailbox_virtual_patterns) *includes, - ARRAY_TYPE(mailbox_virtual_patterns) *excludes); - -#endif
--- a/src/plugins/fts/fts-plugin.c Fri Jul 22 13:13:29 2011 +0300 +++ b/src/plugins/fts/fts-plugin.c Fri Jul 22 13:21:59 2011 +0300 @@ -2,6 +2,7 @@ #include "lib.h" #include "mail-storage-hooks.h" +#include "fts-storage.h" #include "fts-plugin.h" #include <stdlib.h> @@ -9,6 +10,7 @@ const char *fts_plugin_version = DOVECOT_VERSION; static struct mail_storage_hooks fts_mail_storage_hooks = { + .mailbox_list_created = fts_mailbox_list_created, .mailbox_allocated = fts_mailbox_allocated, .mail_allocated = fts_mail_allocated };
--- a/src/plugins/fts/fts-plugin.h Fri Jul 22 13:13:29 2011 +0300 +++ b/src/plugins/fts/fts-plugin.h Fri Jul 22 13:21:59 2011 +0300 @@ -1,9 +1,6 @@ #ifndef FTS_PLUGIN_H #define FTS_PLUGIN_H -void fts_mailbox_allocated(struct mailbox *box); -void fts_mail_allocated(struct mail *mail); - void fts_plugin_init(struct module *module); void fts_plugin_deinit(void);
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/plugins/fts/fts-search-serialize.c Fri Jul 22 13:21:59 2011 +0300 @@ -0,0 +1,99 @@ +/* Copyright (c) 2011 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "buffer.h" +#include "mail-search.h" +#include "fts-search-serialize.h" + +#define HAVE_SUBARGS(arg) \ + ((arg)->type == SEARCH_SUB || (arg)->type == SEARCH_OR) + +void fts_search_serialize(buffer_t *buf, const struct mail_search_arg *args) +{ + char chr; + + for (; args != NULL; args = args->next) { + chr = (args->match_always ? 1 : 0) | + (args->nonmatch_always ? 2 : 0); + buffer_append_c(buf, chr); + + if (HAVE_SUBARGS(args)) + fts_search_serialize(buf, args->value.subargs); + } +} + +static void fts_search_deserialize_idx(struct mail_search_arg *args, + const buffer_t *buf, unsigned int *idx) +{ + const char *data = buf->data; + + for (; args != NULL; args = args->next) { + i_assert(*idx < buf->used); + + args->match_always = (data[*idx] & 1) != 0; + args->nonmatch_always = (data[*idx] & 2) != 0; + args->result = args->match_always ? 1 : + (args->nonmatch_always ? 0 : -1); + *idx += 1; + + if (HAVE_SUBARGS(args)) { + fts_search_deserialize_idx(args->value.subargs, + buf, idx); + } + } +} + +void fts_search_deserialize(struct mail_search_arg *args, + const buffer_t *buf) +{ + unsigned int idx = 0; + + fts_search_deserialize_idx(args, buf, &idx); + i_assert(idx == buf->used); +} + +static void +fts_search_deserialize_add_idx(struct mail_search_arg *args, + const buffer_t *buf, unsigned int *idx, + bool matches) +{ + const char *data = buf->data; + + for (; args != NULL; args = args->next) { + i_assert(*idx < buf->used); + + if (data[*idx] != 0) { + if (matches) { + args->match_always = TRUE; + args->result = 1; + } else { + args->nonmatch_always = TRUE; + args->result = 0; + } + } + *idx += 1; + + if (HAVE_SUBARGS(args)) { + fts_search_deserialize_add_idx(args->value.subargs, + buf, idx, matches); + } + } +} + +void fts_search_deserialize_add_matches(struct mail_search_arg *args, + const buffer_t *buf) +{ + unsigned int idx = 0; + + fts_search_deserialize_add_idx(args, buf, &idx, TRUE); + i_assert(idx == buf->used); +} + +void fts_search_deserialize_add_nonmatches(struct mail_search_arg *args, + const buffer_t *buf) +{ + unsigned int idx = 0; + + fts_search_deserialize_add_idx(args, buf, &idx, FALSE); + i_assert(idx == buf->used); +}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/plugins/fts/fts-search-serialize.h Fri Jul 22 13:21:59 2011 +0300 @@ -0,0 +1,16 @@ +#ifndef FTS_SEARCH_SERIALIZE_H +#define FTS_SEARCH_SERIALIZE_H + +/* serialize [non]match_always fields (clearing buffer) */ +void fts_search_serialize(buffer_t *buf, const struct mail_search_arg *args); +/* add/remove [non]match_always fields in search args */ +void fts_search_deserialize(struct mail_search_arg *args, + const buffer_t *buf); +/* add match_always=TRUE fields to search args */ +void fts_search_deserialize_add_matches(struct mail_search_arg *args, + const buffer_t *buf); +/* add nonmatch_always=TRUE fields to search args */ +void fts_search_deserialize_add_nonmatches(struct mail_search_arg *args, + const buffer_t *buf); + +#endif
--- a/src/plugins/fts/fts-search.c Fri Jul 22 13:13:29 2011 +0300 +++ b/src/plugins/fts/fts-search.c Fri Jul 22 13:21:59 2011 +0300 @@ -5,12 +5,14 @@ #include "str.h" #include "seq-range-array.h" #include "mail-search.h" -#include "mail-storage-private.h" +#include "../virtual/virtual-storage.h" #include "fts-api-private.h" +#include "fts-search-serialize.h" #include "fts-storage.h" static void -uid_range_to_seqs(struct mailbox *box, const ARRAY_TYPE(seq_range) *uid_range, +uid_range_to_seqs(struct fts_search_context *fctx, + const ARRAY_TYPE(seq_range) *uid_range, ARRAY_TYPE(seq_range) *seq_range) { const struct seq_range *range; @@ -18,156 +20,338 @@ uint32_t seq1, seq2; range = array_get(uid_range, &count); - i_array_init(seq_range, count); + if (!array_is_created(seq_range)) + p_array_init(seq_range, fctx->result_pool, count); for (i = 0; i < count; i++) { - mailbox_get_seq_range(box, range[i].seq1, range[i].seq2, + mailbox_get_seq_range(fctx->box, range[i].seq1, range[i].seq2, &seq1, &seq2); if (seq1 != 0) seq_range_array_add_range(seq_range, seq1, seq2); } } -static void fts_uid_results_to_seq(struct fts_search_context *fctx) +static int fts_search_lookup_level_single(struct fts_search_context *fctx, + struct mail_search_arg *args, + bool and_args) { - ARRAY_TYPE(seq_range) uid_range; + struct fts_search_level *level; + struct fts_result result; + + memset(&result, 0, sizeof(result)); + p_array_init(&result.definite_uids, fctx->result_pool, 32); + p_array_init(&result.maybe_uids, fctx->result_pool, 32); + p_array_init(&result.scores, fctx->result_pool, 32); + + mail_search_args_reset(args, TRUE); + if (fts_backend_lookup(fctx->backend, fctx->box, args, and_args, + &result) < 0) + return -1; + + level = array_append_space(&fctx->levels); + level->args_matches = buffer_create_dynamic(fctx->result_pool, 16); + fts_search_serialize(level->args_matches, args); + + uid_range_to_seqs(fctx, &result.definite_uids, &level->definite_seqs); + uid_range_to_seqs(fctx, &result.maybe_uids, &level->maybe_seqs); + level->score_map = result.scores; + return 0; +} + +static void +level_scores_add_vuids(struct virtual_mailbox *vbox, + struct fts_search_level *level, struct fts_result *br) +{ + const struct fts_score_map *scores; + unsigned int i, count; + ARRAY_TYPE(seq_range) backend_uids; + ARRAY_TYPE(uint32_t) vuids_arr; + const uint32_t *vuids; + struct fts_score_map *score; + + scores = array_get(&br->scores, &count); + t_array_init(&vuids_arr, count); + t_array_init(&backend_uids, 64); + for (i = 0; i < count; i++) + seq_range_array_add(&backend_uids, 0, scores[i].uid); + vbox->vfuncs.get_virtual_uid_map(&vbox->box, br->box, + &backend_uids, &vuids_arr); + + i_assert(array_count(&vuids_arr) == array_count(&br->scores)); + vuids = array_get(&vuids_arr, &count); + for (i = 0; i < count; i++) { + score = array_append_space(&level->score_map); + score->uid = vuids[i]; + score->score = scores[i].score; + } +} - uid_range = fctx->definite_seqs; - uid_range_to_seqs(fctx->t->box, &uid_range, &fctx->definite_seqs); - array_free(&uid_range); +static int +mailbox_cmp_fts_backend(struct mailbox *const *m1, struct mailbox *const *m2) +{ + struct fts_backend *b1, *b2; + + b1 = fts_mailbox_backend(*m1); + b2 = fts_mailbox_backend(*m2); + if (b1 < b2) + return -1; + if (b1 > b2) + return 1; + return 0; +} + +static int +multi_add_lookup_result(struct fts_search_context *fctx, + struct fts_search_level *level, + struct mail_search_arg *args, + struct fts_multi_result *result) +{ + struct virtual_mailbox *vbox = (struct virtual_mailbox *)fctx->box; + ARRAY_TYPE(seq_range) vuids; + size_t orig_size; + unsigned int i; - uid_range = fctx->maybe_seqs; - uid_range_to_seqs(fctx->t->box, &uid_range, &fctx->maybe_seqs); - array_free(&uid_range); + orig_size = level->args_matches->used; + fts_search_serialize(level->args_matches, args); + if (orig_size > 0) { + if (level->args_matches->used != orig_size * 2 || + memcmp(level->args_matches->data, + CONST_PTR_OFFSET(level->args_matches->data, + orig_size), orig_size) != 0) + i_panic("incompatible fts backends for namespaces"); + buffer_set_used_size(level->args_matches, orig_size); + } + + t_array_init(&vuids, 64); + for (i = 0; result->box_results[i].box != NULL; i++) { + struct fts_result *br = &result->box_results[i]; + + array_clear(&vuids); + if (array_is_created(&br->definite_uids)) { + vbox->vfuncs.get_virtual_uids(fctx->box, br->box, + &br->definite_uids, + &vuids); + } + uid_range_to_seqs(fctx, &vuids, &level->definite_seqs); + + array_clear(&vuids); + if (array_is_created(&br->maybe_uids)) { + vbox->vfuncs.get_virtual_uids(fctx->box, br->box, + &br->maybe_uids, &vuids); + } + uid_range_to_seqs(fctx, &vuids, &level->maybe_seqs); + + if (array_is_created(&br->scores)) + level_scores_add_vuids(vbox, level, br); + } + return 0; } -static int fts_search_lookup_arg(struct fts_search_context *fctx, - struct mail_search_arg *arg) +static int fts_search_lookup_level_multi(struct fts_search_context *fctx, + struct mail_search_arg *args, + bool and_args) { - enum fts_lookup_flags flags = 0; - const char *key; + struct virtual_mailbox *vbox = (struct virtual_mailbox *)fctx->box; + ARRAY_TYPE(mailboxes) mailboxes_arr, tmp_mailboxes; + struct mailbox *const *mailboxes; + struct fts_backend *backend; + struct fts_search_level *level; + struct fts_multi_result result; + unsigned int i, j, mailbox_count; + + p_array_init(&mailboxes_arr, fctx->result_pool, 8); + vbox->vfuncs.get_virtual_backend_boxes(fctx->box, &mailboxes_arr, TRUE); + array_sort(&mailboxes_arr, mailbox_cmp_fts_backend); - switch (arg->type) { - case SEARCH_HEADER: - case SEARCH_HEADER_COMPRESS_LWSP: - /* we can filter out messages that don't have the header, - but we can't trust definite results list. */ - flags = FTS_LOOKUP_FLAG_HEADER; - key = arg->value.str; - if (*key == '\0') { - /* we're only checking the existence - of the header. */ - key = t_str_ucase(arg->hdr_field_name); + memset(&result, 0, sizeof(result)); + result.pool = fctx->result_pool; + + level = array_append_space(&fctx->levels); + level->args_matches = buffer_create_dynamic(fctx->result_pool, 16); + p_array_init(&level->score_map, fctx->result_pool, 1); + + mailboxes = array_get(&mailboxes_arr, &mailbox_count); + t_array_init(&tmp_mailboxes, mailbox_count); + for (i = 0; i < mailbox_count; i = j) { + array_clear(&tmp_mailboxes); + array_append(&tmp_mailboxes, &mailboxes[i], 1); + + backend = fts_mailbox_backend(mailboxes[i]); + for (j = i + 1; j < mailbox_count; j++) { + if (fts_mailbox_backend(mailboxes[j]) != backend) + break; + array_append(&tmp_mailboxes, &mailboxes[j], 1); } - break; - case SEARCH_TEXT: - flags = FTS_LOOKUP_FLAG_HEADER; - case SEARCH_BODY: - flags |= FTS_LOOKUP_FLAG_BODY; - key = arg->value.str; - break; - default: - /* can't filter this */ - return 0; + (void)array_append_space(&tmp_mailboxes); + + mail_search_args_reset(args, TRUE); + if (fts_backend_lookup_multi(backend, + array_idx(&tmp_mailboxes, 0), + args, and_args, &result) < 0) + return -1; + + if (multi_add_lookup_result(fctx, level, args, &result) < 0) + return -1; + } + return 0; +} + +static int fts_search_lookup_level(struct fts_search_context *fctx, + struct mail_search_arg *args, + bool and_args) +{ + if (!fctx->virtual_mailbox) { + if (fts_search_lookup_level_single(fctx, args, and_args) < 0) + return -1; + } else T_BEGIN { + if (fts_search_lookup_level_multi(fctx, args, and_args) < 0) + return -1; + } T_END; + + for (; args != NULL; args = args->next) { + if (args->type != SEARCH_OR && args->type != SEARCH_SUB) + continue; + + if (fts_search_lookup_level(fctx, args->value.subargs, + args->type == SEARCH_SUB) < 0) + return -1; + } + return 0; +} + +static void +fts_search_merge_scores_and(ARRAY_TYPE(fts_score_map) *dest, + const ARRAY_TYPE(fts_score_map) *src) +{ + struct fts_score_map *dest_map; + const struct fts_score_map *src_map; + unsigned int desti, srci, dest_count, src_count; + + dest_map = array_get_modifiable(dest, &dest_count); + src_map = array_get(src, &src_count); + + /* arg_scores are summed to current scores. we could drop UIDs that + don't exist in both, but that's just extra work so don't bother */ + for (desti = srci = 0; desti < dest_count && srci < src_count;) { + if (dest_map[desti].uid < src_map[srci].uid) + desti++; + else if (dest_map[desti].uid > src_map[srci].uid) + srci++; + else { + if (dest_map[desti].score < src_map[srci].score) + dest_map[desti].score = src_map[srci].score; + desti++; srci++; + } } - if (arg->match_not) - flags |= FTS_LOOKUP_FLAG_INVERT; +} + +static void +fts_search_merge_scores_or(ARRAY_TYPE(fts_score_map) *dest, + const ARRAY_TYPE(fts_score_map) *src) +{ + ARRAY_TYPE(fts_score_map) src2; + const struct fts_score_map *src_map, *src2_map; + unsigned int srci, src2i, src_count, src2_count; + + t_array_init(&src2, array_count(dest)); + array_append_array(&src2, dest); + array_clear(dest); + + src_map = array_get(src, &src_count); + src2_map = array_get(&src2, &src2_count); + + /* add any missing UIDs to current scores. if any existing UIDs have + lower scores than in arg_scores, increase them. */ + for (srci = src2i = 0; srci < src_count || src2i < src2_count;) { + if (src2i == src2_count || + src_map[srci].uid < src2_map[src2i].uid) { + array_append(dest, &src_map[srci], 1); + srci++; + } else if (srci == src_count || + src_map[srci].uid > src2_map[src2i].uid) { + array_append(dest, &src2_map[src2i], 1); + src2i++; + } else { + i_assert(src_map[srci].uid == src2_map[src2i].uid); + if (src_map[srci].score > src2_map[src2i].score) + array_append(dest, &src_map[srci], 1); + else + array_append(dest, &src2_map[src2i], 1); + srci++; src2i++; + } + } +} - if (!fctx->refreshed) { - if (fts_backend_refresh(fctx->fbox->backend) < 0) - return -1; - fctx->refreshed = TRUE; +static void +fts_search_merge_scores_level(struct fts_search_context *fctx, + struct mail_search_arg *args, unsigned int *idx, + bool and_args, ARRAY_TYPE(fts_score_map) *scores) +{ + const struct fts_search_level *level; + ARRAY_TYPE(fts_score_map) arg_scores; + + i_assert(array_count(scores) == 0); + + /* + The (simplified) args can look like: + + A and B and (C or D) and (E or F) and ... + A or B or (C and D) or (E and F) or ... + + The A op B part's scores are in level->scores. The child args' + scores are in the sub levels' scores. + */ + + level = array_idx(&fctx->levels, *idx); + array_append_array(scores, &level->score_map); + + t_array_init(&arg_scores, 64); + for (; args != NULL; args = args->next) { + if (args->type != SEARCH_OR && args->type != SEARCH_SUB) + continue; + + *idx += 1; + array_clear(&arg_scores); + fts_search_merge_scores_level(fctx, args->value.subargs, idx, + args->type == SEARCH_OR, + &arg_scores); + + if (and_args) + fts_search_merge_scores_and(scores, &arg_scores); + else T_BEGIN { + fts_search_merge_scores_or(scores, &arg_scores); + } T_END; } +} - /* note that the key is in UTF-8 decomposed titlecase */ - fctx->lookup_ctx = fts_backend_lookup_init(fctx->fbox->backend); - fts_backend_lookup_add(fctx->lookup_ctx, key, flags); - return 0; +static void fts_search_merge_scores(struct fts_search_context *fctx) +{ + unsigned int idx = 0; + + fts_search_merge_scores_level(fctx, fctx->args->args, &idx, + TRUE, &fctx->scores->score_map); } void fts_search_lookup(struct fts_search_context *fctx) { - struct mail_search_arg *arg; - int ret; + uint32_t last_uid, seq1, seq2; - if (fctx->best_arg == NULL) - return; - - i_array_init(&fctx->definite_seqs, 64); - i_array_init(&fctx->maybe_seqs, 64); - i_array_init(&fctx->score_map, 64); + i_assert(array_count(&fctx->levels) == 0); + i_assert(fctx->args->simplified); - /* start lookup with the best arg */ - T_BEGIN { - ret = fts_search_lookup_arg(fctx, fctx->best_arg); - } T_END; - /* filter the rest */ - for (arg = fctx->args->args; arg != NULL && ret == 0; arg = arg->next) { - if (arg != fctx->best_arg) { - T_BEGIN { - ret = fts_search_lookup_arg(fctx, arg); - } T_END; - } - } + if (fts_backend_refresh(fctx->backend) < 0) + return; + if (fts_backend_get_last_uid(fctx->backend, fctx->box, &last_uid) < 0) + return; + mailbox_get_seq_range(fctx->box, last_uid+1, (uint32_t)-1, + &seq1, &seq2); + fctx->first_unindexed_seq = seq1 != 0 ? seq1 : (uint32_t)-1; - if (fctx->lookup_ctx != NULL) { - fts_backend_lookup_deinit(&fctx->lookup_ctx, - &fctx->definite_seqs, - &fctx->maybe_seqs, - &fctx->score_map); + fts_search_serialize(fctx->orig_matches, fctx->args->args); + + if (fts_search_lookup_level(fctx, fctx->args->args, TRUE) == 0) { + fctx->fts_lookup_success = TRUE; + fts_search_merge_scores(fctx); } - if (ret == 0) { - fctx->seqs_set = TRUE; - fts_uid_results_to_seq(fctx); - } + fts_search_deserialize(fctx->args->args, fctx->orig_matches); } - -static bool arg_is_better(const struct mail_search_arg *new_arg, - const struct mail_search_arg *old_arg) -{ - if (old_arg == NULL) - return TRUE; - if (new_arg == NULL) - return FALSE; - - /* avoid NOTs */ - if (old_arg->match_not && !new_arg->match_not) - return TRUE; - if (!old_arg->match_not && new_arg->match_not) - return FALSE; - - /* prefer not to use headers. they have a larger possibility of - having lots of identical strings */ - if (old_arg->type == SEARCH_HEADER || - old_arg->type == SEARCH_HEADER_COMPRESS_LWSP) - return TRUE; - else if (new_arg->type == SEARCH_HEADER || - new_arg->type == SEARCH_HEADER_COMPRESS_LWSP) - return FALSE; - - return strlen(new_arg->value.str) > strlen(old_arg->value.str); -} - -static void -fts_search_args_find_best(struct mail_search_arg *args, - struct mail_search_arg **best_arg) -{ - for (; args != NULL; args = args->next) { - switch (args->type) { - case SEARCH_BODY: - case SEARCH_TEXT: - case SEARCH_HEADER: - case SEARCH_HEADER_COMPRESS_LWSP: - if (arg_is_better(args, *best_arg)) - *best_arg = args; - break; - default: - break; - } - } -} - -void fts_search_analyze(struct fts_search_context *fctx) -{ - fts_search_args_find_best(fctx->args->args, &fctx->best_arg); -}
--- a/src/plugins/fts/fts-storage.c Fri Jul 22 13:13:29 2011 +0300 +++ b/src/plugins/fts/fts-storage.c Fri Jul 22 13:21:59 2011 +0300 @@ -1,23 +1,16 @@ /* Copyright (c) 2006-2011 Dovecot authors, see the included COPYING file */ #include "lib.h" -#include "ioloop.h" #include "array.h" -#include "str.h" -#include "unichar.h" -#include "istream.h" -#include "time-util.h" -#include "rfc822-parser.h" -#include "message-parser.h" -#include "message-decoder.h" -#include "mail-namespace.h" -#include "mail-search-build.h" +#include "mail-search.h" #include "mail-storage-private.h" #include "mailbox-list-private.h" +#include "../virtual/virtual-storage.h" #include "fts-api-private.h" -#include "fts-mailbox.h" +#include "fts-build.h" +#include "fts-search-serialize.h" +#include "fts-plugin.h" #include "fts-storage.h" -#include "fts-plugin.h" #include <stdlib.h> @@ -25,642 +18,95 @@ MODULE_CONTEXT(obj, fts_storage_module) #define FTS_MAIL_CONTEXT(obj) \ MODULE_CONTEXT(obj, fts_mail_module) +#define FTS_LIST_CONTEXT(obj) \ + MODULE_CONTEXT(obj, fts_mailbox_list_module) -#define FTS_SEARCH_NONBLOCK_COUNT 50 -#define FTS_BUILD_NOTIFY_INTERVAL_SECS 10 +struct fts_mailbox_list { + union mailbox_list_module_context module_ctx; + struct fts_backend *backend; +}; + +struct fts_mailbox { + union mailbox_module_context module_ctx; + struct fts_backend_update_context *sync_update_ctx; +}; + +struct fts_transaction_context { + union mailbox_transaction_module_context module_ctx; + + struct fts_scores *scores; +}; struct fts_mail { union mail_module_context module_ctx; char score[30]; }; -struct fts_storage_build_context { - struct mail_search_context *search_ctx; - struct mail_search_args *search_args; - struct fts_backend_build_context *build; - - struct timeval search_start_time, last_notify; - - uint32_t uid; - string_t *headers; - char *content_type, *content_disposition; -}; - -struct fts_transaction_context { - union mailbox_transaction_module_context module_ctx; - - struct fts_storage_build_context *build_ctx; - ARRAY_TYPE(fts_score_map) *score_map; - struct mail *mail; - - uint32_t last_uid; - - unsigned int free_mail:1; - unsigned int expunges:1; -}; - static MODULE_CONTEXT_DEFINE_INIT(fts_storage_module, &mail_storage_module_register); static MODULE_CONTEXT_DEFINE_INIT(fts_mail_module, &mail_module_register); - -static void fts_mailbox_free(struct mailbox *box) -{ - struct fts_mailbox *fbox = FTS_CONTEXT(box); - - if (fbox->backend != NULL) - fts_backend_deinit(&fbox->backend); - - fbox->module_ctx.super.free(box); - i_free(fbox); -} - -static int fts_build_mail_flush_headers(struct fts_storage_build_context *ctx) -{ - if (str_len(ctx->headers) == 0) - return 0; - - i_assert(uni_utf8_data_is_valid(ctx->headers->data, ctx->headers->used)); - - fts_backend_build_hdr(ctx->build, ctx->uid); - if (fts_backend_build_more(ctx->build, str_data(ctx->headers), - str_len(ctx->headers)) < 0) - return -1; - - str_truncate(ctx->headers, 0); - return 0; -} - -static void fts_build_parse_content_type(struct fts_storage_build_context *ctx, - const struct message_header_line *hdr) -{ - struct rfc822_parser_context parser; - string_t *content_type; - - rfc822_parser_init(&parser, hdr->full_value, hdr->full_value_len, NULL); - (void)rfc822_skip_lwsp(&parser); - - T_BEGIN { - content_type = t_str_new(64); - if (rfc822_parse_content_type(&parser, content_type) >= 0) { - i_free(ctx->content_type); - ctx->content_type = i_strdup(str_c(content_type)); - } - } T_END; -} - -static void -fts_build_parse_content_disposition(struct fts_storage_build_context *ctx, - const struct message_header_line *hdr) -{ - /* just pass it as-is to backend. */ - i_free(ctx->content_disposition); - ctx->content_disposition = - i_strndup(hdr->full_value, hdr->full_value_len); -} - -static void fts_parse_mail_header(struct fts_storage_build_context *ctx, - const struct message_block *raw_block) -{ - const struct message_header_line *hdr = raw_block->hdr; - - if (strcasecmp(hdr->name, "Content-Type") == 0) - fts_build_parse_content_type(ctx, hdr); - else if (strcasecmp(hdr->name, "Content-Disposition") == 0) - fts_build_parse_content_disposition(ctx, hdr); -} - -static void fts_build_mail_header(struct fts_storage_build_context *ctx, - const struct message_block *block) -{ - const struct message_header_line *hdr = block->hdr; - - /* hdr->full_value is always set because we get the block from - message_decoder */ - str_append(ctx->headers, hdr->name); - str_append_n(ctx->headers, hdr->middle, hdr->middle_len); - str_append_n(ctx->headers, hdr->full_value, hdr->full_value_len); - if (!hdr->no_newline) - str_append_c(ctx->headers, '\n'); -} - -static int -fts_build_mail(struct fts_storage_build_context *ctx, struct mail *mail) -{ - enum message_decoder_flags decoder_flags = MESSAGE_DECODER_FLAG_DTCASE; - struct istream *input; - struct message_parser_ctx *parser; - struct message_decoder_context *decoder; - struct message_block raw_block, block; - struct message_part *prev_part, *parts; - bool skip_body = FALSE, body_part = FALSE; - int ret; - - ctx->uid = mail->uid; - - if (mail_get_stream(mail, NULL, NULL, &input) < 0) - return -1; - - prev_part = NULL; - parser = message_parser_init(pool_datastack_create(), input, - MESSAGE_HEADER_PARSER_FLAG_CLEAN_ONELINE, - 0); - - - if ((ctx->build->backend->flags & FTS_BACKEND_FLAG_BINARY_MIME_PARTS) != 0) - decoder_flags |= MESSAGE_DECODER_FLAG_RETURN_BINARY; - decoder = message_decoder_init(decoder_flags); - for (;;) { - ret = message_parser_parse_next_block(parser, &raw_block); - i_assert(ret != 0); - if (ret < 0) { - if (input->stream_errno == 0) - ret = 0; - break; - } - - if (raw_block.part != prev_part) { - /* body part changed. we're now parsing the end of - boundary, possibly followed by message epilogue */ - if (!skip_body && prev_part != NULL) { - i_assert(body_part); - fts_backend_build_body_end(ctx->build); - } - prev_part = raw_block.part; - i_free_and_null(ctx->content_type); - i_free_and_null(ctx->content_disposition); - - if (raw_block.size != 0) { - /* multipart. skip until beginning of next - part's headers */ - skip_body = TRUE; - } - } - - if (raw_block.hdr != NULL) { - /* always handle headers */ - } else if (raw_block.size == 0) { - /* end of headers */ - const char *content_type = ctx->content_type == NULL ? - "text/plain" : ctx->content_type; +static MODULE_CONTEXT_DEFINE_INIT(fts_mailbox_list_module, + &mailbox_list_module_register); - skip_body = !fts_backend_build_body_begin(ctx->build, - ctx->uid, content_type, - ctx->content_disposition); - body_part = TRUE; - } else { - if (skip_body) - continue; - } - - if (!message_decoder_decode_next_block(decoder, &raw_block, - &block)) - continue; - - if (block.hdr != NULL) { - fts_parse_mail_header(ctx, &raw_block); - fts_build_mail_header(ctx, &block); - } else if (block.size == 0) { - /* end of headers */ - str_append_c(ctx->headers, '\n'); - } else { - i_assert(body_part); - if (fts_backend_build_more(ctx->build, - block.data, block.size) < 0) { - ret = -1; - break; - } - } - } - if (!skip_body && body_part) - fts_backend_build_body_end(ctx->build); - if (message_parser_deinit(&parser, &parts) < 0) - mail_set_cache_corrupted(mail, MAIL_FETCH_MESSAGE_PARTS); - message_decoder_deinit(&decoder); - - if (ret == 0) { - /* Index all headers at the end. This is required for Squat, - because it can handle only incremental UIDs. */ - ret = fts_build_mail_flush_headers(ctx); - } - return ret; -} - -static int fts_build_init_seq(struct fts_search_context *fctx, - struct mailbox_transaction_context *t, - uint32_t seq1, uint32_t seq2, uint32_t last_uid) -{ - struct mail_search_args *search_args; - struct fts_storage_build_context *ctx; - struct fts_backend_build_context *build; - uint32_t last_uid_locked; - - if (fctx->best_arg->type == SEARCH_HEADER || - fctx->best_arg->type == SEARCH_HEADER_COMPRESS_LWSP) { - /* we're not updating the index just for header lookups */ - if (seq1 < fctx->first_nonindexed_seq) - fctx->first_nonindexed_seq = seq1; - return 0; - } - - if (fts_backend_build_init(fctx->fbox->backend, - &last_uid_locked, &build) < 0) - return -1; - if (last_uid != last_uid_locked && last_uid_locked != (uint32_t)-1) { - /* changed, need to get again the sequences */ - last_uid = last_uid_locked; - mailbox_get_seq_range(t->box, last_uid+1, (uint32_t)-1, - &seq1, &seq2); - if (seq1 == 0) { - /* no new messages */ - (void)fts_backend_build_deinit(&build); - return 0; - } - } - - search_args = mail_search_build_init(); - mail_search_build_add_seqset(search_args, seq1, seq2); - - ctx = i_new(struct fts_storage_build_context, 1); - ctx->build = build; - ctx->headers = str_new(default_pool, 512); - ctx->search_ctx = mailbox_search_init(t, search_args, NULL, 0, NULL); - ctx->search_ctx->progress_hidden = TRUE; - ctx->search_args = search_args; - - fctx->build_ctx = ctx; - return 1; -} - -static int fts_build_init_trans(struct fts_search_context *fctx, - struct mailbox_transaction_context *t) -{ - uint32_t last_uid, seq1, seq2; - int ret; - - if (fts_backend_get_last_uid(fctx->fbox->backend, &last_uid) < 0) - return -1; - - mailbox_get_seq_range(t->box, last_uid+1, (uint32_t)-1, &seq1, &seq2); - if (seq1 == 0) { - /* no new messages */ - return 0; - } - - ret = fts_build_init_seq(fctx, t, seq1, seq2, last_uid); - return ret < 0 ? -1 : 0; -} - -static int -fts_build_init_box(struct fts_search_context *fctx, struct mailbox *box, - uint32_t last_uid) -{ - uint32_t seq1, seq2; - - mailbox_get_seq_range(box, last_uid + 1, (uint32_t)-1, &seq1, &seq2); - if (seq1 == 0) - return 0; - - fctx->virtual_ctx.trans = mailbox_transaction_begin(box, 0); - return fts_build_init_seq(fctx, fctx->virtual_ctx.trans, - seq1, seq2, last_uid); -} - -static int mailbox_name_cmp(const struct fts_orig_mailboxes *box1, - const struct fts_orig_mailboxes *box2) -{ - int ret; - - T_BEGIN { - const char *vname1, *vname2; - - vname1 = mailbox_list_get_vname(box1->ns->list, box1->name); - vname2 = mailbox_list_get_vname(box2->ns->list, box2->name); - ret = strcmp(vname1, vname2); - } T_END; - return ret; -} - -static int -fts_backend_uid_map_mailbox_cmp(const struct fts_backend_uid_map *map1, - const struct fts_backend_uid_map *map2) +static void fts_scores_unref(struct fts_scores **_scores) { - return strcmp(map1->mailbox, map2->mailbox); -} - -static int fts_build_init_virtual_next(struct fts_search_context *fctx) -{ - struct fts_search_virtual_context *vctx = &fctx->virtual_ctx; - struct mailbox_status status; - const struct fts_orig_mailboxes *boxes; - const struct fts_backend_uid_map *last_uids; - unsigned int boxi, uidi, box_count, last_uid_count; - int ret, vret = 0; - - if (vctx->pool == NULL) - return 0; - - if (fctx->virtual_ctx.trans != NULL) - (void)mailbox_transaction_commit(&fctx->virtual_ctx.trans); - - boxes = array_get(&vctx->orig_mailboxes, &box_count); - last_uids = array_get(&vctx->last_uids, &last_uid_count); - - boxi = vctx->boxi; - uidi = vctx->uidi; - while (vret == 0 && boxi < box_count && uidi < last_uid_count) { - T_BEGIN { - const char *vname; - - vname = mailbox_list_get_vname(boxes[boxi].ns->list, - boxes[boxi].name); - ret = strcmp(vname, last_uids[uidi].mailbox); - } T_END; - if (ret == 0) { - /* match. check also that uidvalidity matches. */ - mailbox_get_open_status(boxes[boxi].box, - STATUS_UIDVALIDITY, &status); - if (status.uidvalidity != last_uids[uidi].uidvalidity) { - uidi++; - continue; - } - vret = fts_build_init_box(fctx, boxes[boxi].box, - last_uids[uidi].uid); - boxi++; - uidi++; - } else if (ret > 0) { - /* not part of this virtual mailbox */ - uidi++; - } else { - /* no messages indexed in the mailbox */ - vret = fts_build_init_box(fctx, boxes[boxi].box, 0); - boxi++; - } - } - while (vret == 0 && boxi < box_count) { - vret = fts_build_init_box(fctx, boxes[boxi].box, 0); - boxi++; - } - vctx->boxi = boxi; - vctx->uidi = uidi; - return vret; -} - -static const char * -fts_box_get_root(struct mailbox *box, struct mail_namespace **ns_r) -{ - struct mail_namespace *ns = mailbox_get_namespace(box); - const char *name = box->name; - - while (ns->alias_for != NULL) - ns = ns->alias_for; - *ns_r = ns; - - if (*name == '\0' && ns != mailbox_get_namespace(box) && - (ns->flags & NAMESPACE_FLAG_INBOX_USER) != 0) { - /* ugly workaround to allow selecting INBOX from a Maildir/ - when it's not in the inbox=yes namespace. */ - return "INBOX"; - } - return name; -} - -static int fts_build_init_virtual(struct fts_search_context *fctx) -{ - struct fts_search_virtual_context *vctx = &fctx->virtual_ctx; - ARRAY_TYPE(mailboxes) mailboxes; - struct mailbox *const *boxes; - const struct fts_orig_mailboxes *orig_boxes; - struct fts_orig_mailboxes orig_box; - unsigned int i, box_count; - int ret; - - t_array_init(&mailboxes, 64); - fts_mailbox_get_virtual_backend_boxes(fctx->t->box, &mailboxes, TRUE); - boxes = array_get_modifiable(&mailboxes, &box_count); - - vctx->pool = pool_alloconly_create("fts virtual build", 1024); - p_array_init(&vctx->orig_mailboxes, vctx->pool, box_count); - memset(&orig_box, 0, sizeof(orig_box)); - for (i = 0; i < box_count; i++) { - orig_box.box = boxes[i]; - orig_box.name = fts_box_get_root(boxes[i], &orig_box.ns); - array_append(&vctx->orig_mailboxes, &orig_box, 1); - } - - orig_boxes = array_get(&vctx->orig_mailboxes, &box_count); - if (box_count <= 0) { - if (box_count == 0) { - /* empty virtual mailbox */ - return 0; - } - /* virtual mailbox is built from only a single mailbox - (currently). check that directly. */ - fctx->virtual_ctx.trans = - mailbox_transaction_begin(orig_boxes[0].box, 0); - ret = fts_build_init_trans(fctx, fctx->virtual_ctx.trans); - return ret; - } - - /* virtual mailbox is built from multiple mailboxes. figure out which - ones need updating. */ - p_array_init(&vctx->last_uids, vctx->pool, 64); - if (fts_backend_get_all_last_uids(fctx->fbox->backend, vctx->pool, - &vctx->last_uids) < 0) { - pool_unref(&vctx->pool); - return -1; - } - - array_sort(&vctx->orig_mailboxes, mailbox_name_cmp); - array_sort(&vctx->last_uids, fts_backend_uid_map_mailbox_cmp); - - ret = fts_build_init_virtual_next(fctx); - return ret < 0 ? -1 : 0; -} - -static int fts_build_init(struct fts_search_context *fctx) -{ - struct mailbox_status status; - int ret; + struct fts_scores *scores = *_scores; - mailbox_get_open_status(fctx->t->box, STATUS_MESSAGES | STATUS_UIDNEXT, - &status); - if (status.messages == fctx->fbox->last_messages_count && - status.uidnext == fctx->fbox->last_uidnext) { - /* no new messages since last check */ - return 0; - } - - if (fctx->fbox->virtual && - (fctx->fbox->backend->flags & FTS_BACKEND_FLAG_VIRTUAL_LOOKUPS) != 0) - ret = fts_build_init_virtual(fctx); - else - ret = fts_build_init_trans(fctx, fctx->t); - if (ret == 0 && fctx->build_ctx == NULL) { - /* index was up-to-date */ - fctx->fbox->last_messages_count = status.messages; - fctx->fbox->last_uidnext = status.uidnext; - } - return ret; -} - -static int fts_build_deinit(struct fts_storage_build_context **_ctx) -{ - struct fts_storage_build_context *ctx = *_ctx; - struct mailbox *box = ctx->search_ctx->transaction->box; - struct fts_mailbox *fbox = FTS_CONTEXT(box); - struct mailbox_status status; - int ret = 0; - - *_ctx = NULL; - - if (mailbox_search_deinit(&ctx->search_ctx) < 0) - ret = -1; - - if (fts_backend_build_deinit(&ctx->build) < 0) - ret = -1; - - if (ret == 0) { - mailbox_get_open_status(box, STATUS_MESSAGES | STATUS_UIDNEXT, - &status); - fbox->last_messages_count = status.messages; - fbox->last_uidnext = status.uidnext; - } - - if (ioloop_time - ctx->search_start_time.tv_sec >= - FTS_BUILD_NOTIFY_INTERVAL_SECS && - box->storage->callbacks.notify_ok != NULL) { - /* we notified at least once */ - box->storage->callbacks. - notify_ok(box, "Mailbox indexing finished", - box->storage->callback_context); + *_scores = NULL; + if (--scores->refcount == 0) { + array_free(&scores->score_map); + i_free(scores); } - - str_free(&ctx->headers); - mail_search_args_unref(&ctx->search_args); - i_free(ctx->content_type); - i_free(ctx->content_disposition); - i_free(ctx); - return ret; -} - -static void -fts_build_notify(struct fts_storage_build_context *ctx, uint32_t seq) -{ - struct mailbox *box = ctx->search_ctx->transaction->box; - - if (ctx->last_notify.tv_sec == 0) { - /* set the search time in here, in case a plugin - already spent some time indexing the mailbox */ - ctx->search_start_time = ioloop_timeval; - } else if (box->storage->callbacks.notify_ok != NULL) { - double completed_frac; - unsigned int eta_secs; - const struct seq_range *range; - uint32_t seq_diff; - - range = array_idx(&ctx->search_args->args->value.seqset, 0); - - seq_diff = range->seq2 - range->seq1; - - if (seq_diff != 0) { - completed_frac = (double)(seq - range->seq1) / seq_diff; - - if (completed_frac >= 0.000001) { - unsigned int elapsed_msecs, est_total_msecs; - - elapsed_msecs = timeval_diff_msecs(&ioloop_timeval, - &ctx->search_start_time); - est_total_msecs = elapsed_msecs / completed_frac; - eta_secs = (est_total_msecs - elapsed_msecs) / 1000; - } else { - eta_secs = 0; - } - } else { - completed_frac = 0.0; - eta_secs = 0; - } - - T_BEGIN { - const char *text; - - text = t_strdup_printf("Indexed %d%% of the mailbox, " - "ETA %d:%02d", (int)(completed_frac * 100.0), - eta_secs/60, eta_secs%60); - box->storage->callbacks. - notify_ok(box, text, - box->storage->callback_context); - } T_END; - } - ctx->last_notify = ioloop_timeval; -} - -static int fts_build_more(struct fts_storage_build_context *ctx) -{ - struct mail *mail = NULL; - unsigned int count = 0; - int ret; - - while (mailbox_search_next(ctx->search_ctx, &mail)) { - T_BEGIN { - ret = fts_build_mail(ctx, mail); - } T_END; - - if (ret < 0) - return -1; - - if (++count == FTS_SEARCH_NONBLOCK_COUNT) - return 0; - } - - if (ioloop_time - ctx->last_notify.tv_sec >= - FTS_BUILD_NOTIFY_INTERVAL_SECS && mail != NULL) - fts_build_notify(ctx, mail->seq); - return 1; -} - -static void fts_search_init_lookup(struct mail_search_context *ctx, - struct fts_search_context *fctx) -{ - fts_search_lookup(fctx); - - if (fctx->seqs_set && - strcmp(ctx->transaction->box->storage->name, "virtual") != 0) { - ctx->progress_max = array_count(&fctx->definite_seqs) + - array_count(&fctx->maybe_seqs); - } - ctx->progress_cur = 0; } static bool fts_try_build_init(struct mail_search_context *ctx, struct fts_search_context *fctx) { - if (fctx->best_arg == NULL) { - /* don't use FTS for this search */ - fctx->build_initialized = TRUE; - return TRUE; - } - - if (fts_backend_is_building(fctx->fbox->backend)) { + if (fts_backend_is_updating(fctx->backend)) { /* this process is already building the indexes */ return FALSE; } fctx->build_initialized = TRUE; - if (fts_build_init(fctx) < 0) { - fctx->fbox->backend = NULL; - return TRUE; - } - - if (fctx->build_ctx == NULL) { + switch (fts_build_init(fctx->backend, ctx->transaction->box, FALSE, + &fctx->build_ctx)) { + case -1: + break; + case 0: /* the index was up to date */ - fts_search_init_lookup(ctx, fctx); - } else { - /* hide "searching" notifications */ + fts_search_lookup(fctx); + break; + case 1: + /* hide "searching" notifications while building index */ ctx->progress_hidden = TRUE; + break; } return TRUE; } +static bool fts_want_build_args(const struct mail_search_arg *args) +{ + /* we want to update index only when searching from message body. + it's not worth the wait for searching only from headers, which + could be in cache file already */ + for (; args != NULL; args = args->next) { + switch (args->type) { + case SEARCH_OR: + case SEARCH_SUB: + case SEARCH_INTHREAD: + if (fts_want_build_args(args->value.subargs)) + return TRUE; + break; + case SEARCH_BODY: + case SEARCH_TEXT: + return TRUE; + default: + break; + } + } + return FALSE; +} + static struct mail_search_context * fts_mailbox_search_init(struct mailbox_transaction_context *t, struct mail_search_args *args, @@ -670,52 +116,75 @@ { struct fts_transaction_context *ft = FTS_CONTEXT(t); struct fts_mailbox *fbox = FTS_CONTEXT(t->box); + struct fts_mailbox_list *flist = FTS_LIST_CONTEXT(t->box->list); struct mail_search_context *ctx; struct fts_search_context *fctx; ctx = fbox->module_ctx.super.search_init(t, args, sort_program, wanted_fields, wanted_headers); + if (!fts_backend_can_lookup(flist->backend, args->args)) + return ctx; + fctx = i_new(struct fts_search_context, 1); - fctx->fbox = fbox; + fctx->box = t->box; + fctx->backend = flist->backend; fctx->t = t; fctx->args = args; - fctx->first_nonindexed_seq = (uint32_t)-1; + fctx->result_pool = pool_alloconly_create("fts results", 1024*32); + fctx->orig_matches = buffer_create_dynamic(default_pool, 64); + i_array_init(&fctx->levels, 8); + fctx->scores = i_new(struct fts_scores, 1); + fctx->scores->refcount = 1; + i_array_init(&fctx->scores->score_map, 64); MODULE_CONTEXT_SET(ctx, fts_storage_module, fctx); - if (fbox->backend == NULL) - return ctx; + fctx->virtual_mailbox = + strcmp(t->box->storage->name, VIRTUAL_STORAGE_NAME) == 0; - ft->score_map = &fctx->score_map; + /* transaction contains the last search's scores. they can be + queried later with mail_get_special() */ + if (ft->scores != NULL) + fts_scores_unref(&ft->scores); + ft->scores = fctx->scores; + ft->scores->refcount++; - fts_search_analyze(fctx); - (void)fts_try_build_init(ctx, fctx); + if (fts_want_build_args(args->args)) + (void)fts_try_build_init(ctx, fctx); + else { + fctx->build_initialized = TRUE; + fts_search_lookup(fctx); + } return ctx; } -static int fts_mailbox_search_build_more(struct mail_search_context *ctx) +static bool fts_mailbox_build_continue(struct mail_search_context *ctx) { struct fts_search_context *fctx = FTS_CONTEXT(ctx); - int ret = 1; + int ret; + + if (fctx == NULL) + return TRUE; - while (fctx->build_ctx != NULL) { + if (!fctx->build_initialized) { + /* we're still waiting for this process (but another command) + to finish building the indexes */ + if (!fts_try_build_init(ctx, fctx)) + return FALSE; + } + + if (fctx->build_ctx != NULL) { /* this command is still building the indexes */ ret = fts_build_more(fctx->build_ctx); if (ret == 0) - return 0; - - /* finished / error */ + return FALSE; ctx->progress_hidden = FALSE; if (fts_build_deinit(&fctx->build_ctx) < 0) ret = -1; - if (ret > 0) { - if (fts_build_init_virtual_next(fctx) == 0) { - /* all finished */ - fts_search_init_lookup(ctx, fctx); - } - } + if (ret > 0) + fts_search_lookup(fctx); } - return ret; + return TRUE; } static bool @@ -723,212 +192,89 @@ struct mail **mail_r, bool *tryagain_r) { struct fts_mailbox *fbox = FTS_CONTEXT(ctx->transaction->box); - struct fts_search_context *fctx = FTS_CONTEXT(ctx); - if (!fctx->build_initialized) { - /* we're still waiting for this process (but another command) - to finish building the indexes */ - if (!fts_try_build_init(ctx, fctx)) { - *tryagain_r = TRUE; - return FALSE; - } - } - - if (fts_mailbox_search_build_more(ctx) == 0) { + if (!fts_mailbox_build_continue(ctx)) { *tryagain_r = TRUE; return FALSE; } - /* if we're here, the indexes are either built or they're not used */ return fbox->module_ctx.super. search_next_nonblock(ctx, mail_r, tryagain_r); } static void -fts_mailbox_search_args_definite_set(struct fts_search_context *fctx) -{ - struct mail_search_arg *arg; - - for (arg = fctx->args->args; arg != NULL; arg = arg->next) { - switch (arg->type) { - case SEARCH_TEXT: - case SEARCH_BODY: - arg->result = 1; - break; - default: - break; - } - } -} - -static bool search_nonindexed(struct mail_search_context *ctx) +fts_search_apply_results_level(struct mail_search_context *ctx, + struct mail_search_arg *args, unsigned int *idx) { struct fts_search_context *fctx = FTS_CONTEXT(ctx); - struct fts_mailbox *fbox = FTS_CONTEXT(ctx->transaction->box); - struct mailbox_status status; + const struct fts_search_level *level; - mailbox_get_open_status(ctx->transaction->box, - STATUS_MESSAGES, &status); + level = array_idx(&fctx->levels, *idx); - fctx->seqs_set = FALSE; - ctx->seq = fctx->first_nonindexed_seq - 1; - ctx->progress_cur = ctx->seq; - ctx->progress_max = status.messages; - return fbox->module_ctx.super.search_next_update_seq(ctx); + if (array_is_created(&level->definite_seqs) && + seq_range_exists(&level->definite_seqs, ctx->seq)) + fts_search_deserialize_add_matches(args, level->args_matches); + else if (!array_is_created(&level->maybe_seqs) || + !seq_range_exists(&level->maybe_seqs, ctx->seq)) + fts_search_deserialize_add_nonmatches(args, level->args_matches); + + for (; args != NULL; args = args->next) { + if (args->type != SEARCH_OR && args->type != SEARCH_SUB) + continue; + + *idx += 1; + fts_search_apply_results_level(ctx, args->value.subargs, idx); + } } static bool fts_mailbox_search_next_update_seq(struct mail_search_context *ctx) { struct fts_mailbox *fbox = FTS_CONTEXT(ctx->transaction->box); struct fts_search_context *fctx = FTS_CONTEXT(ctx); - struct seq_range *def_range, *maybe_range, *range; - unsigned int def_count, maybe_count; - uint32_t wanted_seq; - bool use_maybe, ret; - - if (!fctx->seqs_set) - return fbox->module_ctx.super.search_next_update_seq(ctx); - - wanted_seq = ctx->seq + 1; - /* fts_search_lookup() was called successfully */ - for (;;) { - def_range = array_get_modifiable(&fctx->definite_seqs, - &def_count); - maybe_range = array_get_modifiable(&fctx->maybe_seqs, - &maybe_count); - /* if we're ahead of current positions, skip them */ - while (fctx->definite_idx < def_count && - wanted_seq > def_range[fctx->definite_idx].seq2) - fctx->definite_idx++; - while (fctx->maybe_idx < maybe_count && - wanted_seq > maybe_range[fctx->maybe_idx].seq2) - fctx->maybe_idx++; + unsigned int idx; - /* use whichever is lower of definite/maybe */ - if (fctx->definite_idx == def_count) { - if (fctx->maybe_idx == maybe_count) { - /* look for the non-indexed mails */ - if (fctx->first_nonindexed_seq == (uint32_t)-1) - return FALSE; - return search_nonindexed(ctx); - } - use_maybe = TRUE; - } else if (fctx->maybe_idx == maybe_count) { - use_maybe = FALSE; - } else { - use_maybe = maybe_range[fctx->maybe_idx].seq1 < - def_range[fctx->definite_idx].seq2; - } - - if (use_maybe) - range = maybe_range + fctx->maybe_idx; - else - range = def_range + fctx->definite_idx; - - i_assert(range->seq1 <= range->seq2); - if (wanted_seq > range->seq1) { - /* current sequence is already larger than where - range begins, so use the current sequence. */ - range->seq1 = wanted_seq+1; - } else { - wanted_seq = range->seq1; - range->seq1++; - } - if (range->seq1 > range->seq2) - range->seq2 = 0; - - /* ctx->seq points to previous sequence we want */ - ctx->seq = wanted_seq - 1; - ret = fbox->module_ctx.super.search_next_update_seq(ctx); - if (!ret || wanted_seq == ctx->seq) - break; - wanted_seq = ctx->seq; - mail_search_args_reset(ctx->args->args, FALSE); + if (fctx == NULL || !fctx->fts_lookup_success) { + /* fts lookup not done for this search */ + return fbox->module_ctx.super.search_next_update_seq(ctx); } - if (!use_maybe) { - /* we have definite results, update args */ - fts_mailbox_search_args_definite_set(fctx); + /* restore original [non]matches */ + fts_search_deserialize(ctx->args->args, fctx->orig_matches); + + if (!fbox->module_ctx.super.search_next_update_seq(ctx)) + return FALSE; + + if (ctx->seq >= fctx->first_unindexed_seq) { + /* we've not indexed this far */ + return TRUE; } - if (ctx->seq + 1 >= fctx->first_nonindexed_seq) { - /* this is a virtual mailbox and we're searching headers. - some mailboxes had more messages indexed than others. - to avoid duplicates or jumping around, ignore the rest of - the search results and just go through the messages in - order. */ - return search_nonindexed(ctx); - } - - ctx->progress_cur = fctx->definite_idx + fctx->maybe_idx; - return ret; + /* apply [non]matches based on the FTS lookup results */ + idx = 0; + fts_search_apply_results_level(ctx, ctx->args->args, &idx); + return TRUE; } -static bool -fts_mailbox_search_next_update_seq_virtual(struct mail_search_context *ctx) +static int fts_mailbox_search_deinit(struct mail_search_context *ctx) { struct fts_mailbox *fbox = FTS_CONTEXT(ctx->transaction->box); struct fts_search_context *fctx = FTS_CONTEXT(ctx); - while (fbox->module_ctx.super.search_next_update_seq(ctx)) { - if (!fctx->seqs_set) - return TRUE; - - /* virtual mailbox searches don't return sequences sorted. - just check if the suggested sequence exists. */ - if (seq_range_exists(&fctx->definite_seqs, ctx->seq)) { - fts_mailbox_search_args_definite_set(fctx); - return TRUE; + if (fctx != NULL) { + if (fctx->build_ctx != NULL) { + /* the search was cancelled */ + (void)fts_build_deinit(&fctx->build_ctx); } - if (seq_range_exists(&fctx->maybe_seqs, ctx->seq)) - return TRUE; - mail_search_args_reset(ctx->args->args, FALSE); - } - return FALSE; -} - -static int fts_mailbox_search_deinit(struct mail_search_context *ctx) -{ - struct fts_transaction_context *ft = FTS_CONTEXT(ctx->transaction); - struct fts_mailbox *fbox = FTS_CONTEXT(ctx->transaction->box); - struct fts_search_context *fctx = FTS_CONTEXT(ctx); - - if (ft->score_map == &fctx->score_map) - ft->score_map = NULL; - if (fctx->build_ctx != NULL) { - /* the search was cancelled */ - (void)fts_build_deinit(&fctx->build_ctx); + buffer_free(&fctx->orig_matches); + array_free(&fctx->levels); + pool_unref(&fctx->result_pool); + fts_scores_unref(&fctx->scores); + i_free(fctx); } - - if (array_is_created(&fctx->definite_seqs)) - array_free(&fctx->definite_seqs); - if (array_is_created(&fctx->maybe_seqs)) - array_free(&fctx->maybe_seqs); - if (array_is_created(&fctx->score_map)) - array_free(&fctx->score_map); - if (fctx->virtual_ctx.trans != NULL) - (void)mailbox_transaction_commit(&fctx->virtual_ctx.trans); - if (fctx->virtual_ctx.pool != NULL) - pool_unref(&fctx->virtual_ctx.pool); - i_free(fctx); return fbox->module_ctx.super.search_deinit(ctx); } -static void fts_mail_expunge(struct mail *_mail) -{ - struct mail_private *mail = (struct mail_private *)_mail; - struct fts_mail *fmail = FTS_MAIL_CONTEXT(mail); - struct fts_mailbox *fbox = FTS_CONTEXT(_mail->box); - struct fts_transaction_context *ft = FTS_CONTEXT(_mail->transaction); - - ft->expunges = TRUE; - if (fbox->backend != NULL) - fts_backend_expunge(fbox->backend, _mail); - - fmail->module_ctx.super.expunge(_mail); -} - static int fts_score_cmp(const uint32_t *uid, const struct fts_score_map *score) { return *uid < score->uid ? -1 : @@ -943,11 +289,10 @@ struct fts_transaction_context *ft = FTS_CONTEXT(_mail->transaction); const struct fts_score_map *scores; - if (field != MAIL_FETCH_SEARCH_SCORE || ft->score_map == NULL || - !array_is_created(ft->score_map)) + if (field != MAIL_FETCH_SEARCH_SCORE || ft->scores == NULL) scores = NULL; else { - scores = array_bsearch(ft->score_map, &_mail->uid, + scores = array_bsearch(&ft->scores->score_map, &_mail->uid, fts_score_cmp); } if (scores != NULL) { @@ -968,27 +313,17 @@ struct fts_mailbox *fbox = FTS_CONTEXT(_mail->box); struct fts_mail *fmail; - if (fbox == NULL || fbox->backend == NULL) + if (fbox == NULL) return; fmail = p_new(mail->pool, struct fts_mail, 1); fmail->module_ctx.super = *v; mail->vlast = &fmail->module_ctx.super; - v->expunge = fts_mail_expunge; v->get_special = fts_mail_get_special; MODULE_CONTEXT_SET(mail, fts_mail_module, fmail); } -static void fts_box_backends_init(struct mailbox *box) -{ - struct fts_mailbox *fbox = FTS_CONTEXT(box); - - fbox->backend = fts_backend_init(fbox->env, box); - if (fbox->backend == NULL && box->storage->set->mail_debug) - i_debug("fts: Backend not enabled by the fts setting"); -} - static struct mailbox_transaction_context * fts_transaction_begin(struct mailbox *box, enum mailbox_transaction_flags flags) @@ -999,100 +334,77 @@ ft = i_new(struct fts_transaction_context, 1); - /* the backend creation is delayed until the first transaction is - started. at that point the mailbox has been synced at least once. */ - if (!fbox->backend_set) { - fts_box_backends_init(box); - fbox->backend_set = TRUE; - } - t = fbox->module_ctx.super.transaction_begin(box, flags); MODULE_CONTEXT_SET(t, fts_storage_module, ft); return t; } -static void -fts_storage_build_context_deinit(struct fts_storage_build_context *build_ctx) -{ - (void)fts_backend_build_deinit(&build_ctx->build); - str_free(&build_ctx->headers); - i_free(build_ctx); -} - -static void -fts_transaction_finish(struct mailbox *box, struct fts_transaction_context *ft, - bool committed) -{ - struct fts_mailbox *fbox = FTS_CONTEXT(box); - - if (ft->expunges) - fts_backend_expunge_finish(fbox->backend, box, committed); - i_free(ft); -} - static void fts_transaction_rollback(struct mailbox_transaction_context *t) { - struct mailbox *box = t->box; - struct fts_mailbox *fbox = FTS_CONTEXT(box); + struct fts_mailbox *fbox = FTS_CONTEXT(t->box); struct fts_transaction_context *ft = FTS_CONTEXT(t); - if (ft->build_ctx != NULL) { - fts_storage_build_context_deinit(ft->build_ctx); - ft->build_ctx = NULL; - } - if (ft->free_mail) - mail_free(&ft->mail); - + if (ft->scores != NULL) + fts_scores_unref(&ft->scores); fbox->module_ctx.super.transaction_rollback(t); - fts_transaction_finish(box, ft, FALSE); } static int fts_transaction_commit(struct mailbox_transaction_context *t, struct mail_transaction_commit_changes *changes_r) { - struct mailbox *box = t->box; + struct fts_mailbox *fbox = FTS_CONTEXT(t->box); + struct fts_transaction_context *ft = FTS_CONTEXT(t); + + if (ft->scores != NULL) + fts_scores_unref(&ft->scores); + return fbox->module_ctx.super.transaction_commit(t, changes_r); +} + +static void fts_mailbox_sync_notify(struct mailbox *box, uint32_t uid, + enum mailbox_sync_type sync_type) +{ + struct fts_mailbox_list *flist = FTS_LIST_CONTEXT(box->list); struct fts_mailbox *fbox = FTS_CONTEXT(box); - struct fts_transaction_context *ft = FTS_CONTEXT(t); - int ret; - if (ft->build_ctx != NULL) { - fts_storage_build_context_deinit(ft->build_ctx); - ft->build_ctx = NULL; + if (fbox->module_ctx.super.sync_notify != NULL) + fbox->module_ctx.super.sync_notify(box, uid, sync_type); + + if (sync_type != MAILBOX_SYNC_TYPE_EXPUNGE) { + if (uid == 0 && fbox->sync_update_ctx != NULL) { + /* this sync is finished */ + (void)fts_backend_update_deinit(&fbox->sync_update_ctx); + } + return; } - if (ft->free_mail) - mail_free(&ft->mail); - ret = fbox->module_ctx.super.transaction_commit(t, changes_r); - fts_transaction_finish(box, ft, ret == 0); - return ret; + if (fbox->sync_update_ctx == NULL) { + fbox->sync_update_ctx = fts_backend_update_init(flist->backend); + fts_backend_update_set_mailbox(fbox->sync_update_ctx, box); + } + fts_backend_update_expunge(fbox->sync_update_ctx, uid); } static int fts_update(struct mailbox *box) { - struct mailbox_transaction_context *t; - struct mail_search_args *search_args; - struct mail_search_arg *arg; - struct mail_search_context *ctx; - struct fts_search_context *fctx; + struct fts_storage_build_context *build_ctx; + struct fts_mailbox_list *flist = FTS_LIST_CONTEXT(box->list); int ret = 0; - t = mailbox_transaction_begin(box, 0); - search_args = mail_search_build_init(); - mail_search_build_add_all(search_args); - arg = mail_search_build_add(search_args, SEARCH_BODY); - arg->value.str = "xyzzy"; + if ((ret = fts_build_init(flist->backend, box, + TRUE, &build_ctx)) <= 0) { + if (box->storage->set->mail_debug) + i_debug("%s: FTS index is up to date", box->vname); + return ret; + } - ctx = mailbox_search_init(t, search_args, NULL, 0, NULL); - mail_search_args_unref(&search_args); + if (box->storage->set->mail_debug) + i_debug("%s: Updating FTS index", box->vname); - fctx = FTS_CONTEXT(ctx); - if (fctx->build_initialized) { - while ((ret = fts_mailbox_search_build_more(ctx)) == 0) ; - } - (void)mailbox_search_deinit(&ctx); - (void)mailbox_transaction_commit(&t); + while ((ret = fts_build_more(build_ctx)) == 0) ; + if (fts_build_deinit(&build_ctx) < 0) + ret = -1; return ret < 0 ? -1 : 0; } @@ -1108,40 +420,92 @@ return -1; ctx = NULL; - return !precache ? 0 : fts_update(box); + if (precache) { + if (fts_update(box) < 0) { + mail_storage_set_critical(box->storage, + "FTS index update for mailbox %s failed", + box->vname); + return -1; + } + } + return 0; } -static void fts_mailbox_init(struct mailbox *box, const char *env) +void fts_mailbox_allocated(struct mailbox *box) { + struct fts_mailbox_list *flist = FTS_LIST_CONTEXT(box->list); struct mailbox_vfuncs *v = box->vlast; struct fts_mailbox *fbox; - fbox = i_new(struct fts_mailbox, 1); - fbox->virtual = strcmp(box->storage->name, "virtual") == 0; - fbox->env = env; + if (flist == NULL) + return; + + fbox = p_new(box->pool, struct fts_mailbox, 1); fbox->module_ctx.super = *v; box->vlast = &fbox->module_ctx.super; - v->free = fts_mailbox_free; v->search_init = fts_mailbox_search_init; v->search_next_nonblock = fts_mailbox_search_next_nonblock; - v->search_next_update_seq = fbox->virtual ? - fts_mailbox_search_next_update_seq_virtual : - fts_mailbox_search_next_update_seq; + v->search_next_update_seq = fts_mailbox_search_next_update_seq; v->search_deinit = fts_mailbox_search_deinit; v->transaction_begin = fts_transaction_begin; v->transaction_rollback = fts_transaction_rollback; v->transaction_commit = fts_transaction_commit; + v->sync_notify = fts_mailbox_sync_notify; v->sync_deinit = fts_sync_deinit; MODULE_CONTEXT_SET(box, fts_storage_module, fbox); } -void fts_mailbox_allocated(struct mailbox *box) +static void fts_mailbox_list_deinit(struct mailbox_list *list) +{ + struct fts_mailbox_list *flist = FTS_LIST_CONTEXT(list); + + fts_backend_deinit(&flist->backend); + flist->module_ctx.super.deinit(list); +} + +void fts_mailbox_list_created(struct mailbox_list *list) { - const char *env; + struct fts_backend *backend; + const char *name, *path, *error; + + name = mail_user_plugin_getenv(list->ns->user, "fts"); + if (name == NULL) { + if (list->mail_set->mail_debug) + i_debug("fts: No fts setting - plugin disabled"); + return; + } - env = mail_user_plugin_getenv(box->storage->user, "fts"); - if (env != NULL) - fts_mailbox_init(box, env); + path = mailbox_list_get_path(list, NULL, + MAILBOX_LIST_PATH_TYPE_INDEX); + if (*path == '\0') { + if (list->mail_set->mail_debug) { + i_debug("fts: Indexes disabled for namespace '%s'", + list->ns->prefix); + } + return; + } + + if (fts_backend_init(name, list->ns, &error, &backend) < 0) { + i_error("fts: Failed to initialize backend '%s': %s", + name, error); + } else { + struct fts_mailbox_list *flist; + struct mailbox_list_vfuncs *v = list->vlast; + + flist = p_new(list->pool, struct fts_mailbox_list, 1); + flist->module_ctx.super = *v; + flist->backend = backend; + list->vlast = &flist->module_ctx.super; + v->deinit = fts_mailbox_list_deinit; + MODULE_CONTEXT_SET(list, fts_mailbox_list_module, flist); + } } + +struct fts_backend *fts_mailbox_backend(struct mailbox *box) +{ + struct fts_mailbox_list *flist = FTS_LIST_CONTEXT(box->list); + + return flist->backend; +}
--- a/src/plugins/fts/fts-storage.h Fri Jul 22 13:13:29 2011 +0300 +++ b/src/plugins/fts/fts-storage.h Fri Jul 22 13:21:59 2011 +0300 @@ -1,53 +1,42 @@ #ifndef FTS_STORAGE_H #define FTS_STORAGE_H -struct fts_mailbox { - union mailbox_module_context module_ctx; - struct fts_backend *backend; +#include "mail-storage-private.h" +#include "fts-api.h" - unsigned int last_messages_count, last_uidnext; - - const char *env; - unsigned int virtual:1; - unsigned int backend_set:1; +struct fts_scores { + int refcount; + ARRAY_TYPE(fts_score_map) score_map; }; -struct fts_orig_mailboxes { - const char *name; - struct mail_namespace *ns; - struct mailbox *box; -}; - -struct fts_search_virtual_context { - pool_t pool; - - struct mailbox_transaction_context *trans; - ARRAY_DEFINE(orig_mailboxes, struct fts_orig_mailboxes); - ARRAY_TYPE(fts_backend_uid_map) last_uids; - - unsigned int boxi, uidi; +struct fts_search_level { + ARRAY_TYPE(seq_range) definite_seqs, maybe_seqs; + buffer_t *args_matches; + ARRAY_TYPE(fts_score_map) score_map; }; struct fts_search_context { union mail_search_module_context module_ctx; - struct fts_mailbox *fbox; + struct fts_backend *backend; + struct mailbox *box; struct mailbox_transaction_context *t; struct mail_search_args *args; - struct mail_search_arg *best_arg; + + pool_t result_pool; + ARRAY_DEFINE(levels, struct fts_search_level); + buffer_t *orig_matches; - struct fts_backend_lookup_context *lookup_ctx; - ARRAY_TYPE(seq_range) definite_seqs, maybe_seqs; - ARRAY_TYPE(fts_score_map) score_map; - unsigned int definite_idx, maybe_idx; - uint32_t first_nonindexed_seq; + uint32_t first_unindexed_seq; + + /* final scores, combined from all levels */ + struct fts_scores *scores; struct fts_storage_build_context *build_ctx; - struct fts_search_virtual_context virtual_ctx; + unsigned int virtual_mailbox:1; unsigned int build_initialized:1; - unsigned int seqs_set:1; - unsigned int refreshed:1; + unsigned int fts_lookup_success:1; }; /* Figure out if we want to use full text search indexes and update @@ -55,5 +44,11 @@ void fts_search_analyze(struct fts_search_context *fctx); /* Perform the actual index lookup and update definite_uids and maybe_uids. */ void fts_search_lookup(struct fts_search_context *fctx); +/* Returns FTS backend for the given mailbox. */ +struct fts_backend *fts_mailbox_backend(struct mailbox *box); + +void fts_mail_allocated(struct mail *mail); +void fts_mailbox_allocated(struct mailbox *box); +void fts_mailbox_list_created(struct mailbox_list *list); #endif
--- a/src/plugins/virtual/virtual-storage.c Fri Jul 22 13:13:29 2011 +0300 +++ b/src/plugins/virtual/virtual-storage.c Fri Jul 22 13:21:59 2011 +0300 @@ -233,12 +233,7 @@ VIRTUAL_INDEX_PREFIX); mbox->storage = storage; - mbox->vseq_lookup_prev_mailbox = i_strdup(""); - - mbox->virtual_ext_id = - mail_index_ext_register(mbox->box.index, "virtual", 0, - sizeof(struct virtual_mail_index_record), - sizeof(uint32_t)); + mbox->virtual_ext_id = (uint32_t)-1; return &mbox->box; } @@ -262,7 +257,6 @@ array_free(&bboxes[i]->sync_pending_removes); array_free(&bboxes[i]->uids); } - i_free_and_null(mbox->vseq_lookup_prev_mailbox); } static int virtual_mailbox_exists(struct mailbox *box, @@ -295,7 +289,14 @@ virtual_mailbox_close_internal(mbox); return -1; } - return index_storage_mailbox_open(box, FALSE); + if (index_storage_mailbox_open(box, FALSE) < 0) + return -1; + + mbox->virtual_ext_id = + mail_index_ext_register(mbox->box.index, "virtual", 0, + sizeof(struct virtual_mail_index_record), + sizeof(uint32_t)); + return 0; } static void virtual_mailbox_close(struct mailbox *box) @@ -371,46 +372,77 @@ } } -static int virtual_backend_uidmap_cmp(const uint32_t *uid, - const struct virtual_backend_uidmap *map) -{ - return *uid < map->real_uid ? -1 : - *uid > map->real_uid ? 1 : 0; -} - -static bool -virtual_get_virtual_uid(struct mailbox *box, const char *backend_mailbox, - uint32_t backend_uidvalidity, - uint32_t backend_uid, uint32_t *uid_r) +static void +virtual_get_virtual_uids(struct mailbox *box, + struct mailbox *backend_mailbox, + const ARRAY_TYPE(seq_range) *backend_uids, + ARRAY_TYPE(seq_range) *virtual_uids_r) { struct virtual_mailbox *mbox = (struct virtual_mailbox *)box; struct virtual_backend_box *bbox; - struct mailbox_status status; const struct virtual_backend_uidmap *uids; + struct seq_range_iter iter; + unsigned int n, i, count; + uint32_t uid; - if (strcmp(mbox->vseq_lookup_prev_mailbox, backend_mailbox) == 0) - bbox = mbox->vseq_lookup_prev_bbox; + if (mbox->lookup_prev_bbox != NULL && + strcmp(mbox->lookup_prev_bbox->box->vname, backend_mailbox->vname) == 0) + bbox = mbox->lookup_prev_bbox; else { - i_free(mbox->vseq_lookup_prev_mailbox); - mbox->vseq_lookup_prev_mailbox = i_strdup(backend_mailbox); - - bbox = virtual_backend_box_lookup_name(mbox, backend_mailbox); - mbox->vseq_lookup_prev_bbox = bbox; + bbox = virtual_backend_box_lookup_name(mbox, backend_mailbox->vname); + mbox->lookup_prev_bbox = bbox; } if (bbox == NULL) - return FALSE; + return; - mailbox_get_open_status(bbox->box, STATUS_UIDVALIDITY, &status); - if (status.uidvalidity != backend_uidvalidity) - return FALSE; + uids = array_get(&bbox->uids, &count); i = 0; + seq_range_array_iter_init(&iter, backend_uids); n = 0; + while (seq_range_array_iter_nth(&iter, n++, &uid)) { + while (i < count && uids[i].real_uid < uid) i++; + if (i < count && uids[i].real_uid == uid) { + seq_range_array_add(virtual_uids_r, 0, + uids[i].virtual_uid); + i++; + } + } +} - uids = array_bsearch(&bbox->uids, &backend_uid, - virtual_backend_uidmap_cmp); - if (uids == NULL) - return FALSE; +static void +virtual_get_virtual_uid_map(struct mailbox *box, + struct mailbox *backend_mailbox, + const ARRAY_TYPE(seq_range) *backend_uids, + ARRAY_TYPE(uint32_t) *virtual_uids_r) +{ + struct virtual_mailbox *mbox = (struct virtual_mailbox *)box; + struct virtual_backend_box *bbox; + const struct virtual_backend_uidmap *uids; + struct seq_range_iter iter; + unsigned int n, i, count; + uint32_t uid; - *uid_r = uids->virtual_uid; - return TRUE; + if (mbox->lookup_prev_bbox != NULL && + strcmp(mbox->lookup_prev_bbox->box->vname, backend_mailbox->vname) == 0) + bbox = mbox->lookup_prev_bbox; + else { + bbox = virtual_backend_box_lookup_name(mbox, backend_mailbox->vname); + mbox->lookup_prev_bbox = bbox; + } + if (bbox == NULL) + return; + + uids = array_get(&bbox->uids, &count); i = 0; + seq_range_array_iter_init(&iter, backend_uids); n = 0; + while (seq_range_array_iter_nth(&iter, n++, &uid)) { + while (i < count && uids[i].real_uid < uid) i++; + if (i == count || uids[i].real_uid > uid) { + uint32_t zero = 0; + + array_append(virtual_uids_r, &zero, 1); + } else { + array_append(virtual_uids_r, &uids[i].virtual_uid, 1); + i++; + } + } } static void @@ -429,17 +461,6 @@ } } -static void -virtual_get_virtual_box_patterns(struct mailbox *box, - ARRAY_TYPE(mailbox_virtual_patterns) *includes, - ARRAY_TYPE(mailbox_virtual_patterns) *excludes) -{ - struct virtual_mailbox *mbox = (struct virtual_mailbox *)box; - - array_append_array(includes, &mbox->list_include_patterns); - array_append_array(excludes, &mbox->list_exclude_patterns); -} - static bool virtual_is_inconsistent(struct mailbox *box) { struct virtual_mailbox *mbox = (struct virtual_mailbox *)box; @@ -511,7 +532,7 @@ }; struct virtual_mailbox_vfuncs virtual_mailbox_vfuncs = { - virtual_get_virtual_uid, - virtual_get_virtual_backend_boxes, - virtual_get_virtual_box_patterns + virtual_get_virtual_uids, + virtual_get_virtual_uid_map, + virtual_get_virtual_backend_boxes };
--- a/src/plugins/virtual/virtual-storage.h Fri Jul 22 13:13:29 2011 +0300 +++ b/src/plugins/virtual/virtual-storage.h Fri Jul 22 13:21:59 2011 +0300 @@ -101,16 +101,21 @@ ARRAY_DEFINE_TYPE(virtual_backend_box, struct virtual_backend_box *); struct virtual_mailbox_vfuncs { - bool (*get_virtual_uid)(struct mailbox *box, - const char *backend_mailbox, - uint32_t backend_uidvalidity, - uint32_t backend_uid, uint32_t *uid_r); + /* convert backend UIDs to virtual UIDs. if some backend UID doesn't + exist in mailbox, it's simply ignored */ + void (*get_virtual_uids)(struct mailbox *box, + struct mailbox *backend_mailbox, + const ARRAY_TYPE(seq_range) *backend_uids, + ARRAY_TYPE(seq_range) *virtual_uids_r); + /* like get_virtual_uids(), but if a backend UID doesn't exist, + convert it to 0. */ + void (*get_virtual_uid_map)(struct mailbox *box, + struct mailbox *backend_mailbox, + const ARRAY_TYPE(seq_range) *backend_uids, + ARRAY_TYPE(uint32_t) *virtual_uids_r); void (*get_virtual_backend_boxes)(struct mailbox *box, ARRAY_TYPE(mailboxes) *mailboxes, bool only_with_msgs); - void (*get_virtual_box_patterns)(struct mailbox *box, - ARRAY_TYPE(mailbox_virtual_patterns) *includes, - ARRAY_TYPE(mailbox_virtual_patterns) *excludes); }; struct virtual_mailbox { @@ -124,8 +129,7 @@ uint32_t highest_mailbox_id; uint32_t search_args_crc32; - char *vseq_lookup_prev_mailbox; - struct virtual_backend_box *vseq_lookup_prev_bbox; + struct virtual_backend_box *lookup_prev_bbox; uint32_t sync_virtual_next_uid; /* Mailboxes this virtual mailbox consists of, sorted by mailbox_id */