Mercurial > dovecot > original-hg > dovecot-1.2

annotate src/plugins/fts-solr/fts-backend-solr.c @ 8484:db78eab2ba5d HEAD

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Initial support for getting full text search working nicely with virtual mailboxes.
author Timo Sirainen <tss@iki.fi>
date Sun, 23 Nov 2008 03:59:55 +0200
parents e7929190cd32
children ee1ecdda8a6b
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
7990
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
1 /* Copyright (c) 2006-2008 Dovecot authors, see the included COPYING file */
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
2
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
3 #include "lib.h"
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
4 #include "array.h"
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
5 #include "str.h"
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
6 #include "mail-storage-private.h"
8109
e7929190cd32 fts-solr: Fixed compiling with recent struct mail_user changes.
Timo Sirainen <tss@iki.fi>
parents: 8006
diff changeset
7 #include "mail-namespace.h"
7990
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
8 #include "solr-connection.h"
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
9 #include "fts-solr-plugin.h"
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
10
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
11 #include <curl/curl.h>
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
12
8001
3fbfdda3e5d3 fts-solr: Send data to Solr in larger packets. Commit after expunges.
Timo Sirainen <tss@iki.fi>
parents: 8000
diff changeset
13 #define SOLR_CMDBUF_SIZE (1024*64)
8484
db78eab2ba5d Initial support for getting full text search working nicely with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents: 8109
diff changeset
14 #define SOLR_MAX_ROWS 100000
8001
3fbfdda3e5d3 fts-solr: Send data to Solr in larger packets. Commit after expunges.
Timo Sirainen <tss@iki.fi>
parents: 8000
diff changeset
15
7990
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
16 struct solr_fts_backend_build_context {
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
17 struct fts_backend_build_context ctx;
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
18
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
19 struct solr_connection_post *post;
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
20 uint32_t prev_uid, uid_validity;
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
21 string_t *cmd;
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
22 bool headers;
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
23 };
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
24
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
25 static struct solr_connection *solr_conn = NULL;
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
26
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
27 static void solr_quote_str(string_t *dest, const char *str)
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
28 {
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
29 solr_connection_quote_str(solr_conn, dest, str);
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
30 }
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
31
8000
61be6a4722c5 fts-solr: Encode entities when indexing data.
Timo Sirainen <tss@iki.fi>
parents: 7999
diff changeset
32 static void
61be6a4722c5 fts-solr: Encode entities when indexing data.
Timo Sirainen <tss@iki.fi>
parents: 7999
diff changeset
33 xml_encode_data(string_t *dest, const unsigned char *data, unsigned int len)
7990
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
34 {
8000
61be6a4722c5 fts-solr: Encode entities when indexing data.
Timo Sirainen <tss@iki.fi>
parents: 7999
diff changeset
35 unsigned int i;
61be6a4722c5 fts-solr: Encode entities when indexing data.
Timo Sirainen <tss@iki.fi>
parents: 7999
diff changeset
36
61be6a4722c5 fts-solr: Encode entities when indexing data.
Timo Sirainen <tss@iki.fi>
parents: 7999
diff changeset
37 for (i = 0; i < len; i++) {
61be6a4722c5 fts-solr: Encode entities when indexing data.
Timo Sirainen <tss@iki.fi>
parents: 7999
diff changeset
38 switch (data[i]) {
7990
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
39 case '&':
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
40 str_append(dest, "&amp;");
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
41 break;
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
42 case '<':
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
43 str_append(dest, "&lt;");
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
44 break;
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
45 case '>':
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
46 str_append(dest, "&gt;");
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
47 break;
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
48 default:
8000
61be6a4722c5 fts-solr: Encode entities when indexing data.
Timo Sirainen <tss@iki.fi>
parents: 7999
diff changeset
49 str_append_c(dest, data[i]);
7990
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
50 break;
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
51 }
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
52 }
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
53 }
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
54
8000
61be6a4722c5 fts-solr: Encode entities when indexing data.
Timo Sirainen <tss@iki.fi>
parents: 7999
diff changeset
55 static void xml_encode(string_t *dest, const char *str)
61be6a4722c5 fts-solr: Encode entities when indexing data.
Timo Sirainen <tss@iki.fi>
parents: 7999
diff changeset
56 {
61be6a4722c5 fts-solr: Encode entities when indexing data.
Timo Sirainen <tss@iki.fi>
parents: 7999
diff changeset
57 xml_encode_data(dest, (const unsigned char *)str, strlen(str));
61be6a4722c5 fts-solr: Encode entities when indexing data.
Timo Sirainen <tss@iki.fi>
parents: 7999
diff changeset
58 }
61be6a4722c5 fts-solr: Encode entities when indexing data.
Timo Sirainen <tss@iki.fi>
parents: 7999
diff changeset
59
7990
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
60 static struct fts_backend *
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
61 fts_backend_solr_init(struct mailbox *box ATTR_UNUSED)
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
62 {
8002
e814614ffb65 fts-solr: Added break-imap-search setting to use Solr with SEARCH TEXT/BODY
Timo Sirainen <tss@iki.fi>
parents: 8001
diff changeset
63 const struct fts_solr_settings *set = &fts_solr_settings;
7990
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
64 struct fts_backend *backend;
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
65
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
66 if (solr_conn == NULL)
8002
e814614ffb65 fts-solr: Added break-imap-search setting to use Solr with SEARCH TEXT/BODY
Timo Sirainen <tss@iki.fi>
parents: 8001
diff changeset
67 solr_conn = solr_connection_init(set->url, set->debug);
7990
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
68
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
69 backend = i_new(struct fts_backend, 1);
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
70 *backend = fts_backend_solr;
8002
e814614ffb65 fts-solr: Added break-imap-search setting to use Solr with SEARCH TEXT/BODY
Timo Sirainen <tss@iki.fi>
parents: 8001
diff changeset
71
e814614ffb65 fts-solr: Added break-imap-search setting to use Solr with SEARCH TEXT/BODY
Timo Sirainen <tss@iki.fi>
parents: 8001
diff changeset
72 if (set->substring_search)
e814614ffb65 fts-solr: Added break-imap-search setting to use Solr with SEARCH TEXT/BODY
Timo Sirainen <tss@iki.fi>
parents: 8001
diff changeset
73 backend->flags |= FTS_BACKEND_FLAG_SUBSTRING_LOOKUPS;
7990
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
74 return backend;
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
75 }
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
76
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
77 static void fts_backend_solr_deinit(struct fts_backend *backend)
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
78 {
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
79 i_free(backend);
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
80 }
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
81
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
82 static int fts_backend_solr_get_last_uid(struct fts_backend *backend,
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
83 uint32_t *last_uid_r)
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
84 {
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
85 struct mailbox_status status;
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
86 ARRAY_TYPE(seq_range) uids;
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
87 const struct seq_range *uidvals;
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
88 unsigned int count;
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
89 string_t *str;
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
90
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
91 str = t_str_new(256);
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
92 str_append(str, "fl=uid&rows=1&sort=uid%20desc&q=");
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
93
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
94 mailbox_get_status(backend->box, STATUS_UIDVALIDITY, &status);
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
95 str_printfa(str, "uidv:%u%%20box:", status.uidvalidity);
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
96 solr_quote_str(str, backend->box->name);
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
97 str_append(str, "%20user:");
8109
e7929190cd32 fts-solr: Fixed compiling with recent struct mail_user changes.
Timo Sirainen <tss@iki.fi>
parents: 8006
diff changeset
98 solr_quote_str(str, backend->box->storage->ns->user->username);
7990
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
99
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
100 t_array_init(&uids, 1);
8484
db78eab2ba5d Initial support for getting full text search working nicely with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents: 8109
diff changeset
101 if (solr_connection_select(solr_conn, str_c(str),
db78eab2ba5d Initial support for getting full text search working nicely with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents: 8109
diff changeset
102 NULL, NULL, &uids, NULL) < 0)
7990
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
103 return -1;
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
104
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
105 uidvals = array_get(&uids, &count);
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
106 if (count == 0) {
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
107 /* nothing indexed yet for this mailbox */
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
108 *last_uid_r = 0;
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
109 } else if (count == 1 && uidvals[0].seq1 == uidvals[0].seq2) {
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
110 *last_uid_r = uidvals[0].seq1;
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
111 } else {
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
112 i_error("fts_solr: Last UID lookup returned multiple rows");
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
113 return -1;
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
114 }
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
115 return 0;
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
116 }
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
117
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
118 static int
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
119 fts_backend_solr_build_init(struct fts_backend *backend, uint32_t *last_uid_r,
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
120 struct fts_backend_build_context **ctx_r)
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
121 {
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
122 struct solr_fts_backend_build_context *ctx;
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
123 struct mailbox_status status;
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
124
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
125 *last_uid_r = (uint32_t)-1;
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
126
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
127 ctx = i_new(struct solr_fts_backend_build_context, 1);
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
128 ctx->ctx.backend = backend;
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
129 ctx->post = solr_connection_post_begin(solr_conn);
8001
3fbfdda3e5d3 fts-solr: Send data to Solr in larger packets. Commit after expunges.
Timo Sirainen <tss@iki.fi>
parents: 8000
diff changeset
130 ctx->cmd = str_new(default_pool, SOLR_CMDBUF_SIZE);
7990
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
131
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
132 mailbox_get_status(backend->box, STATUS_UIDVALIDITY, &status);
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
133 ctx->uid_validity = status.uidvalidity;
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
134
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
135 *ctx_r = &ctx->ctx;
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
136 return 0;
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
137 }
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
138
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
139 static int
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
140 fts_backend_solr_build_more(struct fts_backend_build_context *_ctx,
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
141 uint32_t uid, const unsigned char *data,
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
142 size_t size, bool headers)
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
143 {
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
144 struct solr_fts_backend_build_context *ctx =
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
145 (struct solr_fts_backend_build_context *)_ctx;
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
146 struct mailbox *box = _ctx->backend->box;
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
147 string_t *cmd = ctx->cmd;
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
148
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
149 /* body comes first, then headers */
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
150 if (ctx->prev_uid != uid) {
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
151 /* uid changed */
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
152 if (ctx->prev_uid == 0)
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
153 str_append(cmd, "<add>");
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
154 else
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
155 str_append(cmd, "</field></doc>");
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
156 ctx->prev_uid = uid;
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
157
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
158 str_printfa(cmd, "<doc>"
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
159 "<field name=\"uid\">%u</field>"
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
160 "<field name=\"uidv\">%u</field>",
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
161 uid, ctx->uid_validity);
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
162
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
163 str_append(cmd, "<field name=\"box\">");
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
164 xml_encode(cmd, box->name);
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
165 str_append(cmd, "</field><field name=\"user\">");
8109
e7929190cd32 fts-solr: Fixed compiling with recent struct mail_user changes.
Timo Sirainen <tss@iki.fi>
parents: 8006
diff changeset
166 xml_encode(cmd, box->storage->ns->user->username);
7990
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
167
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
168 str_printfa(cmd, "</field><field name=\"id\">%u/%u/",
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
169 uid, ctx->uid_validity);
8109
e7929190cd32 fts-solr: Fixed compiling with recent struct mail_user changes.
Timo Sirainen <tss@iki.fi>
parents: 8006
diff changeset
170 xml_encode(cmd, box->storage->ns->user->username);
7990
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
171 str_append_c(cmd, '/');
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
172 xml_encode(cmd, box->name);
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
173 str_append(cmd, "</field>");
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
174
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
175 ctx->headers = headers;
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
176 if (headers) {
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
177 str_append(cmd, "<field name=\"hdr\">");
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
178 } else {
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
179 str_append(cmd, "<field name=\"body\">");
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
180 }
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
181 } else if (headers && !ctx->headers) {
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
182 str_append(cmd, "</field><field name=\"hdr\">");
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
183 } else {
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
184 i_assert(!(!headers && ctx->headers));
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
185 }
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
186
8000
61be6a4722c5 fts-solr: Encode entities when indexing data.
Timo Sirainen <tss@iki.fi>
parents: 7999
diff changeset
187 xml_encode_data(cmd, data, size);
8001
3fbfdda3e5d3 fts-solr: Send data to Solr in larger packets. Commit after expunges.
Timo Sirainen <tss@iki.fi>
parents: 8000
diff changeset
188 if (str_len(cmd) > SOLR_CMDBUF_SIZE-128) {
3fbfdda3e5d3 fts-solr: Send data to Solr in larger packets. Commit after expunges.
Timo Sirainen <tss@iki.fi>
parents: 8000
diff changeset
189 solr_connection_post_more(ctx->post, str_data(cmd),
3fbfdda3e5d3 fts-solr: Send data to Solr in larger packets. Commit after expunges.
Timo Sirainen <tss@iki.fi>
parents: 8000
diff changeset
190 str_len(cmd));
3fbfdda3e5d3 fts-solr: Send data to Solr in larger packets. Commit after expunges.
Timo Sirainen <tss@iki.fi>
parents: 8000
diff changeset
191 str_truncate(cmd, 0);
3fbfdda3e5d3 fts-solr: Send data to Solr in larger packets. Commit after expunges.
Timo Sirainen <tss@iki.fi>
parents: 8000
diff changeset
192 }
7990
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
193 return 0;
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
194 }
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
195
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
196 static int
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
197 fts_backend_solr_build_deinit(struct fts_backend_build_context *_ctx)
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
198 {
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
199 struct solr_fts_backend_build_context *ctx =
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
200 (struct solr_fts_backend_build_context *)_ctx;
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
201 int ret = 0;
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
202
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
203 if (ctx->prev_uid != 0) {
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
204 str_append(ctx->cmd, "</field></doc></add>");
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
205 solr_connection_post_more(ctx->post, str_data(ctx->cmd),
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
206 str_len(ctx->cmd));
7999
1ceb49f2eb50 fts-solr: Error handling fixes.
Timo Sirainen <tss@iki.fi>
parents: 7990
diff changeset
207 ret = solr_connection_post_end(ctx->post);
7990
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
208 /* commit and wait until the documents we just indexed are
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
209 visible to the following search */
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
210 if (solr_connection_post(solr_conn,
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
211 "<commit waitFlush=\"false\" "
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
212 "waitSearcher=\"true\"/>") < 0)
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
213 ret = -1;
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
214 }
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
215 str_free(&ctx->cmd);
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
216 i_free(ctx);
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
217 return ret;
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
218 }
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
219
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
220 static void
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
221 fts_backend_solr_expunge(struct fts_backend *backend ATTR_UNUSED,
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
222 struct mail *mail)
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
223 {
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
224 struct mailbox_status status;
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
225
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
226 mailbox_get_status(mail->box, STATUS_UIDVALIDITY, &status);
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
227
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
228 T_BEGIN {
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
229 string_t *cmd;
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
230
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
231 cmd = t_str_new(256);
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
232 str_printfa(cmd, "<delete><id>%u/%u/",
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
233 mail->uid, status.uidvalidity);
8109
e7929190cd32 fts-solr: Fixed compiling with recent struct mail_user changes.
Timo Sirainen <tss@iki.fi>
parents: 8006
diff changeset
234 xml_encode(cmd, mail->box->storage->ns->user->username);
7990
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
235 str_append_c(cmd, '/');
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
236 xml_encode(cmd, mail->box->name);
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
237 str_append(cmd, "</id></delete>");
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
238
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
239 (void)solr_connection_post(solr_conn, str_c(cmd));
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
240 } T_END;
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
241 }
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
242
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
243 static void
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
244 fts_backend_solr_expunge_finish(struct fts_backend *backend ATTR_UNUSED,
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
245 struct mailbox *box ATTR_UNUSED,
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
246 bool committed ATTR_UNUSED)
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
247 {
8001
3fbfdda3e5d3 fts-solr: Send data to Solr in larger packets. Commit after expunges.
Timo Sirainen <tss@iki.fi>
parents: 8000
diff changeset
248 solr_connection_post(solr_conn,
3fbfdda3e5d3 fts-solr: Send data to Solr in larger packets. Commit after expunges.
Timo Sirainen <tss@iki.fi>
parents: 8000
diff changeset
249 "<commit waitFlush=\"false\" waitSearcher=\"false\"/>");
7990
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
250 }
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
251
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
252 static int fts_backend_solr_lock(struct fts_backend *backend ATTR_UNUSED)
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
253 {
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
254 return 1;
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
255 }
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
256
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
257 static void fts_backend_solr_unlock(struct fts_backend *backend ATTR_UNUSED)
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
258 {
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
259 }
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
260
8484
db78eab2ba5d Initial support for getting full text search working nicely with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents: 8109
diff changeset
261 static bool solr_virtual_uid_map(const char *mailbox, uint32_t uidvalidity,
db78eab2ba5d Initial support for getting full text search working nicely with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents: 8109
diff changeset
262 uint32_t *uid, void *context)
db78eab2ba5d Initial support for getting full text search working nicely with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents: 8109
diff changeset
263 {
db78eab2ba5d Initial support for getting full text search working nicely with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents: 8109
diff changeset
264 struct mailbox *box = context;
db78eab2ba5d Initial support for getting full text search working nicely with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents: 8109
diff changeset
265
db78eab2ba5d Initial support for getting full text search working nicely with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents: 8109
diff changeset
266 return mailbox_get_virtual_uid(box, mailbox, uidvalidity,
db78eab2ba5d Initial support for getting full text search working nicely with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents: 8109
diff changeset
267 *uid, uid);
db78eab2ba5d Initial support for getting full text search working nicely with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents: 8109
diff changeset
268 }
db78eab2ba5d Initial support for getting full text search working nicely with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents: 8109
diff changeset
269
8004
6418d4c0ff16 fts: Added a new lookup API where the backend can look up all the fields
Timo Sirainen <tss@iki.fi>
parents: 8002
diff changeset
270 static int fts_backend_solr_lookup(struct fts_backend_lookup_context *ctx,
6418d4c0ff16 fts: Added a new lookup API where the backend can look up all the fields
Timo Sirainen <tss@iki.fi>
parents: 8002
diff changeset
271 ARRAY_TYPE(seq_range) *definite_uids,
8006
0a3b5ddb0869 Added support for sorting by X-SCORE. Currently it's only set by fts-solr.
Timo Sirainen <tss@iki.fi>
parents: 8004
diff changeset
272 ARRAY_TYPE(seq_range) *maybe_uids,
0a3b5ddb0869 Added support for sorting by X-SCORE. Currently it's only set by fts-solr.
Timo Sirainen <tss@iki.fi>
parents: 8004
diff changeset
273 ARRAY_TYPE(fts_score_map) *scores)
7990
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
274 {
8004
6418d4c0ff16 fts: Added a new lookup API where the backend can look up all the fields
Timo Sirainen <tss@iki.fi>
parents: 8002
diff changeset
275 struct mailbox *box = ctx->backend->box;
6418d4c0ff16 fts: Added a new lookup API where the backend can look up all the fields
Timo Sirainen <tss@iki.fi>
parents: 8002
diff changeset
276 const struct fts_backend_lookup_field *fields;
6418d4c0ff16 fts: Added a new lookup API where the backend can look up all the fields
Timo Sirainen <tss@iki.fi>
parents: 8002
diff changeset
277 unsigned int i, count;
7990
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
278 struct mailbox_status status;
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
279 string_t *str;
8484
db78eab2ba5d Initial support for getting full text search working nicely with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents: 8109
diff changeset
280 bool virtual;
7990
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
281
8484
db78eab2ba5d Initial support for getting full text search working nicely with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents: 8109
diff changeset
282 virtual = strcmp(box->storage->name, "virtual") == 0;
8004
6418d4c0ff16 fts: Added a new lookup API where the backend can look up all the fields
Timo Sirainen <tss@iki.fi>
parents: 8002
diff changeset
283 mailbox_get_status(box, STATUS_UIDVALIDITY, &status);
7990
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
284
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
285 str = t_str_new(256);
8484
db78eab2ba5d Initial support for getting full text search working nicely with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents: 8109
diff changeset
286 if (!virtual) {
db78eab2ba5d Initial support for getting full text search working nicely with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents: 8109
diff changeset
287 str_printfa(str, "fl=uid,score&rows=%u&sort=uid%%20asc&q=",
db78eab2ba5d Initial support for getting full text search working nicely with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents: 8109
diff changeset
288 status.uidnext);
db78eab2ba5d Initial support for getting full text search working nicely with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents: 8109
diff changeset
289 } else {
db78eab2ba5d Initial support for getting full text search working nicely with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents: 8109
diff changeset
290 str_printfa(str, "fl=uid,score,box,uidv&rows=%u"
db78eab2ba5d Initial support for getting full text search working nicely with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents: 8109
diff changeset
291 "&sort=box%%20asc,uid%%20asc&q=",
db78eab2ba5d Initial support for getting full text search working nicely with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents: 8109
diff changeset
292 SOLR_MAX_ROWS);
db78eab2ba5d Initial support for getting full text search working nicely with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents: 8109
diff changeset
293 }
8004
6418d4c0ff16 fts: Added a new lookup API where the backend can look up all the fields
Timo Sirainen <tss@iki.fi>
parents: 8002
diff changeset
294
6418d4c0ff16 fts: Added a new lookup API where the backend can look up all the fields
Timo Sirainen <tss@iki.fi>
parents: 8002
diff changeset
295 /* build a lucene search query from the fields */
6418d4c0ff16 fts: Added a new lookup API where the backend can look up all the fields
Timo Sirainen <tss@iki.fi>
parents: 8002
diff changeset
296 fields = array_get(&ctx->fields, &count);
6418d4c0ff16 fts: Added a new lookup API where the backend can look up all the fields
Timo Sirainen <tss@iki.fi>
parents: 8002
diff changeset
297 for (i = 0; i < count; i++) {
6418d4c0ff16 fts: Added a new lookup API where the backend can look up all the fields
Timo Sirainen <tss@iki.fi>
parents: 8002
diff changeset
298 if (i > 0)
6418d4c0ff16 fts: Added a new lookup API where the backend can look up all the fields
Timo Sirainen <tss@iki.fi>
parents: 8002
diff changeset
299 str_append(str, "%20");
6418d4c0ff16 fts: Added a new lookup API where the backend can look up all the fields
Timo Sirainen <tss@iki.fi>
parents: 8002
diff changeset
300
6418d4c0ff16 fts: Added a new lookup API where the backend can look up all the fields
Timo Sirainen <tss@iki.fi>
parents: 8002
diff changeset
301 if ((fields[i].flags & FTS_LOOKUP_FLAG_INVERT) != 0)
6418d4c0ff16 fts: Added a new lookup API where the backend can look up all the fields
Timo Sirainen <tss@iki.fi>
parents: 8002
diff changeset
302 str_append_c(str, '-');
6418d4c0ff16 fts: Added a new lookup API where the backend can look up all the fields
Timo Sirainen <tss@iki.fi>
parents: 8002
diff changeset
303
6418d4c0ff16 fts: Added a new lookup API where the backend can look up all the fields
Timo Sirainen <tss@iki.fi>
parents: 8002
diff changeset
304 if ((fields[i].flags & FTS_LOOKUP_FLAG_HEADER) == 0) {
6418d4c0ff16 fts: Added a new lookup API where the backend can look up all the fields
Timo Sirainen <tss@iki.fi>
parents: 8002
diff changeset
305 /* body only */
6418d4c0ff16 fts: Added a new lookup API where the backend can look up all the fields
Timo Sirainen <tss@iki.fi>
parents: 8002
diff changeset
306 i_assert((fields[i].flags & FTS_LOOKUP_FLAG_BODY) != 0);
6418d4c0ff16 fts: Added a new lookup API where the backend can look up all the fields
Timo Sirainen <tss@iki.fi>
parents: 8002
diff changeset
307 str_append(str, "body:");
6418d4c0ff16 fts: Added a new lookup API where the backend can look up all the fields
Timo Sirainen <tss@iki.fi>
parents: 8002
diff changeset
308 } else if ((fields[i].flags & FTS_LOOKUP_FLAG_BODY) == 0) {
6418d4c0ff16 fts: Added a new lookup API where the backend can look up all the fields
Timo Sirainen <tss@iki.fi>
parents: 8002
diff changeset
309 /* header only */
6418d4c0ff16 fts: Added a new lookup API where the backend can look up all the fields
Timo Sirainen <tss@iki.fi>
parents: 8002
diff changeset
310 str_append(str, "hdr:");
6418d4c0ff16 fts: Added a new lookup API where the backend can look up all the fields
Timo Sirainen <tss@iki.fi>
parents: 8002
diff changeset
311 } else {
6418d4c0ff16 fts: Added a new lookup API where the backend can look up all the fields
Timo Sirainen <tss@iki.fi>
parents: 8002
diff changeset
312 /* both */
6418d4c0ff16 fts: Added a new lookup API where the backend can look up all the fields
Timo Sirainen <tss@iki.fi>
parents: 8002
diff changeset
313 str_append(str, "any:");
6418d4c0ff16 fts: Added a new lookup API where the backend can look up all the fields
Timo Sirainen <tss@iki.fi>
parents: 8002
diff changeset
314 }
6418d4c0ff16 fts: Added a new lookup API where the backend can look up all the fields
Timo Sirainen <tss@iki.fi>
parents: 8002
diff changeset
315 solr_quote_str(str, fields[i].key);
7990
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
316 }
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
317
8004
6418d4c0ff16 fts: Added a new lookup API where the backend can look up all the fields
Timo Sirainen <tss@iki.fi>
parents: 8002
diff changeset
318 /* use a separate filter query for selecting the mailbox. it shouldn't
6418d4c0ff16 fts: Added a new lookup API where the backend can look up all the fields
Timo Sirainen <tss@iki.fi>
parents: 8002
diff changeset
319 affect the score and there could be some caching benefits too. */
8484
db78eab2ba5d Initial support for getting full text search working nicely with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents: 8109
diff changeset
320 str_append(str, "&fq=user:");
8109
e7929190cd32 fts-solr: Fixed compiling with recent struct mail_user changes.
Timo Sirainen <tss@iki.fi>
parents: 8006
diff changeset
321 solr_quote_str(str, box->storage->ns->user->username);
7990
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
322
8484
db78eab2ba5d Initial support for getting full text search working nicely with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents: 8109
diff changeset
323 /* FIXME: limit what mailboxes to search with virtual storage */
db78eab2ba5d Initial support for getting full text search working nicely with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents: 8109
diff changeset
324 if (!virtual) {
db78eab2ba5d Initial support for getting full text search working nicely with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents: 8109
diff changeset
325 str_printfa(str, "%%20uidv:%u%%20box:", status.uidvalidity);
db78eab2ba5d Initial support for getting full text search working nicely with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents: 8109
diff changeset
326 solr_quote_str(str, box->name);
db78eab2ba5d Initial support for getting full text search working nicely with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents: 8109
diff changeset
327 }
db78eab2ba5d Initial support for getting full text search working nicely with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents: 8109
diff changeset
328
7990
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
329 array_clear(maybe_uids);
8484
db78eab2ba5d Initial support for getting full text search working nicely with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents: 8109
diff changeset
330 if (!virtual) {
db78eab2ba5d Initial support for getting full text search working nicely with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents: 8109
diff changeset
331 return solr_connection_select(solr_conn, str_c(str), NULL, NULL,
db78eab2ba5d Initial support for getting full text search working nicely with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents: 8109
diff changeset
332 definite_uids, scores);
db78eab2ba5d Initial support for getting full text search working nicely with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents: 8109
diff changeset
333 } else {
db78eab2ba5d Initial support for getting full text search working nicely with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents: 8109
diff changeset
334 return solr_connection_select(solr_conn, str_c(str),
db78eab2ba5d Initial support for getting full text search working nicely with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents: 8109
diff changeset
335 solr_virtual_uid_map, box,
db78eab2ba5d Initial support for getting full text search working nicely with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents: 8109
diff changeset
336 definite_uids, scores);
db78eab2ba5d Initial support for getting full text search working nicely with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents: 8109
diff changeset
337 }
7990
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
338 }
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
339
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
340 struct fts_backend fts_backend_solr = {
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
341 MEMBER(name) "solr",
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
342 MEMBER(flags) 0,
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
343
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
344 {
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
345 fts_backend_solr_init,
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
346 fts_backend_solr_deinit,
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
347 fts_backend_solr_get_last_uid,
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
348 fts_backend_solr_build_init,
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
349 fts_backend_solr_build_more,
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
350 fts_backend_solr_build_deinit,
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
351 fts_backend_solr_expunge,
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
352 fts_backend_solr_expunge_finish,
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
353 fts_backend_solr_lock,
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
354 fts_backend_solr_unlock,
8004
6418d4c0ff16 fts: Added a new lookup API where the backend can look up all the fields
Timo Sirainen <tss@iki.fi>
parents: 8002
diff changeset
355 NULL,
6418d4c0ff16 fts: Added a new lookup API where the backend can look up all the fields
Timo Sirainen <tss@iki.fi>
parents: 8002
diff changeset
356 NULL,
6418d4c0ff16 fts: Added a new lookup API where the backend can look up all the fields
Timo Sirainen <tss@iki.fi>
parents: 8002
diff changeset
357 fts_backend_solr_lookup
7990
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
358 }
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
359 };