Mercurial > dovecot > original-hg > dovecot-1.2

annotate src/plugins/fts-solr/fts-backend-solr.c @ 8006:0a3b5ddb0869 HEAD

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Added support for sorting by X-SCORE. Currently it's only set by fts-solr.
author Timo Sirainen <tss@iki.fi>
date Sun, 13 Jul 2008 19:09:55 +0300
parents 6418d4c0ff16
children e7929190cd32
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
7990
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
1 /* Copyright (c) 2006-2008 Dovecot authors, see the included COPYING file */
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
2
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
3 #include "lib.h"
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
4 #include "array.h"
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
5 #include "str.h"
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
6 #include "mail-storage-private.h"
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
7 #include "solr-connection.h"
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
8 #include "fts-solr-plugin.h"
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
9
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
10 #include <curl/curl.h>
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
11
8001
3fbfdda3e5d3 fts-solr: Send data to Solr in larger packets. Commit after expunges.
Timo Sirainen <tss@iki.fi>
parents: 8000
diff changeset
12 #define SOLR_CMDBUF_SIZE (1024*64)
3fbfdda3e5d3 fts-solr: Send data to Solr in larger packets. Commit after expunges.
Timo Sirainen <tss@iki.fi>
parents: 8000
diff changeset
13
7990
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
14 struct solr_fts_backend_build_context {
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
15 struct fts_backend_build_context ctx;
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
16
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
17 struct solr_connection_post *post;
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
18 uint32_t prev_uid, uid_validity;
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
19 string_t *cmd;
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
20 bool headers;
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
21 };
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
22
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
23 static struct solr_connection *solr_conn = NULL;
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
24
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
25 static void solr_quote_str(string_t *dest, const char *str)
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
26 {
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
27 solr_connection_quote_str(solr_conn, dest, str);
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
28 }
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
29
8000
61be6a4722c5 fts-solr: Encode entities when indexing data.
Timo Sirainen <tss@iki.fi>
parents: 7999
diff changeset
30 static void
61be6a4722c5 fts-solr: Encode entities when indexing data.
Timo Sirainen <tss@iki.fi>
parents: 7999
diff changeset
31 xml_encode_data(string_t *dest, const unsigned char *data, unsigned int len)
7990
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
32 {
8000
61be6a4722c5 fts-solr: Encode entities when indexing data.
Timo Sirainen <tss@iki.fi>
parents: 7999
diff changeset
33 unsigned int i;
61be6a4722c5 fts-solr: Encode entities when indexing data.
Timo Sirainen <tss@iki.fi>
parents: 7999
diff changeset
34
61be6a4722c5 fts-solr: Encode entities when indexing data.
Timo Sirainen <tss@iki.fi>
parents: 7999
diff changeset
35 for (i = 0; i < len; i++) {
61be6a4722c5 fts-solr: Encode entities when indexing data.
Timo Sirainen <tss@iki.fi>
parents: 7999
diff changeset
36 switch (data[i]) {
7990
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
37 case '&':
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
38 str_append(dest, "&amp;");
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
39 break;
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
40 case '<':
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
41 str_append(dest, "&lt;");
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
42 break;
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
43 case '>':
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
44 str_append(dest, "&gt;");
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
45 break;
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
46 default:
8000
61be6a4722c5 fts-solr: Encode entities when indexing data.
Timo Sirainen <tss@iki.fi>
parents: 7999
diff changeset
47 str_append_c(dest, data[i]);
7990
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
48 break;
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
49 }
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
50 }
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
51 }
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
52
8000
61be6a4722c5 fts-solr: Encode entities when indexing data.
Timo Sirainen <tss@iki.fi>
parents: 7999
diff changeset
53 static void xml_encode(string_t *dest, const char *str)
61be6a4722c5 fts-solr: Encode entities when indexing data.
Timo Sirainen <tss@iki.fi>
parents: 7999
diff changeset
54 {
61be6a4722c5 fts-solr: Encode entities when indexing data.
Timo Sirainen <tss@iki.fi>
parents: 7999
diff changeset
55 xml_encode_data(dest, (const unsigned char *)str, strlen(str));
61be6a4722c5 fts-solr: Encode entities when indexing data.
Timo Sirainen <tss@iki.fi>
parents: 7999
diff changeset
56 }
61be6a4722c5 fts-solr: Encode entities when indexing data.
Timo Sirainen <tss@iki.fi>
parents: 7999
diff changeset
57
7990
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
58 static struct fts_backend *
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
59 fts_backend_solr_init(struct mailbox *box ATTR_UNUSED)
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
60 {
8002
e814614ffb65 fts-solr: Added break-imap-search setting to use Solr with SEARCH TEXT/BODY
Timo Sirainen <tss@iki.fi>
parents: 8001
diff changeset
61 const struct fts_solr_settings *set = &fts_solr_settings;
7990
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
62 struct fts_backend *backend;
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
63
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
64 if (solr_conn == NULL)
8002
e814614ffb65 fts-solr: Added break-imap-search setting to use Solr with SEARCH TEXT/BODY
Timo Sirainen <tss@iki.fi>
parents: 8001
diff changeset
65 solr_conn = solr_connection_init(set->url, set->debug);
7990
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
66
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
67 backend = i_new(struct fts_backend, 1);
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
68 *backend = fts_backend_solr;
8002
e814614ffb65 fts-solr: Added break-imap-search setting to use Solr with SEARCH TEXT/BODY
Timo Sirainen <tss@iki.fi>
parents: 8001
diff changeset
69
e814614ffb65 fts-solr: Added break-imap-search setting to use Solr with SEARCH TEXT/BODY
Timo Sirainen <tss@iki.fi>
parents: 8001
diff changeset
70 if (set->substring_search)
e814614ffb65 fts-solr: Added break-imap-search setting to use Solr with SEARCH TEXT/BODY
Timo Sirainen <tss@iki.fi>
parents: 8001
diff changeset
71 backend->flags |= FTS_BACKEND_FLAG_SUBSTRING_LOOKUPS;
7990
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
72 return backend;
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
73 }
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
74
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
75 static void fts_backend_solr_deinit(struct fts_backend *backend)
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
76 {
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
77 i_free(backend);
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
78 }
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
79
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
80 static int fts_backend_solr_get_last_uid(struct fts_backend *backend,
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
81 uint32_t *last_uid_r)
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
82 {
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
83 struct mailbox_status status;
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
84 ARRAY_TYPE(seq_range) uids;
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
85 const struct seq_range *uidvals;
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
86 unsigned int count;
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
87 string_t *str;
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
88
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
89 str = t_str_new(256);
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
90 str_append(str, "fl=uid&rows=1&sort=uid%20desc&q=");
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
91
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
92 mailbox_get_status(backend->box, STATUS_UIDVALIDITY, &status);
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
93 str_printfa(str, "uidv:%u%%20box:", status.uidvalidity);
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
94 solr_quote_str(str, backend->box->name);
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
95 str_append(str, "%20user:");
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
96 solr_quote_str(str, backend->box->storage->user);
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
97
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
98 t_array_init(&uids, 1);
8006
0a3b5ddb0869 Added support for sorting by X-SCORE. Currently it's only set by fts-solr.
Timo Sirainen <tss@iki.fi>
parents: 8004
diff changeset
99 if (solr_connection_select(solr_conn, str_c(str), &uids, NULL) < 0)
7990
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
100 return -1;
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
101
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
102 uidvals = array_get(&uids, &count);
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
103 if (count == 0) {
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
104 /* nothing indexed yet for this mailbox */
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
105 *last_uid_r = 0;
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
106 } else if (count == 1 && uidvals[0].seq1 == uidvals[0].seq2) {
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
107 *last_uid_r = uidvals[0].seq1;
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
108 } else {
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
109 i_error("fts_solr: Last UID lookup returned multiple rows");
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
110 return -1;
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
111 }
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
112 return 0;
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
113 }
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
114
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
115 static int
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
116 fts_backend_solr_build_init(struct fts_backend *backend, uint32_t *last_uid_r,
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
117 struct fts_backend_build_context **ctx_r)
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
118 {
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
119 struct solr_fts_backend_build_context *ctx;
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
120 struct mailbox_status status;
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
121
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
122 *last_uid_r = (uint32_t)-1;
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
123
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
124 ctx = i_new(struct solr_fts_backend_build_context, 1);
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
125 ctx->ctx.backend = backend;
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
126 ctx->post = solr_connection_post_begin(solr_conn);
8001
3fbfdda3e5d3 fts-solr: Send data to Solr in larger packets. Commit after expunges.
Timo Sirainen <tss@iki.fi>
parents: 8000
diff changeset
127 ctx->cmd = str_new(default_pool, SOLR_CMDBUF_SIZE);
7990
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
128
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
129 mailbox_get_status(backend->box, STATUS_UIDVALIDITY, &status);
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
130 ctx->uid_validity = status.uidvalidity;
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
131
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
132 *ctx_r = &ctx->ctx;
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
133 return 0;
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
134 }
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
135
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
136 static int
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
137 fts_backend_solr_build_more(struct fts_backend_build_context *_ctx,
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
138 uint32_t uid, const unsigned char *data,
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
139 size_t size, bool headers)
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
140 {
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
141 struct solr_fts_backend_build_context *ctx =
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
142 (struct solr_fts_backend_build_context *)_ctx;
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
143 struct mailbox *box = _ctx->backend->box;
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
144 string_t *cmd = ctx->cmd;
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
145
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
146 /* body comes first, then headers */
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
147 if (ctx->prev_uid != uid) {
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
148 /* uid changed */
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
149 if (ctx->prev_uid == 0)
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
150 str_append(cmd, "<add>");
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
151 else
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
152 str_append(cmd, "</field></doc>");
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
153 ctx->prev_uid = uid;
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
154
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
155 str_printfa(cmd, "<doc>"
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
156 "<field name=\"uid\">%u</field>"
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
157 "<field name=\"uidv\">%u</field>",
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
158 uid, ctx->uid_validity);
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
159
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
160 str_append(cmd, "<field name=\"box\">");
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
161 xml_encode(cmd, box->name);
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
162 str_append(cmd, "</field><field name=\"user\">");
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
163 xml_encode(cmd, box->storage->user);
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
164
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
165 str_printfa(cmd, "</field><field name=\"id\">%u/%u/",
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
166 uid, ctx->uid_validity);
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
167 xml_encode(cmd, box->storage->user);
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
168 str_append_c(cmd, '/');
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
169 xml_encode(cmd, box->name);
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
170 str_append(cmd, "</field>");
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
171
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
172 ctx->headers = headers;
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
173 if (headers) {
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
174 str_append(cmd, "<field name=\"hdr\">");
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
175 } else {
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
176 str_append(cmd, "<field name=\"body\">");
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
177 }
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
178 } else if (headers && !ctx->headers) {
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
179 str_append(cmd, "</field><field name=\"hdr\">");
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
180 } else {
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
181 i_assert(!(!headers && ctx->headers));
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
182 }
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
183
8000
61be6a4722c5 fts-solr: Encode entities when indexing data.
Timo Sirainen <tss@iki.fi>
parents: 7999
diff changeset
184 xml_encode_data(cmd, data, size);
8001
3fbfdda3e5d3 fts-solr: Send data to Solr in larger packets. Commit after expunges.
Timo Sirainen <tss@iki.fi>
parents: 8000
diff changeset
185 if (str_len(cmd) > SOLR_CMDBUF_SIZE-128) {
3fbfdda3e5d3 fts-solr: Send data to Solr in larger packets. Commit after expunges.
Timo Sirainen <tss@iki.fi>
parents: 8000
diff changeset
186 solr_connection_post_more(ctx->post, str_data(cmd),
3fbfdda3e5d3 fts-solr: Send data to Solr in larger packets. Commit after expunges.
Timo Sirainen <tss@iki.fi>
parents: 8000
diff changeset
187 str_len(cmd));
3fbfdda3e5d3 fts-solr: Send data to Solr in larger packets. Commit after expunges.
Timo Sirainen <tss@iki.fi>
parents: 8000
diff changeset
188 str_truncate(cmd, 0);
3fbfdda3e5d3 fts-solr: Send data to Solr in larger packets. Commit after expunges.
Timo Sirainen <tss@iki.fi>
parents: 8000
diff changeset
189 }
7990
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
190 return 0;
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
191 }
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
192
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
193 static int
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
194 fts_backend_solr_build_deinit(struct fts_backend_build_context *_ctx)
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
195 {
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
196 struct solr_fts_backend_build_context *ctx =
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
197 (struct solr_fts_backend_build_context *)_ctx;
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
198 int ret = 0;
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
199
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
200 if (ctx->prev_uid != 0) {
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
201 str_append(ctx->cmd, "</field></doc></add>");
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
202 solr_connection_post_more(ctx->post, str_data(ctx->cmd),
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
203 str_len(ctx->cmd));
7999
1ceb49f2eb50 fts-solr: Error handling fixes.
Timo Sirainen <tss@iki.fi>
parents: 7990
diff changeset
204 ret = solr_connection_post_end(ctx->post);
7990
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
205 /* commit and wait until the documents we just indexed are
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
206 visible to the following search */
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
207 if (solr_connection_post(solr_conn,
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
208 "<commit waitFlush=\"false\" "
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
209 "waitSearcher=\"true\"/>") < 0)
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
210 ret = -1;
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
211 }
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
212 str_free(&ctx->cmd);
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
213 i_free(ctx);
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
214 return ret;
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
215 }
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
216
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
217 static void
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
218 fts_backend_solr_expunge(struct fts_backend *backend ATTR_UNUSED,
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
219 struct mail *mail)
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
220 {
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
221 struct mailbox_status status;
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
222
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
223 mailbox_get_status(mail->box, STATUS_UIDVALIDITY, &status);
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
224
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
225 T_BEGIN {
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
226 string_t *cmd;
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
227
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
228 cmd = t_str_new(256);
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
229 str_printfa(cmd, "<delete><id>%u/%u/",
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
230 mail->uid, status.uidvalidity);
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
231 xml_encode(cmd, mail->box->storage->user);
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
232 str_append_c(cmd, '/');
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
233 xml_encode(cmd, mail->box->name);
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
234 str_append(cmd, "</id></delete>");
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
235
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
236 (void)solr_connection_post(solr_conn, str_c(cmd));
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
237 } T_END;
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
238 }
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
239
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
240 static void
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
241 fts_backend_solr_expunge_finish(struct fts_backend *backend ATTR_UNUSED,
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
242 struct mailbox *box ATTR_UNUSED,
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
243 bool committed ATTR_UNUSED)
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
244 {
8001
3fbfdda3e5d3 fts-solr: Send data to Solr in larger packets. Commit after expunges.
Timo Sirainen <tss@iki.fi>
parents: 8000
diff changeset
245 solr_connection_post(solr_conn,
3fbfdda3e5d3 fts-solr: Send data to Solr in larger packets. Commit after expunges.
Timo Sirainen <tss@iki.fi>
parents: 8000
diff changeset
246 "<commit waitFlush=\"false\" waitSearcher=\"false\"/>");
7990
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
247 }
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
248
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
249 static int fts_backend_solr_lock(struct fts_backend *backend ATTR_UNUSED)
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
250 {
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
251 return 1;
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
252 }
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
253
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
254 static void fts_backend_solr_unlock(struct fts_backend *backend ATTR_UNUSED)
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
255 {
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
256 }
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
257
8004
6418d4c0ff16 fts: Added a new lookup API where the backend can look up all the fields
Timo Sirainen <tss@iki.fi>
parents: 8002
diff changeset
258 static int fts_backend_solr_lookup(struct fts_backend_lookup_context *ctx,
6418d4c0ff16 fts: Added a new lookup API where the backend can look up all the fields
Timo Sirainen <tss@iki.fi>
parents: 8002
diff changeset
259 ARRAY_TYPE(seq_range) *definite_uids,
8006
0a3b5ddb0869 Added support for sorting by X-SCORE. Currently it's only set by fts-solr.
Timo Sirainen <tss@iki.fi>
parents: 8004
diff changeset
260 ARRAY_TYPE(seq_range) *maybe_uids,
0a3b5ddb0869 Added support for sorting by X-SCORE. Currently it's only set by fts-solr.
Timo Sirainen <tss@iki.fi>
parents: 8004
diff changeset
261 ARRAY_TYPE(fts_score_map) *scores)
7990
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
262 {
8004
6418d4c0ff16 fts: Added a new lookup API where the backend can look up all the fields
Timo Sirainen <tss@iki.fi>
parents: 8002
diff changeset
263 struct mailbox *box = ctx->backend->box;
6418d4c0ff16 fts: Added a new lookup API where the backend can look up all the fields
Timo Sirainen <tss@iki.fi>
parents: 8002
diff changeset
264 const struct fts_backend_lookup_field *fields;
6418d4c0ff16 fts: Added a new lookup API where the backend can look up all the fields
Timo Sirainen <tss@iki.fi>
parents: 8002
diff changeset
265 unsigned int i, count;
7990
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
266 struct mailbox_status status;
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
267 string_t *str;
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
268
8004
6418d4c0ff16 fts: Added a new lookup API where the backend can look up all the fields
Timo Sirainen <tss@iki.fi>
parents: 8002
diff changeset
269 mailbox_get_status(box, STATUS_UIDVALIDITY, &status);
7990
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
270
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
271 str = t_str_new(256);
8006
0a3b5ddb0869 Added support for sorting by X-SCORE. Currently it's only set by fts-solr.
Timo Sirainen <tss@iki.fi>
parents: 8004
diff changeset
272 str_printfa(str, "fl=uid,score&rows=%u&sort=uid%%20asc&q=",
0a3b5ddb0869 Added support for sorting by X-SCORE. Currently it's only set by fts-solr.
Timo Sirainen <tss@iki.fi>
parents: 8004
diff changeset
273 status.uidnext);
8004
6418d4c0ff16 fts: Added a new lookup API where the backend can look up all the fields
Timo Sirainen <tss@iki.fi>
parents: 8002
diff changeset
274
6418d4c0ff16 fts: Added a new lookup API where the backend can look up all the fields
Timo Sirainen <tss@iki.fi>
parents: 8002
diff changeset
275 /* build a lucene search query from the fields */
6418d4c0ff16 fts: Added a new lookup API where the backend can look up all the fields
Timo Sirainen <tss@iki.fi>
parents: 8002
diff changeset
276 fields = array_get(&ctx->fields, &count);
6418d4c0ff16 fts: Added a new lookup API where the backend can look up all the fields
Timo Sirainen <tss@iki.fi>
parents: 8002
diff changeset
277 for (i = 0; i < count; i++) {
6418d4c0ff16 fts: Added a new lookup API where the backend can look up all the fields
Timo Sirainen <tss@iki.fi>
parents: 8002
diff changeset
278 if (i > 0)
6418d4c0ff16 fts: Added a new lookup API where the backend can look up all the fields
Timo Sirainen <tss@iki.fi>
parents: 8002
diff changeset
279 str_append(str, "%20");
6418d4c0ff16 fts: Added a new lookup API where the backend can look up all the fields
Timo Sirainen <tss@iki.fi>
parents: 8002
diff changeset
280
6418d4c0ff16 fts: Added a new lookup API where the backend can look up all the fields
Timo Sirainen <tss@iki.fi>
parents: 8002
diff changeset
281 if ((fields[i].flags & FTS_LOOKUP_FLAG_INVERT) != 0)
6418d4c0ff16 fts: Added a new lookup API where the backend can look up all the fields
Timo Sirainen <tss@iki.fi>
parents: 8002
diff changeset
282 str_append_c(str, '-');
6418d4c0ff16 fts: Added a new lookup API where the backend can look up all the fields
Timo Sirainen <tss@iki.fi>
parents: 8002
diff changeset
283
6418d4c0ff16 fts: Added a new lookup API where the backend can look up all the fields
Timo Sirainen <tss@iki.fi>
parents: 8002
diff changeset
284 if ((fields[i].flags & FTS_LOOKUP_FLAG_HEADER) == 0) {
6418d4c0ff16 fts: Added a new lookup API where the backend can look up all the fields
Timo Sirainen <tss@iki.fi>
parents: 8002
diff changeset
285 /* body only */
6418d4c0ff16 fts: Added a new lookup API where the backend can look up all the fields
Timo Sirainen <tss@iki.fi>
parents: 8002
diff changeset
286 i_assert((fields[i].flags & FTS_LOOKUP_FLAG_BODY) != 0);
6418d4c0ff16 fts: Added a new lookup API where the backend can look up all the fields
Timo Sirainen <tss@iki.fi>
parents: 8002
diff changeset
287 str_append(str, "body:");
6418d4c0ff16 fts: Added a new lookup API where the backend can look up all the fields
Timo Sirainen <tss@iki.fi>
parents: 8002
diff changeset
288 } else if ((fields[i].flags & FTS_LOOKUP_FLAG_BODY) == 0) {
6418d4c0ff16 fts: Added a new lookup API where the backend can look up all the fields
Timo Sirainen <tss@iki.fi>
parents: 8002
diff changeset
289 /* header only */
6418d4c0ff16 fts: Added a new lookup API where the backend can look up all the fields
Timo Sirainen <tss@iki.fi>
parents: 8002
diff changeset
290 str_append(str, "hdr:");
6418d4c0ff16 fts: Added a new lookup API where the backend can look up all the fields
Timo Sirainen <tss@iki.fi>
parents: 8002
diff changeset
291 } else {
6418d4c0ff16 fts: Added a new lookup API where the backend can look up all the fields
Timo Sirainen <tss@iki.fi>
parents: 8002
diff changeset
292 /* both */
6418d4c0ff16 fts: Added a new lookup API where the backend can look up all the fields
Timo Sirainen <tss@iki.fi>
parents: 8002
diff changeset
293 str_append(str, "any:");
6418d4c0ff16 fts: Added a new lookup API where the backend can look up all the fields
Timo Sirainen <tss@iki.fi>
parents: 8002
diff changeset
294 }
6418d4c0ff16 fts: Added a new lookup API where the backend can look up all the fields
Timo Sirainen <tss@iki.fi>
parents: 8002
diff changeset
295 solr_quote_str(str, fields[i].key);
7990
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
296 }
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
297
8004
6418d4c0ff16 fts: Added a new lookup API where the backend can look up all the fields
Timo Sirainen <tss@iki.fi>
parents: 8002
diff changeset
298 /* use a separate filter query for selecting the mailbox. it shouldn't
6418d4c0ff16 fts: Added a new lookup API where the backend can look up all the fields
Timo Sirainen <tss@iki.fi>
parents: 8002
diff changeset
299 affect the score and there could be some caching benefits too. */
6418d4c0ff16 fts: Added a new lookup API where the backend can look up all the fields
Timo Sirainen <tss@iki.fi>
parents: 8002
diff changeset
300 str_printfa(str, "&fq=uidv:%u%%20box:", status.uidvalidity);
6418d4c0ff16 fts: Added a new lookup API where the backend can look up all the fields
Timo Sirainen <tss@iki.fi>
parents: 8002
diff changeset
301 solr_quote_str(str, box->name);
7990
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
302 str_append(str, "%20user:");
8004
6418d4c0ff16 fts: Added a new lookup API where the backend can look up all the fields
Timo Sirainen <tss@iki.fi>
parents: 8002
diff changeset
303 solr_quote_str(str, box->storage->user);
7990
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
304
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
305 array_clear(maybe_uids);
8006
0a3b5ddb0869 Added support for sorting by X-SCORE. Currently it's only set by fts-solr.
Timo Sirainen <tss@iki.fi>
parents: 8004
diff changeset
306 return solr_connection_select(solr_conn, str_c(str),
0a3b5ddb0869 Added support for sorting by X-SCORE. Currently it's only set by fts-solr.
Timo Sirainen <tss@iki.fi>
parents: 8004
diff changeset
307 definite_uids, scores);
7990
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
308 }
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
309
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
310 struct fts_backend fts_backend_solr = {
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
311 MEMBER(name) "solr",
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
312 MEMBER(flags) 0,
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
313
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
314 {
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
315 fts_backend_solr_init,
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
316 fts_backend_solr_deinit,
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
317 fts_backend_solr_get_last_uid,
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
318 fts_backend_solr_build_init,
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
319 fts_backend_solr_build_more,
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
320 fts_backend_solr_build_deinit,
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
321 fts_backend_solr_expunge,
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
322 fts_backend_solr_expunge_finish,
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
323 fts_backend_solr_lock,
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
324 fts_backend_solr_unlock,
8004
6418d4c0ff16 fts: Added a new lookup API where the backend can look up all the fields
Timo Sirainen <tss@iki.fi>
parents: 8002
diff changeset
325 NULL,
6418d4c0ff16 fts: Added a new lookup API where the backend can look up all the fields
Timo Sirainen <tss@iki.fi>
parents: 8002
diff changeset
326 NULL,
6418d4c0ff16 fts: Added a new lookup API where the backend can look up all the fields
Timo Sirainen <tss@iki.fi>
parents: 8002
diff changeset
327 fts_backend_solr_lookup
7990
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
328 }
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
329 };