Mercurial > dovecot > original-hg > dovecot-1.2

annotate src/plugins/fts-solr/fts-backend-solr.c @ 8493:149c6ccc3df4 HEAD

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
fts-solr: Use '!' as escape character in usernames. Escape '/' and '!'.
author Timo Sirainen <tss@iki.fi>
date Sat, 29 Nov 2008 19:24:59 +0200
parents ee1ecdda8a6b
children ffb37c392166
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
7990
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
1 /* Copyright (c) 2006-2008 Dovecot authors, see the included COPYING file */
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
2
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
3 #include "lib.h"
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
4 #include "array.h"
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
5 #include "str.h"
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
6 #include "mail-storage-private.h"
8109
e7929190cd32 fts-solr: Fixed compiling with recent struct mail_user changes.
Timo Sirainen <tss@iki.fi>
parents: 8006
diff changeset
7 #include "mail-namespace.h"
7990
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
8 #include "solr-connection.h"
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
9 #include "fts-solr-plugin.h"
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
10
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
11 #include <curl/curl.h>
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
12
8001
3fbfdda3e5d3 fts-solr: Send data to Solr in larger packets. Commit after expunges.
Timo Sirainen <tss@iki.fi>
parents: 8000
diff changeset
13 #define SOLR_CMDBUF_SIZE (1024*64)
8484
db78eab2ba5d Initial support for getting full text search working nicely with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents: 8109
diff changeset
14 #define SOLR_MAX_ROWS 100000
8001
3fbfdda3e5d3 fts-solr: Send data to Solr in larger packets. Commit after expunges.
Timo Sirainen <tss@iki.fi>
parents: 8000
diff changeset
15
8493
149c6ccc3df4 fts-solr: Use '!' as escape character in usernames. Escape '/' and '!'.
Timo Sirainen <tss@iki.fi>
parents: 8492
diff changeset
16 struct solr_fts_backend {
149c6ccc3df4 fts-solr: Use '!' as escape character in usernames. Escape '/' and '!'.
Timo Sirainen <tss@iki.fi>
parents: 8492
diff changeset
17 struct fts_backend backend;
149c6ccc3df4 fts-solr: Use '!' as escape character in usernames. Escape '/' and '!'.
Timo Sirainen <tss@iki.fi>
parents: 8492
diff changeset
18 char *id_username;
149c6ccc3df4 fts-solr: Use '!' as escape character in usernames. Escape '/' and '!'.
Timo Sirainen <tss@iki.fi>
parents: 8492
diff changeset
19 };
149c6ccc3df4 fts-solr: Use '!' as escape character in usernames. Escape '/' and '!'.
Timo Sirainen <tss@iki.fi>
parents: 8492
diff changeset
20
7990
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
21 struct solr_fts_backend_build_context {
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
22 struct fts_backend_build_context ctx;
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
23
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
24 struct solr_connection_post *post;
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
25 uint32_t prev_uid, uid_validity;
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
26 string_t *cmd;
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
27 bool headers;
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
28 };
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
29
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
30 static struct solr_connection *solr_conn = NULL;
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
31
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
32 static void solr_quote_str(string_t *dest, const char *str)
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
33 {
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
34 solr_connection_quote_str(solr_conn, dest, str);
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
35 }
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
36
8000
61be6a4722c5 fts-solr: Encode entities when indexing data.
Timo Sirainen <tss@iki.fi>
parents: 7999
diff changeset
37 static void
61be6a4722c5 fts-solr: Encode entities when indexing data.
Timo Sirainen <tss@iki.fi>
parents: 7999
diff changeset
38 xml_encode_data(string_t *dest, const unsigned char *data, unsigned int len)
7990
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
39 {
8000
61be6a4722c5 fts-solr: Encode entities when indexing data.
Timo Sirainen <tss@iki.fi>
parents: 7999
diff changeset
40 unsigned int i;
61be6a4722c5 fts-solr: Encode entities when indexing data.
Timo Sirainen <tss@iki.fi>
parents: 7999
diff changeset
41
61be6a4722c5 fts-solr: Encode entities when indexing data.
Timo Sirainen <tss@iki.fi>
parents: 7999
diff changeset
42 for (i = 0; i < len; i++) {
61be6a4722c5 fts-solr: Encode entities when indexing data.
Timo Sirainen <tss@iki.fi>
parents: 7999
diff changeset
43 switch (data[i]) {
7990
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
44 case '&':
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
45 str_append(dest, "&amp;");
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
46 break;
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
47 case '<':
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
48 str_append(dest, "&lt;");
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
49 break;
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
50 case '>':
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
51 str_append(dest, "&gt;");
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
52 break;
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
53 default:
8000
61be6a4722c5 fts-solr: Encode entities when indexing data.
Timo Sirainen <tss@iki.fi>
parents: 7999
diff changeset
54 str_append_c(dest, data[i]);
7990
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
55 break;
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
56 }
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
57 }
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
58 }
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
59
8000
61be6a4722c5 fts-solr: Encode entities when indexing data.
Timo Sirainen <tss@iki.fi>
parents: 7999
diff changeset
60 static void xml_encode(string_t *dest, const char *str)
61be6a4722c5 fts-solr: Encode entities when indexing data.
Timo Sirainen <tss@iki.fi>
parents: 7999
diff changeset
61 {
61be6a4722c5 fts-solr: Encode entities when indexing data.
Timo Sirainen <tss@iki.fi>
parents: 7999
diff changeset
62 xml_encode_data(dest, (const unsigned char *)str, strlen(str));
61be6a4722c5 fts-solr: Encode entities when indexing data.
Timo Sirainen <tss@iki.fi>
parents: 7999
diff changeset
63 }
61be6a4722c5 fts-solr: Encode entities when indexing data.
Timo Sirainen <tss@iki.fi>
parents: 7999
diff changeset
64
8493
149c6ccc3df4 fts-solr: Use '!' as escape character in usernames. Escape '/' and '!'.
Timo Sirainen <tss@iki.fi>
parents: 8492
diff changeset
65 static const char *solr_escape_id_str(const char *str)
149c6ccc3df4 fts-solr: Use '!' as escape character in usernames. Escape '/' and '!'.
Timo Sirainen <tss@iki.fi>
parents: 8492
diff changeset
66 {
149c6ccc3df4 fts-solr: Use '!' as escape character in usernames. Escape '/' and '!'.
Timo Sirainen <tss@iki.fi>
parents: 8492
diff changeset
67 string_t *tmp;
149c6ccc3df4 fts-solr: Use '!' as escape character in usernames. Escape '/' and '!'.
Timo Sirainen <tss@iki.fi>
parents: 8492
diff changeset
68 const char *p;
149c6ccc3df4 fts-solr: Use '!' as escape character in usernames. Escape '/' and '!'.
Timo Sirainen <tss@iki.fi>
parents: 8492
diff changeset
69
149c6ccc3df4 fts-solr: Use '!' as escape character in usernames. Escape '/' and '!'.
Timo Sirainen <tss@iki.fi>
parents: 8492
diff changeset
70 for (p = str; *p != '\0'; p++) {
149c6ccc3df4 fts-solr: Use '!' as escape character in usernames. Escape '/' and '!'.
Timo Sirainen <tss@iki.fi>
parents: 8492
diff changeset
71 if (*p == '/' || *p == '!')
149c6ccc3df4 fts-solr: Use '!' as escape character in usernames. Escape '/' and '!'.
Timo Sirainen <tss@iki.fi>
parents: 8492
diff changeset
72 break;
149c6ccc3df4 fts-solr: Use '!' as escape character in usernames. Escape '/' and '!'.
Timo Sirainen <tss@iki.fi>
parents: 8492
diff changeset
73 }
149c6ccc3df4 fts-solr: Use '!' as escape character in usernames. Escape '/' and '!'.
Timo Sirainen <tss@iki.fi>
parents: 8492
diff changeset
74 if (*p == '\0')
149c6ccc3df4 fts-solr: Use '!' as escape character in usernames. Escape '/' and '!'.
Timo Sirainen <tss@iki.fi>
parents: 8492
diff changeset
75 return str;
149c6ccc3df4 fts-solr: Use '!' as escape character in usernames. Escape '/' and '!'.
Timo Sirainen <tss@iki.fi>
parents: 8492
diff changeset
76
149c6ccc3df4 fts-solr: Use '!' as escape character in usernames. Escape '/' and '!'.
Timo Sirainen <tss@iki.fi>
parents: 8492
diff changeset
77 tmp = t_str_new(64);
149c6ccc3df4 fts-solr: Use '!' as escape character in usernames. Escape '/' and '!'.
Timo Sirainen <tss@iki.fi>
parents: 8492
diff changeset
78 for (p = str; *p != '\0'; p++) {
149c6ccc3df4 fts-solr: Use '!' as escape character in usernames. Escape '/' and '!'.
Timo Sirainen <tss@iki.fi>
parents: 8492
diff changeset
79 switch (*p) {
149c6ccc3df4 fts-solr: Use '!' as escape character in usernames. Escape '/' and '!'.
Timo Sirainen <tss@iki.fi>
parents: 8492
diff changeset
80 case '/':
149c6ccc3df4 fts-solr: Use '!' as escape character in usernames. Escape '/' and '!'.
Timo Sirainen <tss@iki.fi>
parents: 8492
diff changeset
81 str_append(tmp, "!\\");
149c6ccc3df4 fts-solr: Use '!' as escape character in usernames. Escape '/' and '!'.
Timo Sirainen <tss@iki.fi>
parents: 8492
diff changeset
82 break;
149c6ccc3df4 fts-solr: Use '!' as escape character in usernames. Escape '/' and '!'.
Timo Sirainen <tss@iki.fi>
parents: 8492
diff changeset
83 case '!':
149c6ccc3df4 fts-solr: Use '!' as escape character in usernames. Escape '/' and '!'.
Timo Sirainen <tss@iki.fi>
parents: 8492
diff changeset
84 str_append(tmp, "!!");
149c6ccc3df4 fts-solr: Use '!' as escape character in usernames. Escape '/' and '!'.
Timo Sirainen <tss@iki.fi>
parents: 8492
diff changeset
85 break;
149c6ccc3df4 fts-solr: Use '!' as escape character in usernames. Escape '/' and '!'.
Timo Sirainen <tss@iki.fi>
parents: 8492
diff changeset
86 default:
149c6ccc3df4 fts-solr: Use '!' as escape character in usernames. Escape '/' and '!'.
Timo Sirainen <tss@iki.fi>
parents: 8492
diff changeset
87 str_append_c(tmp, *p);
149c6ccc3df4 fts-solr: Use '!' as escape character in usernames. Escape '/' and '!'.
Timo Sirainen <tss@iki.fi>
parents: 8492
diff changeset
88 break;
149c6ccc3df4 fts-solr: Use '!' as escape character in usernames. Escape '/' and '!'.
Timo Sirainen <tss@iki.fi>
parents: 8492
diff changeset
89 }
149c6ccc3df4 fts-solr: Use '!' as escape character in usernames. Escape '/' and '!'.
Timo Sirainen <tss@iki.fi>
parents: 8492
diff changeset
90 }
149c6ccc3df4 fts-solr: Use '!' as escape character in usernames. Escape '/' and '!'.
Timo Sirainen <tss@iki.fi>
parents: 8492
diff changeset
91 return str_c(tmp);
149c6ccc3df4 fts-solr: Use '!' as escape character in usernames. Escape '/' and '!'.
Timo Sirainen <tss@iki.fi>
parents: 8492
diff changeset
92 }
149c6ccc3df4 fts-solr: Use '!' as escape character in usernames. Escape '/' and '!'.
Timo Sirainen <tss@iki.fi>
parents: 8492
diff changeset
93
7990
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
94 static struct fts_backend *
8493
149c6ccc3df4 fts-solr: Use '!' as escape character in usernames. Escape '/' and '!'.
Timo Sirainen <tss@iki.fi>
parents: 8492
diff changeset
95 fts_backend_solr_init(struct mailbox *box)
7990
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
96 {
8002
e814614ffb65 fts-solr: Added break-imap-search setting to use Solr with SEARCH TEXT/BODY
Timo Sirainen <tss@iki.fi>
parents: 8001
diff changeset
97 const struct fts_solr_settings *set = &fts_solr_settings;
8493
149c6ccc3df4 fts-solr: Use '!' as escape character in usernames. Escape '/' and '!'.
Timo Sirainen <tss@iki.fi>
parents: 8492
diff changeset
98 struct solr_fts_backend *backend;
149c6ccc3df4 fts-solr: Use '!' as escape character in usernames. Escape '/' and '!'.
Timo Sirainen <tss@iki.fi>
parents: 8492
diff changeset
99 const char *username = box->storage->ns->user->username;
7990
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
100
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
101 if (solr_conn == NULL)
8002
e814614ffb65 fts-solr: Added break-imap-search setting to use Solr with SEARCH TEXT/BODY
Timo Sirainen <tss@iki.fi>
parents: 8001
diff changeset
102 solr_conn = solr_connection_init(set->url, set->debug);
7990
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
103
8493
149c6ccc3df4 fts-solr: Use '!' as escape character in usernames. Escape '/' and '!'.
Timo Sirainen <tss@iki.fi>
parents: 8492
diff changeset
104 backend = i_new(struct solr_fts_backend, 1);
149c6ccc3df4 fts-solr: Use '!' as escape character in usernames. Escape '/' and '!'.
Timo Sirainen <tss@iki.fi>
parents: 8492
diff changeset
105 backend->id_username = i_strdup(solr_escape_id_str(username));
149c6ccc3df4 fts-solr: Use '!' as escape character in usernames. Escape '/' and '!'.
Timo Sirainen <tss@iki.fi>
parents: 8492
diff changeset
106 backend->backend = fts_backend_solr;
8002
e814614ffb65 fts-solr: Added break-imap-search setting to use Solr with SEARCH TEXT/BODY
Timo Sirainen <tss@iki.fi>
parents: 8001
diff changeset
107
e814614ffb65 fts-solr: Added break-imap-search setting to use Solr with SEARCH TEXT/BODY
Timo Sirainen <tss@iki.fi>
parents: 8001
diff changeset
108 if (set->substring_search)
8493
149c6ccc3df4 fts-solr: Use '!' as escape character in usernames. Escape '/' and '!'.
Timo Sirainen <tss@iki.fi>
parents: 8492
diff changeset
109 backend->backend.flags |= FTS_BACKEND_FLAG_SUBSTRING_LOOKUPS;
149c6ccc3df4 fts-solr: Use '!' as escape character in usernames. Escape '/' and '!'.
Timo Sirainen <tss@iki.fi>
parents: 8492
diff changeset
110 return &backend->backend;
7990
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
111 }
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
112
8493
149c6ccc3df4 fts-solr: Use '!' as escape character in usernames. Escape '/' and '!'.
Timo Sirainen <tss@iki.fi>
parents: 8492
diff changeset
113 static void fts_backend_solr_deinit(struct fts_backend *_backend)
7990
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
114 {
8493
149c6ccc3df4 fts-solr: Use '!' as escape character in usernames. Escape '/' and '!'.
Timo Sirainen <tss@iki.fi>
parents: 8492
diff changeset
115 struct solr_fts_backend *backend = (struct solr_fts_backend *)_backend;
149c6ccc3df4 fts-solr: Use '!' as escape character in usernames. Escape '/' and '!'.
Timo Sirainen <tss@iki.fi>
parents: 8492
diff changeset
116
149c6ccc3df4 fts-solr: Use '!' as escape character in usernames. Escape '/' and '!'.
Timo Sirainen <tss@iki.fi>
parents: 8492
diff changeset
117 i_free(backend->id_username);
7990
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
118 i_free(backend);
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
119 }
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
120
8493
149c6ccc3df4 fts-solr: Use '!' as escape character in usernames. Escape '/' and '!'.
Timo Sirainen <tss@iki.fi>
parents: 8492
diff changeset
121 static const char *fts_backend_solr_username(struct fts_backend *_backend)
149c6ccc3df4 fts-solr: Use '!' as escape character in usernames. Escape '/' and '!'.
Timo Sirainen <tss@iki.fi>
parents: 8492
diff changeset
122 {
149c6ccc3df4 fts-solr: Use '!' as escape character in usernames. Escape '/' and '!'.
Timo Sirainen <tss@iki.fi>
parents: 8492
diff changeset
123 struct solr_fts_backend *backend = (struct solr_fts_backend *)_backend;
149c6ccc3df4 fts-solr: Use '!' as escape character in usernames. Escape '/' and '!'.
Timo Sirainen <tss@iki.fi>
parents: 8492
diff changeset
124
149c6ccc3df4 fts-solr: Use '!' as escape character in usernames. Escape '/' and '!'.
Timo Sirainen <tss@iki.fi>
parents: 8492
diff changeset
125 return backend->id_username;
149c6ccc3df4 fts-solr: Use '!' as escape character in usernames. Escape '/' and '!'.
Timo Sirainen <tss@iki.fi>
parents: 8492
diff changeset
126 }
149c6ccc3df4 fts-solr: Use '!' as escape character in usernames. Escape '/' and '!'.
Timo Sirainen <tss@iki.fi>
parents: 8492
diff changeset
127
8492
ee1ecdda8a6b fts-solr: Keep last_uid field updated containing the mailbox's last added UID.
Timo Sirainen <tss@iki.fi>
parents: 8484
diff changeset
128 static int fts_backend_solr_get_last_uid_fallback(struct fts_backend *backend,
ee1ecdda8a6b fts-solr: Keep last_uid field updated containing the mailbox's last added UID.
Timo Sirainen <tss@iki.fi>
parents: 8484
diff changeset
129 uint32_t *last_uid_r)
7990
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
130 {
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
131 struct mailbox_status status;
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
132 ARRAY_TYPE(seq_range) uids;
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
133 const struct seq_range *uidvals;
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
134 unsigned int count;
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
135 string_t *str;
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
136
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
137 str = t_str_new(256);
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
138 str_append(str, "fl=uid&rows=1&sort=uid%20desc&q=");
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
139
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
140 mailbox_get_status(backend->box, STATUS_UIDVALIDITY, &status);
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
141 str_printfa(str, "uidv:%u%%20box:", status.uidvalidity);
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
142 solr_quote_str(str, backend->box->name);
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
143 str_append(str, "%20user:");
8109
e7929190cd32 fts-solr: Fixed compiling with recent struct mail_user changes.
Timo Sirainen <tss@iki.fi>
parents: 8006
diff changeset
144 solr_quote_str(str, backend->box->storage->ns->user->username);
7990
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
145
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
146 t_array_init(&uids, 1);
8484
db78eab2ba5d Initial support for getting full text search working nicely with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents: 8109
diff changeset
147 if (solr_connection_select(solr_conn, str_c(str),
db78eab2ba5d Initial support for getting full text search working nicely with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents: 8109
diff changeset
148 NULL, NULL, &uids, NULL) < 0)
7990
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
149 return -1;
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
150
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
151 uidvals = array_get(&uids, &count);
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
152 if (count == 0) {
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
153 /* nothing indexed yet for this mailbox */
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
154 *last_uid_r = 0;
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
155 } else if (count == 1 && uidvals[0].seq1 == uidvals[0].seq2) {
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
156 *last_uid_r = uidvals[0].seq1;
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
157 } else {
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
158 i_error("fts_solr: Last UID lookup returned multiple rows");
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
159 return -1;
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
160 }
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
161 return 0;
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
162 }
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
163
8492
ee1ecdda8a6b fts-solr: Keep last_uid field updated containing the mailbox's last added UID.
Timo Sirainen <tss@iki.fi>
parents: 8484
diff changeset
164 static int fts_backend_solr_get_last_uid(struct fts_backend *backend,
ee1ecdda8a6b fts-solr: Keep last_uid field updated containing the mailbox's last added UID.
Timo Sirainen <tss@iki.fi>
parents: 8484
diff changeset
165 uint32_t *last_uid_r)
ee1ecdda8a6b fts-solr: Keep last_uid field updated containing the mailbox's last added UID.
Timo Sirainen <tss@iki.fi>
parents: 8484
diff changeset
166 {
ee1ecdda8a6b fts-solr: Keep last_uid field updated containing the mailbox's last added UID.
Timo Sirainen <tss@iki.fi>
parents: 8484
diff changeset
167 struct mailbox_status status;
ee1ecdda8a6b fts-solr: Keep last_uid field updated containing the mailbox's last added UID.
Timo Sirainen <tss@iki.fi>
parents: 8484
diff changeset
168 ARRAY_TYPE(seq_range) uids;
ee1ecdda8a6b fts-solr: Keep last_uid field updated containing the mailbox's last added UID.
Timo Sirainen <tss@iki.fi>
parents: 8484
diff changeset
169 const struct seq_range *uidvals;
ee1ecdda8a6b fts-solr: Keep last_uid field updated containing the mailbox's last added UID.
Timo Sirainen <tss@iki.fi>
parents: 8484
diff changeset
170 unsigned int count;
ee1ecdda8a6b fts-solr: Keep last_uid field updated containing the mailbox's last added UID.
Timo Sirainen <tss@iki.fi>
parents: 8484
diff changeset
171 string_t *str;
ee1ecdda8a6b fts-solr: Keep last_uid field updated containing the mailbox's last added UID.
Timo Sirainen <tss@iki.fi>
parents: 8484
diff changeset
172
ee1ecdda8a6b fts-solr: Keep last_uid field updated containing the mailbox's last added UID.
Timo Sirainen <tss@iki.fi>
parents: 8484
diff changeset
173 str = t_str_new(256);
ee1ecdda8a6b fts-solr: Keep last_uid field updated containing the mailbox's last added UID.
Timo Sirainen <tss@iki.fi>
parents: 8484
diff changeset
174 str_append(str, "fl=uid&rows=1&q=last_uid:TRUE%20");
ee1ecdda8a6b fts-solr: Keep last_uid field updated containing the mailbox's last added UID.
Timo Sirainen <tss@iki.fi>
parents: 8484
diff changeset
175
ee1ecdda8a6b fts-solr: Keep last_uid field updated containing the mailbox's last added UID.
Timo Sirainen <tss@iki.fi>
parents: 8484
diff changeset
176 mailbox_get_status(backend->box, STATUS_UIDVALIDITY, &status);
ee1ecdda8a6b fts-solr: Keep last_uid field updated containing the mailbox's last added UID.
Timo Sirainen <tss@iki.fi>
parents: 8484
diff changeset
177 str_printfa(str, "uidv:%u%%20box:", status.uidvalidity);
ee1ecdda8a6b fts-solr: Keep last_uid field updated containing the mailbox's last added UID.
Timo Sirainen <tss@iki.fi>
parents: 8484
diff changeset
178 solr_quote_str(str, backend->box->name);
ee1ecdda8a6b fts-solr: Keep last_uid field updated containing the mailbox's last added UID.
Timo Sirainen <tss@iki.fi>
parents: 8484
diff changeset
179 str_append(str, "%20user:");
ee1ecdda8a6b fts-solr: Keep last_uid field updated containing the mailbox's last added UID.
Timo Sirainen <tss@iki.fi>
parents: 8484
diff changeset
180 solr_quote_str(str, backend->box->storage->ns->user->username);
ee1ecdda8a6b fts-solr: Keep last_uid field updated containing the mailbox's last added UID.
Timo Sirainen <tss@iki.fi>
parents: 8484
diff changeset
181
ee1ecdda8a6b fts-solr: Keep last_uid field updated containing the mailbox's last added UID.
Timo Sirainen <tss@iki.fi>
parents: 8484
diff changeset
182 t_array_init(&uids, 1);
ee1ecdda8a6b fts-solr: Keep last_uid field updated containing the mailbox's last added UID.
Timo Sirainen <tss@iki.fi>
parents: 8484
diff changeset
183 if (solr_connection_select(solr_conn, str_c(str),
ee1ecdda8a6b fts-solr: Keep last_uid field updated containing the mailbox's last added UID.
Timo Sirainen <tss@iki.fi>
parents: 8484
diff changeset
184 NULL, NULL, &uids, NULL) < 0)
ee1ecdda8a6b fts-solr: Keep last_uid field updated containing the mailbox's last added UID.
Timo Sirainen <tss@iki.fi>
parents: 8484
diff changeset
185 return -1;
ee1ecdda8a6b fts-solr: Keep last_uid field updated containing the mailbox's last added UID.
Timo Sirainen <tss@iki.fi>
parents: 8484
diff changeset
186
ee1ecdda8a6b fts-solr: Keep last_uid field updated containing the mailbox's last added UID.
Timo Sirainen <tss@iki.fi>
parents: 8484
diff changeset
187 uidvals = array_get(&uids, &count);
ee1ecdda8a6b fts-solr: Keep last_uid field updated containing the mailbox's last added UID.
Timo Sirainen <tss@iki.fi>
parents: 8484
diff changeset
188 if (count == 0) {
ee1ecdda8a6b fts-solr: Keep last_uid field updated containing the mailbox's last added UID.
Timo Sirainen <tss@iki.fi>
parents: 8484
diff changeset
189 /* either nothing is indexed or we're converting from an
ee1ecdda8a6b fts-solr: Keep last_uid field updated containing the mailbox's last added UID.
Timo Sirainen <tss@iki.fi>
parents: 8484
diff changeset
190 older database format without the last_uid fields */
ee1ecdda8a6b fts-solr: Keep last_uid field updated containing the mailbox's last added UID.
Timo Sirainen <tss@iki.fi>
parents: 8484
diff changeset
191 return fts_backend_solr_get_last_uid_fallback(backend,
ee1ecdda8a6b fts-solr: Keep last_uid field updated containing the mailbox's last added UID.
Timo Sirainen <tss@iki.fi>
parents: 8484
diff changeset
192 last_uid_r);
ee1ecdda8a6b fts-solr: Keep last_uid field updated containing the mailbox's last added UID.
Timo Sirainen <tss@iki.fi>
parents: 8484
diff changeset
193 } else if (count == 1 && uidvals[0].seq1 == uidvals[0].seq2) {
ee1ecdda8a6b fts-solr: Keep last_uid field updated containing the mailbox's last added UID.
Timo Sirainen <tss@iki.fi>
parents: 8484
diff changeset
194 *last_uid_r = uidvals[0].seq1;
ee1ecdda8a6b fts-solr: Keep last_uid field updated containing the mailbox's last added UID.
Timo Sirainen <tss@iki.fi>
parents: 8484
diff changeset
195 } else {
ee1ecdda8a6b fts-solr: Keep last_uid field updated containing the mailbox's last added UID.
Timo Sirainen <tss@iki.fi>
parents: 8484
diff changeset
196 i_error("fts_solr: Last UID lookup returned multiple rows");
ee1ecdda8a6b fts-solr: Keep last_uid field updated containing the mailbox's last added UID.
Timo Sirainen <tss@iki.fi>
parents: 8484
diff changeset
197 return -1;
ee1ecdda8a6b fts-solr: Keep last_uid field updated containing the mailbox's last added UID.
Timo Sirainen <tss@iki.fi>
parents: 8484
diff changeset
198 }
ee1ecdda8a6b fts-solr: Keep last_uid field updated containing the mailbox's last added UID.
Timo Sirainen <tss@iki.fi>
parents: 8484
diff changeset
199 return 0;
ee1ecdda8a6b fts-solr: Keep last_uid field updated containing the mailbox's last added UID.
Timo Sirainen <tss@iki.fi>
parents: 8484
diff changeset
200 }
ee1ecdda8a6b fts-solr: Keep last_uid field updated containing the mailbox's last added UID.
Timo Sirainen <tss@iki.fi>
parents: 8484
diff changeset
201
7990
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
202 static int
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
203 fts_backend_solr_build_init(struct fts_backend *backend, uint32_t *last_uid_r,
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
204 struct fts_backend_build_context **ctx_r)
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
205 {
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
206 struct solr_fts_backend_build_context *ctx;
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
207 struct mailbox_status status;
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
208
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
209 *last_uid_r = (uint32_t)-1;
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
210
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
211 ctx = i_new(struct solr_fts_backend_build_context, 1);
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
212 ctx->ctx.backend = backend;
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
213 ctx->post = solr_connection_post_begin(solr_conn);
8001
3fbfdda3e5d3 fts-solr: Send data to Solr in larger packets. Commit after expunges.
Timo Sirainen <tss@iki.fi>
parents: 8000
diff changeset
214 ctx->cmd = str_new(default_pool, SOLR_CMDBUF_SIZE);
7990
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
215
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
216 mailbox_get_status(backend->box, STATUS_UIDVALIDITY, &status);
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
217 ctx->uid_validity = status.uidvalidity;
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
218
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
219 *ctx_r = &ctx->ctx;
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
220 return 0;
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
221 }
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
222
8492
ee1ecdda8a6b fts-solr: Keep last_uid field updated containing the mailbox's last added UID.
Timo Sirainen <tss@iki.fi>
parents: 8484
diff changeset
223 static void
ee1ecdda8a6b fts-solr: Keep last_uid field updated containing the mailbox's last added UID.
Timo Sirainen <tss@iki.fi>
parents: 8484
diff changeset
224 fts_backend_solr_add_doc_prefix(struct solr_fts_backend_build_context *ctx,
ee1ecdda8a6b fts-solr: Keep last_uid field updated containing the mailbox's last added UID.
Timo Sirainen <tss@iki.fi>
parents: 8484
diff changeset
225 uint32_t uid)
ee1ecdda8a6b fts-solr: Keep last_uid field updated containing the mailbox's last added UID.
Timo Sirainen <tss@iki.fi>
parents: 8484
diff changeset
226 {
ee1ecdda8a6b fts-solr: Keep last_uid field updated containing the mailbox's last added UID.
Timo Sirainen <tss@iki.fi>
parents: 8484
diff changeset
227 struct mailbox *box = ctx->ctx.backend->box;
ee1ecdda8a6b fts-solr: Keep last_uid field updated containing the mailbox's last added UID.
Timo Sirainen <tss@iki.fi>
parents: 8484
diff changeset
228
ee1ecdda8a6b fts-solr: Keep last_uid field updated containing the mailbox's last added UID.
Timo Sirainen <tss@iki.fi>
parents: 8484
diff changeset
229 str_printfa(ctx->cmd, "<doc>"
ee1ecdda8a6b fts-solr: Keep last_uid field updated containing the mailbox's last added UID.
Timo Sirainen <tss@iki.fi>
parents: 8484
diff changeset
230 "<field name=\"uid\">%u</field>"
ee1ecdda8a6b fts-solr: Keep last_uid field updated containing the mailbox's last added UID.
Timo Sirainen <tss@iki.fi>
parents: 8484
diff changeset
231 "<field name=\"uidv\">%u</field>",
ee1ecdda8a6b fts-solr: Keep last_uid field updated containing the mailbox's last added UID.
Timo Sirainen <tss@iki.fi>
parents: 8484
diff changeset
232 uid, ctx->uid_validity);
ee1ecdda8a6b fts-solr: Keep last_uid field updated containing the mailbox's last added UID.
Timo Sirainen <tss@iki.fi>
parents: 8484
diff changeset
233
ee1ecdda8a6b fts-solr: Keep last_uid field updated containing the mailbox's last added UID.
Timo Sirainen <tss@iki.fi>
parents: 8484
diff changeset
234 str_append(ctx->cmd, "<field name=\"box\">");
ee1ecdda8a6b fts-solr: Keep last_uid field updated containing the mailbox's last added UID.
Timo Sirainen <tss@iki.fi>
parents: 8484
diff changeset
235 xml_encode(ctx->cmd, box->name);
ee1ecdda8a6b fts-solr: Keep last_uid field updated containing the mailbox's last added UID.
Timo Sirainen <tss@iki.fi>
parents: 8484
diff changeset
236 str_append(ctx->cmd, "</field><field name=\"user\">");
ee1ecdda8a6b fts-solr: Keep last_uid field updated containing the mailbox's last added UID.
Timo Sirainen <tss@iki.fi>
parents: 8484
diff changeset
237 xml_encode(ctx->cmd, box->storage->ns->user->username);
ee1ecdda8a6b fts-solr: Keep last_uid field updated containing the mailbox's last added UID.
Timo Sirainen <tss@iki.fi>
parents: 8484
diff changeset
238 str_append(ctx->cmd, "</field>");
ee1ecdda8a6b fts-solr: Keep last_uid field updated containing the mailbox's last added UID.
Timo Sirainen <tss@iki.fi>
parents: 8484
diff changeset
239 }
ee1ecdda8a6b fts-solr: Keep last_uid field updated containing the mailbox's last added UID.
Timo Sirainen <tss@iki.fi>
parents: 8484
diff changeset
240
7990
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
241 static int
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
242 fts_backend_solr_build_more(struct fts_backend_build_context *_ctx,
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
243 uint32_t uid, const unsigned char *data,
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
244 size_t size, bool headers)
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
245 {
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
246 struct solr_fts_backend_build_context *ctx =
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
247 (struct solr_fts_backend_build_context *)_ctx;
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
248 struct mailbox *box = _ctx->backend->box;
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
249 string_t *cmd = ctx->cmd;
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
250
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
251 /* body comes first, then headers */
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
252 if (ctx->prev_uid != uid) {
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
253 /* uid changed */
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
254 if (ctx->prev_uid == 0)
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
255 str_append(cmd, "<add>");
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
256 else
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
257 str_append(cmd, "</field></doc>");
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
258 ctx->prev_uid = uid;
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
259
8492
ee1ecdda8a6b fts-solr: Keep last_uid field updated containing the mailbox's last added UID.
Timo Sirainen <tss@iki.fi>
parents: 8484
diff changeset
260 fts_backend_solr_add_doc_prefix(ctx, uid);
ee1ecdda8a6b fts-solr: Keep last_uid field updated containing the mailbox's last added UID.
Timo Sirainen <tss@iki.fi>
parents: 8484
diff changeset
261 str_printfa(cmd, "<field name=\"id\">%u/%u/",
7990
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
262 uid, ctx->uid_validity);
8493
149c6ccc3df4 fts-solr: Use '!' as escape character in usernames. Escape '/' and '!'.
Timo Sirainen <tss@iki.fi>
parents: 8492
diff changeset
263 xml_encode(cmd, fts_backend_solr_username(ctx->ctx.backend));
7990
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
264 str_append_c(cmd, '/');
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
265 xml_encode(cmd, box->name);
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
266 str_append(cmd, "</field>");
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
267
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
268 ctx->headers = headers;
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
269 if (headers) {
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
270 str_append(cmd, "<field name=\"hdr\">");
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
271 } else {
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
272 str_append(cmd, "<field name=\"body\">");
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
273 }
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
274 } else if (headers && !ctx->headers) {
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
275 str_append(cmd, "</field><field name=\"hdr\">");
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
276 } else {
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
277 i_assert(!(!headers && ctx->headers));
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
278 }
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
279
8000
61be6a4722c5 fts-solr: Encode entities when indexing data.
Timo Sirainen <tss@iki.fi>
parents: 7999
diff changeset
280 xml_encode_data(cmd, data, size);
8001
3fbfdda3e5d3 fts-solr: Send data to Solr in larger packets. Commit after expunges.
Timo Sirainen <tss@iki.fi>
parents: 8000
diff changeset
281 if (str_len(cmd) > SOLR_CMDBUF_SIZE-128) {
3fbfdda3e5d3 fts-solr: Send data to Solr in larger packets. Commit after expunges.
Timo Sirainen <tss@iki.fi>
parents: 8000
diff changeset
282 solr_connection_post_more(ctx->post, str_data(cmd),
3fbfdda3e5d3 fts-solr: Send data to Solr in larger packets. Commit after expunges.
Timo Sirainen <tss@iki.fi>
parents: 8000
diff changeset
283 str_len(cmd));
3fbfdda3e5d3 fts-solr: Send data to Solr in larger packets. Commit after expunges.
Timo Sirainen <tss@iki.fi>
parents: 8000
diff changeset
284 str_truncate(cmd, 0);
3fbfdda3e5d3 fts-solr: Send data to Solr in larger packets. Commit after expunges.
Timo Sirainen <tss@iki.fi>
parents: 8000
diff changeset
285 }
7990
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
286 return 0;
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
287 }
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
288
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
289 static int
8492
ee1ecdda8a6b fts-solr: Keep last_uid field updated containing the mailbox's last added UID.
Timo Sirainen <tss@iki.fi>
parents: 8484
diff changeset
290 fts_backed_solr_build_commit(struct solr_fts_backend_build_context *ctx)
ee1ecdda8a6b fts-solr: Keep last_uid field updated containing the mailbox's last added UID.
Timo Sirainen <tss@iki.fi>
parents: 8484
diff changeset
291 {
ee1ecdda8a6b fts-solr: Keep last_uid field updated containing the mailbox's last added UID.
Timo Sirainen <tss@iki.fi>
parents: 8484
diff changeset
292 struct mailbox *box = ctx->ctx.backend->box;
ee1ecdda8a6b fts-solr: Keep last_uid field updated containing the mailbox's last added UID.
Timo Sirainen <tss@iki.fi>
parents: 8484
diff changeset
293 int ret;
ee1ecdda8a6b fts-solr: Keep last_uid field updated containing the mailbox's last added UID.
Timo Sirainen <tss@iki.fi>
parents: 8484
diff changeset
294
ee1ecdda8a6b fts-solr: Keep last_uid field updated containing the mailbox's last added UID.
Timo Sirainen <tss@iki.fi>
parents: 8484
diff changeset
295 if (ctx->prev_uid == 0)
ee1ecdda8a6b fts-solr: Keep last_uid field updated containing the mailbox's last added UID.
Timo Sirainen <tss@iki.fi>
parents: 8484
diff changeset
296 return 0;
ee1ecdda8a6b fts-solr: Keep last_uid field updated containing the mailbox's last added UID.
Timo Sirainen <tss@iki.fi>
parents: 8484
diff changeset
297
ee1ecdda8a6b fts-solr: Keep last_uid field updated containing the mailbox's last added UID.
Timo Sirainen <tss@iki.fi>
parents: 8484
diff changeset
298 str_append(ctx->cmd, "</field></doc>");
ee1ecdda8a6b fts-solr: Keep last_uid field updated containing the mailbox's last added UID.
Timo Sirainen <tss@iki.fi>
parents: 8484
diff changeset
299
ee1ecdda8a6b fts-solr: Keep last_uid field updated containing the mailbox's last added UID.
Timo Sirainen <tss@iki.fi>
parents: 8484
diff changeset
300 /* Update the mailbox's last_uid field, replacing the existing
ee1ecdda8a6b fts-solr: Keep last_uid field updated containing the mailbox's last added UID.
Timo Sirainen <tss@iki.fi>
parents: 8484
diff changeset
301 document. Note that since there is no locking, it's possible that
ee1ecdda8a6b fts-solr: Keep last_uid field updated containing the mailbox's last added UID.
Timo Sirainen <tss@iki.fi>
parents: 8484
diff changeset
302 if another session is indexing at the same time, the last_uid value
ee1ecdda8a6b fts-solr: Keep last_uid field updated containing the mailbox's last added UID.
Timo Sirainen <tss@iki.fi>
parents: 8484
diff changeset
303 may shrink. This doesn't really matter, we'll simply do more work
ee1ecdda8a6b fts-solr: Keep last_uid field updated containing the mailbox's last added UID.
Timo Sirainen <tss@iki.fi>
parents: 8484
diff changeset
304 in future by reindexing some messages. */
ee1ecdda8a6b fts-solr: Keep last_uid field updated containing the mailbox's last added UID.
Timo Sirainen <tss@iki.fi>
parents: 8484
diff changeset
305 fts_backend_solr_add_doc_prefix(ctx, ctx->prev_uid);
ee1ecdda8a6b fts-solr: Keep last_uid field updated containing the mailbox's last added UID.
Timo Sirainen <tss@iki.fi>
parents: 8484
diff changeset
306 str_printfa(ctx->cmd, "<field name=\"last_uid\">TRUE</field>"
ee1ecdda8a6b fts-solr: Keep last_uid field updated containing the mailbox's last added UID.
Timo Sirainen <tss@iki.fi>
parents: 8484
diff changeset
307 "<field name=\"id\">L/%u/", ctx->uid_validity);
8493
149c6ccc3df4 fts-solr: Use '!' as escape character in usernames. Escape '/' and '!'.
Timo Sirainen <tss@iki.fi>
parents: 8492
diff changeset
308 xml_encode(ctx->cmd, fts_backend_solr_username(ctx->ctx.backend));
8492
ee1ecdda8a6b fts-solr: Keep last_uid field updated containing the mailbox's last added UID.
Timo Sirainen <tss@iki.fi>
parents: 8484
diff changeset
309 str_append_c(ctx->cmd, '/');
ee1ecdda8a6b fts-solr: Keep last_uid field updated containing the mailbox's last added UID.
Timo Sirainen <tss@iki.fi>
parents: 8484
diff changeset
310 xml_encode(ctx->cmd, box->name);
ee1ecdda8a6b fts-solr: Keep last_uid field updated containing the mailbox's last added UID.
Timo Sirainen <tss@iki.fi>
parents: 8484
diff changeset
311 str_append(ctx->cmd, "</field></doc></add>");
ee1ecdda8a6b fts-solr: Keep last_uid field updated containing the mailbox's last added UID.
Timo Sirainen <tss@iki.fi>
parents: 8484
diff changeset
312
ee1ecdda8a6b fts-solr: Keep last_uid field updated containing the mailbox's last added UID.
Timo Sirainen <tss@iki.fi>
parents: 8484
diff changeset
313 solr_connection_post_more(ctx->post, str_data(ctx->cmd),
ee1ecdda8a6b fts-solr: Keep last_uid field updated containing the mailbox's last added UID.
Timo Sirainen <tss@iki.fi>
parents: 8484
diff changeset
314 str_len(ctx->cmd));
ee1ecdda8a6b fts-solr: Keep last_uid field updated containing the mailbox's last added UID.
Timo Sirainen <tss@iki.fi>
parents: 8484
diff changeset
315 ret = solr_connection_post_end(ctx->post);
ee1ecdda8a6b fts-solr: Keep last_uid field updated containing the mailbox's last added UID.
Timo Sirainen <tss@iki.fi>
parents: 8484
diff changeset
316 /* commit and wait until the documents we just indexed are
ee1ecdda8a6b fts-solr: Keep last_uid field updated containing the mailbox's last added UID.
Timo Sirainen <tss@iki.fi>
parents: 8484
diff changeset
317 visible to the following search */
ee1ecdda8a6b fts-solr: Keep last_uid field updated containing the mailbox's last added UID.
Timo Sirainen <tss@iki.fi>
parents: 8484
diff changeset
318 if (solr_connection_post(solr_conn, "<commit waitFlush=\"false\" "
ee1ecdda8a6b fts-solr: Keep last_uid field updated containing the mailbox's last added UID.
Timo Sirainen <tss@iki.fi>
parents: 8484
diff changeset
319 "waitSearcher=\"true\"/>") < 0)
ee1ecdda8a6b fts-solr: Keep last_uid field updated containing the mailbox's last added UID.
Timo Sirainen <tss@iki.fi>
parents: 8484
diff changeset
320 ret = -1;
ee1ecdda8a6b fts-solr: Keep last_uid field updated containing the mailbox's last added UID.
Timo Sirainen <tss@iki.fi>
parents: 8484
diff changeset
321 return ret;
ee1ecdda8a6b fts-solr: Keep last_uid field updated containing the mailbox's last added UID.
Timo Sirainen <tss@iki.fi>
parents: 8484
diff changeset
322 }
ee1ecdda8a6b fts-solr: Keep last_uid field updated containing the mailbox's last added UID.
Timo Sirainen <tss@iki.fi>
parents: 8484
diff changeset
323
ee1ecdda8a6b fts-solr: Keep last_uid field updated containing the mailbox's last added UID.
Timo Sirainen <tss@iki.fi>
parents: 8484
diff changeset
324 static int
7990
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
325 fts_backend_solr_build_deinit(struct fts_backend_build_context *_ctx)
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
326 {
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
327 struct solr_fts_backend_build_context *ctx =
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
328 (struct solr_fts_backend_build_context *)_ctx;
8492
ee1ecdda8a6b fts-solr: Keep last_uid field updated containing the mailbox's last added UID.
Timo Sirainen <tss@iki.fi>
parents: 8484
diff changeset
329 int ret;
7990
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
330
8492
ee1ecdda8a6b fts-solr: Keep last_uid field updated containing the mailbox's last added UID.
Timo Sirainen <tss@iki.fi>
parents: 8484
diff changeset
331 ret = fts_backed_solr_build_commit(ctx);
7990
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
332 str_free(&ctx->cmd);
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
333 i_free(ctx);
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
334 return ret;
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
335 }
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
336
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
337 static void
8493
149c6ccc3df4 fts-solr: Use '!' as escape character in usernames. Escape '/' and '!'.
Timo Sirainen <tss@iki.fi>
parents: 8492
diff changeset
338 fts_backend_solr_expunge(struct fts_backend *backend, struct mail *mail)
7990
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
339 {
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
340 struct mailbox_status status;
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
341
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
342 mailbox_get_status(mail->box, STATUS_UIDVALIDITY, &status);
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
343
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
344 T_BEGIN {
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
345 string_t *cmd;
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
346
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
347 cmd = t_str_new(256);
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
348 str_printfa(cmd, "<delete><id>%u/%u/",
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
349 mail->uid, status.uidvalidity);
8493
149c6ccc3df4 fts-solr: Use '!' as escape character in usernames. Escape '/' and '!'.
Timo Sirainen <tss@iki.fi>
parents: 8492
diff changeset
350 xml_encode(cmd, fts_backend_solr_username(backend));
7990
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
351 str_append_c(cmd, '/');
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
352 xml_encode(cmd, mail->box->name);
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
353 str_append(cmd, "</id></delete>");
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
354
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
355 (void)solr_connection_post(solr_conn, str_c(cmd));
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
356 } T_END;
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
357 }
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
358
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
359 static void
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
360 fts_backend_solr_expunge_finish(struct fts_backend *backend ATTR_UNUSED,
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
361 struct mailbox *box ATTR_UNUSED,
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
362 bool committed ATTR_UNUSED)
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
363 {
8001
3fbfdda3e5d3 fts-solr: Send data to Solr in larger packets. Commit after expunges.
Timo Sirainen <tss@iki.fi>
parents: 8000
diff changeset
364 solr_connection_post(solr_conn,
3fbfdda3e5d3 fts-solr: Send data to Solr in larger packets. Commit after expunges.
Timo Sirainen <tss@iki.fi>
parents: 8000
diff changeset
365 "<commit waitFlush=\"false\" waitSearcher=\"false\"/>");
7990
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
366 }
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
367
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
368 static int fts_backend_solr_lock(struct fts_backend *backend ATTR_UNUSED)
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
369 {
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
370 return 1;
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
371 }
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
372
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
373 static void fts_backend_solr_unlock(struct fts_backend *backend ATTR_UNUSED)
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
374 {
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
375 }
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
376
8484
db78eab2ba5d Initial support for getting full text search working nicely with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents: 8109
diff changeset
377 static bool solr_virtual_uid_map(const char *mailbox, uint32_t uidvalidity,
db78eab2ba5d Initial support for getting full text search working nicely with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents: 8109
diff changeset
378 uint32_t *uid, void *context)
db78eab2ba5d Initial support for getting full text search working nicely with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents: 8109
diff changeset
379 {
db78eab2ba5d Initial support for getting full text search working nicely with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents: 8109
diff changeset
380 struct mailbox *box = context;
db78eab2ba5d Initial support for getting full text search working nicely with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents: 8109
diff changeset
381
db78eab2ba5d Initial support for getting full text search working nicely with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents: 8109
diff changeset
382 return mailbox_get_virtual_uid(box, mailbox, uidvalidity,
db78eab2ba5d Initial support for getting full text search working nicely with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents: 8109
diff changeset
383 *uid, uid);
db78eab2ba5d Initial support for getting full text search working nicely with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents: 8109
diff changeset
384 }
db78eab2ba5d Initial support for getting full text search working nicely with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents: 8109
diff changeset
385
8004
6418d4c0ff16 fts: Added a new lookup API where the backend can look up all the fields
Timo Sirainen <tss@iki.fi>
parents: 8002
diff changeset
386 static int fts_backend_solr_lookup(struct fts_backend_lookup_context *ctx,
6418d4c0ff16 fts: Added a new lookup API where the backend can look up all the fields
Timo Sirainen <tss@iki.fi>
parents: 8002
diff changeset
387 ARRAY_TYPE(seq_range) *definite_uids,
8006
0a3b5ddb0869 Added support for sorting by X-SCORE. Currently it's only set by fts-solr.
Timo Sirainen <tss@iki.fi>
parents: 8004
diff changeset
388 ARRAY_TYPE(seq_range) *maybe_uids,
0a3b5ddb0869 Added support for sorting by X-SCORE. Currently it's only set by fts-solr.
Timo Sirainen <tss@iki.fi>
parents: 8004
diff changeset
389 ARRAY_TYPE(fts_score_map) *scores)
7990
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
390 {
8004
6418d4c0ff16 fts: Added a new lookup API where the backend can look up all the fields
Timo Sirainen <tss@iki.fi>
parents: 8002
diff changeset
391 struct mailbox *box = ctx->backend->box;
6418d4c0ff16 fts: Added a new lookup API where the backend can look up all the fields
Timo Sirainen <tss@iki.fi>
parents: 8002
diff changeset
392 const struct fts_backend_lookup_field *fields;
6418d4c0ff16 fts: Added a new lookup API where the backend can look up all the fields
Timo Sirainen <tss@iki.fi>
parents: 8002
diff changeset
393 unsigned int i, count;
7990
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
394 struct mailbox_status status;
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
395 string_t *str;
8484
db78eab2ba5d Initial support for getting full text search working nicely with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents: 8109
diff changeset
396 bool virtual;
7990
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
397
8484
db78eab2ba5d Initial support for getting full text search working nicely with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents: 8109
diff changeset
398 virtual = strcmp(box->storage->name, "virtual") == 0;
8004
6418d4c0ff16 fts: Added a new lookup API where the backend can look up all the fields
Timo Sirainen <tss@iki.fi>
parents: 8002
diff changeset
399 mailbox_get_status(box, STATUS_UIDVALIDITY, &status);
7990
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
400
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
401 str = t_str_new(256);
8484
db78eab2ba5d Initial support for getting full text search working nicely with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents: 8109
diff changeset
402 if (!virtual) {
db78eab2ba5d Initial support for getting full text search working nicely with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents: 8109
diff changeset
403 str_printfa(str, "fl=uid,score&rows=%u&sort=uid%%20asc&q=",
db78eab2ba5d Initial support for getting full text search working nicely with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents: 8109
diff changeset
404 status.uidnext);
db78eab2ba5d Initial support for getting full text search working nicely with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents: 8109
diff changeset
405 } else {
db78eab2ba5d Initial support for getting full text search working nicely with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents: 8109
diff changeset
406 str_printfa(str, "fl=uid,score,box,uidv&rows=%u"
db78eab2ba5d Initial support for getting full text search working nicely with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents: 8109
diff changeset
407 "&sort=box%%20asc,uid%%20asc&q=",
db78eab2ba5d Initial support for getting full text search working nicely with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents: 8109
diff changeset
408 SOLR_MAX_ROWS);
db78eab2ba5d Initial support for getting full text search working nicely with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents: 8109
diff changeset
409 }
8004
6418d4c0ff16 fts: Added a new lookup API where the backend can look up all the fields
Timo Sirainen <tss@iki.fi>
parents: 8002
diff changeset
410
6418d4c0ff16 fts: Added a new lookup API where the backend can look up all the fields
Timo Sirainen <tss@iki.fi>
parents: 8002
diff changeset
411 /* build a lucene search query from the fields */
6418d4c0ff16 fts: Added a new lookup API where the backend can look up all the fields
Timo Sirainen <tss@iki.fi>
parents: 8002
diff changeset
412 fields = array_get(&ctx->fields, &count);
6418d4c0ff16 fts: Added a new lookup API where the backend can look up all the fields
Timo Sirainen <tss@iki.fi>
parents: 8002
diff changeset
413 for (i = 0; i < count; i++) {
6418d4c0ff16 fts: Added a new lookup API where the backend can look up all the fields
Timo Sirainen <tss@iki.fi>
parents: 8002
diff changeset
414 if (i > 0)
6418d4c0ff16 fts: Added a new lookup API where the backend can look up all the fields
Timo Sirainen <tss@iki.fi>
parents: 8002
diff changeset
415 str_append(str, "%20");
6418d4c0ff16 fts: Added a new lookup API where the backend can look up all the fields
Timo Sirainen <tss@iki.fi>
parents: 8002
diff changeset
416
6418d4c0ff16 fts: Added a new lookup API where the backend can look up all the fields
Timo Sirainen <tss@iki.fi>
parents: 8002
diff changeset
417 if ((fields[i].flags & FTS_LOOKUP_FLAG_INVERT) != 0)
6418d4c0ff16 fts: Added a new lookup API where the backend can look up all the fields
Timo Sirainen <tss@iki.fi>
parents: 8002
diff changeset
418 str_append_c(str, '-');
6418d4c0ff16 fts: Added a new lookup API where the backend can look up all the fields
Timo Sirainen <tss@iki.fi>
parents: 8002
diff changeset
419
6418d4c0ff16 fts: Added a new lookup API where the backend can look up all the fields
Timo Sirainen <tss@iki.fi>
parents: 8002
diff changeset
420 if ((fields[i].flags & FTS_LOOKUP_FLAG_HEADER) == 0) {
6418d4c0ff16 fts: Added a new lookup API where the backend can look up all the fields
Timo Sirainen <tss@iki.fi>
parents: 8002
diff changeset
421 /* body only */
6418d4c0ff16 fts: Added a new lookup API where the backend can look up all the fields
Timo Sirainen <tss@iki.fi>
parents: 8002
diff changeset
422 i_assert((fields[i].flags & FTS_LOOKUP_FLAG_BODY) != 0);
6418d4c0ff16 fts: Added a new lookup API where the backend can look up all the fields
Timo Sirainen <tss@iki.fi>
parents: 8002
diff changeset
423 str_append(str, "body:");
6418d4c0ff16 fts: Added a new lookup API where the backend can look up all the fields
Timo Sirainen <tss@iki.fi>
parents: 8002
diff changeset
424 } else if ((fields[i].flags & FTS_LOOKUP_FLAG_BODY) == 0) {
6418d4c0ff16 fts: Added a new lookup API where the backend can look up all the fields
Timo Sirainen <tss@iki.fi>
parents: 8002
diff changeset
425 /* header only */
6418d4c0ff16 fts: Added a new lookup API where the backend can look up all the fields
Timo Sirainen <tss@iki.fi>
parents: 8002
diff changeset
426 str_append(str, "hdr:");
6418d4c0ff16 fts: Added a new lookup API where the backend can look up all the fields
Timo Sirainen <tss@iki.fi>
parents: 8002
diff changeset
427 } else {
6418d4c0ff16 fts: Added a new lookup API where the backend can look up all the fields
Timo Sirainen <tss@iki.fi>
parents: 8002
diff changeset
428 /* both */
6418d4c0ff16 fts: Added a new lookup API where the backend can look up all the fields
Timo Sirainen <tss@iki.fi>
parents: 8002
diff changeset
429 str_append(str, "any:");
6418d4c0ff16 fts: Added a new lookup API where the backend can look up all the fields
Timo Sirainen <tss@iki.fi>
parents: 8002
diff changeset
430 }
6418d4c0ff16 fts: Added a new lookup API where the backend can look up all the fields
Timo Sirainen <tss@iki.fi>
parents: 8002
diff changeset
431 solr_quote_str(str, fields[i].key);
7990
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
432 }
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
433
8004
6418d4c0ff16 fts: Added a new lookup API where the backend can look up all the fields
Timo Sirainen <tss@iki.fi>
parents: 8002
diff changeset
434 /* use a separate filter query for selecting the mailbox. it shouldn't
6418d4c0ff16 fts: Added a new lookup API where the backend can look up all the fields
Timo Sirainen <tss@iki.fi>
parents: 8002
diff changeset
435 affect the score and there could be some caching benefits too. */
8484
db78eab2ba5d Initial support for getting full text search working nicely with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents: 8109
diff changeset
436 str_append(str, "&fq=user:");
8109
e7929190cd32 fts-solr: Fixed compiling with recent struct mail_user changes.
Timo Sirainen <tss@iki.fi>
parents: 8006
diff changeset
437 solr_quote_str(str, box->storage->ns->user->username);
7990
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
438
8484
db78eab2ba5d Initial support for getting full text search working nicely with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents: 8109
diff changeset
439 /* FIXME: limit what mailboxes to search with virtual storage */
db78eab2ba5d Initial support for getting full text search working nicely with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents: 8109
diff changeset
440 if (!virtual) {
db78eab2ba5d Initial support for getting full text search working nicely with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents: 8109
diff changeset
441 str_printfa(str, "%%20uidv:%u%%20box:", status.uidvalidity);
db78eab2ba5d Initial support for getting full text search working nicely with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents: 8109
diff changeset
442 solr_quote_str(str, box->name);
db78eab2ba5d Initial support for getting full text search working nicely with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents: 8109
diff changeset
443 }
db78eab2ba5d Initial support for getting full text search working nicely with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents: 8109
diff changeset
444
7990
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
445 array_clear(maybe_uids);
8484
db78eab2ba5d Initial support for getting full text search working nicely with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents: 8109
diff changeset
446 if (!virtual) {
db78eab2ba5d Initial support for getting full text search working nicely with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents: 8109
diff changeset
447 return solr_connection_select(solr_conn, str_c(str), NULL, NULL,
db78eab2ba5d Initial support for getting full text search working nicely with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents: 8109
diff changeset
448 definite_uids, scores);
db78eab2ba5d Initial support for getting full text search working nicely with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents: 8109
diff changeset
449 } else {
db78eab2ba5d Initial support for getting full text search working nicely with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents: 8109
diff changeset
450 return solr_connection_select(solr_conn, str_c(str),
db78eab2ba5d Initial support for getting full text search working nicely with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents: 8109
diff changeset
451 solr_virtual_uid_map, box,
db78eab2ba5d Initial support for getting full text search working nicely with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents: 8109
diff changeset
452 definite_uids, scores);
db78eab2ba5d Initial support for getting full text search working nicely with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents: 8109
diff changeset
453 }
7990
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
454 }
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
455
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
456 struct fts_backend fts_backend_solr = {
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
457 MEMBER(name) "solr",
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
458 MEMBER(flags) 0,
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
459
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
460 {
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
461 fts_backend_solr_init,
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
462 fts_backend_solr_deinit,
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
463 fts_backend_solr_get_last_uid,
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
464 fts_backend_solr_build_init,
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
465 fts_backend_solr_build_more,
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
466 fts_backend_solr_build_deinit,
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
467 fts_backend_solr_expunge,
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
468 fts_backend_solr_expunge_finish,
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
469 fts_backend_solr_lock,
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
470 fts_backend_solr_unlock,
8004
6418d4c0ff16 fts: Added a new lookup API where the backend can look up all the fields
Timo Sirainen <tss@iki.fi>
parents: 8002
diff changeset
471 NULL,
6418d4c0ff16 fts: Added a new lookup API where the backend can look up all the fields
Timo Sirainen <tss@iki.fi>
parents: 8002
diff changeset
472 NULL,
6418d4c0ff16 fts: Added a new lookup API where the backend can look up all the fields
Timo Sirainen <tss@iki.fi>
parents: 8002
diff changeset
473 fts_backend_solr_lookup
7990
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
474 }
662172573fe1 Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
475 };