Mercurial > dovecot > original-hg > dovecot-1.2

comparison src/plugins/fts-solr/fts-backend-solr.c @ 8499:252b29ac5f43 HEAD

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
fts: Added support for handling multiple namespaces.
author Timo Sirainen <tss@iki.fi>
date Sun, 30 Nov 2008 01:27:19 +0200
parents ffb37c392166
children 3efcdc45d111
comparison
equal deleted inserted replaced
8498:fb5fedcf4deb 8499:252b29ac5f43
11 11
12 #include <ctype.h> 12 #include <ctype.h>
13 13
14 #define SOLR_CMDBUF_SIZE (1024*64) 14 #define SOLR_CMDBUF_SIZE (1024*64)
15 #define SOLR_MAX_ROWS 100000 15 #define SOLR_MAX_ROWS 100000
16 #define FTS_SOLR_MAX_BOX_INC_PATTERNS 5
17 #define FTS_SOLR_MAX_BOX_EXC_PATTERNS 5
16 18
17 struct solr_fts_backend { 19 struct solr_fts_backend {
18 struct fts_backend backend; 20 struct fts_backend backend;
19 char *id_username; 21 char *id_username, *id_namespace;
22 struct mail_namespace *default_ns;
20 }; 23 };
21 24
22 struct solr_fts_backend_build_context { 25 struct solr_fts_backend_build_context {
23 struct fts_backend_build_context ctx; 26 struct fts_backend_build_context ctx;
24 27
26 uint32_t prev_uid, uid_validity; 29 uint32_t prev_uid, uid_validity;
27 string_t *cmd; 30 string_t *cmd;
28 bool headers; 31 bool headers;
29 }; 32 };
30 33
34 struct solr_virtual_uid_map_context {
35 struct fts_backend *backend;
36 struct mailbox *box;
37 string_t *vname;
38 };
39
31 struct fts_backend_solr_get_last_uids_context { 40 struct fts_backend_solr_get_last_uids_context {
41 struct fts_backend *backend;
32 pool_t pool; 42 pool_t pool;
33 ARRAY_TYPE(fts_backend_uid_map) *last_uids; 43 ARRAY_TYPE(fts_backend_uid_map) *last_uids;
44
45 struct mailbox *box;
46 string_t *vname;
34 }; 47 };
35 48
36 static struct solr_connection *solr_conn = NULL; 49 static struct solr_connection *solr_conn = NULL;
37
38 static void solr_quote_str(string_t *dest, const char *str)
39 {
40 solr_connection_quote_str(solr_conn, dest, str);
41 }
42 50
43 static void 51 static void
44 xml_encode_data(string_t *dest, const unsigned char *data, unsigned int len) 52 xml_encode_data(string_t *dest, const unsigned char *data, unsigned int len)
45 { 53 {
46 unsigned int i; 54 unsigned int i;
95 } 103 }
96 } 104 }
97 return str_c(tmp); 105 return str_c(tmp);
98 } 106 }
99 107
108 static void solr_quote(string_t *dest, const char *str)
109 {
110 str_append_c(dest, '"');
111 str_append(dest, str_escape(str));
112 str_append_c(dest, '"');
113 }
114
115 static void solr_quote_http(string_t *dest, const char *str)
116 {
117 str_append(dest, "%22");
118 solr_connection_http_escape(solr_conn, dest, str);
119 str_append(dest, "%22");
120 }
121
100 static struct fts_backend * 122 static struct fts_backend *
101 fts_backend_solr_init(struct mailbox *box) 123 fts_backend_solr_init(struct mailbox *box)
102 { 124 {
103 const struct fts_solr_settings *set = &fts_solr_settings; 125 const struct fts_solr_settings *set = &fts_solr_settings;
104 struct solr_fts_backend *backend; 126 struct solr_fts_backend *backend;
105 const char *username = box->storage->ns->user->username; 127 struct mail_namespace *ns = box->storage->ns;
128 const char *str;
106 129
107 if (solr_conn == NULL) 130 if (solr_conn == NULL)
108 solr_conn = solr_connection_init(set->url, set->debug); 131 solr_conn = solr_connection_init(set->url, set->debug);
109 132
110 backend = i_new(struct solr_fts_backend, 1); 133 backend = i_new(struct solr_fts_backend, 1);
111 backend->id_username = i_strdup(solr_escape_id_str(username)); 134 str = fts_solr_settings.default_ns_prefix;
135 if (str != NULL) {
136 backend->default_ns =
137 mail_namespace_find_prefix(ns->user->namespaces, str);
138 if (backend->default_ns == NULL) {
139 i_fatal("fts_solr: default_ns setting points to "
140 "nonexisting namespace");
141 }
142 } else {
143 backend->default_ns =
144 mail_namespace_find_inbox(ns->user->namespaces);
145 }
146 str = solr_escape_id_str(ns->user->username);
147 backend->id_username = i_strdup(str);
148 if (box->storage->ns != backend->default_ns) {
149 str = solr_escape_id_str(ns->prefix);
150 backend->id_namespace = i_strdup(str);
151 }
112 backend->backend = fts_backend_solr; 152 backend->backend = fts_backend_solr;
113 153
114 if (set->substring_search) 154 if (set->substring_search)
115 backend->backend.flags |= FTS_BACKEND_FLAG_SUBSTRING_LOOKUPS; 155 backend->backend.flags |= FTS_BACKEND_FLAG_SUBSTRING_LOOKUPS;
116 return &backend->backend; 156 return &backend->backend;
118 158
119 static void fts_backend_solr_deinit(struct fts_backend *_backend) 159 static void fts_backend_solr_deinit(struct fts_backend *_backend)
120 { 160 {
121 struct solr_fts_backend *backend = (struct solr_fts_backend *)_backend; 161 struct solr_fts_backend *backend = (struct solr_fts_backend *)_backend;
122 162
163 i_free(backend->id_namespace);
123 i_free(backend->id_username); 164 i_free(backend->id_username);
124 i_free(backend); 165 i_free(backend);
125 } 166 }
126 167
127 static const char *fts_backend_solr_username(struct fts_backend *_backend) 168 static void
169 solr_add_ns_query(string_t *str, struct fts_backend *_backend,
170 struct mail_namespace *ns)
128 { 171 {
129 struct solr_fts_backend *backend = (struct solr_fts_backend *)_backend; 172 struct solr_fts_backend *backend = (struct solr_fts_backend *)_backend;
130 173
131 return backend->id_username; 174 if (ns == backend->default_ns || *ns->prefix == '\0')
175 str_append(str, " -ns:[* TO *]");
176 else {
177 str_append(str, " ns:");
178 solr_quote(str, ns->prefix);
179 }
180 }
181
182 static void
183 solr_add_ns_query_http(string_t *str, struct fts_backend *backend,
184 struct mail_namespace *ns)
185 {
186 string_t *tmp;
187
188 tmp = t_str_new(64);
189 solr_add_ns_query(tmp, backend, ns);
190 solr_connection_http_escape(solr_conn, str, str_c(tmp));
132 } 191 }
133 192
134 static int fts_backend_solr_get_last_uid_fallback(struct fts_backend *backend, 193 static int fts_backend_solr_get_last_uid_fallback(struct fts_backend *backend,
135 uint32_t *last_uid_r) 194 uint32_t *last_uid_r)
136 { 195 {
196 struct mailbox *box = backend->box;
137 struct mailbox_status status; 197 struct mailbox_status status;
138 ARRAY_TYPE(seq_range) uids; 198 ARRAY_TYPE(seq_range) uids;
139 const struct seq_range *uidvals; 199 const struct seq_range *uidvals;
140 unsigned int count; 200 unsigned int count;
141 string_t *str; 201 string_t *str;
142 202
143 str = t_str_new(256); 203 str = t_str_new(256);
144 str_append(str, "fl=uid&rows=1&sort=uid%20desc&q="); 204 str_append(str, "fl=uid&rows=1&sort=uid+desc&q=");
145 205
146 mailbox_get_status(backend->box, STATUS_UIDVALIDITY, &status); 206 mailbox_get_status(box, STATUS_UIDVALIDITY, &status);
147 str_printfa(str, "uidv:%u%%20box:", status.uidvalidity); 207 str_printfa(str, "uidv:%u+box:", status.uidvalidity);
148 solr_quote_str(str, backend->box->name); 208 solr_quote_http(str, box->name);
149 str_append(str, "%20user:"); 209 solr_add_ns_query_http(str, backend, box->storage->ns);
150 solr_quote_str(str, backend->box->storage->ns->user->username); 210 str_append(str, "+user:");
211 solr_quote_http(str, box->storage->ns->user->username);
151 212
152 t_array_init(&uids, 1); 213 t_array_init(&uids, 1);
153 if (solr_connection_select(solr_conn, str_c(str), 214 if (solr_connection_select(solr_conn, str_c(str),
154 NULL, NULL, &uids, NULL) < 0) 215 NULL, NULL, &uids, NULL) < 0)
155 return -1; 216 return -1;
168 } 229 }
169 230
170 static int fts_backend_solr_get_last_uid(struct fts_backend *backend, 231 static int fts_backend_solr_get_last_uid(struct fts_backend *backend,
171 uint32_t *last_uid_r) 232 uint32_t *last_uid_r)
172 { 233 {
234 struct mailbox *box = backend->box;
173 struct mailbox_status status; 235 struct mailbox_status status;
174 ARRAY_TYPE(seq_range) uids; 236 ARRAY_TYPE(seq_range) uids;
175 const struct seq_range *uidvals; 237 const struct seq_range *uidvals;
176 unsigned int count; 238 unsigned int count;
177 string_t *str; 239 string_t *str;
178 240
179 str = t_str_new(256); 241 str = t_str_new(256);
180 str_append(str, "fl=uid&rows=1&q=last_uid:TRUE%20"); 242 str_append(str, "fl=uid&rows=1&q=last_uid:TRUE+");
181 243
182 mailbox_get_status(backend->box, STATUS_UIDVALIDITY, &status); 244 mailbox_get_status(box, STATUS_UIDVALIDITY, &status);
183 str_printfa(str, "uidv:%u%%20box:", status.uidvalidity); 245 str_printfa(str, "uidv:%u+box:", status.uidvalidity);
184 solr_quote_str(str, backend->box->name); 246 solr_quote_http(str, box->name);
185 str_append(str, "%20user:"); 247 solr_add_ns_query_http(str, backend, box->storage->ns);
186 solr_quote_str(str, backend->box->storage->ns->user->username); 248 str_append(str, "+user:");
249 solr_quote_http(str, box->storage->ns->user->username);
187 250
188 t_array_init(&uids, 1); 251 t_array_init(&uids, 1);
189 if (solr_connection_select(solr_conn, str_c(str), 252 if (solr_connection_select(solr_conn, str_c(str),
190 NULL, NULL, &uids, NULL) < 0) 253 NULL, NULL, &uids, NULL) < 0)
191 return -1; 254 return -1;
203 return -1; 266 return -1;
204 } 267 }
205 return 0; 268 return 0;
206 } 269 }
207 270
271 static const char *
272 solr_get_vmailbox(struct fts_backend *_backend,
273 struct mailbox *box, const char *ns_prefix,
274 const char *mailbox, string_t *dest)
275 {
276 struct solr_fts_backend *backend = (struct solr_fts_backend *)_backend;
277 struct mail_namespace *namespaces = box->storage->ns->user->namespaces;
278 struct mail_namespace *ns;
279
280 if (ns_prefix == NULL)
281 ns = backend->default_ns;
282 else {
283 ns = mail_namespace_find_prefix(namespaces, ns_prefix);
284 if (ns == NULL)
285 return FALSE;
286 }
287 return mail_namespace_get_vname(ns, dest, mailbox);
288 }
289
208 static bool 290 static bool
209 solr_virtual_get_last_uids(const char *mailbox, uint32_t uidvalidity, 291 solr_virtual_get_last_uids(const char *ns_prefix, const char *mailbox,
210 uint32_t *uid, void *context) 292 uint32_t uidvalidity, uint32_t *uid, void *context)
211 { 293 {
212 struct fts_backend_solr_get_last_uids_context *ctx = context; 294 struct fts_backend_solr_get_last_uids_context *ctx = context;
213 struct fts_backend_uid_map *map; 295 struct fts_backend_uid_map *map;
296 const char *vname;
297
298 vname = solr_get_vmailbox(ctx->backend, ctx->box, ns_prefix,
299 mailbox, ctx->vname);
214 300
215 map = array_append_space(ctx->last_uids); 301 map = array_append_space(ctx->last_uids);
216 map->mailbox = p_strdup(ctx->pool, mailbox); 302 map->mailbox = p_strdup(ctx->pool, vname);
217 map->uidvalidity = uidvalidity; 303 map->uidvalidity = uidvalidity;
218 map->uid = *uid; 304 map->uid = *uid;
219 return FALSE; 305 return FALSE;
220 } 306 }
221 307
222 static void add_pattern_as_solr(string_t *str, const char *pattern) 308 static void
223 { 309 solr_add_pattern(string_t *str, const struct mailbox_virtual_pattern *pattern)
224 const char *p; 310 {
311 const char *name, *p;
312
313 name = pattern->pattern;
314 if (!mail_namespace_update_name(pattern->ns, &name))
315 name = mail_namespace_fix_sep(pattern->ns, name);
225 316
226 /* first check if there are any wildcards in the pattern */ 317 /* first check if there are any wildcards in the pattern */
227 for (p = pattern; *p != '\0'; p++) { 318 for (p = name; *p != '\0'; p++) {
228 if (*p == '%' || *p == '*') 319 if (*p == '%' || *p == '*')
229 break; 320 break;
230 } 321 }
231 if (*p == '\0') { 322 if (*p == '\0') {
232 /* full mailbox name */ 323 /* full mailbox name */
233 str_append_c(str, '"'); 324 solr_quote(str, name);
234 str_append(str, str_escape(pattern));
235 str_append_c(str, '"');
236 return; 325 return;
237 } 326 }
238 327
239 /* there are at least some wildcards. */ 328 /* there are at least some wildcards. */
240 for (p = pattern; *p != '\0'; p++) { 329 for (p = name; *p != '\0'; p++) {
241 if (*p == '%' || *p == '*') { 330 if (*p == '%' || *p == '*') {
242 if (p == pattern || (p[-1] != '%' && p[-1] != '*')) 331 if (p == name || (p[-1] != '%' && p[-1] != '*'))
243 str_append_c(str, '*'); 332 str_append_c(str, '*');
244 } else { 333 } else {
245 if (!i_isalnum(*p)) 334 if (!i_isalnum(*p))
246 str_append_c(str, '\\'); 335 str_append_c(str, '\\');
247 str_append_c(str, *p); 336 str_append_c(str, *p);
248 } 337 }
249 } 338 }
250 } 339 }
251 340
252 static void 341 static void
253 fts_backend_solr_filter_mailboxes(struct solr_connection *solr_conn, 342 fts_backend_solr_filter_mailboxes(struct fts_backend *_backend,
254 string_t *str, struct mailbox *box) 343 string_t *str, struct mailbox *box)
255 { 344 {
256 ARRAY_TYPE(const_string) includes_arr, excludes_arr; 345 struct solr_fts_backend *backend = (struct solr_fts_backend *)_backend;
257 const char *const *includes, *const *excludes; 346 ARRAY_TYPE(mailbox_virtual_patterns) includes_arr, excludes_arr;
258 unsigned int i, inc_count, exc_count; 347 const struct mailbox_virtual_pattern *includes, *excludes;
348 unsigned int i, inc_count, exc_count, len;
259 string_t *fq; 349 string_t *fq;
260 350
261 t_array_init(&includes_arr, 16); 351 t_array_init(&includes_arr, 16);
262 t_array_init(&excludes_arr, 16); 352 t_array_init(&excludes_arr, 16);
263 mailbox_get_virtual_box_patterns(box, &includes_arr, &excludes_arr); 353 mailbox_get_virtual_box_patterns(box, &includes_arr, &excludes_arr);
267 357
268 /* First see if there are any patterns that begin with a wildcard. 358 /* First see if there are any patterns that begin with a wildcard.
269 Solr doesn't allow them, so in that case we'll need to return 359 Solr doesn't allow them, so in that case we'll need to return
270 all mailboxes. */ 360 all mailboxes. */
271 for (i = 0; i < inc_count; i++) { 361 for (i = 0; i < inc_count; i++) {
272 if (*includes[i] == '*' || *includes[i] == '%') 362 if (*includes[i].pattern == '*' ||
363 *includes[i].pattern == '%')
273 break; 364 break;
274 } 365 }
275 366
276 fq = t_str_new(128); 367 fq = t_str_new(128);
277 if (i == inc_count) { 368 if (i == inc_count && inc_count <= FTS_SOLR_MAX_BOX_INC_PATTERNS) {
278 /* we can filter what mailboxes we want returned */ 369 /* we can filter what mailboxes we want returned */
279 str_append_c(fq, '('); 370 str_append_c(fq, '(');
280 for (i = 0; i < inc_count; i++) { 371 for (i = 0; i < inc_count; i++) {
281 if (i != 0) 372 if (i != 0)
282 str_append(fq, " OR "); 373 str_append(fq, " OR ");
374 str_append_c(fq, '(');
283 str_append(fq, "box:"); 375 str_append(fq, "box:");
284 add_pattern_as_solr(fq, includes[i]); 376 solr_add_pattern(fq, &includes[i]);
377 solr_add_ns_query(fq, _backend, includes[i].ns);
378 str_append_c(fq, ')');
285 } 379 }
286 str_append_c(fq, ')'); 380 str_append_c(fq, ')');
287 } 381 }
382 exc_count = I_MIN(FTS_SOLR_MAX_BOX_EXC_PATTERNS, exc_count);
288 for (i = 0; i < exc_count; i++) { 383 for (i = 0; i < exc_count; i++) {
289 if (str_len(fq) > 0) 384 if (str_len(fq) > len)
290 str_append_c(fq, ' '); 385 str_append_c(fq, ' ');
386 str_append_c(fq, '(');
291 str_append(fq, "-box:"); 387 str_append(fq, "-box:");
292 add_pattern_as_solr(fq, excludes[i]); 388 solr_add_pattern(fq, &excludes[i]);
389 if (excludes[i].ns == backend->default_ns) {
390 str_append(fq, " OR NOT");
391 solr_add_ns_query(fq, _backend, excludes[i].ns);
392 } else if (*excludes[i].ns->prefix != '\0') {
393 str_append(fq, " OR -ns:");
394 solr_quote(fq, excludes[i].ns->prefix);
395 }
396 str_append_c(fq, ')');
293 } 397 }
294 if (str_len(fq) > 0) { 398 if (str_len(fq) > 0) {
295 str_append(str, "&fq="); 399 str_append(str, "&fq=");
296 solr_connection_http_escape(solr_conn, str, str_c(fq)); 400 solr_connection_http_escape(solr_conn, str, str_c(fq));
297 } 401 }
303 { 407 {
304 struct fts_backend_solr_get_last_uids_context ctx; 408 struct fts_backend_solr_get_last_uids_context ctx;
305 string_t *str; 409 string_t *str;
306 410
307 memset(&ctx, 0, sizeof(ctx)); 411 memset(&ctx, 0, sizeof(ctx));
412 ctx.backend = backend;
308 ctx.pool = pool; 413 ctx.pool = pool;
309 ctx.last_uids = last_uids; 414 ctx.last_uids = last_uids;
415 ctx.box = backend->box;
416 ctx.vname = t_str_new(256);
310 417
311 str = t_str_new(256); 418 str = t_str_new(256);
312 str_printfa(str, "fl=uid,box,uidv&rows=%u&q=last_uid:TRUE%%20user:", 419 str_printfa(str, "fl=uid,box,uidv,ns&rows=%u&q=last_uid:TRUE+user:",
313 SOLR_MAX_ROWS); 420 SOLR_MAX_ROWS);
314 solr_quote_str(str, backend->box->storage->ns->user->username); 421 solr_quote_http(str, backend->box->storage->ns->user->username);
315 fts_backend_solr_filter_mailboxes(solr_conn, str, backend->box); 422 fts_backend_solr_filter_mailboxes(backend, str, backend->box);
316 423
317 return solr_connection_select(solr_conn, str_c(str), 424 return solr_connection_select(solr_conn, str_c(str),
318 solr_virtual_get_last_uids, &ctx, 425 solr_virtual_get_last_uids, &ctx,
319 NULL, NULL); 426 NULL, NULL);
320 } 427 }
341 448
342 static void 449 static void
343 fts_backend_solr_add_doc_prefix(struct solr_fts_backend_build_context *ctx, 450 fts_backend_solr_add_doc_prefix(struct solr_fts_backend_build_context *ctx,
344 uint32_t uid) 451 uint32_t uid)
345 { 452 {
453 struct solr_fts_backend *backend =
454 (struct solr_fts_backend *)ctx->ctx.backend;
346 struct mailbox *box = ctx->ctx.backend->box; 455 struct mailbox *box = ctx->ctx.backend->box;
347 456
348 str_printfa(ctx->cmd, "<doc>" 457 str_printfa(ctx->cmd, "<doc>"
349 "<field name=\"uid\">%u</field>" 458 "<field name=\"uid\">%u</field>"
350 "<field name=\"uidv\">%u</field>", 459 "<field name=\"uidv\">%u</field>",
351 uid, ctx->uid_validity); 460 uid, ctx->uid_validity);
352 461
462 if (box->storage->ns != backend->default_ns) {
463 str_append(ctx->cmd, "<field name=\"ns\">");
464 xml_encode(ctx->cmd, box->storage->ns->prefix);
465 str_append(ctx->cmd, "</field>");
466 }
353 str_append(ctx->cmd, "<field name=\"box\">"); 467 str_append(ctx->cmd, "<field name=\"box\">");
354 xml_encode(ctx->cmd, box->name); 468 xml_encode(ctx->cmd, box->name);
355 str_append(ctx->cmd, "</field><field name=\"user\">"); 469 str_append(ctx->cmd, "</field><field name=\"user\">");
356 xml_encode(ctx->cmd, box->storage->ns->user->username); 470 xml_encode(ctx->cmd, box->storage->ns->user->username);
357 str_append(ctx->cmd, "</field>"); 471 str_append(ctx->cmd, "</field>");
472 }
473
474 static void xml_encode_id(string_t *str, struct fts_backend *_backend,
475 uint32_t uid, uint32_t uid_validity,
476 const char *mailbox)
477 {
478 struct solr_fts_backend *backend = (struct solr_fts_backend *)_backend;
479
480 if (uid != 0)
481 str_printfa(str, "%u/", uid);
482 else
483 str_append(str, "L/");
484 if (backend->id_namespace != NULL) {
485 xml_encode(str, backend->id_namespace);
486 str_append_c(str, '/');
487 }
488 str_printfa(str, "%u/", uid_validity);
489 xml_encode(str, backend->id_username);
490 str_append_c(str, '/');
491 xml_encode(str, mailbox);
358 } 492 }
359 493
360 static int 494 static int
361 fts_backend_solr_build_more(struct fts_backend_build_context *_ctx, 495 fts_backend_solr_build_more(struct fts_backend_build_context *_ctx,
362 uint32_t uid, const unsigned char *data, 496 uint32_t uid, const unsigned char *data,
377 str_append(cmd, "</field></doc>"); 511 str_append(cmd, "</field></doc>");
378 } 512 }
379 ctx->prev_uid = uid; 513 ctx->prev_uid = uid;
380 514
381 fts_backend_solr_add_doc_prefix(ctx, uid); 515 fts_backend_solr_add_doc_prefix(ctx, uid);
382 str_printfa(cmd, "<field name=\"id\">%u/%u/", 516 str_printfa(cmd, "<field name=\"id\">");
383 uid, ctx->uid_validity); 517 xml_encode_id(cmd, _ctx->backend, uid, ctx->uid_validity,
384 xml_encode(cmd, fts_backend_solr_username(ctx->ctx.backend)); 518 box->name);
385 str_append_c(cmd, '/');
386 xml_encode(cmd, box->name);
387 str_append(cmd, "</field>"); 519 str_append(cmd, "</field>");
388 520
389 ctx->headers = headers; 521 ctx->headers = headers;
390 if (headers) { 522 if (headers) {
391 str_append(cmd, "<field name=\"hdr\">"); 523 str_append(cmd, "<field name=\"hdr\">");
423 if another session is indexing at the same time, the last_uid value 555 if another session is indexing at the same time, the last_uid value
424 may shrink. This doesn't really matter, we'll simply do more work 556 may shrink. This doesn't really matter, we'll simply do more work
425 in future by reindexing some messages. */ 557 in future by reindexing some messages. */
426 fts_backend_solr_add_doc_prefix(ctx, ctx->prev_uid); 558 fts_backend_solr_add_doc_prefix(ctx, ctx->prev_uid);
427 str_printfa(ctx->cmd, "<field name=\"last_uid\">TRUE</field>" 559 str_printfa(ctx->cmd, "<field name=\"last_uid\">TRUE</field>"
428 "<field name=\"id\">L/%u/", ctx->uid_validity); 560 "<field name=\"id\">");
429 xml_encode(ctx->cmd, fts_backend_solr_username(ctx->ctx.backend)); 561 xml_encode_id(ctx->cmd, ctx->ctx.backend, 0, ctx->uid_validity,
430 str_append_c(ctx->cmd, '/'); 562 box->name);
431 xml_encode(ctx->cmd, box->name);
432 str_append(ctx->cmd, "</field></doc></add>"); 563 str_append(ctx->cmd, "</field></doc></add>");
433 564
434 solr_connection_post_more(ctx->post, str_data(ctx->cmd), 565 solr_connection_post_more(ctx->post, str_data(ctx->cmd),
435 str_len(ctx->cmd)); 566 str_len(ctx->cmd));
436 ret = solr_connection_post_end(ctx->post); 567 ret = solr_connection_post_end(ctx->post);
464 595
465 T_BEGIN { 596 T_BEGIN {
466 string_t *cmd; 597 string_t *cmd;
467 598
468 cmd = t_str_new(256); 599 cmd = t_str_new(256);
469 str_printfa(cmd, "<delete><id>%u/%u/", 600 str_append(cmd, "<delete><id>");
470 mail->uid, status.uidvalidity); 601 xml_encode_id(cmd, backend, mail->uid, status.uidvalidity,
471 xml_encode(cmd, fts_backend_solr_username(backend)); 602 mail->box->name);
472 str_append_c(cmd, '/');
473 xml_encode(cmd, mail->box->name);
474 str_append(cmd, "</id></delete>"); 603 str_append(cmd, "</id></delete>");
475 604
476 (void)solr_connection_post(solr_conn, str_c(cmd)); 605 (void)solr_connection_post(solr_conn, str_c(cmd));
477 } T_END; 606 } T_END;
478 } 607 }
493 622
494 static void fts_backend_solr_unlock(struct fts_backend *backend ATTR_UNUSED) 623 static void fts_backend_solr_unlock(struct fts_backend *backend ATTR_UNUSED)
495 { 624 {
496 } 625 }
497 626
498 static bool solr_virtual_uid_map(const char *mailbox, uint32_t uidvalidity, 627 static bool solr_virtual_uid_map(const char *ns_prefix, const char *mailbox,
499 uint32_t *uid, void *context) 628 uint32_t uidvalidity, uint32_t *uid,
500 { 629 void *context)
501 struct mailbox *box = context; 630 {
502 631 struct solr_virtual_uid_map_context *ctx = context;
503 return mailbox_get_virtual_uid(box, mailbox, uidvalidity, 632 const char *vname;
504 *uid, uid); 633
634 vname = solr_get_vmailbox(ctx->backend, ctx->box, ns_prefix,
635 mailbox, ctx->vname);
636 return mailbox_get_virtual_uid(ctx->box, vname, uidvalidity, *uid, uid);
505 } 637 }
506 638
507 static int fts_backend_solr_lookup(struct fts_backend_lookup_context *ctx, 639 static int fts_backend_solr_lookup(struct fts_backend_lookup_context *ctx,
508 ARRAY_TYPE(seq_range) *definite_uids, 640 ARRAY_TYPE(seq_range) *definite_uids,
509 ARRAY_TYPE(seq_range) *maybe_uids, 641 ARRAY_TYPE(seq_range) *maybe_uids,
510 ARRAY_TYPE(fts_score_map) *scores) 642 ARRAY_TYPE(fts_score_map) *scores)
511 { 643 {
512 struct mailbox *box = ctx->backend->box; 644 struct mailbox *box = ctx->backend->box;
645 struct solr_virtual_uid_map_context uid_map_ctx;
513 const struct fts_backend_lookup_field *fields; 646 const struct fts_backend_lookup_field *fields;
514 unsigned int i, count; 647 unsigned int i, count;
515 struct mailbox_status status; 648 struct mailbox_status status;
516 string_t *str; 649 string_t *str;
517 bool virtual; 650 bool virtual;
519 virtual = strcmp(box->storage->name, "virtual") == 0; 652 virtual = strcmp(box->storage->name, "virtual") == 0;
520 mailbox_get_status(box, STATUS_UIDVALIDITY, &status); 653 mailbox_get_status(box, STATUS_UIDVALIDITY, &status);
521 654
522 str = t_str_new(256); 655 str = t_str_new(256);
523 if (!virtual) { 656 if (!virtual) {
524 str_printfa(str, "fl=uid,score&rows=%u&sort=uid%%20asc&q=", 657 str_printfa(str, "fl=uid,score&rows=%u&sort=uid+asc&q=",
525 status.uidnext); 658 status.uidnext);
526 } else { 659 } else {
527 str_printfa(str, "fl=uid,score,box,uidv&rows=%u" 660 str_printfa(str, "fl=uid,score,box,uidv,ns&rows=%u"
528 "&sort=box%%20asc,uid%%20asc&q=", 661 "&sort=box+asc,uid+asc&q=",
529 SOLR_MAX_ROWS); 662 SOLR_MAX_ROWS);
530 } 663 }
531 664
532 /* build a lucene search query from the fields */ 665 /* build a lucene search query from the fields */
533 fields = array_get(&ctx->fields, &count); 666 fields = array_get(&ctx->fields, &count);
534 for (i = 0; i < count; i++) { 667 for (i = 0; i < count; i++) {
535 if (i > 0) 668 if (i > 0)
536 str_append(str, "%20"); 669 str_append_c(str, '+');
537 670
538 if ((fields[i].flags & FTS_LOOKUP_FLAG_INVERT) != 0) 671 if ((fields[i].flags & FTS_LOOKUP_FLAG_INVERT) != 0)
539 str_append_c(str, '-'); 672 str_append_c(str, '-');
540 673
541 if ((fields[i].flags & FTS_LOOKUP_FLAG_HEADER) == 0) { 674 if ((fields[i].flags & FTS_LOOKUP_FLAG_HEADER) == 0) {
547 str_append(str, "hdr:"); 680 str_append(str, "hdr:");
548 } else { 681 } else {
549 /* both */ 682 /* both */
550 str_append(str, "any:"); 683 str_append(str, "any:");
551 } 684 }
552 solr_quote_str(str, fields[i].key); 685 solr_quote_http(str, fields[i].key);
553 } 686 }
554 687
555 /* use a separate filter query for selecting the mailbox. it shouldn't 688 /* use a separate filter query for selecting the mailbox. it shouldn't
556 affect the score and there could be some caching benefits too. */ 689 affect the score and there could be some caching benefits too. */
557 str_append(str, "&fq=user:"); 690 str_append(str, "&fq=user:");
558 solr_quote_str(str, box->storage->ns->user->username); 691 solr_quote_http(str, box->storage->ns->user->username);
559 if (virtual) 692 if (virtual)
560 fts_backend_solr_filter_mailboxes(solr_conn, str, box); 693 fts_backend_solr_filter_mailboxes(ctx->backend, str, box);
561 else { 694 else {
562 str_printfa(str, "%%20uidv:%u%%20box:", status.uidvalidity); 695 str_printfa(str, "+uidv:%u+box:", status.uidvalidity);
563 solr_quote_str(str, box->name); 696 solr_quote_http(str, box->name);
697 solr_add_ns_query_http(str, ctx->backend, box->storage->ns);
564 } 698 }
565 699
566 array_clear(maybe_uids); 700 array_clear(maybe_uids);
567 if (!virtual) { 701 if (!virtual) {
568 return solr_connection_select(solr_conn, str_c(str), NULL, NULL, 702 return solr_connection_select(solr_conn, str_c(str), NULL, NULL,
569 definite_uids, scores); 703 definite_uids, scores);
570 } else { 704 } else {
705 memset(&uid_map_ctx, 0, sizeof(uid_map_ctx));
706 uid_map_ctx.backend = ctx->backend;
707 uid_map_ctx.box = box;
708 uid_map_ctx.vname = t_str_new(256);
571 return solr_connection_select(solr_conn, str_c(str), 709 return solr_connection_select(solr_conn, str_c(str),
572 solr_virtual_uid_map, box, 710 solr_virtual_uid_map,
711 &uid_map_ctx,
573 definite_uids, scores); 712 definite_uids, scores);
574 } 713 }
575 } 714 }
576 715
577 struct fts_backend fts_backend_solr = { 716 struct fts_backend fts_backend_solr = {