Mercurial > dovecot > original-hg > dovecot-1.2

comparison src/plugins/fts-solr/fts-backend-solr.c @ 8950:ec1ae90af21a HEAD

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
fts: Fixes to how virtual mailboxes are searched.
author Timo Sirainen <tss@iki.fi>
date Wed, 15 Apr 2009 19:48:55 -0400
parents 7e639229c2c8
children d21bd1a1710a
comparison
equal deleted inserted replaced
8949:508bbbd4e6f6 8950:ec1ae90af21a
16 #define FTS_SOLR_MAX_BOX_INC_PATTERNS 5 16 #define FTS_SOLR_MAX_BOX_INC_PATTERNS 5
17 #define FTS_SOLR_MAX_BOX_EXC_PATTERNS 5 17 #define FTS_SOLR_MAX_BOX_EXC_PATTERNS 5
18 18
19 struct solr_fts_backend { 19 struct solr_fts_backend {
20 struct fts_backend backend; 20 struct fts_backend backend;
21 char *id_username, *id_namespace; 21 char *id_username, *id_namespace, *id_box_name;
22 struct mail_namespace *default_ns; 22 struct mail_namespace *default_ns;
23 }; 23 };
24 24
25 struct solr_fts_backend_build_context { 25 struct solr_fts_backend_build_context {
26 struct fts_backend_build_context ctx; 26 struct fts_backend_build_context ctx;
45 struct mailbox *box; 45 struct mailbox *box;
46 string_t *vname; 46 string_t *vname;
47 }; 47 };
48 48
49 static struct solr_connection *solr_conn = NULL; 49 static struct solr_connection *solr_conn = NULL;
50
51 static void fts_box_name_get_root(struct mail_namespace **ns, const char **name)
52 {
53 struct mail_namespace *orig_ns = *ns;
54
55 while ((*ns)->alias_for != NULL)
56 *ns = (*ns)->alias_for;
57
58 if (**name == '\0' && *ns != orig_ns &&
59 ((*ns)->flags & NAMESPACE_FLAG_INBOX) != 0) {
60 /* ugly workaround to allow selecting INBOX from a Maildir/
61 when it's not in the inbox=yes namespace. */
62 *name = "INBOX";
63 }
64 }
65
66 static const char *
67 fts_box_get_root(struct mailbox *box, struct mail_namespace **ns_r)
68 {
69 struct mail_namespace *ns = box->storage->ns;
70 const char *name = box->name;
71
72 fts_box_name_get_root(&ns, &name);
73 *ns_r = ns;
74 return name;
75 }
50 76
51 static void 77 static void
52 xml_encode_data(string_t *dest, const unsigned char *data, unsigned int len) 78 xml_encode_data(string_t *dest, const unsigned char *data, unsigned int len)
53 { 79 {
54 unsigned int i; 80 unsigned int i;
134 static struct fts_backend * 160 static struct fts_backend *
135 fts_backend_solr_init(struct mailbox *box) 161 fts_backend_solr_init(struct mailbox *box)
136 { 162 {
137 const struct fts_solr_settings *set = &fts_solr_settings; 163 const struct fts_solr_settings *set = &fts_solr_settings;
138 struct solr_fts_backend *backend; 164 struct solr_fts_backend *backend;
139 struct mail_namespace *ns = box->storage->ns; 165 struct mail_namespace *ns;
140 const char *str; 166 const char *str, *box_name;
141 167
142 while (ns->alias_for != NULL) 168
143 ns = ns->alias_for; 169 box_name = fts_box_get_root(box, &ns);
170 i_assert(*box_name != '\0');
144 171
145 if (solr_conn == NULL) 172 if (solr_conn == NULL)
146 solr_conn = solr_connection_init(set->url, set->debug); 173 solr_conn = solr_connection_init(set->url, set->debug);
147 174
148 backend = i_new(struct solr_fts_backend, 1); 175 backend = i_new(struct solr_fts_backend, 1);
165 backend->id_username = i_strdup(str); 192 backend->id_username = i_strdup(str);
166 if (ns != backend->default_ns) { 193 if (ns != backend->default_ns) {
167 str = solr_escape_id_str(ns->prefix); 194 str = solr_escape_id_str(ns->prefix);
168 backend->id_namespace = i_strdup(str); 195 backend->id_namespace = i_strdup(str);
169 } 196 }
197 backend->id_box_name = i_strdup(box_name);
170 backend->backend = fts_backend_solr; 198 backend->backend = fts_backend_solr;
171 199
172 if (set->substring_search) 200 if (set->substring_search)
173 backend->backend.flags |= FTS_BACKEND_FLAG_SUBSTRING_LOOKUPS; 201 backend->backend.flags |= FTS_BACKEND_FLAG_SUBSTRING_LOOKUPS;
174 return &backend->backend; 202 return &backend->backend;
176 204
177 static void fts_backend_solr_deinit(struct fts_backend *_backend) 205 static void fts_backend_solr_deinit(struct fts_backend *_backend)
178 { 206 {
179 struct solr_fts_backend *backend = (struct solr_fts_backend *)_backend; 207 struct solr_fts_backend *backend = (struct solr_fts_backend *)_backend;
180 208
209 i_free(backend->id_box_name);
181 i_free(backend->id_namespace); 210 i_free(backend->id_namespace);
182 i_free(backend->id_username); 211 i_free(backend->id_username);
183 i_free(backend); 212 i_free(backend);
184 } 213 }
185 214
219 248
220 static int fts_backend_solr_get_last_uid_fallback(struct fts_backend *backend, 249 static int fts_backend_solr_get_last_uid_fallback(struct fts_backend *backend,
221 uint32_t *last_uid_r) 250 uint32_t *last_uid_r)
222 { 251 {
223 struct mailbox *box = backend->box; 252 struct mailbox *box = backend->box;
253 struct mail_namespace *ns;
224 struct mailbox_status status; 254 struct mailbox_status status;
225 ARRAY_TYPE(seq_range) uids; 255 ARRAY_TYPE(seq_range) uids;
226 const struct seq_range *uidvals; 256 const struct seq_range *uidvals;
257 const char *box_name;
227 unsigned int count; 258 unsigned int count;
228 string_t *str; 259 string_t *str;
229 260
230 str = t_str_new(256); 261 str = t_str_new(256);
231 str_append(str, "fl=uid&rows=1&sort=uid+desc&q="); 262 str_append(str, "fl=uid&rows=1&sort=uid+desc&q=");
232 263
264 box_name = fts_box_get_root(box, &ns);
265
233 mailbox_get_status(box, STATUS_UIDVALIDITY, &status); 266 mailbox_get_status(box, STATUS_UIDVALIDITY, &status);
234 str_printfa(str, "uidv:%u+box:", status.uidvalidity); 267 str_printfa(str, "uidv:%u+box:", status.uidvalidity);
235 solr_quote_http(str, box->name); 268 solr_quote_http(str, box_name);
236 solr_add_ns_query_http(str, backend, box->storage->ns); 269 solr_add_ns_query_http(str, backend, ns);
237 str_append(str, "+user:"); 270 str_append(str, "+user:");
238 solr_quote_http(str, box->storage->ns->user->username); 271 solr_quote_http(str, ns->user->username);
239 272
240 t_array_init(&uids, 1); 273 t_array_init(&uids, 1);
241 if (solr_connection_select(solr_conn, str_c(str), 274 if (solr_connection_select(solr_conn, str_c(str),
242 NULL, NULL, &uids, NULL) < 0) 275 NULL, NULL, &uids, NULL) < 0)
243 return -1; 276 return -1;
257 290
258 static int fts_backend_solr_get_last_uid(struct fts_backend *backend, 291 static int fts_backend_solr_get_last_uid(struct fts_backend *backend,
259 uint32_t *last_uid_r) 292 uint32_t *last_uid_r)
260 { 293 {
261 struct mailbox *box = backend->box; 294 struct mailbox *box = backend->box;
295 struct mail_namespace *ns;
262 struct mailbox_status status; 296 struct mailbox_status status;
263 ARRAY_TYPE(seq_range) uids; 297 ARRAY_TYPE(seq_range) uids;
264 const struct seq_range *uidvals; 298 const struct seq_range *uidvals;
299 const char *box_name;
265 unsigned int count; 300 unsigned int count;
266 string_t *str; 301 string_t *str;
267 302
268 str = t_str_new(256); 303 str = t_str_new(256);
269 str_append(str, "fl=uid&rows=1&q=last_uid:TRUE+"); 304 str_append(str, "fl=uid&rows=1&q=last_uid:TRUE+");
270 305
306 box_name = fts_box_get_root(box, &ns);
307
271 mailbox_get_status(box, STATUS_UIDVALIDITY, &status); 308 mailbox_get_status(box, STATUS_UIDVALIDITY, &status);
272 str_printfa(str, "uidv:%u+box:", status.uidvalidity); 309 str_printfa(str, "uidv:%u+box:", status.uidvalidity);
273 solr_quote_http(str, box->name); 310 solr_quote_http(str, box_name);
274 solr_add_ns_query_http(str, backend, box->storage->ns); 311 solr_add_ns_query_http(str, backend, ns);
275 str_append(str, "+user:"); 312 str_append(str, "+user:");
276 solr_quote_http(str, box->storage->ns->user->username); 313 solr_quote_http(str, ns->user->username);
277 314
278 t_array_init(&uids, 1); 315 t_array_init(&uids, 1);
279 if (solr_connection_select(solr_conn, str_c(str), 316 if (solr_connection_select(solr_conn, str_c(str),
280 NULL, NULL, &uids, NULL) < 0) 317 NULL, NULL, &uids, NULL) < 0)
281 return -1; 318 return -1;
329 } 366 }
330 367
331 static void 368 static void
332 solr_add_pattern(string_t *str, const struct mailbox_virtual_pattern *pattern) 369 solr_add_pattern(string_t *str, const struct mailbox_virtual_pattern *pattern)
333 { 370 {
371 struct mail_namespace *ns = pattern->ns;
334 const char *name, *p; 372 const char *name, *p;
335 373
336 name = pattern->pattern; 374 name = pattern->pattern;
337 if (!mail_namespace_update_name(pattern->ns, &name)) 375 if (!mail_namespace_update_name(pattern->ns, &name))
338 name = mail_namespace_fix_sep(pattern->ns, name); 376 name = mail_namespace_fix_sep(pattern->ns, name);
377
378 fts_box_name_get_root(&ns, &name);
339 379
340 if (strcmp(name, "*") == 0) { 380 if (strcmp(name, "*") == 0) {
341 str_append(str, "[* TO *]"); 381 str_append(str, "[* TO *]");
342 return; 382 return;
343 } 383 }
476 uint32_t uid) 516 uint32_t uid)
477 { 517 {
478 struct solr_fts_backend *backend = 518 struct solr_fts_backend *backend =
479 (struct solr_fts_backend *)ctx->ctx.backend; 519 (struct solr_fts_backend *)ctx->ctx.backend;
480 struct mailbox *box = ctx->ctx.backend->box; 520 struct mailbox *box = ctx->ctx.backend->box;
481 struct mail_namespace *ns = box->storage->ns; 521 struct mail_namespace *ns;
522 const char *box_name;
482 523
483 str_printfa(ctx->cmd, "<doc>" 524 str_printfa(ctx->cmd, "<doc>"
484 "<field name=\"uid\">%u</field>" 525 "<field name=\"uid\">%u</field>"
485 "<field name=\"uidv\">%u</field>", 526 "<field name=\"uidv\">%u</field>",
486 uid, ctx->uid_validity); 527 uid, ctx->uid_validity);
487 528
488 while (ns->alias_for != NULL) 529 box_name = fts_box_get_root(box, &ns);
489 ns = ns->alias_for;
490 530
491 if (ns != backend->default_ns) { 531 if (ns != backend->default_ns) {
492 str_append(ctx->cmd, "<field name=\"ns\">"); 532 str_append(ctx->cmd, "<field name=\"ns\">");
493 xml_encode(ctx->cmd, ns->prefix); 533 xml_encode(ctx->cmd, ns->prefix);
494 str_append(ctx->cmd, "</field>"); 534 str_append(ctx->cmd, "</field>");
495 } 535 }
496 str_append(ctx->cmd, "<field name=\"box\">"); 536 str_append(ctx->cmd, "<field name=\"box\">");
497 xml_encode(ctx->cmd, box->name); 537 xml_encode(ctx->cmd, box_name);
498 str_append(ctx->cmd, "</field><field name=\"user\">"); 538 str_append(ctx->cmd, "</field><field name=\"user\">");
499 xml_encode(ctx->cmd, ns->user->username); 539 xml_encode(ctx->cmd, ns->user->username);
500 str_append(ctx->cmd, "</field>"); 540 str_append(ctx->cmd, "</field>");
501 } 541 }
502 542
503 static void xml_encode_id(string_t *str, struct fts_backend *_backend, 543 static void xml_encode_id(string_t *str, struct fts_backend *_backend,
504 uint32_t uid, uint32_t uid_validity, 544 uint32_t uid, uint32_t uid_validity)
505 const char *mailbox)
506 { 545 {
507 struct solr_fts_backend *backend = (struct solr_fts_backend *)_backend; 546 struct solr_fts_backend *backend = (struct solr_fts_backend *)_backend;
508 547
509 if (uid != 0) 548 if (uid != 0)
510 str_printfa(str, "%u/", uid); 549 str_printfa(str, "%u/", uid);
515 str_append_c(str, '/'); 554 str_append_c(str, '/');
516 } 555 }
517 str_printfa(str, "%u/", uid_validity); 556 str_printfa(str, "%u/", uid_validity);
518 xml_encode(str, backend->id_username); 557 xml_encode(str, backend->id_username);
519 str_append_c(str, '/'); 558 str_append_c(str, '/');
520 xml_encode(str, mailbox); 559 xml_encode(str, backend->id_box_name);
521 } 560 }
522 561
523 static int 562 static int
524 fts_backend_solr_build_more(struct fts_backend_build_context *_ctx, 563 fts_backend_solr_build_more(struct fts_backend_build_context *_ctx,
525 uint32_t uid, const unsigned char *data, 564 uint32_t uid, const unsigned char *data,
526 size_t size, bool headers) 565 size_t size, bool headers)
527 { 566 {
528 struct solr_fts_backend_build_context *ctx = 567 struct solr_fts_backend_build_context *ctx =
529 (struct solr_fts_backend_build_context *)_ctx; 568 (struct solr_fts_backend_build_context *)_ctx;
530 struct mailbox *box = _ctx->backend->box;
531 string_t *cmd = ctx->cmd; 569 string_t *cmd = ctx->cmd;
532 570
533 /* body comes first, then headers */ 571 /* body comes first, then headers */
534 if (ctx->prev_uid != uid) { 572 if (ctx->prev_uid != uid) {
535 /* uid changed */ 573 /* uid changed */
541 } 579 }
542 ctx->prev_uid = uid; 580 ctx->prev_uid = uid;
543 581
544 fts_backend_solr_add_doc_prefix(ctx, uid); 582 fts_backend_solr_add_doc_prefix(ctx, uid);
545 str_printfa(cmd, "<field name=\"id\">"); 583 str_printfa(cmd, "<field name=\"id\">");
546 xml_encode_id(cmd, _ctx->backend, uid, ctx->uid_validity, 584 xml_encode_id(cmd, _ctx->backend, uid, ctx->uid_validity);
547 box->name);
548 str_append(cmd, "</field>"); 585 str_append(cmd, "</field>");
549 586
550 ctx->headers = headers; 587 ctx->headers = headers;
551 if (headers) { 588 if (headers) {
552 str_append(cmd, "<field name=\"hdr\">"); 589 str_append(cmd, "<field name=\"hdr\">");
569 } 606 }
570 607
571 static int 608 static int
572 fts_backed_solr_build_commit(struct solr_fts_backend_build_context *ctx) 609 fts_backed_solr_build_commit(struct solr_fts_backend_build_context *ctx)
573 { 610 {
574 struct mailbox *box = ctx->ctx.backend->box;
575 int ret; 611 int ret;
576 612
577 if (ctx->post == NULL) 613 if (ctx->post == NULL)
578 return 0; 614 return 0;
579 615
585 may shrink. This doesn't really matter, we'll simply do more work 621 may shrink. This doesn't really matter, we'll simply do more work
586 in future by reindexing some messages. */ 622 in future by reindexing some messages. */
587 fts_backend_solr_add_doc_prefix(ctx, ctx->prev_uid); 623 fts_backend_solr_add_doc_prefix(ctx, ctx->prev_uid);
588 str_printfa(ctx->cmd, "<field name=\"last_uid\">TRUE</field>" 624 str_printfa(ctx->cmd, "<field name=\"last_uid\">TRUE</field>"
589 "<field name=\"id\">"); 625 "<field name=\"id\">");
590 xml_encode_id(ctx->cmd, ctx->ctx.backend, 0, ctx->uid_validity, 626 xml_encode_id(ctx->cmd, ctx->ctx.backend, 0, ctx->uid_validity);
591 box->name);
592 str_append(ctx->cmd, "</field></doc></add>"); 627 str_append(ctx->cmd, "</field></doc></add>");
593 628
594 solr_connection_post_more(ctx->post, str_data(ctx->cmd), 629 solr_connection_post_more(ctx->post, str_data(ctx->cmd),
595 str_len(ctx->cmd)); 630 str_len(ctx->cmd));
596 ret = solr_connection_post_end(ctx->post); 631 ret = solr_connection_post_end(ctx->post);
625 T_BEGIN { 660 T_BEGIN {
626 string_t *cmd; 661 string_t *cmd;
627 662
628 cmd = t_str_new(256); 663 cmd = t_str_new(256);
629 str_append(cmd, "<delete><id>"); 664 str_append(cmd, "<delete><id>");
630 xml_encode_id(cmd, backend, mail->uid, status.uidvalidity, 665 xml_encode_id(cmd, backend, mail->uid, status.uidvalidity);
631 mail->box->name);
632 str_append(cmd, "</id></delete>"); 666 str_append(cmd, "</id></delete>");
633 667
634 (void)solr_connection_post(solr_conn, str_c(cmd)); 668 (void)solr_connection_post(solr_conn, str_c(cmd));
635 } T_END; 669 } T_END;
636 } 670 }
658 void *context) 692 void *context)
659 { 693 {
660 struct solr_virtual_uid_map_context *ctx = context; 694 struct solr_virtual_uid_map_context *ctx = context;
661 struct mail_namespace *ns; 695 struct mail_namespace *ns;
662 const char *vname; 696 const char *vname;
697 bool convert_inbox;
663 698
664 ns = solr_get_namespaces(ctx->backend, ctx->box, ns_prefix); 699 ns = solr_get_namespaces(ctx->backend, ctx->box, ns_prefix);
700 convert_inbox = (ns->flags & NAMESPACE_FLAG_INBOX) != 0 &&
701 strcmp(mailbox, "INBOX") == 0;
665 for (; ns != NULL; ns = ns->alias_chain_next) { 702 for (; ns != NULL; ns = ns->alias_chain_next) {
666 vname = mail_namespace_get_vname(ns, ctx->vname, mailbox); 703 vname = convert_inbox ? ns->prefix :
704 mail_namespace_get_vname(ns, ctx->vname, mailbox);
667 if (mailbox_get_virtual_uid(ctx->box, vname, uidvalidity, 705 if (mailbox_get_virtual_uid(ctx->box, vname, uidvalidity,
668 *uid, uid)) 706 *uid, uid))
669 return TRUE; 707 return TRUE;
670 } 708 }
671 return FALSE; 709 return FALSE;
675 ARRAY_TYPE(seq_range) *definite_uids, 713 ARRAY_TYPE(seq_range) *definite_uids,
676 ARRAY_TYPE(seq_range) *maybe_uids, 714 ARRAY_TYPE(seq_range) *maybe_uids,
677 ARRAY_TYPE(fts_score_map) *scores) 715 ARRAY_TYPE(fts_score_map) *scores)
678 { 716 {
679 struct mailbox *box = ctx->backend->box; 717 struct mailbox *box = ctx->backend->box;
718 struct mail_namespace *ns;
680 struct solr_virtual_uid_map_context uid_map_ctx; 719 struct solr_virtual_uid_map_context uid_map_ctx;
681 const struct fts_backend_lookup_field *fields; 720 const struct fts_backend_lookup_field *fields;
721 const char *box_name;
682 unsigned int i, count; 722 unsigned int i, count;
683 struct mailbox_status status; 723 struct mailbox_status status;
684 string_t *str; 724 string_t *str;
685 bool virtual; 725 bool virtual;
686 726
725 str_append(str, "&fq=%2Buser:"); 765 str_append(str, "&fq=%2Buser:");
726 solr_quote_http(str, box->storage->ns->user->username); 766 solr_quote_http(str, box->storage->ns->user->username);
727 if (virtual) 767 if (virtual)
728 fts_backend_solr_filter_mailboxes(ctx->backend, str, box); 768 fts_backend_solr_filter_mailboxes(ctx->backend, str, box);
729 else { 769 else {
770 box_name = fts_box_get_root(box, &ns);
730 str_printfa(str, "+%%2Buidv:%u+%%2Bbox:", status.uidvalidity); 771 str_printfa(str, "+%%2Buidv:%u+%%2Bbox:", status.uidvalidity);
731 solr_quote_http(str, box->name); 772 solr_quote_http(str, box_name);
732 solr_add_ns_query_http(str, ctx->backend, box->storage->ns); 773 solr_add_ns_query_http(str, ctx->backend, ns);
733 } 774 }
734 775
735 array_clear(maybe_uids); 776 array_clear(maybe_uids);
736 if (!virtual) { 777 if (!virtual) {
737 return solr_connection_select(solr_conn, str_c(str), NULL, NULL, 778 return solr_connection_select(solr_conn, str_c(str), NULL, NULL,