Mercurial > dovecot > original-hg > dovecot-1.2
annotate src/plugins/fts-solr/fts-backend-solr.c @ 9606:ac0855b1bbea HEAD
fts-solr: Fixes to previous data encoding changes.
Patch by Erik Hetzner
author | Timo Sirainen <tss@iki.fi> |
---|---|
date | Mon, 23 Aug 2010 20:14:43 +0100 |
parents | cf0da2cd31fb |
children |
rev | line source |
---|---|
9532
00cd9aacd03c
Updated copyright notices to include year 2010.
Timo Sirainen <tss@iki.fi>
parents:
9292
diff
changeset
|
1 /* Copyright (c) 2006-2010 Dovecot authors, see the included COPYING file */ |
7990
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
2 |
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
3 #include "lib.h" |
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
4 #include "array.h" |
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
5 #include "str.h" |
8495
ffb37c392166
Finished making full text search indexes work fast with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents:
8493
diff
changeset
|
6 #include "strescape.h" |
9604
cf0da2cd31fb
fts-solr: Replace characters not valid for XML with replacement char.
Timo Sirainen <tss@iki.fi>
parents:
9532
diff
changeset
|
7 #include "unichar.h" |
7990
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
8 #include "mail-storage-private.h" |
8109
e7929190cd32
fts-solr: Fixed compiling with recent struct mail_user changes.
Timo Sirainen <tss@iki.fi>
parents:
8006
diff
changeset
|
9 #include "mail-namespace.h" |
7990
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
10 #include "solr-connection.h" |
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
11 #include "fts-solr-plugin.h" |
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
12 |
8495
ffb37c392166
Finished making full text search indexes work fast with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents:
8493
diff
changeset
|
13 #include <ctype.h> |
7990
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
14 |
8001
3fbfdda3e5d3
fts-solr: Send data to Solr in larger packets. Commit after expunges.
Timo Sirainen <tss@iki.fi>
parents:
8000
diff
changeset
|
15 #define SOLR_CMDBUF_SIZE (1024*64) |
8484
db78eab2ba5d
Initial support for getting full text search working nicely with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents:
8109
diff
changeset
|
16 #define SOLR_MAX_ROWS 100000 |
8499
252b29ac5f43
fts: Added support for handling multiple namespaces.
Timo Sirainen <tss@iki.fi>
parents:
8495
diff
changeset
|
17 #define FTS_SOLR_MAX_BOX_INC_PATTERNS 5 |
252b29ac5f43
fts: Added support for handling multiple namespaces.
Timo Sirainen <tss@iki.fi>
parents:
8495
diff
changeset
|
18 #define FTS_SOLR_MAX_BOX_EXC_PATTERNS 5 |
8001
3fbfdda3e5d3
fts-solr: Send data to Solr in larger packets. Commit after expunges.
Timo Sirainen <tss@iki.fi>
parents:
8000
diff
changeset
|
19 |
8493
149c6ccc3df4
fts-solr: Use '!' as escape character in usernames. Escape '/' and '!'.
Timo Sirainen <tss@iki.fi>
parents:
8492
diff
changeset
|
20 struct solr_fts_backend { |
149c6ccc3df4
fts-solr: Use '!' as escape character in usernames. Escape '/' and '!'.
Timo Sirainen <tss@iki.fi>
parents:
8492
diff
changeset
|
21 struct fts_backend backend; |
8950
ec1ae90af21a
fts: Fixes to how virtual mailboxes are searched.
Timo Sirainen <tss@iki.fi>
parents:
8946
diff
changeset
|
22 char *id_username, *id_namespace, *id_box_name; |
8499
252b29ac5f43
fts: Added support for handling multiple namespaces.
Timo Sirainen <tss@iki.fi>
parents:
8495
diff
changeset
|
23 struct mail_namespace *default_ns; |
8493
149c6ccc3df4
fts-solr: Use '!' as escape character in usernames. Escape '/' and '!'.
Timo Sirainen <tss@iki.fi>
parents:
8492
diff
changeset
|
24 }; |
149c6ccc3df4
fts-solr: Use '!' as escape character in usernames. Escape '/' and '!'.
Timo Sirainen <tss@iki.fi>
parents:
8492
diff
changeset
|
25 |
7990
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
26 struct solr_fts_backend_build_context { |
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
27 struct fts_backend_build_context ctx; |
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
28 |
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
29 struct solr_connection_post *post; |
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
30 uint32_t prev_uid, uid_validity; |
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
31 string_t *cmd; |
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
32 bool headers; |
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
33 }; |
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
34 |
8499
252b29ac5f43
fts: Added support for handling multiple namespaces.
Timo Sirainen <tss@iki.fi>
parents:
8495
diff
changeset
|
35 struct solr_virtual_uid_map_context { |
252b29ac5f43
fts: Added support for handling multiple namespaces.
Timo Sirainen <tss@iki.fi>
parents:
8495
diff
changeset
|
36 struct fts_backend *backend; |
252b29ac5f43
fts: Added support for handling multiple namespaces.
Timo Sirainen <tss@iki.fi>
parents:
8495
diff
changeset
|
37 struct mailbox *box; |
252b29ac5f43
fts: Added support for handling multiple namespaces.
Timo Sirainen <tss@iki.fi>
parents:
8495
diff
changeset
|
38 string_t *vname; |
252b29ac5f43
fts: Added support for handling multiple namespaces.
Timo Sirainen <tss@iki.fi>
parents:
8495
diff
changeset
|
39 }; |
252b29ac5f43
fts: Added support for handling multiple namespaces.
Timo Sirainen <tss@iki.fi>
parents:
8495
diff
changeset
|
40 |
8495
ffb37c392166
Finished making full text search indexes work fast with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents:
8493
diff
changeset
|
41 struct fts_backend_solr_get_last_uids_context { |
8499
252b29ac5f43
fts: Added support for handling multiple namespaces.
Timo Sirainen <tss@iki.fi>
parents:
8495
diff
changeset
|
42 struct fts_backend *backend; |
8495
ffb37c392166
Finished making full text search indexes work fast with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents:
8493
diff
changeset
|
43 pool_t pool; |
ffb37c392166
Finished making full text search indexes work fast with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents:
8493
diff
changeset
|
44 ARRAY_TYPE(fts_backend_uid_map) *last_uids; |
8499
252b29ac5f43
fts: Added support for handling multiple namespaces.
Timo Sirainen <tss@iki.fi>
parents:
8495
diff
changeset
|
45 |
252b29ac5f43
fts: Added support for handling multiple namespaces.
Timo Sirainen <tss@iki.fi>
parents:
8495
diff
changeset
|
46 struct mailbox *box; |
252b29ac5f43
fts: Added support for handling multiple namespaces.
Timo Sirainen <tss@iki.fi>
parents:
8495
diff
changeset
|
47 string_t *vname; |
8495
ffb37c392166
Finished making full text search indexes work fast with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents:
8493
diff
changeset
|
48 }; |
ffb37c392166
Finished making full text search indexes work fast with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents:
8493
diff
changeset
|
49 |
7990
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
50 static struct solr_connection *solr_conn = NULL; |
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
51 |
8950
ec1ae90af21a
fts: Fixes to how virtual mailboxes are searched.
Timo Sirainen <tss@iki.fi>
parents:
8946
diff
changeset
|
52 static void fts_box_name_get_root(struct mail_namespace **ns, const char **name) |
ec1ae90af21a
fts: Fixes to how virtual mailboxes are searched.
Timo Sirainen <tss@iki.fi>
parents:
8946
diff
changeset
|
53 { |
ec1ae90af21a
fts: Fixes to how virtual mailboxes are searched.
Timo Sirainen <tss@iki.fi>
parents:
8946
diff
changeset
|
54 struct mail_namespace *orig_ns = *ns; |
ec1ae90af21a
fts: Fixes to how virtual mailboxes are searched.
Timo Sirainen <tss@iki.fi>
parents:
8946
diff
changeset
|
55 |
ec1ae90af21a
fts: Fixes to how virtual mailboxes are searched.
Timo Sirainen <tss@iki.fi>
parents:
8946
diff
changeset
|
56 while ((*ns)->alias_for != NULL) |
ec1ae90af21a
fts: Fixes to how virtual mailboxes are searched.
Timo Sirainen <tss@iki.fi>
parents:
8946
diff
changeset
|
57 *ns = (*ns)->alias_for; |
ec1ae90af21a
fts: Fixes to how virtual mailboxes are searched.
Timo Sirainen <tss@iki.fi>
parents:
8946
diff
changeset
|
58 |
ec1ae90af21a
fts: Fixes to how virtual mailboxes are searched.
Timo Sirainen <tss@iki.fi>
parents:
8946
diff
changeset
|
59 if (**name == '\0' && *ns != orig_ns && |
ec1ae90af21a
fts: Fixes to how virtual mailboxes are searched.
Timo Sirainen <tss@iki.fi>
parents:
8946
diff
changeset
|
60 ((*ns)->flags & NAMESPACE_FLAG_INBOX) != 0) { |
ec1ae90af21a
fts: Fixes to how virtual mailboxes are searched.
Timo Sirainen <tss@iki.fi>
parents:
8946
diff
changeset
|
61 /* ugly workaround to allow selecting INBOX from a Maildir/ |
ec1ae90af21a
fts: Fixes to how virtual mailboxes are searched.
Timo Sirainen <tss@iki.fi>
parents:
8946
diff
changeset
|
62 when it's not in the inbox=yes namespace. */ |
ec1ae90af21a
fts: Fixes to how virtual mailboxes are searched.
Timo Sirainen <tss@iki.fi>
parents:
8946
diff
changeset
|
63 *name = "INBOX"; |
ec1ae90af21a
fts: Fixes to how virtual mailboxes are searched.
Timo Sirainen <tss@iki.fi>
parents:
8946
diff
changeset
|
64 } |
ec1ae90af21a
fts: Fixes to how virtual mailboxes are searched.
Timo Sirainen <tss@iki.fi>
parents:
8946
diff
changeset
|
65 } |
ec1ae90af21a
fts: Fixes to how virtual mailboxes are searched.
Timo Sirainen <tss@iki.fi>
parents:
8946
diff
changeset
|
66 |
ec1ae90af21a
fts: Fixes to how virtual mailboxes are searched.
Timo Sirainen <tss@iki.fi>
parents:
8946
diff
changeset
|
67 static const char * |
ec1ae90af21a
fts: Fixes to how virtual mailboxes are searched.
Timo Sirainen <tss@iki.fi>
parents:
8946
diff
changeset
|
68 fts_box_get_root(struct mailbox *box, struct mail_namespace **ns_r) |
ec1ae90af21a
fts: Fixes to how virtual mailboxes are searched.
Timo Sirainen <tss@iki.fi>
parents:
8946
diff
changeset
|
69 { |
ec1ae90af21a
fts: Fixes to how virtual mailboxes are searched.
Timo Sirainen <tss@iki.fi>
parents:
8946
diff
changeset
|
70 struct mail_namespace *ns = box->storage->ns; |
ec1ae90af21a
fts: Fixes to how virtual mailboxes are searched.
Timo Sirainen <tss@iki.fi>
parents:
8946
diff
changeset
|
71 const char *name = box->name; |
ec1ae90af21a
fts: Fixes to how virtual mailboxes are searched.
Timo Sirainen <tss@iki.fi>
parents:
8946
diff
changeset
|
72 |
ec1ae90af21a
fts: Fixes to how virtual mailboxes are searched.
Timo Sirainen <tss@iki.fi>
parents:
8946
diff
changeset
|
73 fts_box_name_get_root(&ns, &name); |
ec1ae90af21a
fts: Fixes to how virtual mailboxes are searched.
Timo Sirainen <tss@iki.fi>
parents:
8946
diff
changeset
|
74 *ns_r = ns; |
ec1ae90af21a
fts: Fixes to how virtual mailboxes are searched.
Timo Sirainen <tss@iki.fi>
parents:
8946
diff
changeset
|
75 return name; |
ec1ae90af21a
fts: Fixes to how virtual mailboxes are searched.
Timo Sirainen <tss@iki.fi>
parents:
8946
diff
changeset
|
76 } |
ec1ae90af21a
fts: Fixes to how virtual mailboxes are searched.
Timo Sirainen <tss@iki.fi>
parents:
8946
diff
changeset
|
77 |
9604
cf0da2cd31fb
fts-solr: Replace characters not valid for XML with replacement char.
Timo Sirainen <tss@iki.fi>
parents:
9532
diff
changeset
|
78 static bool is_valid_xml_char(unichar_t chr) |
cf0da2cd31fb
fts-solr: Replace characters not valid for XML with replacement char.
Timo Sirainen <tss@iki.fi>
parents:
9532
diff
changeset
|
79 { |
cf0da2cd31fb
fts-solr: Replace characters not valid for XML with replacement char.
Timo Sirainen <tss@iki.fi>
parents:
9532
diff
changeset
|
80 /* Valid characters in XML: |
cf0da2cd31fb
fts-solr: Replace characters not valid for XML with replacement char.
Timo Sirainen <tss@iki.fi>
parents:
9532
diff
changeset
|
81 |
cf0da2cd31fb
fts-solr: Replace characters not valid for XML with replacement char.
Timo Sirainen <tss@iki.fi>
parents:
9532
diff
changeset
|
82 #x9 | #xA | #xD | [#x20-#xD7FF] | [#xE000-#xFFFD] | |
cf0da2cd31fb
fts-solr: Replace characters not valid for XML with replacement char.
Timo Sirainen <tss@iki.fi>
parents:
9532
diff
changeset
|
83 [#x10000-#x10FFFF] |
cf0da2cd31fb
fts-solr: Replace characters not valid for XML with replacement char.
Timo Sirainen <tss@iki.fi>
parents:
9532
diff
changeset
|
84 |
cf0da2cd31fb
fts-solr: Replace characters not valid for XML with replacement char.
Timo Sirainen <tss@iki.fi>
parents:
9532
diff
changeset
|
85 This function gets called only for #x80 and higher */ |
cf0da2cd31fb
fts-solr: Replace characters not valid for XML with replacement char.
Timo Sirainen <tss@iki.fi>
parents:
9532
diff
changeset
|
86 if (chr > 0xd7ff && chr < 0xe000) |
cf0da2cd31fb
fts-solr: Replace characters not valid for XML with replacement char.
Timo Sirainen <tss@iki.fi>
parents:
9532
diff
changeset
|
87 return FALSE; |
cf0da2cd31fb
fts-solr: Replace characters not valid for XML with replacement char.
Timo Sirainen <tss@iki.fi>
parents:
9532
diff
changeset
|
88 if (chr > 0xfffd && chr < 0x10000) |
cf0da2cd31fb
fts-solr: Replace characters not valid for XML with replacement char.
Timo Sirainen <tss@iki.fi>
parents:
9532
diff
changeset
|
89 return FALSE; |
cf0da2cd31fb
fts-solr: Replace characters not valid for XML with replacement char.
Timo Sirainen <tss@iki.fi>
parents:
9532
diff
changeset
|
90 return chr < 0x10ffff; |
cf0da2cd31fb
fts-solr: Replace characters not valid for XML with replacement char.
Timo Sirainen <tss@iki.fi>
parents:
9532
diff
changeset
|
91 } |
cf0da2cd31fb
fts-solr: Replace characters not valid for XML with replacement char.
Timo Sirainen <tss@iki.fi>
parents:
9532
diff
changeset
|
92 |
8000
61be6a4722c5
fts-solr: Encode entities when indexing data.
Timo Sirainen <tss@iki.fi>
parents:
7999
diff
changeset
|
93 static void |
61be6a4722c5
fts-solr: Encode entities when indexing data.
Timo Sirainen <tss@iki.fi>
parents:
7999
diff
changeset
|
94 xml_encode_data(string_t *dest, const unsigned char *data, unsigned int len) |
7990
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
95 { |
9604
cf0da2cd31fb
fts-solr: Replace characters not valid for XML with replacement char.
Timo Sirainen <tss@iki.fi>
parents:
9532
diff
changeset
|
96 unichar_t chr; |
8000
61be6a4722c5
fts-solr: Encode entities when indexing data.
Timo Sirainen <tss@iki.fi>
parents:
7999
diff
changeset
|
97 unsigned int i; |
61be6a4722c5
fts-solr: Encode entities when indexing data.
Timo Sirainen <tss@iki.fi>
parents:
7999
diff
changeset
|
98 |
61be6a4722c5
fts-solr: Encode entities when indexing data.
Timo Sirainen <tss@iki.fi>
parents:
7999
diff
changeset
|
99 for (i = 0; i < len; i++) { |
61be6a4722c5
fts-solr: Encode entities when indexing data.
Timo Sirainen <tss@iki.fi>
parents:
7999
diff
changeset
|
100 switch (data[i]) { |
7990
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
101 case '&': |
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
102 str_append(dest, "&"); |
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
103 break; |
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
104 case '<': |
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
105 str_append(dest, "<"); |
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
106 break; |
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
107 case '>': |
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
108 str_append(dest, ">"); |
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
109 break; |
8848
dd441977c578
fts-solr: Solr breaks when it sees control characters, so replace them with spaces.
Timo Sirainen <tss@iki.fi>
parents:
8590
diff
changeset
|
110 case '\t': |
dd441977c578
fts-solr: Solr breaks when it sees control characters, so replace them with spaces.
Timo Sirainen <tss@iki.fi>
parents:
8590
diff
changeset
|
111 case '\n': |
dd441977c578
fts-solr: Solr breaks when it sees control characters, so replace them with spaces.
Timo Sirainen <tss@iki.fi>
parents:
8590
diff
changeset
|
112 case '\r': |
dd441977c578
fts-solr: Solr breaks when it sees control characters, so replace them with spaces.
Timo Sirainen <tss@iki.fi>
parents:
8590
diff
changeset
|
113 /* exceptions to the following control char check */ |
dd441977c578
fts-solr: Solr breaks when it sees control characters, so replace them with spaces.
Timo Sirainen <tss@iki.fi>
parents:
8590
diff
changeset
|
114 str_append_c(dest, data[i]); |
dd441977c578
fts-solr: Solr breaks when it sees control characters, so replace them with spaces.
Timo Sirainen <tss@iki.fi>
parents:
8590
diff
changeset
|
115 break; |
7990
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
116 default: |
8848
dd441977c578
fts-solr: Solr breaks when it sees control characters, so replace them with spaces.
Timo Sirainen <tss@iki.fi>
parents:
8590
diff
changeset
|
117 if (data[i] < 32) { |
dd441977c578
fts-solr: Solr breaks when it sees control characters, so replace them with spaces.
Timo Sirainen <tss@iki.fi>
parents:
8590
diff
changeset
|
118 /* SOLR doesn't like control characters. |
dd441977c578
fts-solr: Solr breaks when it sees control characters, so replace them with spaces.
Timo Sirainen <tss@iki.fi>
parents:
8590
diff
changeset
|
119 replace them with spaces. */ |
dd441977c578
fts-solr: Solr breaks when it sees control characters, so replace them with spaces.
Timo Sirainen <tss@iki.fi>
parents:
8590
diff
changeset
|
120 str_append_c(dest, ' '); |
9604
cf0da2cd31fb
fts-solr: Replace characters not valid for XML with replacement char.
Timo Sirainen <tss@iki.fi>
parents:
9532
diff
changeset
|
121 } else if (data[i] >= 0x80) { |
cf0da2cd31fb
fts-solr: Replace characters not valid for XML with replacement char.
Timo Sirainen <tss@iki.fi>
parents:
9532
diff
changeset
|
122 /* make sure the character is valid for XML |
cf0da2cd31fb
fts-solr: Replace characters not valid for XML with replacement char.
Timo Sirainen <tss@iki.fi>
parents:
9532
diff
changeset
|
123 so we don't get XML parser errors */ |
cf0da2cd31fb
fts-solr: Replace characters not valid for XML with replacement char.
Timo Sirainen <tss@iki.fi>
parents:
9532
diff
changeset
|
124 unsigned int char_len = |
9606
ac0855b1bbea
fts-solr: Fixes to previous data encoding changes.
Timo Sirainen <tss@iki.fi>
parents:
9604
diff
changeset
|
125 uni_utf8_char_bytes(data[i]); |
9604
cf0da2cd31fb
fts-solr: Replace characters not valid for XML with replacement char.
Timo Sirainen <tss@iki.fi>
parents:
9532
diff
changeset
|
126 if (i + char_len <= len && |
9606
ac0855b1bbea
fts-solr: Fixes to previous data encoding changes.
Timo Sirainen <tss@iki.fi>
parents:
9604
diff
changeset
|
127 uni_utf8_get_char_n(data + i, char_len, &chr) == 1 && |
9604
cf0da2cd31fb
fts-solr: Replace characters not valid for XML with replacement char.
Timo Sirainen <tss@iki.fi>
parents:
9532
diff
changeset
|
128 is_valid_xml_char(chr)) |
cf0da2cd31fb
fts-solr: Replace characters not valid for XML with replacement char.
Timo Sirainen <tss@iki.fi>
parents:
9532
diff
changeset
|
129 str_append_n(dest, data + i, char_len); |
cf0da2cd31fb
fts-solr: Replace characters not valid for XML with replacement char.
Timo Sirainen <tss@iki.fi>
parents:
9532
diff
changeset
|
130 else { |
cf0da2cd31fb
fts-solr: Replace characters not valid for XML with replacement char.
Timo Sirainen <tss@iki.fi>
parents:
9532
diff
changeset
|
131 str_append_n(dest, utf8_replacement_char, |
cf0da2cd31fb
fts-solr: Replace characters not valid for XML with replacement char.
Timo Sirainen <tss@iki.fi>
parents:
9532
diff
changeset
|
132 UTF8_REPLACEMENT_CHAR_LEN); |
cf0da2cd31fb
fts-solr: Replace characters not valid for XML with replacement char.
Timo Sirainen <tss@iki.fi>
parents:
9532
diff
changeset
|
133 } |
cf0da2cd31fb
fts-solr: Replace characters not valid for XML with replacement char.
Timo Sirainen <tss@iki.fi>
parents:
9532
diff
changeset
|
134 i += char_len - 1; |
8848
dd441977c578
fts-solr: Solr breaks when it sees control characters, so replace them with spaces.
Timo Sirainen <tss@iki.fi>
parents:
8590
diff
changeset
|
135 } else { |
dd441977c578
fts-solr: Solr breaks when it sees control characters, so replace them with spaces.
Timo Sirainen <tss@iki.fi>
parents:
8590
diff
changeset
|
136 str_append_c(dest, data[i]); |
dd441977c578
fts-solr: Solr breaks when it sees control characters, so replace them with spaces.
Timo Sirainen <tss@iki.fi>
parents:
8590
diff
changeset
|
137 } |
7990
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
138 break; |
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
139 } |
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
140 } |
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
141 } |
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
142 |
8000
61be6a4722c5
fts-solr: Encode entities when indexing data.
Timo Sirainen <tss@iki.fi>
parents:
7999
diff
changeset
|
143 static void xml_encode(string_t *dest, const char *str) |
61be6a4722c5
fts-solr: Encode entities when indexing data.
Timo Sirainen <tss@iki.fi>
parents:
7999
diff
changeset
|
144 { |
61be6a4722c5
fts-solr: Encode entities when indexing data.
Timo Sirainen <tss@iki.fi>
parents:
7999
diff
changeset
|
145 xml_encode_data(dest, (const unsigned char *)str, strlen(str)); |
61be6a4722c5
fts-solr: Encode entities when indexing data.
Timo Sirainen <tss@iki.fi>
parents:
7999
diff
changeset
|
146 } |
61be6a4722c5
fts-solr: Encode entities when indexing data.
Timo Sirainen <tss@iki.fi>
parents:
7999
diff
changeset
|
147 |
8493
149c6ccc3df4
fts-solr: Use '!' as escape character in usernames. Escape '/' and '!'.
Timo Sirainen <tss@iki.fi>
parents:
8492
diff
changeset
|
148 static const char *solr_escape_id_str(const char *str) |
149c6ccc3df4
fts-solr: Use '!' as escape character in usernames. Escape '/' and '!'.
Timo Sirainen <tss@iki.fi>
parents:
8492
diff
changeset
|
149 { |
149c6ccc3df4
fts-solr: Use '!' as escape character in usernames. Escape '/' and '!'.
Timo Sirainen <tss@iki.fi>
parents:
8492
diff
changeset
|
150 string_t *tmp; |
149c6ccc3df4
fts-solr: Use '!' as escape character in usernames. Escape '/' and '!'.
Timo Sirainen <tss@iki.fi>
parents:
8492
diff
changeset
|
151 const char *p; |
149c6ccc3df4
fts-solr: Use '!' as escape character in usernames. Escape '/' and '!'.
Timo Sirainen <tss@iki.fi>
parents:
8492
diff
changeset
|
152 |
149c6ccc3df4
fts-solr: Use '!' as escape character in usernames. Escape '/' and '!'.
Timo Sirainen <tss@iki.fi>
parents:
8492
diff
changeset
|
153 for (p = str; *p != '\0'; p++) { |
149c6ccc3df4
fts-solr: Use '!' as escape character in usernames. Escape '/' and '!'.
Timo Sirainen <tss@iki.fi>
parents:
8492
diff
changeset
|
154 if (*p == '/' || *p == '!') |
149c6ccc3df4
fts-solr: Use '!' as escape character in usernames. Escape '/' and '!'.
Timo Sirainen <tss@iki.fi>
parents:
8492
diff
changeset
|
155 break; |
149c6ccc3df4
fts-solr: Use '!' as escape character in usernames. Escape '/' and '!'.
Timo Sirainen <tss@iki.fi>
parents:
8492
diff
changeset
|
156 } |
149c6ccc3df4
fts-solr: Use '!' as escape character in usernames. Escape '/' and '!'.
Timo Sirainen <tss@iki.fi>
parents:
8492
diff
changeset
|
157 if (*p == '\0') |
149c6ccc3df4
fts-solr: Use '!' as escape character in usernames. Escape '/' and '!'.
Timo Sirainen <tss@iki.fi>
parents:
8492
diff
changeset
|
158 return str; |
149c6ccc3df4
fts-solr: Use '!' as escape character in usernames. Escape '/' and '!'.
Timo Sirainen <tss@iki.fi>
parents:
8492
diff
changeset
|
159 |
149c6ccc3df4
fts-solr: Use '!' as escape character in usernames. Escape '/' and '!'.
Timo Sirainen <tss@iki.fi>
parents:
8492
diff
changeset
|
160 tmp = t_str_new(64); |
149c6ccc3df4
fts-solr: Use '!' as escape character in usernames. Escape '/' and '!'.
Timo Sirainen <tss@iki.fi>
parents:
8492
diff
changeset
|
161 for (p = str; *p != '\0'; p++) { |
149c6ccc3df4
fts-solr: Use '!' as escape character in usernames. Escape '/' and '!'.
Timo Sirainen <tss@iki.fi>
parents:
8492
diff
changeset
|
162 switch (*p) { |
149c6ccc3df4
fts-solr: Use '!' as escape character in usernames. Escape '/' and '!'.
Timo Sirainen <tss@iki.fi>
parents:
8492
diff
changeset
|
163 case '/': |
149c6ccc3df4
fts-solr: Use '!' as escape character in usernames. Escape '/' and '!'.
Timo Sirainen <tss@iki.fi>
parents:
8492
diff
changeset
|
164 str_append(tmp, "!\\"); |
149c6ccc3df4
fts-solr: Use '!' as escape character in usernames. Escape '/' and '!'.
Timo Sirainen <tss@iki.fi>
parents:
8492
diff
changeset
|
165 break; |
149c6ccc3df4
fts-solr: Use '!' as escape character in usernames. Escape '/' and '!'.
Timo Sirainen <tss@iki.fi>
parents:
8492
diff
changeset
|
166 case '!': |
149c6ccc3df4
fts-solr: Use '!' as escape character in usernames. Escape '/' and '!'.
Timo Sirainen <tss@iki.fi>
parents:
8492
diff
changeset
|
167 str_append(tmp, "!!"); |
149c6ccc3df4
fts-solr: Use '!' as escape character in usernames. Escape '/' and '!'.
Timo Sirainen <tss@iki.fi>
parents:
8492
diff
changeset
|
168 break; |
149c6ccc3df4
fts-solr: Use '!' as escape character in usernames. Escape '/' and '!'.
Timo Sirainen <tss@iki.fi>
parents:
8492
diff
changeset
|
169 default: |
149c6ccc3df4
fts-solr: Use '!' as escape character in usernames. Escape '/' and '!'.
Timo Sirainen <tss@iki.fi>
parents:
8492
diff
changeset
|
170 str_append_c(tmp, *p); |
149c6ccc3df4
fts-solr: Use '!' as escape character in usernames. Escape '/' and '!'.
Timo Sirainen <tss@iki.fi>
parents:
8492
diff
changeset
|
171 break; |
149c6ccc3df4
fts-solr: Use '!' as escape character in usernames. Escape '/' and '!'.
Timo Sirainen <tss@iki.fi>
parents:
8492
diff
changeset
|
172 } |
149c6ccc3df4
fts-solr: Use '!' as escape character in usernames. Escape '/' and '!'.
Timo Sirainen <tss@iki.fi>
parents:
8492
diff
changeset
|
173 } |
149c6ccc3df4
fts-solr: Use '!' as escape character in usernames. Escape '/' and '!'.
Timo Sirainen <tss@iki.fi>
parents:
8492
diff
changeset
|
174 return str_c(tmp); |
149c6ccc3df4
fts-solr: Use '!' as escape character in usernames. Escape '/' and '!'.
Timo Sirainen <tss@iki.fi>
parents:
8492
diff
changeset
|
175 } |
149c6ccc3df4
fts-solr: Use '!' as escape character in usernames. Escape '/' and '!'.
Timo Sirainen <tss@iki.fi>
parents:
8492
diff
changeset
|
176 |
8499
252b29ac5f43
fts: Added support for handling multiple namespaces.
Timo Sirainen <tss@iki.fi>
parents:
8495
diff
changeset
|
177 static void solr_quote(string_t *dest, const char *str) |
252b29ac5f43
fts: Added support for handling multiple namespaces.
Timo Sirainen <tss@iki.fi>
parents:
8495
diff
changeset
|
178 { |
252b29ac5f43
fts: Added support for handling multiple namespaces.
Timo Sirainen <tss@iki.fi>
parents:
8495
diff
changeset
|
179 str_append_c(dest, '"'); |
252b29ac5f43
fts: Added support for handling multiple namespaces.
Timo Sirainen <tss@iki.fi>
parents:
8495
diff
changeset
|
180 str_append(dest, str_escape(str)); |
252b29ac5f43
fts: Added support for handling multiple namespaces.
Timo Sirainen <tss@iki.fi>
parents:
8495
diff
changeset
|
181 str_append_c(dest, '"'); |
252b29ac5f43
fts: Added support for handling multiple namespaces.
Timo Sirainen <tss@iki.fi>
parents:
8495
diff
changeset
|
182 } |
252b29ac5f43
fts: Added support for handling multiple namespaces.
Timo Sirainen <tss@iki.fi>
parents:
8495
diff
changeset
|
183 |
252b29ac5f43
fts: Added support for handling multiple namespaces.
Timo Sirainen <tss@iki.fi>
parents:
8495
diff
changeset
|
184 static void solr_quote_http(string_t *dest, const char *str) |
252b29ac5f43
fts: Added support for handling multiple namespaces.
Timo Sirainen <tss@iki.fi>
parents:
8495
diff
changeset
|
185 { |
252b29ac5f43
fts: Added support for handling multiple namespaces.
Timo Sirainen <tss@iki.fi>
parents:
8495
diff
changeset
|
186 str_append(dest, "%22"); |
252b29ac5f43
fts: Added support for handling multiple namespaces.
Timo Sirainen <tss@iki.fi>
parents:
8495
diff
changeset
|
187 solr_connection_http_escape(solr_conn, dest, str); |
252b29ac5f43
fts: Added support for handling multiple namespaces.
Timo Sirainen <tss@iki.fi>
parents:
8495
diff
changeset
|
188 str_append(dest, "%22"); |
252b29ac5f43
fts: Added support for handling multiple namespaces.
Timo Sirainen <tss@iki.fi>
parents:
8495
diff
changeset
|
189 } |
252b29ac5f43
fts: Added support for handling multiple namespaces.
Timo Sirainen <tss@iki.fi>
parents:
8495
diff
changeset
|
190 |
7990
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
191 static struct fts_backend * |
8493
149c6ccc3df4
fts-solr: Use '!' as escape character in usernames. Escape '/' and '!'.
Timo Sirainen <tss@iki.fi>
parents:
8492
diff
changeset
|
192 fts_backend_solr_init(struct mailbox *box) |
7990
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
193 { |
8002
e814614ffb65
fts-solr: Added break-imap-search setting to use Solr with SEARCH TEXT/BODY
Timo Sirainen <tss@iki.fi>
parents:
8001
diff
changeset
|
194 const struct fts_solr_settings *set = &fts_solr_settings; |
8493
149c6ccc3df4
fts-solr: Use '!' as escape character in usernames. Escape '/' and '!'.
Timo Sirainen <tss@iki.fi>
parents:
8492
diff
changeset
|
195 struct solr_fts_backend *backend; |
8950
ec1ae90af21a
fts: Fixes to how virtual mailboxes are searched.
Timo Sirainen <tss@iki.fi>
parents:
8946
diff
changeset
|
196 struct mail_namespace *ns; |
ec1ae90af21a
fts: Fixes to how virtual mailboxes are searched.
Timo Sirainen <tss@iki.fi>
parents:
8946
diff
changeset
|
197 const char *str, *box_name; |
7990
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
198 |
8950
ec1ae90af21a
fts: Fixes to how virtual mailboxes are searched.
Timo Sirainen <tss@iki.fi>
parents:
8946
diff
changeset
|
199 |
ec1ae90af21a
fts: Fixes to how virtual mailboxes are searched.
Timo Sirainen <tss@iki.fi>
parents:
8946
diff
changeset
|
200 box_name = fts_box_get_root(box, &ns); |
ec1ae90af21a
fts: Fixes to how virtual mailboxes are searched.
Timo Sirainen <tss@iki.fi>
parents:
8946
diff
changeset
|
201 i_assert(*box_name != '\0'); |
8500
3efcdc45d111
Added alias_for setting for namespaces. Fixes namespace issues with fts.
Timo Sirainen <tss@iki.fi>
parents:
8499
diff
changeset
|
202 |
7990
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
203 if (solr_conn == NULL) |
8002
e814614ffb65
fts-solr: Added break-imap-search setting to use Solr with SEARCH TEXT/BODY
Timo Sirainen <tss@iki.fi>
parents:
8001
diff
changeset
|
204 solr_conn = solr_connection_init(set->url, set->debug); |
7990
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
205 |
8493
149c6ccc3df4
fts-solr: Use '!' as escape character in usernames. Escape '/' and '!'.
Timo Sirainen <tss@iki.fi>
parents:
8492
diff
changeset
|
206 backend = i_new(struct solr_fts_backend, 1); |
8499
252b29ac5f43
fts: Added support for handling multiple namespaces.
Timo Sirainen <tss@iki.fi>
parents:
8495
diff
changeset
|
207 str = fts_solr_settings.default_ns_prefix; |
252b29ac5f43
fts: Added support for handling multiple namespaces.
Timo Sirainen <tss@iki.fi>
parents:
8495
diff
changeset
|
208 if (str != NULL) { |
252b29ac5f43
fts: Added support for handling multiple namespaces.
Timo Sirainen <tss@iki.fi>
parents:
8495
diff
changeset
|
209 backend->default_ns = |
252b29ac5f43
fts: Added support for handling multiple namespaces.
Timo Sirainen <tss@iki.fi>
parents:
8495
diff
changeset
|
210 mail_namespace_find_prefix(ns->user->namespaces, str); |
252b29ac5f43
fts: Added support for handling multiple namespaces.
Timo Sirainen <tss@iki.fi>
parents:
8495
diff
changeset
|
211 if (backend->default_ns == NULL) { |
252b29ac5f43
fts: Added support for handling multiple namespaces.
Timo Sirainen <tss@iki.fi>
parents:
8495
diff
changeset
|
212 i_fatal("fts_solr: default_ns setting points to " |
252b29ac5f43
fts: Added support for handling multiple namespaces.
Timo Sirainen <tss@iki.fi>
parents:
8495
diff
changeset
|
213 "nonexisting namespace"); |
252b29ac5f43
fts: Added support for handling multiple namespaces.
Timo Sirainen <tss@iki.fi>
parents:
8495
diff
changeset
|
214 } |
252b29ac5f43
fts: Added support for handling multiple namespaces.
Timo Sirainen <tss@iki.fi>
parents:
8495
diff
changeset
|
215 } else { |
252b29ac5f43
fts: Added support for handling multiple namespaces.
Timo Sirainen <tss@iki.fi>
parents:
8495
diff
changeset
|
216 backend->default_ns = |
252b29ac5f43
fts: Added support for handling multiple namespaces.
Timo Sirainen <tss@iki.fi>
parents:
8495
diff
changeset
|
217 mail_namespace_find_inbox(ns->user->namespaces); |
252b29ac5f43
fts: Added support for handling multiple namespaces.
Timo Sirainen <tss@iki.fi>
parents:
8495
diff
changeset
|
218 } |
8500
3efcdc45d111
Added alias_for setting for namespaces. Fixes namespace issues with fts.
Timo Sirainen <tss@iki.fi>
parents:
8499
diff
changeset
|
219 while (backend->default_ns->alias_for != NULL) |
3efcdc45d111
Added alias_for setting for namespaces. Fixes namespace issues with fts.
Timo Sirainen <tss@iki.fi>
parents:
8499
diff
changeset
|
220 backend->default_ns = backend->default_ns->alias_for; |
3efcdc45d111
Added alias_for setting for namespaces. Fixes namespace issues with fts.
Timo Sirainen <tss@iki.fi>
parents:
8499
diff
changeset
|
221 |
8499
252b29ac5f43
fts: Added support for handling multiple namespaces.
Timo Sirainen <tss@iki.fi>
parents:
8495
diff
changeset
|
222 str = solr_escape_id_str(ns->user->username); |
252b29ac5f43
fts: Added support for handling multiple namespaces.
Timo Sirainen <tss@iki.fi>
parents:
8495
diff
changeset
|
223 backend->id_username = i_strdup(str); |
8500
3efcdc45d111
Added alias_for setting for namespaces. Fixes namespace issues with fts.
Timo Sirainen <tss@iki.fi>
parents:
8499
diff
changeset
|
224 if (ns != backend->default_ns) { |
8499
252b29ac5f43
fts: Added support for handling multiple namespaces.
Timo Sirainen <tss@iki.fi>
parents:
8495
diff
changeset
|
225 str = solr_escape_id_str(ns->prefix); |
252b29ac5f43
fts: Added support for handling multiple namespaces.
Timo Sirainen <tss@iki.fi>
parents:
8495
diff
changeset
|
226 backend->id_namespace = i_strdup(str); |
252b29ac5f43
fts: Added support for handling multiple namespaces.
Timo Sirainen <tss@iki.fi>
parents:
8495
diff
changeset
|
227 } |
8950
ec1ae90af21a
fts: Fixes to how virtual mailboxes are searched.
Timo Sirainen <tss@iki.fi>
parents:
8946
diff
changeset
|
228 backend->id_box_name = i_strdup(box_name); |
8493
149c6ccc3df4
fts-solr: Use '!' as escape character in usernames. Escape '/' and '!'.
Timo Sirainen <tss@iki.fi>
parents:
8492
diff
changeset
|
229 backend->backend = fts_backend_solr; |
8002
e814614ffb65
fts-solr: Added break-imap-search setting to use Solr with SEARCH TEXT/BODY
Timo Sirainen <tss@iki.fi>
parents:
8001
diff
changeset
|
230 |
e814614ffb65
fts-solr: Added break-imap-search setting to use Solr with SEARCH TEXT/BODY
Timo Sirainen <tss@iki.fi>
parents:
8001
diff
changeset
|
231 if (set->substring_search) |
8493
149c6ccc3df4
fts-solr: Use '!' as escape character in usernames. Escape '/' and '!'.
Timo Sirainen <tss@iki.fi>
parents:
8492
diff
changeset
|
232 backend->backend.flags |= FTS_BACKEND_FLAG_SUBSTRING_LOOKUPS; |
149c6ccc3df4
fts-solr: Use '!' as escape character in usernames. Escape '/' and '!'.
Timo Sirainen <tss@iki.fi>
parents:
8492
diff
changeset
|
233 return &backend->backend; |
7990
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
234 } |
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
235 |
8493
149c6ccc3df4
fts-solr: Use '!' as escape character in usernames. Escape '/' and '!'.
Timo Sirainen <tss@iki.fi>
parents:
8492
diff
changeset
|
236 static void fts_backend_solr_deinit(struct fts_backend *_backend) |
7990
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
237 { |
8493
149c6ccc3df4
fts-solr: Use '!' as escape character in usernames. Escape '/' and '!'.
Timo Sirainen <tss@iki.fi>
parents:
8492
diff
changeset
|
238 struct solr_fts_backend *backend = (struct solr_fts_backend *)_backend; |
149c6ccc3df4
fts-solr: Use '!' as escape character in usernames. Escape '/' and '!'.
Timo Sirainen <tss@iki.fi>
parents:
8492
diff
changeset
|
239 |
8950
ec1ae90af21a
fts: Fixes to how virtual mailboxes are searched.
Timo Sirainen <tss@iki.fi>
parents:
8946
diff
changeset
|
240 i_free(backend->id_box_name); |
8499
252b29ac5f43
fts: Added support for handling multiple namespaces.
Timo Sirainen <tss@iki.fi>
parents:
8495
diff
changeset
|
241 i_free(backend->id_namespace); |
8493
149c6ccc3df4
fts-solr: Use '!' as escape character in usernames. Escape '/' and '!'.
Timo Sirainen <tss@iki.fi>
parents:
8492
diff
changeset
|
242 i_free(backend->id_username); |
7990
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
243 i_free(backend); |
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
244 } |
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
245 |
8499
252b29ac5f43
fts: Added support for handling multiple namespaces.
Timo Sirainen <tss@iki.fi>
parents:
8495
diff
changeset
|
246 static void |
252b29ac5f43
fts: Added support for handling multiple namespaces.
Timo Sirainen <tss@iki.fi>
parents:
8495
diff
changeset
|
247 solr_add_ns_query(string_t *str, struct fts_backend *_backend, |
8945
43b68bd6023a
fts-solr: Fixes with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents:
8879
diff
changeset
|
248 struct mail_namespace *ns, bool neg) |
8493
149c6ccc3df4
fts-solr: Use '!' as escape character in usernames. Escape '/' and '!'.
Timo Sirainen <tss@iki.fi>
parents:
8492
diff
changeset
|
249 { |
149c6ccc3df4
fts-solr: Use '!' as escape character in usernames. Escape '/' and '!'.
Timo Sirainen <tss@iki.fi>
parents:
8492
diff
changeset
|
250 struct solr_fts_backend *backend = (struct solr_fts_backend *)_backend; |
149c6ccc3df4
fts-solr: Use '!' as escape character in usernames. Escape '/' and '!'.
Timo Sirainen <tss@iki.fi>
parents:
8492
diff
changeset
|
251 |
8500
3efcdc45d111
Added alias_for setting for namespaces. Fixes namespace issues with fts.
Timo Sirainen <tss@iki.fi>
parents:
8499
diff
changeset
|
252 while (ns->alias_for != NULL) |
3efcdc45d111
Added alias_for setting for namespaces. Fixes namespace issues with fts.
Timo Sirainen <tss@iki.fi>
parents:
8499
diff
changeset
|
253 ns = ns->alias_for; |
3efcdc45d111
Added alias_for setting for namespaces. Fixes namespace issues with fts.
Timo Sirainen <tss@iki.fi>
parents:
8499
diff
changeset
|
254 |
8945
43b68bd6023a
fts-solr: Fixes with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents:
8879
diff
changeset
|
255 if (ns == backend->default_ns || *ns->prefix == '\0') { |
43b68bd6023a
fts-solr: Fixes with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents:
8879
diff
changeset
|
256 if (!neg) |
43b68bd6023a
fts-solr: Fixes with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents:
8879
diff
changeset
|
257 str_append(str, " -ns:[* TO *]"); |
43b68bd6023a
fts-solr: Fixes with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents:
8879
diff
changeset
|
258 else |
43b68bd6023a
fts-solr: Fixes with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents:
8879
diff
changeset
|
259 str_append(str, " +ns:[* TO *]"); |
43b68bd6023a
fts-solr: Fixes with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents:
8879
diff
changeset
|
260 } else { |
43b68bd6023a
fts-solr: Fixes with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents:
8879
diff
changeset
|
261 if (!neg) |
43b68bd6023a
fts-solr: Fixes with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents:
8879
diff
changeset
|
262 str_append(str, " +ns:"); |
43b68bd6023a
fts-solr: Fixes with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents:
8879
diff
changeset
|
263 else |
43b68bd6023a
fts-solr: Fixes with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents:
8879
diff
changeset
|
264 str_append(str, " -ns:"); |
8499
252b29ac5f43
fts: Added support for handling multiple namespaces.
Timo Sirainen <tss@iki.fi>
parents:
8495
diff
changeset
|
265 solr_quote(str, ns->prefix); |
252b29ac5f43
fts: Added support for handling multiple namespaces.
Timo Sirainen <tss@iki.fi>
parents:
8495
diff
changeset
|
266 } |
252b29ac5f43
fts: Added support for handling multiple namespaces.
Timo Sirainen <tss@iki.fi>
parents:
8495
diff
changeset
|
267 } |
252b29ac5f43
fts: Added support for handling multiple namespaces.
Timo Sirainen <tss@iki.fi>
parents:
8495
diff
changeset
|
268 |
252b29ac5f43
fts: Added support for handling multiple namespaces.
Timo Sirainen <tss@iki.fi>
parents:
8495
diff
changeset
|
269 static void |
252b29ac5f43
fts: Added support for handling multiple namespaces.
Timo Sirainen <tss@iki.fi>
parents:
8495
diff
changeset
|
270 solr_add_ns_query_http(string_t *str, struct fts_backend *backend, |
252b29ac5f43
fts: Added support for handling multiple namespaces.
Timo Sirainen <tss@iki.fi>
parents:
8495
diff
changeset
|
271 struct mail_namespace *ns) |
252b29ac5f43
fts: Added support for handling multiple namespaces.
Timo Sirainen <tss@iki.fi>
parents:
8495
diff
changeset
|
272 { |
252b29ac5f43
fts: Added support for handling multiple namespaces.
Timo Sirainen <tss@iki.fi>
parents:
8495
diff
changeset
|
273 string_t *tmp; |
252b29ac5f43
fts: Added support for handling multiple namespaces.
Timo Sirainen <tss@iki.fi>
parents:
8495
diff
changeset
|
274 |
252b29ac5f43
fts: Added support for handling multiple namespaces.
Timo Sirainen <tss@iki.fi>
parents:
8495
diff
changeset
|
275 tmp = t_str_new(64); |
8945
43b68bd6023a
fts-solr: Fixes with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents:
8879
diff
changeset
|
276 solr_add_ns_query(tmp, backend, ns, FALSE); |
8499
252b29ac5f43
fts: Added support for handling multiple namespaces.
Timo Sirainen <tss@iki.fi>
parents:
8495
diff
changeset
|
277 solr_connection_http_escape(solr_conn, str, str_c(tmp)); |
8493
149c6ccc3df4
fts-solr: Use '!' as escape character in usernames. Escape '/' and '!'.
Timo Sirainen <tss@iki.fi>
parents:
8492
diff
changeset
|
278 } |
149c6ccc3df4
fts-solr: Use '!' as escape character in usernames. Escape '/' and '!'.
Timo Sirainen <tss@iki.fi>
parents:
8492
diff
changeset
|
279 |
8492
ee1ecdda8a6b
fts-solr: Keep last_uid field updated containing the mailbox's last added UID.
Timo Sirainen <tss@iki.fi>
parents:
8484
diff
changeset
|
280 static int fts_backend_solr_get_last_uid_fallback(struct fts_backend *backend, |
ee1ecdda8a6b
fts-solr: Keep last_uid field updated containing the mailbox's last added UID.
Timo Sirainen <tss@iki.fi>
parents:
8484
diff
changeset
|
281 uint32_t *last_uid_r) |
7990
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
282 { |
8499
252b29ac5f43
fts: Added support for handling multiple namespaces.
Timo Sirainen <tss@iki.fi>
parents:
8495
diff
changeset
|
283 struct mailbox *box = backend->box; |
8950
ec1ae90af21a
fts: Fixes to how virtual mailboxes are searched.
Timo Sirainen <tss@iki.fi>
parents:
8946
diff
changeset
|
284 struct mail_namespace *ns; |
7990
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
285 struct mailbox_status status; |
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
286 ARRAY_TYPE(seq_range) uids; |
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
287 const struct seq_range *uidvals; |
8950
ec1ae90af21a
fts: Fixes to how virtual mailboxes are searched.
Timo Sirainen <tss@iki.fi>
parents:
8946
diff
changeset
|
288 const char *box_name; |
7990
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
289 unsigned int count; |
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
290 string_t *str; |
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
291 |
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
292 str = t_str_new(256); |
8499
252b29ac5f43
fts: Added support for handling multiple namespaces.
Timo Sirainen <tss@iki.fi>
parents:
8495
diff
changeset
|
293 str_append(str, "fl=uid&rows=1&sort=uid+desc&q="); |
7990
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
294 |
8950
ec1ae90af21a
fts: Fixes to how virtual mailboxes are searched.
Timo Sirainen <tss@iki.fi>
parents:
8946
diff
changeset
|
295 box_name = fts_box_get_root(box, &ns); |
ec1ae90af21a
fts: Fixes to how virtual mailboxes are searched.
Timo Sirainen <tss@iki.fi>
parents:
8946
diff
changeset
|
296 |
8499
252b29ac5f43
fts: Added support for handling multiple namespaces.
Timo Sirainen <tss@iki.fi>
parents:
8495
diff
changeset
|
297 mailbox_get_status(box, STATUS_UIDVALIDITY, &status); |
252b29ac5f43
fts: Added support for handling multiple namespaces.
Timo Sirainen <tss@iki.fi>
parents:
8495
diff
changeset
|
298 str_printfa(str, "uidv:%u+box:", status.uidvalidity); |
8950
ec1ae90af21a
fts: Fixes to how virtual mailboxes are searched.
Timo Sirainen <tss@iki.fi>
parents:
8946
diff
changeset
|
299 solr_quote_http(str, box_name); |
ec1ae90af21a
fts: Fixes to how virtual mailboxes are searched.
Timo Sirainen <tss@iki.fi>
parents:
8946
diff
changeset
|
300 solr_add_ns_query_http(str, backend, ns); |
8499
252b29ac5f43
fts: Added support for handling multiple namespaces.
Timo Sirainen <tss@iki.fi>
parents:
8495
diff
changeset
|
301 str_append(str, "+user:"); |
8950
ec1ae90af21a
fts: Fixes to how virtual mailboxes are searched.
Timo Sirainen <tss@iki.fi>
parents:
8946
diff
changeset
|
302 solr_quote_http(str, ns->user->username); |
7990
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
303 |
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
304 t_array_init(&uids, 1); |
8484
db78eab2ba5d
Initial support for getting full text search working nicely with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents:
8109
diff
changeset
|
305 if (solr_connection_select(solr_conn, str_c(str), |
db78eab2ba5d
Initial support for getting full text search working nicely with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents:
8109
diff
changeset
|
306 NULL, NULL, &uids, NULL) < 0) |
7990
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
307 return -1; |
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
308 |
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
309 uidvals = array_get(&uids, &count); |
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
310 if (count == 0) { |
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
311 /* nothing indexed yet for this mailbox */ |
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
312 *last_uid_r = 0; |
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
313 } else if (count == 1 && uidvals[0].seq1 == uidvals[0].seq2) { |
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
314 *last_uid_r = uidvals[0].seq1; |
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
315 } else { |
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
316 i_error("fts_solr: Last UID lookup returned multiple rows"); |
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
317 return -1; |
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
318 } |
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
319 return 0; |
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
320 } |
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
321 |
8492
ee1ecdda8a6b
fts-solr: Keep last_uid field updated containing the mailbox's last added UID.
Timo Sirainen <tss@iki.fi>
parents:
8484
diff
changeset
|
322 static int fts_backend_solr_get_last_uid(struct fts_backend *backend, |
ee1ecdda8a6b
fts-solr: Keep last_uid field updated containing the mailbox's last added UID.
Timo Sirainen <tss@iki.fi>
parents:
8484
diff
changeset
|
323 uint32_t *last_uid_r) |
ee1ecdda8a6b
fts-solr: Keep last_uid field updated containing the mailbox's last added UID.
Timo Sirainen <tss@iki.fi>
parents:
8484
diff
changeset
|
324 { |
8499
252b29ac5f43
fts: Added support for handling multiple namespaces.
Timo Sirainen <tss@iki.fi>
parents:
8495
diff
changeset
|
325 struct mailbox *box = backend->box; |
8950
ec1ae90af21a
fts: Fixes to how virtual mailboxes are searched.
Timo Sirainen <tss@iki.fi>
parents:
8946
diff
changeset
|
326 struct mail_namespace *ns; |
8492
ee1ecdda8a6b
fts-solr: Keep last_uid field updated containing the mailbox's last added UID.
Timo Sirainen <tss@iki.fi>
parents:
8484
diff
changeset
|
327 struct mailbox_status status; |
ee1ecdda8a6b
fts-solr: Keep last_uid field updated containing the mailbox's last added UID.
Timo Sirainen <tss@iki.fi>
parents:
8484
diff
changeset
|
328 ARRAY_TYPE(seq_range) uids; |
ee1ecdda8a6b
fts-solr: Keep last_uid field updated containing the mailbox's last added UID.
Timo Sirainen <tss@iki.fi>
parents:
8484
diff
changeset
|
329 const struct seq_range *uidvals; |
8950
ec1ae90af21a
fts: Fixes to how virtual mailboxes are searched.
Timo Sirainen <tss@iki.fi>
parents:
8946
diff
changeset
|
330 const char *box_name; |
8492
ee1ecdda8a6b
fts-solr: Keep last_uid field updated containing the mailbox's last added UID.
Timo Sirainen <tss@iki.fi>
parents:
8484
diff
changeset
|
331 unsigned int count; |
ee1ecdda8a6b
fts-solr: Keep last_uid field updated containing the mailbox's last added UID.
Timo Sirainen <tss@iki.fi>
parents:
8484
diff
changeset
|
332 string_t *str; |
ee1ecdda8a6b
fts-solr: Keep last_uid field updated containing the mailbox's last added UID.
Timo Sirainen <tss@iki.fi>
parents:
8484
diff
changeset
|
333 |
ee1ecdda8a6b
fts-solr: Keep last_uid field updated containing the mailbox's last added UID.
Timo Sirainen <tss@iki.fi>
parents:
8484
diff
changeset
|
334 str = t_str_new(256); |
8499
252b29ac5f43
fts: Added support for handling multiple namespaces.
Timo Sirainen <tss@iki.fi>
parents:
8495
diff
changeset
|
335 str_append(str, "fl=uid&rows=1&q=last_uid:TRUE+"); |
8492
ee1ecdda8a6b
fts-solr: Keep last_uid field updated containing the mailbox's last added UID.
Timo Sirainen <tss@iki.fi>
parents:
8484
diff
changeset
|
336 |
8950
ec1ae90af21a
fts: Fixes to how virtual mailboxes are searched.
Timo Sirainen <tss@iki.fi>
parents:
8946
diff
changeset
|
337 box_name = fts_box_get_root(box, &ns); |
ec1ae90af21a
fts: Fixes to how virtual mailboxes are searched.
Timo Sirainen <tss@iki.fi>
parents:
8946
diff
changeset
|
338 |
8499
252b29ac5f43
fts: Added support for handling multiple namespaces.
Timo Sirainen <tss@iki.fi>
parents:
8495
diff
changeset
|
339 mailbox_get_status(box, STATUS_UIDVALIDITY, &status); |
252b29ac5f43
fts: Added support for handling multiple namespaces.
Timo Sirainen <tss@iki.fi>
parents:
8495
diff
changeset
|
340 str_printfa(str, "uidv:%u+box:", status.uidvalidity); |
8950
ec1ae90af21a
fts: Fixes to how virtual mailboxes are searched.
Timo Sirainen <tss@iki.fi>
parents:
8946
diff
changeset
|
341 solr_quote_http(str, box_name); |
ec1ae90af21a
fts: Fixes to how virtual mailboxes are searched.
Timo Sirainen <tss@iki.fi>
parents:
8946
diff
changeset
|
342 solr_add_ns_query_http(str, backend, ns); |
8499
252b29ac5f43
fts: Added support for handling multiple namespaces.
Timo Sirainen <tss@iki.fi>
parents:
8495
diff
changeset
|
343 str_append(str, "+user:"); |
8950
ec1ae90af21a
fts: Fixes to how virtual mailboxes are searched.
Timo Sirainen <tss@iki.fi>
parents:
8946
diff
changeset
|
344 solr_quote_http(str, ns->user->username); |
8492
ee1ecdda8a6b
fts-solr: Keep last_uid field updated containing the mailbox's last added UID.
Timo Sirainen <tss@iki.fi>
parents:
8484
diff
changeset
|
345 |
ee1ecdda8a6b
fts-solr: Keep last_uid field updated containing the mailbox's last added UID.
Timo Sirainen <tss@iki.fi>
parents:
8484
diff
changeset
|
346 t_array_init(&uids, 1); |
ee1ecdda8a6b
fts-solr: Keep last_uid field updated containing the mailbox's last added UID.
Timo Sirainen <tss@iki.fi>
parents:
8484
diff
changeset
|
347 if (solr_connection_select(solr_conn, str_c(str), |
ee1ecdda8a6b
fts-solr: Keep last_uid field updated containing the mailbox's last added UID.
Timo Sirainen <tss@iki.fi>
parents:
8484
diff
changeset
|
348 NULL, NULL, &uids, NULL) < 0) |
ee1ecdda8a6b
fts-solr: Keep last_uid field updated containing the mailbox's last added UID.
Timo Sirainen <tss@iki.fi>
parents:
8484
diff
changeset
|
349 return -1; |
ee1ecdda8a6b
fts-solr: Keep last_uid field updated containing the mailbox's last added UID.
Timo Sirainen <tss@iki.fi>
parents:
8484
diff
changeset
|
350 |
ee1ecdda8a6b
fts-solr: Keep last_uid field updated containing the mailbox's last added UID.
Timo Sirainen <tss@iki.fi>
parents:
8484
diff
changeset
|
351 uidvals = array_get(&uids, &count); |
ee1ecdda8a6b
fts-solr: Keep last_uid field updated containing the mailbox's last added UID.
Timo Sirainen <tss@iki.fi>
parents:
8484
diff
changeset
|
352 if (count == 0) { |
ee1ecdda8a6b
fts-solr: Keep last_uid field updated containing the mailbox's last added UID.
Timo Sirainen <tss@iki.fi>
parents:
8484
diff
changeset
|
353 /* either nothing is indexed or we're converting from an |
ee1ecdda8a6b
fts-solr: Keep last_uid field updated containing the mailbox's last added UID.
Timo Sirainen <tss@iki.fi>
parents:
8484
diff
changeset
|
354 older database format without the last_uid fields */ |
ee1ecdda8a6b
fts-solr: Keep last_uid field updated containing the mailbox's last added UID.
Timo Sirainen <tss@iki.fi>
parents:
8484
diff
changeset
|
355 return fts_backend_solr_get_last_uid_fallback(backend, |
ee1ecdda8a6b
fts-solr: Keep last_uid field updated containing the mailbox's last added UID.
Timo Sirainen <tss@iki.fi>
parents:
8484
diff
changeset
|
356 last_uid_r); |
ee1ecdda8a6b
fts-solr: Keep last_uid field updated containing the mailbox's last added UID.
Timo Sirainen <tss@iki.fi>
parents:
8484
diff
changeset
|
357 } else if (count == 1 && uidvals[0].seq1 == uidvals[0].seq2) { |
ee1ecdda8a6b
fts-solr: Keep last_uid field updated containing the mailbox's last added UID.
Timo Sirainen <tss@iki.fi>
parents:
8484
diff
changeset
|
358 *last_uid_r = uidvals[0].seq1; |
ee1ecdda8a6b
fts-solr: Keep last_uid field updated containing the mailbox's last added UID.
Timo Sirainen <tss@iki.fi>
parents:
8484
diff
changeset
|
359 } else { |
ee1ecdda8a6b
fts-solr: Keep last_uid field updated containing the mailbox's last added UID.
Timo Sirainen <tss@iki.fi>
parents:
8484
diff
changeset
|
360 i_error("fts_solr: Last UID lookup returned multiple rows"); |
ee1ecdda8a6b
fts-solr: Keep last_uid field updated containing the mailbox's last added UID.
Timo Sirainen <tss@iki.fi>
parents:
8484
diff
changeset
|
361 return -1; |
ee1ecdda8a6b
fts-solr: Keep last_uid field updated containing the mailbox's last added UID.
Timo Sirainen <tss@iki.fi>
parents:
8484
diff
changeset
|
362 } |
ee1ecdda8a6b
fts-solr: Keep last_uid field updated containing the mailbox's last added UID.
Timo Sirainen <tss@iki.fi>
parents:
8484
diff
changeset
|
363 return 0; |
ee1ecdda8a6b
fts-solr: Keep last_uid field updated containing the mailbox's last added UID.
Timo Sirainen <tss@iki.fi>
parents:
8484
diff
changeset
|
364 } |
ee1ecdda8a6b
fts-solr: Keep last_uid field updated containing the mailbox's last added UID.
Timo Sirainen <tss@iki.fi>
parents:
8484
diff
changeset
|
365 |
8500
3efcdc45d111
Added alias_for setting for namespaces. Fixes namespace issues with fts.
Timo Sirainen <tss@iki.fi>
parents:
8499
diff
changeset
|
366 static struct mail_namespace * |
3efcdc45d111
Added alias_for setting for namespaces. Fixes namespace issues with fts.
Timo Sirainen <tss@iki.fi>
parents:
8499
diff
changeset
|
367 solr_get_namespaces(struct fts_backend *_backend, |
3efcdc45d111
Added alias_for setting for namespaces. Fixes namespace issues with fts.
Timo Sirainen <tss@iki.fi>
parents:
8499
diff
changeset
|
368 struct mailbox *box, const char *ns_prefix) |
8499
252b29ac5f43
fts: Added support for handling multiple namespaces.
Timo Sirainen <tss@iki.fi>
parents:
8495
diff
changeset
|
369 { |
252b29ac5f43
fts: Added support for handling multiple namespaces.
Timo Sirainen <tss@iki.fi>
parents:
8495
diff
changeset
|
370 struct solr_fts_backend *backend = (struct solr_fts_backend *)_backend; |
252b29ac5f43
fts: Added support for handling multiple namespaces.
Timo Sirainen <tss@iki.fi>
parents:
8495
diff
changeset
|
371 struct mail_namespace *namespaces = box->storage->ns->user->namespaces; |
252b29ac5f43
fts: Added support for handling multiple namespaces.
Timo Sirainen <tss@iki.fi>
parents:
8495
diff
changeset
|
372 |
252b29ac5f43
fts: Added support for handling multiple namespaces.
Timo Sirainen <tss@iki.fi>
parents:
8495
diff
changeset
|
373 if (ns_prefix == NULL) |
8500
3efcdc45d111
Added alias_for setting for namespaces. Fixes namespace issues with fts.
Timo Sirainen <tss@iki.fi>
parents:
8499
diff
changeset
|
374 return backend->default_ns; |
3efcdc45d111
Added alias_for setting for namespaces. Fixes namespace issues with fts.
Timo Sirainen <tss@iki.fi>
parents:
8499
diff
changeset
|
375 else |
3efcdc45d111
Added alias_for setting for namespaces. Fixes namespace issues with fts.
Timo Sirainen <tss@iki.fi>
parents:
8499
diff
changeset
|
376 return mail_namespace_find_prefix(namespaces, ns_prefix); |
8499
252b29ac5f43
fts: Added support for handling multiple namespaces.
Timo Sirainen <tss@iki.fi>
parents:
8495
diff
changeset
|
377 } |
252b29ac5f43
fts: Added support for handling multiple namespaces.
Timo Sirainen <tss@iki.fi>
parents:
8495
diff
changeset
|
378 |
8495
ffb37c392166
Finished making full text search indexes work fast with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents:
8493
diff
changeset
|
379 static bool |
8499
252b29ac5f43
fts: Added support for handling multiple namespaces.
Timo Sirainen <tss@iki.fi>
parents:
8495
diff
changeset
|
380 solr_virtual_get_last_uids(const char *ns_prefix, const char *mailbox, |
252b29ac5f43
fts: Added support for handling multiple namespaces.
Timo Sirainen <tss@iki.fi>
parents:
8495
diff
changeset
|
381 uint32_t uidvalidity, uint32_t *uid, void *context) |
8495
ffb37c392166
Finished making full text search indexes work fast with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents:
8493
diff
changeset
|
382 { |
ffb37c392166
Finished making full text search indexes work fast with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents:
8493
diff
changeset
|
383 struct fts_backend_solr_get_last_uids_context *ctx = context; |
ffb37c392166
Finished making full text search indexes work fast with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents:
8493
diff
changeset
|
384 struct fts_backend_uid_map *map; |
8500
3efcdc45d111
Added alias_for setting for namespaces. Fixes namespace issues with fts.
Timo Sirainen <tss@iki.fi>
parents:
8499
diff
changeset
|
385 struct mail_namespace *ns; |
8499
252b29ac5f43
fts: Added support for handling multiple namespaces.
Timo Sirainen <tss@iki.fi>
parents:
8495
diff
changeset
|
386 const char *vname; |
252b29ac5f43
fts: Added support for handling multiple namespaces.
Timo Sirainen <tss@iki.fi>
parents:
8495
diff
changeset
|
387 |
8500
3efcdc45d111
Added alias_for setting for namespaces. Fixes namespace issues with fts.
Timo Sirainen <tss@iki.fi>
parents:
8499
diff
changeset
|
388 ns = solr_get_namespaces(ctx->backend, ctx->box, ns_prefix); |
3efcdc45d111
Added alias_for setting for namespaces. Fixes namespace issues with fts.
Timo Sirainen <tss@iki.fi>
parents:
8499
diff
changeset
|
389 for (; ns != NULL; ns = ns->alias_chain_next) { |
3efcdc45d111
Added alias_for setting for namespaces. Fixes namespace issues with fts.
Timo Sirainen <tss@iki.fi>
parents:
8499
diff
changeset
|
390 vname = mail_namespace_get_vname(ns, ctx->vname, mailbox); |
3efcdc45d111
Added alias_for setting for namespaces. Fixes namespace issues with fts.
Timo Sirainen <tss@iki.fi>
parents:
8499
diff
changeset
|
391 map = array_append_space(ctx->last_uids); |
3efcdc45d111
Added alias_for setting for namespaces. Fixes namespace issues with fts.
Timo Sirainen <tss@iki.fi>
parents:
8499
diff
changeset
|
392 map->mailbox = p_strdup(ctx->pool, vname); |
3efcdc45d111
Added alias_for setting for namespaces. Fixes namespace issues with fts.
Timo Sirainen <tss@iki.fi>
parents:
8499
diff
changeset
|
393 map->uidvalidity = uidvalidity; |
3efcdc45d111
Added alias_for setting for namespaces. Fixes namespace issues with fts.
Timo Sirainen <tss@iki.fi>
parents:
8499
diff
changeset
|
394 map->uid = *uid; |
3efcdc45d111
Added alias_for setting for namespaces. Fixes namespace issues with fts.
Timo Sirainen <tss@iki.fi>
parents:
8499
diff
changeset
|
395 } |
8495
ffb37c392166
Finished making full text search indexes work fast with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents:
8493
diff
changeset
|
396 return FALSE; |
ffb37c392166
Finished making full text search indexes work fast with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents:
8493
diff
changeset
|
397 } |
ffb37c392166
Finished making full text search indexes work fast with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents:
8493
diff
changeset
|
398 |
8499
252b29ac5f43
fts: Added support for handling multiple namespaces.
Timo Sirainen <tss@iki.fi>
parents:
8495
diff
changeset
|
399 static void |
252b29ac5f43
fts: Added support for handling multiple namespaces.
Timo Sirainen <tss@iki.fi>
parents:
8495
diff
changeset
|
400 solr_add_pattern(string_t *str, const struct mailbox_virtual_pattern *pattern) |
8495
ffb37c392166
Finished making full text search indexes work fast with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents:
8493
diff
changeset
|
401 { |
8950
ec1ae90af21a
fts: Fixes to how virtual mailboxes are searched.
Timo Sirainen <tss@iki.fi>
parents:
8946
diff
changeset
|
402 struct mail_namespace *ns = pattern->ns; |
8499
252b29ac5f43
fts: Added support for handling multiple namespaces.
Timo Sirainen <tss@iki.fi>
parents:
8495
diff
changeset
|
403 const char *name, *p; |
252b29ac5f43
fts: Added support for handling multiple namespaces.
Timo Sirainen <tss@iki.fi>
parents:
8495
diff
changeset
|
404 |
252b29ac5f43
fts: Added support for handling multiple namespaces.
Timo Sirainen <tss@iki.fi>
parents:
8495
diff
changeset
|
405 name = pattern->pattern; |
252b29ac5f43
fts: Added support for handling multiple namespaces.
Timo Sirainen <tss@iki.fi>
parents:
8495
diff
changeset
|
406 if (!mail_namespace_update_name(pattern->ns, &name)) |
252b29ac5f43
fts: Added support for handling multiple namespaces.
Timo Sirainen <tss@iki.fi>
parents:
8495
diff
changeset
|
407 name = mail_namespace_fix_sep(pattern->ns, name); |
8495
ffb37c392166
Finished making full text search indexes work fast with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents:
8493
diff
changeset
|
408 |
8950
ec1ae90af21a
fts: Fixes to how virtual mailboxes are searched.
Timo Sirainen <tss@iki.fi>
parents:
8946
diff
changeset
|
409 fts_box_name_get_root(&ns, &name); |
ec1ae90af21a
fts: Fixes to how virtual mailboxes are searched.
Timo Sirainen <tss@iki.fi>
parents:
8946
diff
changeset
|
410 |
8945
43b68bd6023a
fts-solr: Fixes with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents:
8879
diff
changeset
|
411 if (strcmp(name, "*") == 0) { |
43b68bd6023a
fts-solr: Fixes with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents:
8879
diff
changeset
|
412 str_append(str, "[* TO *]"); |
43b68bd6023a
fts-solr: Fixes with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents:
8879
diff
changeset
|
413 return; |
43b68bd6023a
fts-solr: Fixes with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents:
8879
diff
changeset
|
414 } |
43b68bd6023a
fts-solr: Fixes with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents:
8879
diff
changeset
|
415 |
8495
ffb37c392166
Finished making full text search indexes work fast with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents:
8493
diff
changeset
|
416 /* first check if there are any wildcards in the pattern */ |
8499
252b29ac5f43
fts: Added support for handling multiple namespaces.
Timo Sirainen <tss@iki.fi>
parents:
8495
diff
changeset
|
417 for (p = name; *p != '\0'; p++) { |
8495
ffb37c392166
Finished making full text search indexes work fast with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents:
8493
diff
changeset
|
418 if (*p == '%' || *p == '*') |
ffb37c392166
Finished making full text search indexes work fast with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents:
8493
diff
changeset
|
419 break; |
ffb37c392166
Finished making full text search indexes work fast with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents:
8493
diff
changeset
|
420 } |
ffb37c392166
Finished making full text search indexes work fast with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents:
8493
diff
changeset
|
421 if (*p == '\0') { |
ffb37c392166
Finished making full text search indexes work fast with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents:
8493
diff
changeset
|
422 /* full mailbox name */ |
8499
252b29ac5f43
fts: Added support for handling multiple namespaces.
Timo Sirainen <tss@iki.fi>
parents:
8495
diff
changeset
|
423 solr_quote(str, name); |
8495
ffb37c392166
Finished making full text search indexes work fast with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents:
8493
diff
changeset
|
424 return; |
ffb37c392166
Finished making full text search indexes work fast with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents:
8493
diff
changeset
|
425 } |
ffb37c392166
Finished making full text search indexes work fast with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents:
8493
diff
changeset
|
426 |
ffb37c392166
Finished making full text search indexes work fast with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents:
8493
diff
changeset
|
427 /* there are at least some wildcards. */ |
8499
252b29ac5f43
fts: Added support for handling multiple namespaces.
Timo Sirainen <tss@iki.fi>
parents:
8495
diff
changeset
|
428 for (p = name; *p != '\0'; p++) { |
8495
ffb37c392166
Finished making full text search indexes work fast with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents:
8493
diff
changeset
|
429 if (*p == '%' || *p == '*') { |
8499
252b29ac5f43
fts: Added support for handling multiple namespaces.
Timo Sirainen <tss@iki.fi>
parents:
8495
diff
changeset
|
430 if (p == name || (p[-1] != '%' && p[-1] != '*')) |
8495
ffb37c392166
Finished making full text search indexes work fast with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents:
8493
diff
changeset
|
431 str_append_c(str, '*'); |
ffb37c392166
Finished making full text search indexes work fast with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents:
8493
diff
changeset
|
432 } else { |
ffb37c392166
Finished making full text search indexes work fast with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents:
8493
diff
changeset
|
433 if (!i_isalnum(*p)) |
ffb37c392166
Finished making full text search indexes work fast with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents:
8493
diff
changeset
|
434 str_append_c(str, '\\'); |
ffb37c392166
Finished making full text search indexes work fast with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents:
8493
diff
changeset
|
435 str_append_c(str, *p); |
ffb37c392166
Finished making full text search indexes work fast with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents:
8493
diff
changeset
|
436 } |
ffb37c392166
Finished making full text search indexes work fast with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents:
8493
diff
changeset
|
437 } |
ffb37c392166
Finished making full text search indexes work fast with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents:
8493
diff
changeset
|
438 } |
ffb37c392166
Finished making full text search indexes work fast with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents:
8493
diff
changeset
|
439 |
ffb37c392166
Finished making full text search indexes work fast with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents:
8493
diff
changeset
|
440 static void |
8499
252b29ac5f43
fts: Added support for handling multiple namespaces.
Timo Sirainen <tss@iki.fi>
parents:
8495
diff
changeset
|
441 fts_backend_solr_filter_mailboxes(struct fts_backend *_backend, |
8495
ffb37c392166
Finished making full text search indexes work fast with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents:
8493
diff
changeset
|
442 string_t *str, struct mailbox *box) |
ffb37c392166
Finished making full text search indexes work fast with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents:
8493
diff
changeset
|
443 { |
8499
252b29ac5f43
fts: Added support for handling multiple namespaces.
Timo Sirainen <tss@iki.fi>
parents:
8495
diff
changeset
|
444 ARRAY_TYPE(mailbox_virtual_patterns) includes_arr, excludes_arr; |
8500
3efcdc45d111
Added alias_for setting for namespaces. Fixes namespace issues with fts.
Timo Sirainen <tss@iki.fi>
parents:
8499
diff
changeset
|
445 struct mail_namespace *ns; |
8499
252b29ac5f43
fts: Added support for handling multiple namespaces.
Timo Sirainen <tss@iki.fi>
parents:
8495
diff
changeset
|
446 const struct mailbox_virtual_pattern *includes, *excludes; |
8519 | 447 unsigned int i, inc_count, exc_count; |
8495
ffb37c392166
Finished making full text search indexes work fast with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents:
8493
diff
changeset
|
448 string_t *fq; |
ffb37c392166
Finished making full text search indexes work fast with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents:
8493
diff
changeset
|
449 |
ffb37c392166
Finished making full text search indexes work fast with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents:
8493
diff
changeset
|
450 t_array_init(&includes_arr, 16); |
ffb37c392166
Finished making full text search indexes work fast with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents:
8493
diff
changeset
|
451 t_array_init(&excludes_arr, 16); |
ffb37c392166
Finished making full text search indexes work fast with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents:
8493
diff
changeset
|
452 mailbox_get_virtual_box_patterns(box, &includes_arr, &excludes_arr); |
ffb37c392166
Finished making full text search indexes work fast with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents:
8493
diff
changeset
|
453 includes = array_get(&includes_arr, &inc_count); |
ffb37c392166
Finished making full text search indexes work fast with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents:
8493
diff
changeset
|
454 excludes = array_get(&excludes_arr, &exc_count); |
ffb37c392166
Finished making full text search indexes work fast with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents:
8493
diff
changeset
|
455 i_assert(inc_count > 0); |
ffb37c392166
Finished making full text search indexes work fast with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents:
8493
diff
changeset
|
456 |
ffb37c392166
Finished making full text search indexes work fast with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents:
8493
diff
changeset
|
457 /* First see if there are any patterns that begin with a wildcard. |
ffb37c392166
Finished making full text search indexes work fast with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents:
8493
diff
changeset
|
458 Solr doesn't allow them, so in that case we'll need to return |
ffb37c392166
Finished making full text search indexes work fast with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents:
8493
diff
changeset
|
459 all mailboxes. */ |
ffb37c392166
Finished making full text search indexes work fast with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents:
8493
diff
changeset
|
460 for (i = 0; i < inc_count; i++) { |
8499
252b29ac5f43
fts: Added support for handling multiple namespaces.
Timo Sirainen <tss@iki.fi>
parents:
8495
diff
changeset
|
461 if (*includes[i].pattern == '*' || |
252b29ac5f43
fts: Added support for handling multiple namespaces.
Timo Sirainen <tss@iki.fi>
parents:
8495
diff
changeset
|
462 *includes[i].pattern == '%') |
8495
ffb37c392166
Finished making full text search indexes work fast with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents:
8493
diff
changeset
|
463 break; |
ffb37c392166
Finished making full text search indexes work fast with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents:
8493
diff
changeset
|
464 } |
ffb37c392166
Finished making full text search indexes work fast with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents:
8493
diff
changeset
|
465 |
ffb37c392166
Finished making full text search indexes work fast with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents:
8493
diff
changeset
|
466 fq = t_str_new(128); |
8499
252b29ac5f43
fts: Added support for handling multiple namespaces.
Timo Sirainen <tss@iki.fi>
parents:
8495
diff
changeset
|
467 if (i == inc_count && inc_count <= FTS_SOLR_MAX_BOX_INC_PATTERNS) { |
8495
ffb37c392166
Finished making full text search indexes work fast with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents:
8493
diff
changeset
|
468 /* we can filter what mailboxes we want returned */ |
ffb37c392166
Finished making full text search indexes work fast with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents:
8493
diff
changeset
|
469 str_append_c(fq, '('); |
ffb37c392166
Finished making full text search indexes work fast with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents:
8493
diff
changeset
|
470 for (i = 0; i < inc_count; i++) { |
ffb37c392166
Finished making full text search indexes work fast with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents:
8493
diff
changeset
|
471 if (i != 0) |
8879
e81dfb69bfc5
fts-solr: Don't double-escape "+" if we're filtering what mailboxes we want.
Timo Sirainen <tss@iki.fi>
parents:
8848
diff
changeset
|
472 str_append(fq, " OR +"); |
8499
252b29ac5f43
fts: Added support for handling multiple namespaces.
Timo Sirainen <tss@iki.fi>
parents:
8495
diff
changeset
|
473 str_append_c(fq, '('); |
8879
e81dfb69bfc5
fts-solr: Don't double-escape "+" if we're filtering what mailboxes we want.
Timo Sirainen <tss@iki.fi>
parents:
8848
diff
changeset
|
474 str_append(fq, "+box:"); |
8499
252b29ac5f43
fts: Added support for handling multiple namespaces.
Timo Sirainen <tss@iki.fi>
parents:
8495
diff
changeset
|
475 solr_add_pattern(fq, &includes[i]); |
8945
43b68bd6023a
fts-solr: Fixes with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents:
8879
diff
changeset
|
476 solr_add_ns_query(fq, _backend, includes[i].ns, FALSE); |
8499
252b29ac5f43
fts: Added support for handling multiple namespaces.
Timo Sirainen <tss@iki.fi>
parents:
8495
diff
changeset
|
477 str_append_c(fq, ')'); |
8495
ffb37c392166
Finished making full text search indexes work fast with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents:
8493
diff
changeset
|
478 } |
ffb37c392166
Finished making full text search indexes work fast with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents:
8493
diff
changeset
|
479 str_append_c(fq, ')'); |
ffb37c392166
Finished making full text search indexes work fast with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents:
8493
diff
changeset
|
480 } |
8499
252b29ac5f43
fts: Added support for handling multiple namespaces.
Timo Sirainen <tss@iki.fi>
parents:
8495
diff
changeset
|
481 exc_count = I_MIN(FTS_SOLR_MAX_BOX_EXC_PATTERNS, exc_count); |
8495
ffb37c392166
Finished making full text search indexes work fast with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents:
8493
diff
changeset
|
482 for (i = 0; i < exc_count; i++) { |
8519 | 483 if (str_len(fq) > 0) |
8495
ffb37c392166
Finished making full text search indexes work fast with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents:
8493
diff
changeset
|
484 str_append_c(fq, ' '); |
8945
43b68bd6023a
fts-solr: Fixes with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents:
8879
diff
changeset
|
485 str_append(fq, "NOT ("); |
43b68bd6023a
fts-solr: Fixes with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents:
8879
diff
changeset
|
486 str_append(fq, "box:"); |
8499
252b29ac5f43
fts: Added support for handling multiple namespaces.
Timo Sirainen <tss@iki.fi>
parents:
8495
diff
changeset
|
487 solr_add_pattern(fq, &excludes[i]); |
8500
3efcdc45d111
Added alias_for setting for namespaces. Fixes namespace issues with fts.
Timo Sirainen <tss@iki.fi>
parents:
8499
diff
changeset
|
488 |
3efcdc45d111
Added alias_for setting for namespaces. Fixes namespace issues with fts.
Timo Sirainen <tss@iki.fi>
parents:
8499
diff
changeset
|
489 for (ns = excludes[i].ns; ns->alias_for != NULL; ) |
3efcdc45d111
Added alias_for setting for namespaces. Fixes namespace issues with fts.
Timo Sirainen <tss@iki.fi>
parents:
8499
diff
changeset
|
490 ns = ns->alias_for; |
8945
43b68bd6023a
fts-solr: Fixes with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents:
8879
diff
changeset
|
491 solr_add_ns_query(fq, _backend, ns, FALSE); |
8499
252b29ac5f43
fts: Added support for handling multiple namespaces.
Timo Sirainen <tss@iki.fi>
parents:
8495
diff
changeset
|
492 str_append_c(fq, ')'); |
8495
ffb37c392166
Finished making full text search indexes work fast with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents:
8493
diff
changeset
|
493 } |
ffb37c392166
Finished making full text search indexes work fast with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents:
8493
diff
changeset
|
494 if (str_len(fq) > 0) { |
ffb37c392166
Finished making full text search indexes work fast with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents:
8493
diff
changeset
|
495 str_append(str, "&fq="); |
ffb37c392166
Finished making full text search indexes work fast with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents:
8493
diff
changeset
|
496 solr_connection_http_escape(solr_conn, str, str_c(fq)); |
ffb37c392166
Finished making full text search indexes work fast with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents:
8493
diff
changeset
|
497 } |
ffb37c392166
Finished making full text search indexes work fast with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents:
8493
diff
changeset
|
498 } |
ffb37c392166
Finished making full text search indexes work fast with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents:
8493
diff
changeset
|
499 |
ffb37c392166
Finished making full text search indexes work fast with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents:
8493
diff
changeset
|
500 static int |
ffb37c392166
Finished making full text search indexes work fast with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents:
8493
diff
changeset
|
501 fts_backend_solr_get_all_last_uids(struct fts_backend *backend, pool_t pool, |
ffb37c392166
Finished making full text search indexes work fast with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents:
8493
diff
changeset
|
502 ARRAY_TYPE(fts_backend_uid_map) *last_uids) |
ffb37c392166
Finished making full text search indexes work fast with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents:
8493
diff
changeset
|
503 { |
ffb37c392166
Finished making full text search indexes work fast with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents:
8493
diff
changeset
|
504 struct fts_backend_solr_get_last_uids_context ctx; |
ffb37c392166
Finished making full text search indexes work fast with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents:
8493
diff
changeset
|
505 string_t *str; |
ffb37c392166
Finished making full text search indexes work fast with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents:
8493
diff
changeset
|
506 |
ffb37c392166
Finished making full text search indexes work fast with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents:
8493
diff
changeset
|
507 memset(&ctx, 0, sizeof(ctx)); |
8499
252b29ac5f43
fts: Added support for handling multiple namespaces.
Timo Sirainen <tss@iki.fi>
parents:
8495
diff
changeset
|
508 ctx.backend = backend; |
8495
ffb37c392166
Finished making full text search indexes work fast with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents:
8493
diff
changeset
|
509 ctx.pool = pool; |
ffb37c392166
Finished making full text search indexes work fast with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents:
8493
diff
changeset
|
510 ctx.last_uids = last_uids; |
8499
252b29ac5f43
fts: Added support for handling multiple namespaces.
Timo Sirainen <tss@iki.fi>
parents:
8495
diff
changeset
|
511 ctx.box = backend->box; |
252b29ac5f43
fts: Added support for handling multiple namespaces.
Timo Sirainen <tss@iki.fi>
parents:
8495
diff
changeset
|
512 ctx.vname = t_str_new(256); |
8495
ffb37c392166
Finished making full text search indexes work fast with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents:
8493
diff
changeset
|
513 |
ffb37c392166
Finished making full text search indexes work fast with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents:
8493
diff
changeset
|
514 str = t_str_new(256); |
8499
252b29ac5f43
fts: Added support for handling multiple namespaces.
Timo Sirainen <tss@iki.fi>
parents:
8495
diff
changeset
|
515 str_printfa(str, "fl=uid,box,uidv,ns&rows=%u&q=last_uid:TRUE+user:", |
8495
ffb37c392166
Finished making full text search indexes work fast with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents:
8493
diff
changeset
|
516 SOLR_MAX_ROWS); |
8499
252b29ac5f43
fts: Added support for handling multiple namespaces.
Timo Sirainen <tss@iki.fi>
parents:
8495
diff
changeset
|
517 solr_quote_http(str, backend->box->storage->ns->user->username); |
252b29ac5f43
fts: Added support for handling multiple namespaces.
Timo Sirainen <tss@iki.fi>
parents:
8495
diff
changeset
|
518 fts_backend_solr_filter_mailboxes(backend, str, backend->box); |
8495
ffb37c392166
Finished making full text search indexes work fast with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents:
8493
diff
changeset
|
519 |
ffb37c392166
Finished making full text search indexes work fast with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents:
8493
diff
changeset
|
520 return solr_connection_select(solr_conn, str_c(str), |
ffb37c392166
Finished making full text search indexes work fast with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents:
8493
diff
changeset
|
521 solr_virtual_get_last_uids, &ctx, |
ffb37c392166
Finished making full text search indexes work fast with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents:
8493
diff
changeset
|
522 NULL, NULL); |
ffb37c392166
Finished making full text search indexes work fast with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents:
8493
diff
changeset
|
523 } |
ffb37c392166
Finished making full text search indexes work fast with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents:
8493
diff
changeset
|
524 |
7990
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
525 static int |
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
526 fts_backend_solr_build_init(struct fts_backend *backend, uint32_t *last_uid_r, |
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
527 struct fts_backend_build_context **ctx_r) |
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
528 { |
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
529 struct solr_fts_backend_build_context *ctx; |
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
530 struct mailbox_status status; |
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
531 |
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
532 *last_uid_r = (uint32_t)-1; |
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
533 |
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
534 ctx = i_new(struct solr_fts_backend_build_context, 1); |
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
535 ctx->ctx.backend = backend; |
8001
3fbfdda3e5d3
fts-solr: Send data to Solr in larger packets. Commit after expunges.
Timo Sirainen <tss@iki.fi>
parents:
8000
diff
changeset
|
536 ctx->cmd = str_new(default_pool, SOLR_CMDBUF_SIZE); |
7990
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
537 |
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
538 mailbox_get_status(backend->box, STATUS_UIDVALIDITY, &status); |
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
539 ctx->uid_validity = status.uidvalidity; |
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
540 |
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
541 *ctx_r = &ctx->ctx; |
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
542 return 0; |
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
543 } |
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
544 |
8492
ee1ecdda8a6b
fts-solr: Keep last_uid field updated containing the mailbox's last added UID.
Timo Sirainen <tss@iki.fi>
parents:
8484
diff
changeset
|
545 static void |
ee1ecdda8a6b
fts-solr: Keep last_uid field updated containing the mailbox's last added UID.
Timo Sirainen <tss@iki.fi>
parents:
8484
diff
changeset
|
546 fts_backend_solr_add_doc_prefix(struct solr_fts_backend_build_context *ctx, |
ee1ecdda8a6b
fts-solr: Keep last_uid field updated containing the mailbox's last added UID.
Timo Sirainen <tss@iki.fi>
parents:
8484
diff
changeset
|
547 uint32_t uid) |
ee1ecdda8a6b
fts-solr: Keep last_uid field updated containing the mailbox's last added UID.
Timo Sirainen <tss@iki.fi>
parents:
8484
diff
changeset
|
548 { |
8499
252b29ac5f43
fts: Added support for handling multiple namespaces.
Timo Sirainen <tss@iki.fi>
parents:
8495
diff
changeset
|
549 struct solr_fts_backend *backend = |
252b29ac5f43
fts: Added support for handling multiple namespaces.
Timo Sirainen <tss@iki.fi>
parents:
8495
diff
changeset
|
550 (struct solr_fts_backend *)ctx->ctx.backend; |
8492
ee1ecdda8a6b
fts-solr: Keep last_uid field updated containing the mailbox's last added UID.
Timo Sirainen <tss@iki.fi>
parents:
8484
diff
changeset
|
551 struct mailbox *box = ctx->ctx.backend->box; |
8950
ec1ae90af21a
fts: Fixes to how virtual mailboxes are searched.
Timo Sirainen <tss@iki.fi>
parents:
8946
diff
changeset
|
552 struct mail_namespace *ns; |
ec1ae90af21a
fts: Fixes to how virtual mailboxes are searched.
Timo Sirainen <tss@iki.fi>
parents:
8946
diff
changeset
|
553 const char *box_name; |
8492
ee1ecdda8a6b
fts-solr: Keep last_uid field updated containing the mailbox's last added UID.
Timo Sirainen <tss@iki.fi>
parents:
8484
diff
changeset
|
554 |
ee1ecdda8a6b
fts-solr: Keep last_uid field updated containing the mailbox's last added UID.
Timo Sirainen <tss@iki.fi>
parents:
8484
diff
changeset
|
555 str_printfa(ctx->cmd, "<doc>" |
ee1ecdda8a6b
fts-solr: Keep last_uid field updated containing the mailbox's last added UID.
Timo Sirainen <tss@iki.fi>
parents:
8484
diff
changeset
|
556 "<field name=\"uid\">%u</field>" |
ee1ecdda8a6b
fts-solr: Keep last_uid field updated containing the mailbox's last added UID.
Timo Sirainen <tss@iki.fi>
parents:
8484
diff
changeset
|
557 "<field name=\"uidv\">%u</field>", |
ee1ecdda8a6b
fts-solr: Keep last_uid field updated containing the mailbox's last added UID.
Timo Sirainen <tss@iki.fi>
parents:
8484
diff
changeset
|
558 uid, ctx->uid_validity); |
ee1ecdda8a6b
fts-solr: Keep last_uid field updated containing the mailbox's last added UID.
Timo Sirainen <tss@iki.fi>
parents:
8484
diff
changeset
|
559 |
8950
ec1ae90af21a
fts: Fixes to how virtual mailboxes are searched.
Timo Sirainen <tss@iki.fi>
parents:
8946
diff
changeset
|
560 box_name = fts_box_get_root(box, &ns); |
8500
3efcdc45d111
Added alias_for setting for namespaces. Fixes namespace issues with fts.
Timo Sirainen <tss@iki.fi>
parents:
8499
diff
changeset
|
561 |
3efcdc45d111
Added alias_for setting for namespaces. Fixes namespace issues with fts.
Timo Sirainen <tss@iki.fi>
parents:
8499
diff
changeset
|
562 if (ns != backend->default_ns) { |
8499
252b29ac5f43
fts: Added support for handling multiple namespaces.
Timo Sirainen <tss@iki.fi>
parents:
8495
diff
changeset
|
563 str_append(ctx->cmd, "<field name=\"ns\">"); |
8500
3efcdc45d111
Added alias_for setting for namespaces. Fixes namespace issues with fts.
Timo Sirainen <tss@iki.fi>
parents:
8499
diff
changeset
|
564 xml_encode(ctx->cmd, ns->prefix); |
8499
252b29ac5f43
fts: Added support for handling multiple namespaces.
Timo Sirainen <tss@iki.fi>
parents:
8495
diff
changeset
|
565 str_append(ctx->cmd, "</field>"); |
252b29ac5f43
fts: Added support for handling multiple namespaces.
Timo Sirainen <tss@iki.fi>
parents:
8495
diff
changeset
|
566 } |
8492
ee1ecdda8a6b
fts-solr: Keep last_uid field updated containing the mailbox's last added UID.
Timo Sirainen <tss@iki.fi>
parents:
8484
diff
changeset
|
567 str_append(ctx->cmd, "<field name=\"box\">"); |
8950
ec1ae90af21a
fts: Fixes to how virtual mailboxes are searched.
Timo Sirainen <tss@iki.fi>
parents:
8946
diff
changeset
|
568 xml_encode(ctx->cmd, box_name); |
8492
ee1ecdda8a6b
fts-solr: Keep last_uid field updated containing the mailbox's last added UID.
Timo Sirainen <tss@iki.fi>
parents:
8484
diff
changeset
|
569 str_append(ctx->cmd, "</field><field name=\"user\">"); |
8500
3efcdc45d111
Added alias_for setting for namespaces. Fixes namespace issues with fts.
Timo Sirainen <tss@iki.fi>
parents:
8499
diff
changeset
|
570 xml_encode(ctx->cmd, ns->user->username); |
8492
ee1ecdda8a6b
fts-solr: Keep last_uid field updated containing the mailbox's last added UID.
Timo Sirainen <tss@iki.fi>
parents:
8484
diff
changeset
|
571 str_append(ctx->cmd, "</field>"); |
ee1ecdda8a6b
fts-solr: Keep last_uid field updated containing the mailbox's last added UID.
Timo Sirainen <tss@iki.fi>
parents:
8484
diff
changeset
|
572 } |
ee1ecdda8a6b
fts-solr: Keep last_uid field updated containing the mailbox's last added UID.
Timo Sirainen <tss@iki.fi>
parents:
8484
diff
changeset
|
573 |
8499
252b29ac5f43
fts: Added support for handling multiple namespaces.
Timo Sirainen <tss@iki.fi>
parents:
8495
diff
changeset
|
574 static void xml_encode_id(string_t *str, struct fts_backend *_backend, |
8950
ec1ae90af21a
fts: Fixes to how virtual mailboxes are searched.
Timo Sirainen <tss@iki.fi>
parents:
8946
diff
changeset
|
575 uint32_t uid, uint32_t uid_validity) |
8499
252b29ac5f43
fts: Added support for handling multiple namespaces.
Timo Sirainen <tss@iki.fi>
parents:
8495
diff
changeset
|
576 { |
252b29ac5f43
fts: Added support for handling multiple namespaces.
Timo Sirainen <tss@iki.fi>
parents:
8495
diff
changeset
|
577 struct solr_fts_backend *backend = (struct solr_fts_backend *)_backend; |
252b29ac5f43
fts: Added support for handling multiple namespaces.
Timo Sirainen <tss@iki.fi>
parents:
8495
diff
changeset
|
578 |
252b29ac5f43
fts: Added support for handling multiple namespaces.
Timo Sirainen <tss@iki.fi>
parents:
8495
diff
changeset
|
579 if (uid != 0) |
252b29ac5f43
fts: Added support for handling multiple namespaces.
Timo Sirainen <tss@iki.fi>
parents:
8495
diff
changeset
|
580 str_printfa(str, "%u/", uid); |
252b29ac5f43
fts: Added support for handling multiple namespaces.
Timo Sirainen <tss@iki.fi>
parents:
8495
diff
changeset
|
581 else |
252b29ac5f43
fts: Added support for handling multiple namespaces.
Timo Sirainen <tss@iki.fi>
parents:
8495
diff
changeset
|
582 str_append(str, "L/"); |
252b29ac5f43
fts: Added support for handling multiple namespaces.
Timo Sirainen <tss@iki.fi>
parents:
8495
diff
changeset
|
583 if (backend->id_namespace != NULL) { |
252b29ac5f43
fts: Added support for handling multiple namespaces.
Timo Sirainen <tss@iki.fi>
parents:
8495
diff
changeset
|
584 xml_encode(str, backend->id_namespace); |
252b29ac5f43
fts: Added support for handling multiple namespaces.
Timo Sirainen <tss@iki.fi>
parents:
8495
diff
changeset
|
585 str_append_c(str, '/'); |
252b29ac5f43
fts: Added support for handling multiple namespaces.
Timo Sirainen <tss@iki.fi>
parents:
8495
diff
changeset
|
586 } |
252b29ac5f43
fts: Added support for handling multiple namespaces.
Timo Sirainen <tss@iki.fi>
parents:
8495
diff
changeset
|
587 str_printfa(str, "%u/", uid_validity); |
252b29ac5f43
fts: Added support for handling multiple namespaces.
Timo Sirainen <tss@iki.fi>
parents:
8495
diff
changeset
|
588 xml_encode(str, backend->id_username); |
252b29ac5f43
fts: Added support for handling multiple namespaces.
Timo Sirainen <tss@iki.fi>
parents:
8495
diff
changeset
|
589 str_append_c(str, '/'); |
8950
ec1ae90af21a
fts: Fixes to how virtual mailboxes are searched.
Timo Sirainen <tss@iki.fi>
parents:
8946
diff
changeset
|
590 xml_encode(str, backend->id_box_name); |
8499
252b29ac5f43
fts: Added support for handling multiple namespaces.
Timo Sirainen <tss@iki.fi>
parents:
8495
diff
changeset
|
591 } |
252b29ac5f43
fts: Added support for handling multiple namespaces.
Timo Sirainen <tss@iki.fi>
parents:
8495
diff
changeset
|
592 |
7990
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
593 static int |
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
594 fts_backend_solr_build_more(struct fts_backend_build_context *_ctx, |
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
595 uint32_t uid, const unsigned char *data, |
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
596 size_t size, bool headers) |
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
597 { |
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
598 struct solr_fts_backend_build_context *ctx = |
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
599 (struct solr_fts_backend_build_context *)_ctx; |
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
600 string_t *cmd = ctx->cmd; |
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
601 |
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
602 /* body comes first, then headers */ |
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
603 if (ctx->prev_uid != uid) { |
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
604 /* uid changed */ |
8495
ffb37c392166
Finished making full text search indexes work fast with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents:
8493
diff
changeset
|
605 if (ctx->post == NULL) { |
ffb37c392166
Finished making full text search indexes work fast with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents:
8493
diff
changeset
|
606 ctx->post = solr_connection_post_begin(solr_conn); |
7990
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
607 str_append(cmd, "<add>"); |
8495
ffb37c392166
Finished making full text search indexes work fast with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents:
8493
diff
changeset
|
608 } else { |
7990
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
609 str_append(cmd, "</field></doc>"); |
8495
ffb37c392166
Finished making full text search indexes work fast with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents:
8493
diff
changeset
|
610 } |
7990
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
611 ctx->prev_uid = uid; |
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
612 |
8492
ee1ecdda8a6b
fts-solr: Keep last_uid field updated containing the mailbox's last added UID.
Timo Sirainen <tss@iki.fi>
parents:
8484
diff
changeset
|
613 fts_backend_solr_add_doc_prefix(ctx, uid); |
8499
252b29ac5f43
fts: Added support for handling multiple namespaces.
Timo Sirainen <tss@iki.fi>
parents:
8495
diff
changeset
|
614 str_printfa(cmd, "<field name=\"id\">"); |
8950
ec1ae90af21a
fts: Fixes to how virtual mailboxes are searched.
Timo Sirainen <tss@iki.fi>
parents:
8946
diff
changeset
|
615 xml_encode_id(cmd, _ctx->backend, uid, ctx->uid_validity); |
7990
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
616 str_append(cmd, "</field>"); |
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
617 |
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
618 ctx->headers = headers; |
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
619 if (headers) { |
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
620 str_append(cmd, "<field name=\"hdr\">"); |
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
621 } else { |
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
622 str_append(cmd, "<field name=\"body\">"); |
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
623 } |
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
624 } else if (headers && !ctx->headers) { |
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
625 str_append(cmd, "</field><field name=\"hdr\">"); |
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
626 } else { |
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
627 i_assert(!(!headers && ctx->headers)); |
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
628 } |
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
629 |
8000
61be6a4722c5
fts-solr: Encode entities when indexing data.
Timo Sirainen <tss@iki.fi>
parents:
7999
diff
changeset
|
630 xml_encode_data(cmd, data, size); |
8001
3fbfdda3e5d3
fts-solr: Send data to Solr in larger packets. Commit after expunges.
Timo Sirainen <tss@iki.fi>
parents:
8000
diff
changeset
|
631 if (str_len(cmd) > SOLR_CMDBUF_SIZE-128) { |
3fbfdda3e5d3
fts-solr: Send data to Solr in larger packets. Commit after expunges.
Timo Sirainen <tss@iki.fi>
parents:
8000
diff
changeset
|
632 solr_connection_post_more(ctx->post, str_data(cmd), |
3fbfdda3e5d3
fts-solr: Send data to Solr in larger packets. Commit after expunges.
Timo Sirainen <tss@iki.fi>
parents:
8000
diff
changeset
|
633 str_len(cmd)); |
3fbfdda3e5d3
fts-solr: Send data to Solr in larger packets. Commit after expunges.
Timo Sirainen <tss@iki.fi>
parents:
8000
diff
changeset
|
634 str_truncate(cmd, 0); |
3fbfdda3e5d3
fts-solr: Send data to Solr in larger packets. Commit after expunges.
Timo Sirainen <tss@iki.fi>
parents:
8000
diff
changeset
|
635 } |
7990
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
636 return 0; |
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
637 } |
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
638 |
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
639 static int |
8492
ee1ecdda8a6b
fts-solr: Keep last_uid field updated containing the mailbox's last added UID.
Timo Sirainen <tss@iki.fi>
parents:
8484
diff
changeset
|
640 fts_backed_solr_build_commit(struct solr_fts_backend_build_context *ctx) |
ee1ecdda8a6b
fts-solr: Keep last_uid field updated containing the mailbox's last added UID.
Timo Sirainen <tss@iki.fi>
parents:
8484
diff
changeset
|
641 { |
ee1ecdda8a6b
fts-solr: Keep last_uid field updated containing the mailbox's last added UID.
Timo Sirainen <tss@iki.fi>
parents:
8484
diff
changeset
|
642 int ret; |
ee1ecdda8a6b
fts-solr: Keep last_uid field updated containing the mailbox's last added UID.
Timo Sirainen <tss@iki.fi>
parents:
8484
diff
changeset
|
643 |
8495
ffb37c392166
Finished making full text search indexes work fast with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents:
8493
diff
changeset
|
644 if (ctx->post == NULL) |
8492
ee1ecdda8a6b
fts-solr: Keep last_uid field updated containing the mailbox's last added UID.
Timo Sirainen <tss@iki.fi>
parents:
8484
diff
changeset
|
645 return 0; |
ee1ecdda8a6b
fts-solr: Keep last_uid field updated containing the mailbox's last added UID.
Timo Sirainen <tss@iki.fi>
parents:
8484
diff
changeset
|
646 |
ee1ecdda8a6b
fts-solr: Keep last_uid field updated containing the mailbox's last added UID.
Timo Sirainen <tss@iki.fi>
parents:
8484
diff
changeset
|
647 str_append(ctx->cmd, "</field></doc>"); |
ee1ecdda8a6b
fts-solr: Keep last_uid field updated containing the mailbox's last added UID.
Timo Sirainen <tss@iki.fi>
parents:
8484
diff
changeset
|
648 |
ee1ecdda8a6b
fts-solr: Keep last_uid field updated containing the mailbox's last added UID.
Timo Sirainen <tss@iki.fi>
parents:
8484
diff
changeset
|
649 /* Update the mailbox's last_uid field, replacing the existing |
ee1ecdda8a6b
fts-solr: Keep last_uid field updated containing the mailbox's last added UID.
Timo Sirainen <tss@iki.fi>
parents:
8484
diff
changeset
|
650 document. Note that since there is no locking, it's possible that |
ee1ecdda8a6b
fts-solr: Keep last_uid field updated containing the mailbox's last added UID.
Timo Sirainen <tss@iki.fi>
parents:
8484
diff
changeset
|
651 if another session is indexing at the same time, the last_uid value |
ee1ecdda8a6b
fts-solr: Keep last_uid field updated containing the mailbox's last added UID.
Timo Sirainen <tss@iki.fi>
parents:
8484
diff
changeset
|
652 may shrink. This doesn't really matter, we'll simply do more work |
ee1ecdda8a6b
fts-solr: Keep last_uid field updated containing the mailbox's last added UID.
Timo Sirainen <tss@iki.fi>
parents:
8484
diff
changeset
|
653 in future by reindexing some messages. */ |
ee1ecdda8a6b
fts-solr: Keep last_uid field updated containing the mailbox's last added UID.
Timo Sirainen <tss@iki.fi>
parents:
8484
diff
changeset
|
654 fts_backend_solr_add_doc_prefix(ctx, ctx->prev_uid); |
ee1ecdda8a6b
fts-solr: Keep last_uid field updated containing the mailbox's last added UID.
Timo Sirainen <tss@iki.fi>
parents:
8484
diff
changeset
|
655 str_printfa(ctx->cmd, "<field name=\"last_uid\">TRUE</field>" |
8499
252b29ac5f43
fts: Added support for handling multiple namespaces.
Timo Sirainen <tss@iki.fi>
parents:
8495
diff
changeset
|
656 "<field name=\"id\">"); |
8950
ec1ae90af21a
fts: Fixes to how virtual mailboxes are searched.
Timo Sirainen <tss@iki.fi>
parents:
8946
diff
changeset
|
657 xml_encode_id(ctx->cmd, ctx->ctx.backend, 0, ctx->uid_validity); |
8492
ee1ecdda8a6b
fts-solr: Keep last_uid field updated containing the mailbox's last added UID.
Timo Sirainen <tss@iki.fi>
parents:
8484
diff
changeset
|
658 str_append(ctx->cmd, "</field></doc></add>"); |
ee1ecdda8a6b
fts-solr: Keep last_uid field updated containing the mailbox's last added UID.
Timo Sirainen <tss@iki.fi>
parents:
8484
diff
changeset
|
659 |
ee1ecdda8a6b
fts-solr: Keep last_uid field updated containing the mailbox's last added UID.
Timo Sirainen <tss@iki.fi>
parents:
8484
diff
changeset
|
660 solr_connection_post_more(ctx->post, str_data(ctx->cmd), |
ee1ecdda8a6b
fts-solr: Keep last_uid field updated containing the mailbox's last added UID.
Timo Sirainen <tss@iki.fi>
parents:
8484
diff
changeset
|
661 str_len(ctx->cmd)); |
ee1ecdda8a6b
fts-solr: Keep last_uid field updated containing the mailbox's last added UID.
Timo Sirainen <tss@iki.fi>
parents:
8484
diff
changeset
|
662 ret = solr_connection_post_end(ctx->post); |
ee1ecdda8a6b
fts-solr: Keep last_uid field updated containing the mailbox's last added UID.
Timo Sirainen <tss@iki.fi>
parents:
8484
diff
changeset
|
663 /* commit and wait until the documents we just indexed are |
ee1ecdda8a6b
fts-solr: Keep last_uid field updated containing the mailbox's last added UID.
Timo Sirainen <tss@iki.fi>
parents:
8484
diff
changeset
|
664 visible to the following search */ |
ee1ecdda8a6b
fts-solr: Keep last_uid field updated containing the mailbox's last added UID.
Timo Sirainen <tss@iki.fi>
parents:
8484
diff
changeset
|
665 if (solr_connection_post(solr_conn, "<commit waitFlush=\"false\" " |
ee1ecdda8a6b
fts-solr: Keep last_uid field updated containing the mailbox's last added UID.
Timo Sirainen <tss@iki.fi>
parents:
8484
diff
changeset
|
666 "waitSearcher=\"true\"/>") < 0) |
ee1ecdda8a6b
fts-solr: Keep last_uid field updated containing the mailbox's last added UID.
Timo Sirainen <tss@iki.fi>
parents:
8484
diff
changeset
|
667 ret = -1; |
ee1ecdda8a6b
fts-solr: Keep last_uid field updated containing the mailbox's last added UID.
Timo Sirainen <tss@iki.fi>
parents:
8484
diff
changeset
|
668 return ret; |
ee1ecdda8a6b
fts-solr: Keep last_uid field updated containing the mailbox's last added UID.
Timo Sirainen <tss@iki.fi>
parents:
8484
diff
changeset
|
669 } |
ee1ecdda8a6b
fts-solr: Keep last_uid field updated containing the mailbox's last added UID.
Timo Sirainen <tss@iki.fi>
parents:
8484
diff
changeset
|
670 |
ee1ecdda8a6b
fts-solr: Keep last_uid field updated containing the mailbox's last added UID.
Timo Sirainen <tss@iki.fi>
parents:
8484
diff
changeset
|
671 static int |
7990
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
672 fts_backend_solr_build_deinit(struct fts_backend_build_context *_ctx) |
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
673 { |
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
674 struct solr_fts_backend_build_context *ctx = |
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
675 (struct solr_fts_backend_build_context *)_ctx; |
8492
ee1ecdda8a6b
fts-solr: Keep last_uid field updated containing the mailbox's last added UID.
Timo Sirainen <tss@iki.fi>
parents:
8484
diff
changeset
|
676 int ret; |
7990
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
677 |
8492
ee1ecdda8a6b
fts-solr: Keep last_uid field updated containing the mailbox's last added UID.
Timo Sirainen <tss@iki.fi>
parents:
8484
diff
changeset
|
678 ret = fts_backed_solr_build_commit(ctx); |
7990
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
679 str_free(&ctx->cmd); |
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
680 i_free(ctx); |
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
681 return ret; |
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
682 } |
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
683 |
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
684 static void |
8493
149c6ccc3df4
fts-solr: Use '!' as escape character in usernames. Escape '/' and '!'.
Timo Sirainen <tss@iki.fi>
parents:
8492
diff
changeset
|
685 fts_backend_solr_expunge(struct fts_backend *backend, struct mail *mail) |
7990
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
686 { |
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
687 struct mailbox_status status; |
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
688 |
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
689 mailbox_get_status(mail->box, STATUS_UIDVALIDITY, &status); |
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
690 |
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
691 T_BEGIN { |
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
692 string_t *cmd; |
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
693 |
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
694 cmd = t_str_new(256); |
8499
252b29ac5f43
fts: Added support for handling multiple namespaces.
Timo Sirainen <tss@iki.fi>
parents:
8495
diff
changeset
|
695 str_append(cmd, "<delete><id>"); |
8950
ec1ae90af21a
fts: Fixes to how virtual mailboxes are searched.
Timo Sirainen <tss@iki.fi>
parents:
8946
diff
changeset
|
696 xml_encode_id(cmd, backend, mail->uid, status.uidvalidity); |
7990
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
697 str_append(cmd, "</id></delete>"); |
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
698 |
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
699 (void)solr_connection_post(solr_conn, str_c(cmd)); |
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
700 } T_END; |
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
701 } |
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
702 |
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
703 static void |
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
704 fts_backend_solr_expunge_finish(struct fts_backend *backend ATTR_UNUSED, |
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
705 struct mailbox *box ATTR_UNUSED, |
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
706 bool committed ATTR_UNUSED) |
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
707 { |
8001
3fbfdda3e5d3
fts-solr: Send data to Solr in larger packets. Commit after expunges.
Timo Sirainen <tss@iki.fi>
parents:
8000
diff
changeset
|
708 solr_connection_post(solr_conn, |
3fbfdda3e5d3
fts-solr: Send data to Solr in larger packets. Commit after expunges.
Timo Sirainen <tss@iki.fi>
parents:
8000
diff
changeset
|
709 "<commit waitFlush=\"false\" waitSearcher=\"false\"/>"); |
7990
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
710 } |
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
711 |
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
712 static int fts_backend_solr_lock(struct fts_backend *backend ATTR_UNUSED) |
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
713 { |
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
714 return 1; |
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
715 } |
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
716 |
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
717 static void fts_backend_solr_unlock(struct fts_backend *backend ATTR_UNUSED) |
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
718 { |
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
719 } |
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
720 |
8499
252b29ac5f43
fts: Added support for handling multiple namespaces.
Timo Sirainen <tss@iki.fi>
parents:
8495
diff
changeset
|
721 static bool solr_virtual_uid_map(const char *ns_prefix, const char *mailbox, |
252b29ac5f43
fts: Added support for handling multiple namespaces.
Timo Sirainen <tss@iki.fi>
parents:
8495
diff
changeset
|
722 uint32_t uidvalidity, uint32_t *uid, |
252b29ac5f43
fts: Added support for handling multiple namespaces.
Timo Sirainen <tss@iki.fi>
parents:
8495
diff
changeset
|
723 void *context) |
8484
db78eab2ba5d
Initial support for getting full text search working nicely with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents:
8109
diff
changeset
|
724 { |
8499
252b29ac5f43
fts: Added support for handling multiple namespaces.
Timo Sirainen <tss@iki.fi>
parents:
8495
diff
changeset
|
725 struct solr_virtual_uid_map_context *ctx = context; |
8500
3efcdc45d111
Added alias_for setting for namespaces. Fixes namespace issues with fts.
Timo Sirainen <tss@iki.fi>
parents:
8499
diff
changeset
|
726 struct mail_namespace *ns; |
8499
252b29ac5f43
fts: Added support for handling multiple namespaces.
Timo Sirainen <tss@iki.fi>
parents:
8495
diff
changeset
|
727 const char *vname; |
8950
ec1ae90af21a
fts: Fixes to how virtual mailboxes are searched.
Timo Sirainen <tss@iki.fi>
parents:
8946
diff
changeset
|
728 bool convert_inbox; |
8484
db78eab2ba5d
Initial support for getting full text search working nicely with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents:
8109
diff
changeset
|
729 |
8500
3efcdc45d111
Added alias_for setting for namespaces. Fixes namespace issues with fts.
Timo Sirainen <tss@iki.fi>
parents:
8499
diff
changeset
|
730 ns = solr_get_namespaces(ctx->backend, ctx->box, ns_prefix); |
8950
ec1ae90af21a
fts: Fixes to how virtual mailboxes are searched.
Timo Sirainen <tss@iki.fi>
parents:
8946
diff
changeset
|
731 convert_inbox = (ns->flags & NAMESPACE_FLAG_INBOX) != 0 && |
ec1ae90af21a
fts: Fixes to how virtual mailboxes are searched.
Timo Sirainen <tss@iki.fi>
parents:
8946
diff
changeset
|
732 strcmp(mailbox, "INBOX") == 0; |
8500
3efcdc45d111
Added alias_for setting for namespaces. Fixes namespace issues with fts.
Timo Sirainen <tss@iki.fi>
parents:
8499
diff
changeset
|
733 for (; ns != NULL; ns = ns->alias_chain_next) { |
8950
ec1ae90af21a
fts: Fixes to how virtual mailboxes are searched.
Timo Sirainen <tss@iki.fi>
parents:
8946
diff
changeset
|
734 vname = convert_inbox ? ns->prefix : |
ec1ae90af21a
fts: Fixes to how virtual mailboxes are searched.
Timo Sirainen <tss@iki.fi>
parents:
8946
diff
changeset
|
735 mail_namespace_get_vname(ns, ctx->vname, mailbox); |
8500
3efcdc45d111
Added alias_for setting for namespaces. Fixes namespace issues with fts.
Timo Sirainen <tss@iki.fi>
parents:
8499
diff
changeset
|
736 if (mailbox_get_virtual_uid(ctx->box, vname, uidvalidity, |
3efcdc45d111
Added alias_for setting for namespaces. Fixes namespace issues with fts.
Timo Sirainen <tss@iki.fi>
parents:
8499
diff
changeset
|
737 *uid, uid)) |
3efcdc45d111
Added alias_for setting for namespaces. Fixes namespace issues with fts.
Timo Sirainen <tss@iki.fi>
parents:
8499
diff
changeset
|
738 return TRUE; |
3efcdc45d111
Added alias_for setting for namespaces. Fixes namespace issues with fts.
Timo Sirainen <tss@iki.fi>
parents:
8499
diff
changeset
|
739 } |
3efcdc45d111
Added alias_for setting for namespaces. Fixes namespace issues with fts.
Timo Sirainen <tss@iki.fi>
parents:
8499
diff
changeset
|
740 return FALSE; |
8484
db78eab2ba5d
Initial support for getting full text search working nicely with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents:
8109
diff
changeset
|
741 } |
db78eab2ba5d
Initial support for getting full text search working nicely with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents:
8109
diff
changeset
|
742 |
8004
6418d4c0ff16
fts: Added a new lookup API where the backend can look up all the fields
Timo Sirainen <tss@iki.fi>
parents:
8002
diff
changeset
|
743 static int fts_backend_solr_lookup(struct fts_backend_lookup_context *ctx, |
6418d4c0ff16
fts: Added a new lookup API where the backend can look up all the fields
Timo Sirainen <tss@iki.fi>
parents:
8002
diff
changeset
|
744 ARRAY_TYPE(seq_range) *definite_uids, |
8006
0a3b5ddb0869
Added support for sorting by X-SCORE. Currently it's only set by fts-solr.
Timo Sirainen <tss@iki.fi>
parents:
8004
diff
changeset
|
745 ARRAY_TYPE(seq_range) *maybe_uids, |
0a3b5ddb0869
Added support for sorting by X-SCORE. Currently it's only set by fts-solr.
Timo Sirainen <tss@iki.fi>
parents:
8004
diff
changeset
|
746 ARRAY_TYPE(fts_score_map) *scores) |
7990
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
747 { |
8004
6418d4c0ff16
fts: Added a new lookup API where the backend can look up all the fields
Timo Sirainen <tss@iki.fi>
parents:
8002
diff
changeset
|
748 struct mailbox *box = ctx->backend->box; |
8950
ec1ae90af21a
fts: Fixes to how virtual mailboxes are searched.
Timo Sirainen <tss@iki.fi>
parents:
8946
diff
changeset
|
749 struct mail_namespace *ns; |
8499
252b29ac5f43
fts: Added support for handling multiple namespaces.
Timo Sirainen <tss@iki.fi>
parents:
8495
diff
changeset
|
750 struct solr_virtual_uid_map_context uid_map_ctx; |
8004
6418d4c0ff16
fts: Added a new lookup API where the backend can look up all the fields
Timo Sirainen <tss@iki.fi>
parents:
8002
diff
changeset
|
751 const struct fts_backend_lookup_field *fields; |
8950
ec1ae90af21a
fts: Fixes to how virtual mailboxes are searched.
Timo Sirainen <tss@iki.fi>
parents:
8946
diff
changeset
|
752 const char *box_name; |
8004
6418d4c0ff16
fts: Added a new lookup API where the backend can look up all the fields
Timo Sirainen <tss@iki.fi>
parents:
8002
diff
changeset
|
753 unsigned int i, count; |
7990
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
754 struct mailbox_status status; |
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
755 string_t *str; |
8484
db78eab2ba5d
Initial support for getting full text search working nicely with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents:
8109
diff
changeset
|
756 bool virtual; |
7990
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
757 |
8484
db78eab2ba5d
Initial support for getting full text search working nicely with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents:
8109
diff
changeset
|
758 virtual = strcmp(box->storage->name, "virtual") == 0; |
8004
6418d4c0ff16
fts: Added a new lookup API where the backend can look up all the fields
Timo Sirainen <tss@iki.fi>
parents:
8002
diff
changeset
|
759 mailbox_get_status(box, STATUS_UIDVALIDITY, &status); |
7990
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
760 |
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
761 str = t_str_new(256); |
8484
db78eab2ba5d
Initial support for getting full text search working nicely with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents:
8109
diff
changeset
|
762 if (!virtual) { |
8499
252b29ac5f43
fts: Added support for handling multiple namespaces.
Timo Sirainen <tss@iki.fi>
parents:
8495
diff
changeset
|
763 str_printfa(str, "fl=uid,score&rows=%u&sort=uid+asc&q=", |
8484
db78eab2ba5d
Initial support for getting full text search working nicely with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents:
8109
diff
changeset
|
764 status.uidnext); |
db78eab2ba5d
Initial support for getting full text search working nicely with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents:
8109
diff
changeset
|
765 } else { |
8499
252b29ac5f43
fts: Added support for handling multiple namespaces.
Timo Sirainen <tss@iki.fi>
parents:
8495
diff
changeset
|
766 str_printfa(str, "fl=uid,score,box,uidv,ns&rows=%u" |
252b29ac5f43
fts: Added support for handling multiple namespaces.
Timo Sirainen <tss@iki.fi>
parents:
8495
diff
changeset
|
767 "&sort=box+asc,uid+asc&q=", |
8484
db78eab2ba5d
Initial support for getting full text search working nicely with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents:
8109
diff
changeset
|
768 SOLR_MAX_ROWS); |
db78eab2ba5d
Initial support for getting full text search working nicely with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents:
8109
diff
changeset
|
769 } |
8004
6418d4c0ff16
fts: Added a new lookup API where the backend can look up all the fields
Timo Sirainen <tss@iki.fi>
parents:
8002
diff
changeset
|
770 |
6418d4c0ff16
fts: Added a new lookup API where the backend can look up all the fields
Timo Sirainen <tss@iki.fi>
parents:
8002
diff
changeset
|
771 /* build a lucene search query from the fields */ |
6418d4c0ff16
fts: Added a new lookup API where the backend can look up all the fields
Timo Sirainen <tss@iki.fi>
parents:
8002
diff
changeset
|
772 fields = array_get(&ctx->fields, &count); |
6418d4c0ff16
fts: Added a new lookup API where the backend can look up all the fields
Timo Sirainen <tss@iki.fi>
parents:
8002
diff
changeset
|
773 for (i = 0; i < count; i++) { |
6418d4c0ff16
fts: Added a new lookup API where the backend can look up all the fields
Timo Sirainen <tss@iki.fi>
parents:
8002
diff
changeset
|
774 if (i > 0) |
8499
252b29ac5f43
fts: Added support for handling multiple namespaces.
Timo Sirainen <tss@iki.fi>
parents:
8495
diff
changeset
|
775 str_append_c(str, '+'); |
8004
6418d4c0ff16
fts: Added a new lookup API where the backend can look up all the fields
Timo Sirainen <tss@iki.fi>
parents:
8002
diff
changeset
|
776 |
6418d4c0ff16
fts: Added a new lookup API where the backend can look up all the fields
Timo Sirainen <tss@iki.fi>
parents:
8002
diff
changeset
|
777 if ((fields[i].flags & FTS_LOOKUP_FLAG_INVERT) != 0) |
6418d4c0ff16
fts: Added a new lookup API where the backend can look up all the fields
Timo Sirainen <tss@iki.fi>
parents:
8002
diff
changeset
|
778 str_append_c(str, '-'); |
6418d4c0ff16
fts: Added a new lookup API where the backend can look up all the fields
Timo Sirainen <tss@iki.fi>
parents:
8002
diff
changeset
|
779 |
6418d4c0ff16
fts: Added a new lookup API where the backend can look up all the fields
Timo Sirainen <tss@iki.fi>
parents:
8002
diff
changeset
|
780 if ((fields[i].flags & FTS_LOOKUP_FLAG_HEADER) == 0) { |
6418d4c0ff16
fts: Added a new lookup API where the backend can look up all the fields
Timo Sirainen <tss@iki.fi>
parents:
8002
diff
changeset
|
781 /* body only */ |
6418d4c0ff16
fts: Added a new lookup API where the backend can look up all the fields
Timo Sirainen <tss@iki.fi>
parents:
8002
diff
changeset
|
782 i_assert((fields[i].flags & FTS_LOOKUP_FLAG_BODY) != 0); |
6418d4c0ff16
fts: Added a new lookup API where the backend can look up all the fields
Timo Sirainen <tss@iki.fi>
parents:
8002
diff
changeset
|
783 str_append(str, "body:"); |
9141
d21bd1a1710a
solr: Don't use "any" copyfield, it doubles the index size.
Timo Sirainen <tss@iki.fi>
parents:
8950
diff
changeset
|
784 solr_quote_http(str, fields[i].key); |
8004
6418d4c0ff16
fts: Added a new lookup API where the backend can look up all the fields
Timo Sirainen <tss@iki.fi>
parents:
8002
diff
changeset
|
785 } else if ((fields[i].flags & FTS_LOOKUP_FLAG_BODY) == 0) { |
6418d4c0ff16
fts: Added a new lookup API where the backend can look up all the fields
Timo Sirainen <tss@iki.fi>
parents:
8002
diff
changeset
|
786 /* header only */ |
6418d4c0ff16
fts: Added a new lookup API where the backend can look up all the fields
Timo Sirainen <tss@iki.fi>
parents:
8002
diff
changeset
|
787 str_append(str, "hdr:"); |
9141
d21bd1a1710a
solr: Don't use "any" copyfield, it doubles the index size.
Timo Sirainen <tss@iki.fi>
parents:
8950
diff
changeset
|
788 solr_quote_http(str, fields[i].key); |
8004
6418d4c0ff16
fts: Added a new lookup API where the backend can look up all the fields
Timo Sirainen <tss@iki.fi>
parents:
8002
diff
changeset
|
789 } else { |
6418d4c0ff16
fts: Added a new lookup API where the backend can look up all the fields
Timo Sirainen <tss@iki.fi>
parents:
8002
diff
changeset
|
790 /* both */ |
9141
d21bd1a1710a
solr: Don't use "any" copyfield, it doubles the index size.
Timo Sirainen <tss@iki.fi>
parents:
8950
diff
changeset
|
791 str_append(str, "(body:"); |
d21bd1a1710a
solr: Don't use "any" copyfield, it doubles the index size.
Timo Sirainen <tss@iki.fi>
parents:
8950
diff
changeset
|
792 solr_quote_http(str, fields[i].key); |
d21bd1a1710a
solr: Don't use "any" copyfield, it doubles the index size.
Timo Sirainen <tss@iki.fi>
parents:
8950
diff
changeset
|
793 str_append(str, "+OR+hdr:"); |
d21bd1a1710a
solr: Don't use "any" copyfield, it doubles the index size.
Timo Sirainen <tss@iki.fi>
parents:
8950
diff
changeset
|
794 solr_quote_http(str, fields[i].key); |
d21bd1a1710a
solr: Don't use "any" copyfield, it doubles the index size.
Timo Sirainen <tss@iki.fi>
parents:
8950
diff
changeset
|
795 str_append_c(str, ')'); |
8004
6418d4c0ff16
fts: Added a new lookup API where the backend can look up all the fields
Timo Sirainen <tss@iki.fi>
parents:
8002
diff
changeset
|
796 } |
7990
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
797 } |
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
798 |
8004
6418d4c0ff16
fts: Added a new lookup API where the backend can look up all the fields
Timo Sirainen <tss@iki.fi>
parents:
8002
diff
changeset
|
799 /* use a separate filter query for selecting the mailbox. it shouldn't |
6418d4c0ff16
fts: Added a new lookup API where the backend can look up all the fields
Timo Sirainen <tss@iki.fi>
parents:
8002
diff
changeset
|
800 affect the score and there could be some caching benefits too. */ |
8505
5cc1c4f3d38b
fts-solr: In filter queries use "+" prefixes to require all the parameters to match.
Timo Sirainen <tss@iki.fi>
parents:
8500
diff
changeset
|
801 str_append(str, "&fq=%2Buser:"); |
8499
252b29ac5f43
fts: Added support for handling multiple namespaces.
Timo Sirainen <tss@iki.fi>
parents:
8495
diff
changeset
|
802 solr_quote_http(str, box->storage->ns->user->username); |
8495
ffb37c392166
Finished making full text search indexes work fast with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents:
8493
diff
changeset
|
803 if (virtual) |
8499
252b29ac5f43
fts: Added support for handling multiple namespaces.
Timo Sirainen <tss@iki.fi>
parents:
8495
diff
changeset
|
804 fts_backend_solr_filter_mailboxes(ctx->backend, str, box); |
8495
ffb37c392166
Finished making full text search indexes work fast with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents:
8493
diff
changeset
|
805 else { |
8950
ec1ae90af21a
fts: Fixes to how virtual mailboxes are searched.
Timo Sirainen <tss@iki.fi>
parents:
8946
diff
changeset
|
806 box_name = fts_box_get_root(box, &ns); |
8505
5cc1c4f3d38b
fts-solr: In filter queries use "+" prefixes to require all the parameters to match.
Timo Sirainen <tss@iki.fi>
parents:
8500
diff
changeset
|
807 str_printfa(str, "+%%2Buidv:%u+%%2Bbox:", status.uidvalidity); |
8950
ec1ae90af21a
fts: Fixes to how virtual mailboxes are searched.
Timo Sirainen <tss@iki.fi>
parents:
8946
diff
changeset
|
808 solr_quote_http(str, box_name); |
ec1ae90af21a
fts: Fixes to how virtual mailboxes are searched.
Timo Sirainen <tss@iki.fi>
parents:
8946
diff
changeset
|
809 solr_add_ns_query_http(str, ctx->backend, ns); |
8484
db78eab2ba5d
Initial support for getting full text search working nicely with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents:
8109
diff
changeset
|
810 } |
db78eab2ba5d
Initial support for getting full text search working nicely with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents:
8109
diff
changeset
|
811 |
7990
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
812 array_clear(maybe_uids); |
8484
db78eab2ba5d
Initial support for getting full text search working nicely with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents:
8109
diff
changeset
|
813 if (!virtual) { |
db78eab2ba5d
Initial support for getting full text search working nicely with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents:
8109
diff
changeset
|
814 return solr_connection_select(solr_conn, str_c(str), NULL, NULL, |
db78eab2ba5d
Initial support for getting full text search working nicely with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents:
8109
diff
changeset
|
815 definite_uids, scores); |
db78eab2ba5d
Initial support for getting full text search working nicely with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents:
8109
diff
changeset
|
816 } else { |
8499
252b29ac5f43
fts: Added support for handling multiple namespaces.
Timo Sirainen <tss@iki.fi>
parents:
8495
diff
changeset
|
817 memset(&uid_map_ctx, 0, sizeof(uid_map_ctx)); |
252b29ac5f43
fts: Added support for handling multiple namespaces.
Timo Sirainen <tss@iki.fi>
parents:
8495
diff
changeset
|
818 uid_map_ctx.backend = ctx->backend; |
252b29ac5f43
fts: Added support for handling multiple namespaces.
Timo Sirainen <tss@iki.fi>
parents:
8495
diff
changeset
|
819 uid_map_ctx.box = box; |
252b29ac5f43
fts: Added support for handling multiple namespaces.
Timo Sirainen <tss@iki.fi>
parents:
8495
diff
changeset
|
820 uid_map_ctx.vname = t_str_new(256); |
8484
db78eab2ba5d
Initial support for getting full text search working nicely with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents:
8109
diff
changeset
|
821 return solr_connection_select(solr_conn, str_c(str), |
8499
252b29ac5f43
fts: Added support for handling multiple namespaces.
Timo Sirainen <tss@iki.fi>
parents:
8495
diff
changeset
|
822 solr_virtual_uid_map, |
252b29ac5f43
fts: Added support for handling multiple namespaces.
Timo Sirainen <tss@iki.fi>
parents:
8495
diff
changeset
|
823 &uid_map_ctx, |
8484
db78eab2ba5d
Initial support for getting full text search working nicely with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents:
8109
diff
changeset
|
824 definite_uids, scores); |
db78eab2ba5d
Initial support for getting full text search working nicely with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents:
8109
diff
changeset
|
825 } |
7990
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
826 } |
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
827 |
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
828 struct fts_backend fts_backend_solr = { |
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
829 MEMBER(name) "solr", |
9292
96b764649cbb
fts: Fixed crashing with Squat.
Timo Sirainen <tss@iki.fi>
parents:
9141
diff
changeset
|
830 MEMBER(flags) FTS_BACKEND_FLAG_VIRTUAL_LOOKUPS, |
7990
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
831 |
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
832 { |
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
833 fts_backend_solr_init, |
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
834 fts_backend_solr_deinit, |
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
835 fts_backend_solr_get_last_uid, |
8495
ffb37c392166
Finished making full text search indexes work fast with virtual mailboxes.
Timo Sirainen <tss@iki.fi>
parents:
8493
diff
changeset
|
836 fts_backend_solr_get_all_last_uids, |
7990
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
837 fts_backend_solr_build_init, |
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
838 fts_backend_solr_build_more, |
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
839 fts_backend_solr_build_deinit, |
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
840 fts_backend_solr_expunge, |
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
841 fts_backend_solr_expunge_finish, |
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
842 fts_backend_solr_lock, |
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
843 fts_backend_solr_unlock, |
8004
6418d4c0ff16
fts: Added a new lookup API where the backend can look up all the fields
Timo Sirainen <tss@iki.fi>
parents:
8002
diff
changeset
|
844 NULL, |
6418d4c0ff16
fts: Added a new lookup API where the backend can look up all the fields
Timo Sirainen <tss@iki.fi>
parents:
8002
diff
changeset
|
845 NULL, |
6418d4c0ff16
fts: Added a new lookup API where the backend can look up all the fields
Timo Sirainen <tss@iki.fi>
parents:
8002
diff
changeset
|
846 fts_backend_solr_lookup |
7990
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
847 } |
662172573fe1
Initial code to support Apache Solr (Lucene indexing server).
Timo Sirainen <tss@iki.fi>
parents:
diff
changeset
|
848 }; |