dovecot/original-hg/dovecot-1.2: src/plugins/fts-solr/fts-backend-solr.c annotate

annotate src/plugins/fts-solr/fts-backend-solr.c @ 9604:cf0da2cd31fb HEAD

fts-solr: Replace characters not valid for XML with replacement char.

author	Timo Sirainen <tss@iki.fi>
date	Fri, 20 Aug 2010 20:38:26 +0100
parents	00cd9aacd03c
children	ac0855b1bbea

rev	line source
9532 00cd9aacd03c Updated copyright notices to include year 2010. Timo Sirainen <tss@iki.fi> parents: 9292 diff changeset	1 /* Copyright (c) 2006-2010 Dovecot authors, see the included COPYING file */
7990 662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	2
662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	3 #include "lib.h"
662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	4 #include "array.h"
662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	5 #include "str.h"
8495 ffb37c392166 Finished making full text search indexes work fast with virtual mailboxes. Timo Sirainen <tss@iki.fi> parents: 8493 diff changeset	6 #include "strescape.h"
9604 cf0da2cd31fb fts-solr: Replace characters not valid for XML with replacement char. Timo Sirainen <tss@iki.fi> parents: 9532 diff changeset	7 #include "unichar.h"
7990 662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	8 #include "mail-storage-private.h"
8109 e7929190cd32 fts-solr: Fixed compiling with recent struct mail_user changes. Timo Sirainen <tss@iki.fi> parents: 8006 diff changeset	9 #include "mail-namespace.h"
7990 662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	10 #include "solr-connection.h"
662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	11 #include "fts-solr-plugin.h"
662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	12
8495 ffb37c392166 Finished making full text search indexes work fast with virtual mailboxes. Timo Sirainen <tss@iki.fi> parents: 8493 diff changeset	13 #include <ctype.h>
7990 662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	14
8001 3fbfdda3e5d3 fts-solr: Send data to Solr in larger packets. Commit after expunges. Timo Sirainen <tss@iki.fi> parents: 8000 diff changeset	15 #define SOLR_CMDBUF_SIZE (1024*64)
8484 db78eab2ba5d Initial support for getting full text search working nicely with virtual mailboxes. Timo Sirainen <tss@iki.fi> parents: 8109 diff changeset	16 #define SOLR_MAX_ROWS 100000
8499 252b29ac5f43 fts: Added support for handling multiple namespaces. Timo Sirainen <tss@iki.fi> parents: 8495 diff changeset	17 #define FTS_SOLR_MAX_BOX_INC_PATTERNS 5
252b29ac5f43 fts: Added support for handling multiple namespaces. Timo Sirainen <tss@iki.fi> parents: 8495 diff changeset	18 #define FTS_SOLR_MAX_BOX_EXC_PATTERNS 5
8001 3fbfdda3e5d3 fts-solr: Send data to Solr in larger packets. Commit after expunges. Timo Sirainen <tss@iki.fi> parents: 8000 diff changeset	19
8493 149c6ccc3df4 fts-solr: Use '!' as escape character in usernames. Escape '/' and '!'. Timo Sirainen <tss@iki.fi> parents: 8492 diff changeset	20 struct solr_fts_backend {
149c6ccc3df4 fts-solr: Use '!' as escape character in usernames. Escape '/' and '!'. Timo Sirainen <tss@iki.fi> parents: 8492 diff changeset	21 struct fts_backend backend;
8950 ec1ae90af21a fts: Fixes to how virtual mailboxes are searched. Timo Sirainen <tss@iki.fi> parents: 8946 diff changeset	22 char id_username, id_namespace, *id_box_name;
8499 252b29ac5f43 fts: Added support for handling multiple namespaces. Timo Sirainen <tss@iki.fi> parents: 8495 diff changeset	23 struct mail_namespace *default_ns;
8493 149c6ccc3df4 fts-solr: Use '!' as escape character in usernames. Escape '/' and '!'. Timo Sirainen <tss@iki.fi> parents: 8492 diff changeset	24 };
149c6ccc3df4 fts-solr: Use '!' as escape character in usernames. Escape '/' and '!'. Timo Sirainen <tss@iki.fi> parents: 8492 diff changeset	25
7990 662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	26 struct solr_fts_backend_build_context {
662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	27 struct fts_backend_build_context ctx;
662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	28
662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	29 struct solr_connection_post *post;
662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	30 uint32_t prev_uid, uid_validity;
662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	31 string_t *cmd;
662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	32 bool headers;
662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	33 };
662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	34
8499 252b29ac5f43 fts: Added support for handling multiple namespaces. Timo Sirainen <tss@iki.fi> parents: 8495 diff changeset	35 struct solr_virtual_uid_map_context {
252b29ac5f43 fts: Added support for handling multiple namespaces. Timo Sirainen <tss@iki.fi> parents: 8495 diff changeset	36 struct fts_backend *backend;
252b29ac5f43 fts: Added support for handling multiple namespaces. Timo Sirainen <tss@iki.fi> parents: 8495 diff changeset	37 struct mailbox *box;
252b29ac5f43 fts: Added support for handling multiple namespaces. Timo Sirainen <tss@iki.fi> parents: 8495 diff changeset	38 string_t *vname;
252b29ac5f43 fts: Added support for handling multiple namespaces. Timo Sirainen <tss@iki.fi> parents: 8495 diff changeset	39 };
252b29ac5f43 fts: Added support for handling multiple namespaces. Timo Sirainen <tss@iki.fi> parents: 8495 diff changeset	40
8495 ffb37c392166 Finished making full text search indexes work fast with virtual mailboxes. Timo Sirainen <tss@iki.fi> parents: 8493 diff changeset	41 struct fts_backend_solr_get_last_uids_context {
8499 252b29ac5f43 fts: Added support for handling multiple namespaces. Timo Sirainen <tss@iki.fi> parents: 8495 diff changeset	42 struct fts_backend *backend;
8495 ffb37c392166 Finished making full text search indexes work fast with virtual mailboxes. Timo Sirainen <tss@iki.fi> parents: 8493 diff changeset	43 pool_t pool;
ffb37c392166 Finished making full text search indexes work fast with virtual mailboxes. Timo Sirainen <tss@iki.fi> parents: 8493 diff changeset	44 ARRAY_TYPE(fts_backend_uid_map) *last_uids;
8499 252b29ac5f43 fts: Added support for handling multiple namespaces. Timo Sirainen <tss@iki.fi> parents: 8495 diff changeset	45
252b29ac5f43 fts: Added support for handling multiple namespaces. Timo Sirainen <tss@iki.fi> parents: 8495 diff changeset	46 struct mailbox *box;
252b29ac5f43 fts: Added support for handling multiple namespaces. Timo Sirainen <tss@iki.fi> parents: 8495 diff changeset	47 string_t *vname;
8495 ffb37c392166 Finished making full text search indexes work fast with virtual mailboxes. Timo Sirainen <tss@iki.fi> parents: 8493 diff changeset	48 };
ffb37c392166 Finished making full text search indexes work fast with virtual mailboxes. Timo Sirainen <tss@iki.fi> parents: 8493 diff changeset	49
7990 662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	50 static struct solr_connection *solr_conn = NULL;
662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	51
8950 ec1ae90af21a fts: Fixes to how virtual mailboxes are searched. Timo Sirainen <tss@iki.fi> parents: 8946 diff changeset	52 static void fts_box_name_get_root(struct mail_namespace ns, const char name)
ec1ae90af21a fts: Fixes to how virtual mailboxes are searched. Timo Sirainen <tss@iki.fi> parents: 8946 diff changeset	53 {
ec1ae90af21a fts: Fixes to how virtual mailboxes are searched. Timo Sirainen <tss@iki.fi> parents: 8946 diff changeset	54 struct mail_namespace orig_ns = ns;
ec1ae90af21a fts: Fixes to how virtual mailboxes are searched. Timo Sirainen <tss@iki.fi> parents: 8946 diff changeset	55
ec1ae90af21a fts: Fixes to how virtual mailboxes are searched. Timo Sirainen <tss@iki.fi> parents: 8946 diff changeset	56 while ((*ns)->alias_for != NULL)
ec1ae90af21a fts: Fixes to how virtual mailboxes are searched. Timo Sirainen <tss@iki.fi> parents: 8946 diff changeset	57 ns = (ns)->alias_for;
ec1ae90af21a fts: Fixes to how virtual mailboxes are searched. Timo Sirainen <tss@iki.fi> parents: 8946 diff changeset	58
ec1ae90af21a fts: Fixes to how virtual mailboxes are searched. Timo Sirainen <tss@iki.fi> parents: 8946 diff changeset	59 if (*name == '\0' && ns != orig_ns &&
ec1ae90af21a fts: Fixes to how virtual mailboxes are searched. Timo Sirainen <tss@iki.fi> parents: 8946 diff changeset	60 ((*ns)->flags & NAMESPACE_FLAG_INBOX) != 0) {
ec1ae90af21a fts: Fixes to how virtual mailboxes are searched. Timo Sirainen <tss@iki.fi> parents: 8946 diff changeset	61 /* ugly workaround to allow selecting INBOX from a Maildir/
ec1ae90af21a fts: Fixes to how virtual mailboxes are searched. Timo Sirainen <tss@iki.fi> parents: 8946 diff changeset	62 when it's not in the inbox=yes namespace. */
ec1ae90af21a fts: Fixes to how virtual mailboxes are searched. Timo Sirainen <tss@iki.fi> parents: 8946 diff changeset	63 *name = "INBOX";
ec1ae90af21a fts: Fixes to how virtual mailboxes are searched. Timo Sirainen <tss@iki.fi> parents: 8946 diff changeset	64 }
ec1ae90af21a fts: Fixes to how virtual mailboxes are searched. Timo Sirainen <tss@iki.fi> parents: 8946 diff changeset	65 }
ec1ae90af21a fts: Fixes to how virtual mailboxes are searched. Timo Sirainen <tss@iki.fi> parents: 8946 diff changeset	66
ec1ae90af21a fts: Fixes to how virtual mailboxes are searched. Timo Sirainen <tss@iki.fi> parents: 8946 diff changeset	67 static const char *
ec1ae90af21a fts: Fixes to how virtual mailboxes are searched. Timo Sirainen <tss@iki.fi> parents: 8946 diff changeset	68 fts_box_get_root(struct mailbox box, struct mail_namespace *ns_r)
ec1ae90af21a fts: Fixes to how virtual mailboxes are searched. Timo Sirainen <tss@iki.fi> parents: 8946 diff changeset	69 {
ec1ae90af21a fts: Fixes to how virtual mailboxes are searched. Timo Sirainen <tss@iki.fi> parents: 8946 diff changeset	70 struct mail_namespace *ns = box->storage->ns;
ec1ae90af21a fts: Fixes to how virtual mailboxes are searched. Timo Sirainen <tss@iki.fi> parents: 8946 diff changeset	71 const char *name = box->name;
ec1ae90af21a fts: Fixes to how virtual mailboxes are searched. Timo Sirainen <tss@iki.fi> parents: 8946 diff changeset	72
ec1ae90af21a fts: Fixes to how virtual mailboxes are searched. Timo Sirainen <tss@iki.fi> parents: 8946 diff changeset	73 fts_box_name_get_root(&ns, &name);
ec1ae90af21a fts: Fixes to how virtual mailboxes are searched. Timo Sirainen <tss@iki.fi> parents: 8946 diff changeset	74 *ns_r = ns;
ec1ae90af21a fts: Fixes to how virtual mailboxes are searched. Timo Sirainen <tss@iki.fi> parents: 8946 diff changeset	75 return name;
ec1ae90af21a fts: Fixes to how virtual mailboxes are searched. Timo Sirainen <tss@iki.fi> parents: 8946 diff changeset	76 }
ec1ae90af21a fts: Fixes to how virtual mailboxes are searched. Timo Sirainen <tss@iki.fi> parents: 8946 diff changeset	77
9604 cf0da2cd31fb fts-solr: Replace characters not valid for XML with replacement char. Timo Sirainen <tss@iki.fi> parents: 9532 diff changeset	78 static bool is_valid_xml_char(unichar_t chr)
cf0da2cd31fb fts-solr: Replace characters not valid for XML with replacement char. Timo Sirainen <tss@iki.fi> parents: 9532 diff changeset	79 {
cf0da2cd31fb fts-solr: Replace characters not valid for XML with replacement char. Timo Sirainen <tss@iki.fi> parents: 9532 diff changeset	80 /* Valid characters in XML:
cf0da2cd31fb fts-solr: Replace characters not valid for XML with replacement char. Timo Sirainen <tss@iki.fi> parents: 9532 diff changeset	81
cf0da2cd31fb fts-solr: Replace characters not valid for XML with replacement char. Timo Sirainen <tss@iki.fi> parents: 9532 diff changeset	82 #x9 \| #xA \| #xD \| [#x20-#xD7FF] \| [#xE000-#xFFFD] \|
cf0da2cd31fb fts-solr: Replace characters not valid for XML with replacement char. Timo Sirainen <tss@iki.fi> parents: 9532 diff changeset	83 [#x10000-#x10FFFF]
cf0da2cd31fb fts-solr: Replace characters not valid for XML with replacement char. Timo Sirainen <tss@iki.fi> parents: 9532 diff changeset	84
cf0da2cd31fb fts-solr: Replace characters not valid for XML with replacement char. Timo Sirainen <tss@iki.fi> parents: 9532 diff changeset	85 This function gets called only for #x80 and higher */
cf0da2cd31fb fts-solr: Replace characters not valid for XML with replacement char. Timo Sirainen <tss@iki.fi> parents: 9532 diff changeset	86 if (chr > 0xd7ff && chr < 0xe000)
cf0da2cd31fb fts-solr: Replace characters not valid for XML with replacement char. Timo Sirainen <tss@iki.fi> parents: 9532 diff changeset	87 return FALSE;
cf0da2cd31fb fts-solr: Replace characters not valid for XML with replacement char. Timo Sirainen <tss@iki.fi> parents: 9532 diff changeset	88 if (chr > 0xfffd && chr < 0x10000)
cf0da2cd31fb fts-solr: Replace characters not valid for XML with replacement char. Timo Sirainen <tss@iki.fi> parents: 9532 diff changeset	89 return FALSE;
cf0da2cd31fb fts-solr: Replace characters not valid for XML with replacement char. Timo Sirainen <tss@iki.fi> parents: 9532 diff changeset	90 return chr < 0x10ffff;
cf0da2cd31fb fts-solr: Replace characters not valid for XML with replacement char. Timo Sirainen <tss@iki.fi> parents: 9532 diff changeset	91 }
cf0da2cd31fb fts-solr: Replace characters not valid for XML with replacement char. Timo Sirainen <tss@iki.fi> parents: 9532 diff changeset	92
8000 61be6a4722c5 fts-solr: Encode entities when indexing data. Timo Sirainen <tss@iki.fi> parents: 7999 diff changeset	93 static void
61be6a4722c5 fts-solr: Encode entities when indexing data. Timo Sirainen <tss@iki.fi> parents: 7999 diff changeset	94 xml_encode_data(string_t dest, const unsigned char data, unsigned int len)
7990 662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	95 {
9604 cf0da2cd31fb fts-solr: Replace characters not valid for XML with replacement char. Timo Sirainen <tss@iki.fi> parents: 9532 diff changeset	96 unichar_t chr;
8000 61be6a4722c5 fts-solr: Encode entities when indexing data. Timo Sirainen <tss@iki.fi> parents: 7999 diff changeset	97 unsigned int i;
61be6a4722c5 fts-solr: Encode entities when indexing data. Timo Sirainen <tss@iki.fi> parents: 7999 diff changeset	98
61be6a4722c5 fts-solr: Encode entities when indexing data. Timo Sirainen <tss@iki.fi> parents: 7999 diff changeset	99 for (i = 0; i < len; i++) {
61be6a4722c5 fts-solr: Encode entities when indexing data. Timo Sirainen <tss@iki.fi> parents: 7999 diff changeset	100 switch (data[i]) {
7990 662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	101 case '&':
662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	102 str_append(dest, "&");
662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	103 break;
662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	104 case '<':
662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	105 str_append(dest, "<");
662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	106 break;
662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	107 case '>':
662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	108 str_append(dest, ">");
662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	109 break;
8848 dd441977c578 fts-solr: Solr breaks when it sees control characters, so replace them with spaces. Timo Sirainen <tss@iki.fi> parents: 8590 diff changeset	110 case '\t':
dd441977c578 fts-solr: Solr breaks when it sees control characters, so replace them with spaces. Timo Sirainen <tss@iki.fi> parents: 8590 diff changeset	111 case '\n':
dd441977c578 fts-solr: Solr breaks when it sees control characters, so replace them with spaces. Timo Sirainen <tss@iki.fi> parents: 8590 diff changeset	112 case '\r':
dd441977c578 fts-solr: Solr breaks when it sees control characters, so replace them with spaces. Timo Sirainen <tss@iki.fi> parents: 8590 diff changeset	113 /* exceptions to the following control char check */
dd441977c578 fts-solr: Solr breaks when it sees control characters, so replace them with spaces. Timo Sirainen <tss@iki.fi> parents: 8590 diff changeset	114 str_append_c(dest, data[i]);
dd441977c578 fts-solr: Solr breaks when it sees control characters, so replace them with spaces. Timo Sirainen <tss@iki.fi> parents: 8590 diff changeset	115 break;
7990 662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	116 default:
8848 dd441977c578 fts-solr: Solr breaks when it sees control characters, so replace them with spaces. Timo Sirainen <tss@iki.fi> parents: 8590 diff changeset	117 if (data[i] < 32) {
dd441977c578 fts-solr: Solr breaks when it sees control characters, so replace them with spaces. Timo Sirainen <tss@iki.fi> parents: 8590 diff changeset	118 /* SOLR doesn't like control characters.
dd441977c578 fts-solr: Solr breaks when it sees control characters, so replace them with spaces. Timo Sirainen <tss@iki.fi> parents: 8590 diff changeset	119 replace them with spaces. */
dd441977c578 fts-solr: Solr breaks when it sees control characters, so replace them with spaces. Timo Sirainen <tss@iki.fi> parents: 8590 diff changeset	120 str_append_c(dest, ' ');
9604 cf0da2cd31fb fts-solr: Replace characters not valid for XML with replacement char. Timo Sirainen <tss@iki.fi> parents: 9532 diff changeset	121 } else if (data[i] >= 0x80) {
cf0da2cd31fb fts-solr: Replace characters not valid for XML with replacement char. Timo Sirainen <tss@iki.fi> parents: 9532 diff changeset	122 /* make sure the character is valid for XML
cf0da2cd31fb fts-solr: Replace characters not valid for XML with replacement char. Timo Sirainen <tss@iki.fi> parents: 9532 diff changeset	123 so we don't get XML parser errors */
cf0da2cd31fb fts-solr: Replace characters not valid for XML with replacement char. Timo Sirainen <tss@iki.fi> parents: 9532 diff changeset	124 unsigned int char_len =
cf0da2cd31fb fts-solr: Replace characters not valid for XML with replacement char. Timo Sirainen <tss@iki.fi> parents: 9532 diff changeset	125 uni_utf8_char_bytes(data[0]);
cf0da2cd31fb fts-solr: Replace characters not valid for XML with replacement char. Timo Sirainen <tss@iki.fi> parents: 9532 diff changeset	126 if (i + char_len <= len &&
cf0da2cd31fb fts-solr: Replace characters not valid for XML with replacement char. Timo Sirainen <tss@iki.fi> parents: 9532 diff changeset	127 uni_utf8_get_char_n(data, len, &chr) == 0 &&
cf0da2cd31fb fts-solr: Replace characters not valid for XML with replacement char. Timo Sirainen <tss@iki.fi> parents: 9532 diff changeset	128 is_valid_xml_char(chr))
cf0da2cd31fb fts-solr: Replace characters not valid for XML with replacement char. Timo Sirainen <tss@iki.fi> parents: 9532 diff changeset	129 str_append_n(dest, data + i, char_len);
cf0da2cd31fb fts-solr: Replace characters not valid for XML with replacement char. Timo Sirainen <tss@iki.fi> parents: 9532 diff changeset	130 else {
cf0da2cd31fb fts-solr: Replace characters not valid for XML with replacement char. Timo Sirainen <tss@iki.fi> parents: 9532 diff changeset	131 str_append_n(dest, utf8_replacement_char,
cf0da2cd31fb fts-solr: Replace characters not valid for XML with replacement char. Timo Sirainen <tss@iki.fi> parents: 9532 diff changeset	132 UTF8_REPLACEMENT_CHAR_LEN);
cf0da2cd31fb fts-solr: Replace characters not valid for XML with replacement char. Timo Sirainen <tss@iki.fi> parents: 9532 diff changeset	133 }
cf0da2cd31fb fts-solr: Replace characters not valid for XML with replacement char. Timo Sirainen <tss@iki.fi> parents: 9532 diff changeset	134 i += char_len - 1;
8848 dd441977c578 fts-solr: Solr breaks when it sees control characters, so replace them with spaces. Timo Sirainen <tss@iki.fi> parents: 8590 diff changeset	135 } else {
dd441977c578 fts-solr: Solr breaks when it sees control characters, so replace them with spaces. Timo Sirainen <tss@iki.fi> parents: 8590 diff changeset	136 str_append_c(dest, data[i]);
dd441977c578 fts-solr: Solr breaks when it sees control characters, so replace them with spaces. Timo Sirainen <tss@iki.fi> parents: 8590 diff changeset	137 }
7990 662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	138 break;
662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	139 }
9604 cf0da2cd31fb fts-solr: Replace characters not valid for XML with replacement char. Timo Sirainen <tss@iki.fi> parents: 9532 diff changeset	140 i += uni_utf8_char_bytes(data[0]);
7990 662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	141 }
662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	142 }
662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	143
8000 61be6a4722c5 fts-solr: Encode entities when indexing data. Timo Sirainen <tss@iki.fi> parents: 7999 diff changeset	144 static void xml_encode(string_t dest, const char str)
61be6a4722c5 fts-solr: Encode entities when indexing data. Timo Sirainen <tss@iki.fi> parents: 7999 diff changeset	145 {
61be6a4722c5 fts-solr: Encode entities when indexing data. Timo Sirainen <tss@iki.fi> parents: 7999 diff changeset	146 xml_encode_data(dest, (const unsigned char *)str, strlen(str));
61be6a4722c5 fts-solr: Encode entities when indexing data. Timo Sirainen <tss@iki.fi> parents: 7999 diff changeset	147 }
61be6a4722c5 fts-solr: Encode entities when indexing data. Timo Sirainen <tss@iki.fi> parents: 7999 diff changeset	148
8493 149c6ccc3df4 fts-solr: Use '!' as escape character in usernames. Escape '/' and '!'. Timo Sirainen <tss@iki.fi> parents: 8492 diff changeset	149 static const char solr_escape_id_str(const char str)
149c6ccc3df4 fts-solr: Use '!' as escape character in usernames. Escape '/' and '!'. Timo Sirainen <tss@iki.fi> parents: 8492 diff changeset	150 {
149c6ccc3df4 fts-solr: Use '!' as escape character in usernames. Escape '/' and '!'. Timo Sirainen <tss@iki.fi> parents: 8492 diff changeset	151 string_t *tmp;
149c6ccc3df4 fts-solr: Use '!' as escape character in usernames. Escape '/' and '!'. Timo Sirainen <tss@iki.fi> parents: 8492 diff changeset	152 const char *p;
149c6ccc3df4 fts-solr: Use '!' as escape character in usernames. Escape '/' and '!'. Timo Sirainen <tss@iki.fi> parents: 8492 diff changeset	153
149c6ccc3df4 fts-solr: Use '!' as escape character in usernames. Escape '/' and '!'. Timo Sirainen <tss@iki.fi> parents: 8492 diff changeset	154 for (p = str; *p != '\0'; p++) {
149c6ccc3df4 fts-solr: Use '!' as escape character in usernames. Escape '/' and '!'. Timo Sirainen <tss@iki.fi> parents: 8492 diff changeset	155 if (p == '/' \|\| p == '!')
149c6ccc3df4 fts-solr: Use '!' as escape character in usernames. Escape '/' and '!'. Timo Sirainen <tss@iki.fi> parents: 8492 diff changeset	156 break;
149c6ccc3df4 fts-solr: Use '!' as escape character in usernames. Escape '/' and '!'. Timo Sirainen <tss@iki.fi> parents: 8492 diff changeset	157 }
149c6ccc3df4 fts-solr: Use '!' as escape character in usernames. Escape '/' and '!'. Timo Sirainen <tss@iki.fi> parents: 8492 diff changeset	158 if (*p == '\0')
149c6ccc3df4 fts-solr: Use '!' as escape character in usernames. Escape '/' and '!'. Timo Sirainen <tss@iki.fi> parents: 8492 diff changeset	159 return str;
149c6ccc3df4 fts-solr: Use '!' as escape character in usernames. Escape '/' and '!'. Timo Sirainen <tss@iki.fi> parents: 8492 diff changeset	160
149c6ccc3df4 fts-solr: Use '!' as escape character in usernames. Escape '/' and '!'. Timo Sirainen <tss@iki.fi> parents: 8492 diff changeset	161 tmp = t_str_new(64);
149c6ccc3df4 fts-solr: Use '!' as escape character in usernames. Escape '/' and '!'. Timo Sirainen <tss@iki.fi> parents: 8492 diff changeset	162 for (p = str; *p != '\0'; p++) {
149c6ccc3df4 fts-solr: Use '!' as escape character in usernames. Escape '/' and '!'. Timo Sirainen <tss@iki.fi> parents: 8492 diff changeset	163 switch (*p) {
149c6ccc3df4 fts-solr: Use '!' as escape character in usernames. Escape '/' and '!'. Timo Sirainen <tss@iki.fi> parents: 8492 diff changeset	164 case '/':
149c6ccc3df4 fts-solr: Use '!' as escape character in usernames. Escape '/' and '!'. Timo Sirainen <tss@iki.fi> parents: 8492 diff changeset	165 str_append(tmp, "!\\");
149c6ccc3df4 fts-solr: Use '!' as escape character in usernames. Escape '/' and '!'. Timo Sirainen <tss@iki.fi> parents: 8492 diff changeset	166 break;
149c6ccc3df4 fts-solr: Use '!' as escape character in usernames. Escape '/' and '!'. Timo Sirainen <tss@iki.fi> parents: 8492 diff changeset	167 case '!':
149c6ccc3df4 fts-solr: Use '!' as escape character in usernames. Escape '/' and '!'. Timo Sirainen <tss@iki.fi> parents: 8492 diff changeset	168 str_append(tmp, "!!");
149c6ccc3df4 fts-solr: Use '!' as escape character in usernames. Escape '/' and '!'. Timo Sirainen <tss@iki.fi> parents: 8492 diff changeset	169 break;
149c6ccc3df4 fts-solr: Use '!' as escape character in usernames. Escape '/' and '!'. Timo Sirainen <tss@iki.fi> parents: 8492 diff changeset	170 default:
149c6ccc3df4 fts-solr: Use '!' as escape character in usernames. Escape '/' and '!'. Timo Sirainen <tss@iki.fi> parents: 8492 diff changeset	171 str_append_c(tmp, *p);
149c6ccc3df4 fts-solr: Use '!' as escape character in usernames. Escape '/' and '!'. Timo Sirainen <tss@iki.fi> parents: 8492 diff changeset	172 break;
149c6ccc3df4 fts-solr: Use '!' as escape character in usernames. Escape '/' and '!'. Timo Sirainen <tss@iki.fi> parents: 8492 diff changeset	173 }
149c6ccc3df4 fts-solr: Use '!' as escape character in usernames. Escape '/' and '!'. Timo Sirainen <tss@iki.fi> parents: 8492 diff changeset	174 }
149c6ccc3df4 fts-solr: Use '!' as escape character in usernames. Escape '/' and '!'. Timo Sirainen <tss@iki.fi> parents: 8492 diff changeset	175 return str_c(tmp);
149c6ccc3df4 fts-solr: Use '!' as escape character in usernames. Escape '/' and '!'. Timo Sirainen <tss@iki.fi> parents: 8492 diff changeset	176 }
149c6ccc3df4 fts-solr: Use '!' as escape character in usernames. Escape '/' and '!'. Timo Sirainen <tss@iki.fi> parents: 8492 diff changeset	177
8499 252b29ac5f43 fts: Added support for handling multiple namespaces. Timo Sirainen <tss@iki.fi> parents: 8495 diff changeset	178 static void solr_quote(string_t dest, const char str)
252b29ac5f43 fts: Added support for handling multiple namespaces. Timo Sirainen <tss@iki.fi> parents: 8495 diff changeset	179 {
252b29ac5f43 fts: Added support for handling multiple namespaces. Timo Sirainen <tss@iki.fi> parents: 8495 diff changeset	180 str_append_c(dest, '"');
252b29ac5f43 fts: Added support for handling multiple namespaces. Timo Sirainen <tss@iki.fi> parents: 8495 diff changeset	181 str_append(dest, str_escape(str));
252b29ac5f43 fts: Added support for handling multiple namespaces. Timo Sirainen <tss@iki.fi> parents: 8495 diff changeset	182 str_append_c(dest, '"');
252b29ac5f43 fts: Added support for handling multiple namespaces. Timo Sirainen <tss@iki.fi> parents: 8495 diff changeset	183 }
252b29ac5f43 fts: Added support for handling multiple namespaces. Timo Sirainen <tss@iki.fi> parents: 8495 diff changeset	184
252b29ac5f43 fts: Added support for handling multiple namespaces. Timo Sirainen <tss@iki.fi> parents: 8495 diff changeset	185 static void solr_quote_http(string_t dest, const char str)
252b29ac5f43 fts: Added support for handling multiple namespaces. Timo Sirainen <tss@iki.fi> parents: 8495 diff changeset	186 {
252b29ac5f43 fts: Added support for handling multiple namespaces. Timo Sirainen <tss@iki.fi> parents: 8495 diff changeset	187 str_append(dest, "%22");
252b29ac5f43 fts: Added support for handling multiple namespaces. Timo Sirainen <tss@iki.fi> parents: 8495 diff changeset	188 solr_connection_http_escape(solr_conn, dest, str);
252b29ac5f43 fts: Added support for handling multiple namespaces. Timo Sirainen <tss@iki.fi> parents: 8495 diff changeset	189 str_append(dest, "%22");
252b29ac5f43 fts: Added support for handling multiple namespaces. Timo Sirainen <tss@iki.fi> parents: 8495 diff changeset	190 }
252b29ac5f43 fts: Added support for handling multiple namespaces. Timo Sirainen <tss@iki.fi> parents: 8495 diff changeset	191
7990 662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	192 static struct fts_backend *
8493 149c6ccc3df4 fts-solr: Use '!' as escape character in usernames. Escape '/' and '!'. Timo Sirainen <tss@iki.fi> parents: 8492 diff changeset	193 fts_backend_solr_init(struct mailbox *box)
7990 662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	194 {
8002 e814614ffb65 fts-solr: Added break-imap-search setting to use Solr with SEARCH TEXT/BODY Timo Sirainen <tss@iki.fi> parents: 8001 diff changeset	195 const struct fts_solr_settings *set = &fts_solr_settings;
8493 149c6ccc3df4 fts-solr: Use '!' as escape character in usernames. Escape '/' and '!'. Timo Sirainen <tss@iki.fi> parents: 8492 diff changeset	196 struct solr_fts_backend *backend;
8950 ec1ae90af21a fts: Fixes to how virtual mailboxes are searched. Timo Sirainen <tss@iki.fi> parents: 8946 diff changeset	197 struct mail_namespace *ns;
ec1ae90af21a fts: Fixes to how virtual mailboxes are searched. Timo Sirainen <tss@iki.fi> parents: 8946 diff changeset	198 const char str, box_name;
7990 662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	199
8950 ec1ae90af21a fts: Fixes to how virtual mailboxes are searched. Timo Sirainen <tss@iki.fi> parents: 8946 diff changeset	200
ec1ae90af21a fts: Fixes to how virtual mailboxes are searched. Timo Sirainen <tss@iki.fi> parents: 8946 diff changeset	201 box_name = fts_box_get_root(box, &ns);
ec1ae90af21a fts: Fixes to how virtual mailboxes are searched. Timo Sirainen <tss@iki.fi> parents: 8946 diff changeset	202 i_assert(*box_name != '\0');
8500 3efcdc45d111 Added alias_for setting for namespaces. Fixes namespace issues with fts. Timo Sirainen <tss@iki.fi> parents: 8499 diff changeset	203
7990 662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	204 if (solr_conn == NULL)
8002 e814614ffb65 fts-solr: Added break-imap-search setting to use Solr with SEARCH TEXT/BODY Timo Sirainen <tss@iki.fi> parents: 8001 diff changeset	205 solr_conn = solr_connection_init(set->url, set->debug);
7990 662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	206
8493 149c6ccc3df4 fts-solr: Use '!' as escape character in usernames. Escape '/' and '!'. Timo Sirainen <tss@iki.fi> parents: 8492 diff changeset	207 backend = i_new(struct solr_fts_backend, 1);
8499 252b29ac5f43 fts: Added support for handling multiple namespaces. Timo Sirainen <tss@iki.fi> parents: 8495 diff changeset	208 str = fts_solr_settings.default_ns_prefix;
252b29ac5f43 fts: Added support for handling multiple namespaces. Timo Sirainen <tss@iki.fi> parents: 8495 diff changeset	209 if (str != NULL) {
252b29ac5f43 fts: Added support for handling multiple namespaces. Timo Sirainen <tss@iki.fi> parents: 8495 diff changeset	210 backend->default_ns =
252b29ac5f43 fts: Added support for handling multiple namespaces. Timo Sirainen <tss@iki.fi> parents: 8495 diff changeset	211 mail_namespace_find_prefix(ns->user->namespaces, str);
252b29ac5f43 fts: Added support for handling multiple namespaces. Timo Sirainen <tss@iki.fi> parents: 8495 diff changeset	212 if (backend->default_ns == NULL) {
252b29ac5f43 fts: Added support for handling multiple namespaces. Timo Sirainen <tss@iki.fi> parents: 8495 diff changeset	213 i_fatal("fts_solr: default_ns setting points to "
252b29ac5f43 fts: Added support for handling multiple namespaces. Timo Sirainen <tss@iki.fi> parents: 8495 diff changeset	214 "nonexisting namespace");
252b29ac5f43 fts: Added support for handling multiple namespaces. Timo Sirainen <tss@iki.fi> parents: 8495 diff changeset	215 }
252b29ac5f43 fts: Added support for handling multiple namespaces. Timo Sirainen <tss@iki.fi> parents: 8495 diff changeset	216 } else {
252b29ac5f43 fts: Added support for handling multiple namespaces. Timo Sirainen <tss@iki.fi> parents: 8495 diff changeset	217 backend->default_ns =
252b29ac5f43 fts: Added support for handling multiple namespaces. Timo Sirainen <tss@iki.fi> parents: 8495 diff changeset	218 mail_namespace_find_inbox(ns->user->namespaces);
252b29ac5f43 fts: Added support for handling multiple namespaces. Timo Sirainen <tss@iki.fi> parents: 8495 diff changeset	219 }
8500 3efcdc45d111 Added alias_for setting for namespaces. Fixes namespace issues with fts. Timo Sirainen <tss@iki.fi> parents: 8499 diff changeset	220 while (backend->default_ns->alias_for != NULL)
3efcdc45d111 Added alias_for setting for namespaces. Fixes namespace issues with fts. Timo Sirainen <tss@iki.fi> parents: 8499 diff changeset	221 backend->default_ns = backend->default_ns->alias_for;
3efcdc45d111 Added alias_for setting for namespaces. Fixes namespace issues with fts. Timo Sirainen <tss@iki.fi> parents: 8499 diff changeset	222
8499 252b29ac5f43 fts: Added support for handling multiple namespaces. Timo Sirainen <tss@iki.fi> parents: 8495 diff changeset	223 str = solr_escape_id_str(ns->user->username);
252b29ac5f43 fts: Added support for handling multiple namespaces. Timo Sirainen <tss@iki.fi> parents: 8495 diff changeset	224 backend->id_username = i_strdup(str);
8500 3efcdc45d111 Added alias_for setting for namespaces. Fixes namespace issues with fts. Timo Sirainen <tss@iki.fi> parents: 8499 diff changeset	225 if (ns != backend->default_ns) {
8499 252b29ac5f43 fts: Added support for handling multiple namespaces. Timo Sirainen <tss@iki.fi> parents: 8495 diff changeset	226 str = solr_escape_id_str(ns->prefix);
252b29ac5f43 fts: Added support for handling multiple namespaces. Timo Sirainen <tss@iki.fi> parents: 8495 diff changeset	227 backend->id_namespace = i_strdup(str);
252b29ac5f43 fts: Added support for handling multiple namespaces. Timo Sirainen <tss@iki.fi> parents: 8495 diff changeset	228 }
8950 ec1ae90af21a fts: Fixes to how virtual mailboxes are searched. Timo Sirainen <tss@iki.fi> parents: 8946 diff changeset	229 backend->id_box_name = i_strdup(box_name);
8493 149c6ccc3df4 fts-solr: Use '!' as escape character in usernames. Escape '/' and '!'. Timo Sirainen <tss@iki.fi> parents: 8492 diff changeset	230 backend->backend = fts_backend_solr;
8002 e814614ffb65 fts-solr: Added break-imap-search setting to use Solr with SEARCH TEXT/BODY Timo Sirainen <tss@iki.fi> parents: 8001 diff changeset	231
e814614ffb65 fts-solr: Added break-imap-search setting to use Solr with SEARCH TEXT/BODY Timo Sirainen <tss@iki.fi> parents: 8001 diff changeset	232 if (set->substring_search)
8493 149c6ccc3df4 fts-solr: Use '!' as escape character in usernames. Escape '/' and '!'. Timo Sirainen <tss@iki.fi> parents: 8492 diff changeset	233 backend->backend.flags \|= FTS_BACKEND_FLAG_SUBSTRING_LOOKUPS;
149c6ccc3df4 fts-solr: Use '!' as escape character in usernames. Escape '/' and '!'. Timo Sirainen <tss@iki.fi> parents: 8492 diff changeset	234 return &backend->backend;
7990 662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	235 }
662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	236
8493 149c6ccc3df4 fts-solr: Use '!' as escape character in usernames. Escape '/' and '!'. Timo Sirainen <tss@iki.fi> parents: 8492 diff changeset	237 static void fts_backend_solr_deinit(struct fts_backend *_backend)
7990 662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	238 {
8493 149c6ccc3df4 fts-solr: Use '!' as escape character in usernames. Escape '/' and '!'. Timo Sirainen <tss@iki.fi> parents: 8492 diff changeset	239 struct solr_fts_backend backend = (struct solr_fts_backend )_backend;
149c6ccc3df4 fts-solr: Use '!' as escape character in usernames. Escape '/' and '!'. Timo Sirainen <tss@iki.fi> parents: 8492 diff changeset	240
8950 ec1ae90af21a fts: Fixes to how virtual mailboxes are searched. Timo Sirainen <tss@iki.fi> parents: 8946 diff changeset	241 i_free(backend->id_box_name);
8499 252b29ac5f43 fts: Added support for handling multiple namespaces. Timo Sirainen <tss@iki.fi> parents: 8495 diff changeset	242 i_free(backend->id_namespace);
8493 149c6ccc3df4 fts-solr: Use '!' as escape character in usernames. Escape '/' and '!'. Timo Sirainen <tss@iki.fi> parents: 8492 diff changeset	243 i_free(backend->id_username);
7990 662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	244 i_free(backend);
662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	245 }
662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	246
8499 252b29ac5f43 fts: Added support for handling multiple namespaces. Timo Sirainen <tss@iki.fi> parents: 8495 diff changeset	247 static void
252b29ac5f43 fts: Added support for handling multiple namespaces. Timo Sirainen <tss@iki.fi> parents: 8495 diff changeset	248 solr_add_ns_query(string_t str, struct fts_backend _backend,
8945 43b68bd6023a fts-solr: Fixes with virtual mailboxes. Timo Sirainen <tss@iki.fi> parents: 8879 diff changeset	249 struct mail_namespace *ns, bool neg)
8493 149c6ccc3df4 fts-solr: Use '!' as escape character in usernames. Escape '/' and '!'. Timo Sirainen <tss@iki.fi> parents: 8492 diff changeset	250 {
149c6ccc3df4 fts-solr: Use '!' as escape character in usernames. Escape '/' and '!'. Timo Sirainen <tss@iki.fi> parents: 8492 diff changeset	251 struct solr_fts_backend backend = (struct solr_fts_backend )_backend;
149c6ccc3df4 fts-solr: Use '!' as escape character in usernames. Escape '/' and '!'. Timo Sirainen <tss@iki.fi> parents: 8492 diff changeset	252
8500 3efcdc45d111 Added alias_for setting for namespaces. Fixes namespace issues with fts. Timo Sirainen <tss@iki.fi> parents: 8499 diff changeset	253 while (ns->alias_for != NULL)
3efcdc45d111 Added alias_for setting for namespaces. Fixes namespace issues with fts. Timo Sirainen <tss@iki.fi> parents: 8499 diff changeset	254 ns = ns->alias_for;
3efcdc45d111 Added alias_for setting for namespaces. Fixes namespace issues with fts. Timo Sirainen <tss@iki.fi> parents: 8499 diff changeset	255
8945 43b68bd6023a fts-solr: Fixes with virtual mailboxes. Timo Sirainen <tss@iki.fi> parents: 8879 diff changeset	256 if (ns == backend->default_ns \|\| *ns->prefix == '\0') {
43b68bd6023a fts-solr: Fixes with virtual mailboxes. Timo Sirainen <tss@iki.fi> parents: 8879 diff changeset	257 if (!neg)
43b68bd6023a fts-solr: Fixes with virtual mailboxes. Timo Sirainen <tss@iki.fi> parents: 8879 diff changeset	258 str_append(str, " -ns:[* TO *]");
43b68bd6023a fts-solr: Fixes with virtual mailboxes. Timo Sirainen <tss@iki.fi> parents: 8879 diff changeset	259 else
43b68bd6023a fts-solr: Fixes with virtual mailboxes. Timo Sirainen <tss@iki.fi> parents: 8879 diff changeset	260 str_append(str, " +ns:[* TO *]");
43b68bd6023a fts-solr: Fixes with virtual mailboxes. Timo Sirainen <tss@iki.fi> parents: 8879 diff changeset	261 } else {
43b68bd6023a fts-solr: Fixes with virtual mailboxes. Timo Sirainen <tss@iki.fi> parents: 8879 diff changeset	262 if (!neg)
43b68bd6023a fts-solr: Fixes with virtual mailboxes. Timo Sirainen <tss@iki.fi> parents: 8879 diff changeset	263 str_append(str, " +ns:");
43b68bd6023a fts-solr: Fixes with virtual mailboxes. Timo Sirainen <tss@iki.fi> parents: 8879 diff changeset	264 else
43b68bd6023a fts-solr: Fixes with virtual mailboxes. Timo Sirainen <tss@iki.fi> parents: 8879 diff changeset	265 str_append(str, " -ns:");
8499 252b29ac5f43 fts: Added support for handling multiple namespaces. Timo Sirainen <tss@iki.fi> parents: 8495 diff changeset	266 solr_quote(str, ns->prefix);
252b29ac5f43 fts: Added support for handling multiple namespaces. Timo Sirainen <tss@iki.fi> parents: 8495 diff changeset	267 }
252b29ac5f43 fts: Added support for handling multiple namespaces. Timo Sirainen <tss@iki.fi> parents: 8495 diff changeset	268 }
252b29ac5f43 fts: Added support for handling multiple namespaces. Timo Sirainen <tss@iki.fi> parents: 8495 diff changeset	269
252b29ac5f43 fts: Added support for handling multiple namespaces. Timo Sirainen <tss@iki.fi> parents: 8495 diff changeset	270 static void
252b29ac5f43 fts: Added support for handling multiple namespaces. Timo Sirainen <tss@iki.fi> parents: 8495 diff changeset	271 solr_add_ns_query_http(string_t str, struct fts_backend backend,
252b29ac5f43 fts: Added support for handling multiple namespaces. Timo Sirainen <tss@iki.fi> parents: 8495 diff changeset	272 struct mail_namespace *ns)
252b29ac5f43 fts: Added support for handling multiple namespaces. Timo Sirainen <tss@iki.fi> parents: 8495 diff changeset	273 {
252b29ac5f43 fts: Added support for handling multiple namespaces. Timo Sirainen <tss@iki.fi> parents: 8495 diff changeset	274 string_t *tmp;
252b29ac5f43 fts: Added support for handling multiple namespaces. Timo Sirainen <tss@iki.fi> parents: 8495 diff changeset	275
252b29ac5f43 fts: Added support for handling multiple namespaces. Timo Sirainen <tss@iki.fi> parents: 8495 diff changeset	276 tmp = t_str_new(64);
8945 43b68bd6023a fts-solr: Fixes with virtual mailboxes. Timo Sirainen <tss@iki.fi> parents: 8879 diff changeset	277 solr_add_ns_query(tmp, backend, ns, FALSE);
8499 252b29ac5f43 fts: Added support for handling multiple namespaces. Timo Sirainen <tss@iki.fi> parents: 8495 diff changeset	278 solr_connection_http_escape(solr_conn, str, str_c(tmp));
8493 149c6ccc3df4 fts-solr: Use '!' as escape character in usernames. Escape '/' and '!'. Timo Sirainen <tss@iki.fi> parents: 8492 diff changeset	279 }
149c6ccc3df4 fts-solr: Use '!' as escape character in usernames. Escape '/' and '!'. Timo Sirainen <tss@iki.fi> parents: 8492 diff changeset	280
8492 ee1ecdda8a6b fts-solr: Keep last_uid field updated containing the mailbox's last added UID. Timo Sirainen <tss@iki.fi> parents: 8484 diff changeset	281 static int fts_backend_solr_get_last_uid_fallback(struct fts_backend *backend,
ee1ecdda8a6b fts-solr: Keep last_uid field updated containing the mailbox's last added UID. Timo Sirainen <tss@iki.fi> parents: 8484 diff changeset	282 uint32_t *last_uid_r)
7990 662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	283 {
8499 252b29ac5f43 fts: Added support for handling multiple namespaces. Timo Sirainen <tss@iki.fi> parents: 8495 diff changeset	284 struct mailbox *box = backend->box;
8950 ec1ae90af21a fts: Fixes to how virtual mailboxes are searched. Timo Sirainen <tss@iki.fi> parents: 8946 diff changeset	285 struct mail_namespace *ns;
7990 662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	286 struct mailbox_status status;
662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	287 ARRAY_TYPE(seq_range) uids;
662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	288 const struct seq_range *uidvals;
8950 ec1ae90af21a fts: Fixes to how virtual mailboxes are searched. Timo Sirainen <tss@iki.fi> parents: 8946 diff changeset	289 const char *box_name;
7990 662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	290 unsigned int count;
662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	291 string_t *str;
662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	292
662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	293 str = t_str_new(256);
8499 252b29ac5f43 fts: Added support for handling multiple namespaces. Timo Sirainen <tss@iki.fi> parents: 8495 diff changeset	294 str_append(str, "fl=uid&rows=1&sort=uid+desc&q=");
7990 662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	295
8950 ec1ae90af21a fts: Fixes to how virtual mailboxes are searched. Timo Sirainen <tss@iki.fi> parents: 8946 diff changeset	296 box_name = fts_box_get_root(box, &ns);
ec1ae90af21a fts: Fixes to how virtual mailboxes are searched. Timo Sirainen <tss@iki.fi> parents: 8946 diff changeset	297
8499 252b29ac5f43 fts: Added support for handling multiple namespaces. Timo Sirainen <tss@iki.fi> parents: 8495 diff changeset	298 mailbox_get_status(box, STATUS_UIDVALIDITY, &status);
252b29ac5f43 fts: Added support for handling multiple namespaces. Timo Sirainen <tss@iki.fi> parents: 8495 diff changeset	299 str_printfa(str, "uidv:%u+box:", status.uidvalidity);
8950 ec1ae90af21a fts: Fixes to how virtual mailboxes are searched. Timo Sirainen <tss@iki.fi> parents: 8946 diff changeset	300 solr_quote_http(str, box_name);
ec1ae90af21a fts: Fixes to how virtual mailboxes are searched. Timo Sirainen <tss@iki.fi> parents: 8946 diff changeset	301 solr_add_ns_query_http(str, backend, ns);
8499 252b29ac5f43 fts: Added support for handling multiple namespaces. Timo Sirainen <tss@iki.fi> parents: 8495 diff changeset	302 str_append(str, "+user:");
8950 ec1ae90af21a fts: Fixes to how virtual mailboxes are searched. Timo Sirainen <tss@iki.fi> parents: 8946 diff changeset	303 solr_quote_http(str, ns->user->username);
7990 662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	304
662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	305 t_array_init(&uids, 1);
8484 db78eab2ba5d Initial support for getting full text search working nicely with virtual mailboxes. Timo Sirainen <tss@iki.fi> parents: 8109 diff changeset	306 if (solr_connection_select(solr_conn, str_c(str),
db78eab2ba5d Initial support for getting full text search working nicely with virtual mailboxes. Timo Sirainen <tss@iki.fi> parents: 8109 diff changeset	307 NULL, NULL, &uids, NULL) < 0)
7990 662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	308 return -1;
662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	309
662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	310 uidvals = array_get(&uids, &count);
662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	311 if (count == 0) {
662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	312 /* nothing indexed yet for this mailbox */
662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	313 *last_uid_r = 0;
662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	314 } else if (count == 1 && uidvals[0].seq1 == uidvals[0].seq2) {
662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	315 *last_uid_r = uidvals[0].seq1;
662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	316 } else {
662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	317 i_error("fts_solr: Last UID lookup returned multiple rows");
662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	318 return -1;
662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	319 }
662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	320 return 0;
662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	321 }
662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	322
8492 ee1ecdda8a6b fts-solr: Keep last_uid field updated containing the mailbox's last added UID. Timo Sirainen <tss@iki.fi> parents: 8484 diff changeset	323 static int fts_backend_solr_get_last_uid(struct fts_backend *backend,
ee1ecdda8a6b fts-solr: Keep last_uid field updated containing the mailbox's last added UID. Timo Sirainen <tss@iki.fi> parents: 8484 diff changeset	324 uint32_t *last_uid_r)
ee1ecdda8a6b fts-solr: Keep last_uid field updated containing the mailbox's last added UID. Timo Sirainen <tss@iki.fi> parents: 8484 diff changeset	325 {
8499 252b29ac5f43 fts: Added support for handling multiple namespaces. Timo Sirainen <tss@iki.fi> parents: 8495 diff changeset	326 struct mailbox *box = backend->box;
8950 ec1ae90af21a fts: Fixes to how virtual mailboxes are searched. Timo Sirainen <tss@iki.fi> parents: 8946 diff changeset	327 struct mail_namespace *ns;
8492 ee1ecdda8a6b fts-solr: Keep last_uid field updated containing the mailbox's last added UID. Timo Sirainen <tss@iki.fi> parents: 8484 diff changeset	328 struct mailbox_status status;
ee1ecdda8a6b fts-solr: Keep last_uid field updated containing the mailbox's last added UID. Timo Sirainen <tss@iki.fi> parents: 8484 diff changeset	329 ARRAY_TYPE(seq_range) uids;
ee1ecdda8a6b fts-solr: Keep last_uid field updated containing the mailbox's last added UID. Timo Sirainen <tss@iki.fi> parents: 8484 diff changeset	330 const struct seq_range *uidvals;
8950 ec1ae90af21a fts: Fixes to how virtual mailboxes are searched. Timo Sirainen <tss@iki.fi> parents: 8946 diff changeset	331 const char *box_name;
8492 ee1ecdda8a6b fts-solr: Keep last_uid field updated containing the mailbox's last added UID. Timo Sirainen <tss@iki.fi> parents: 8484 diff changeset	332 unsigned int count;
ee1ecdda8a6b fts-solr: Keep last_uid field updated containing the mailbox's last added UID. Timo Sirainen <tss@iki.fi> parents: 8484 diff changeset	333 string_t *str;
ee1ecdda8a6b fts-solr: Keep last_uid field updated containing the mailbox's last added UID. Timo Sirainen <tss@iki.fi> parents: 8484 diff changeset	334
ee1ecdda8a6b fts-solr: Keep last_uid field updated containing the mailbox's last added UID. Timo Sirainen <tss@iki.fi> parents: 8484 diff changeset	335 str = t_str_new(256);
8499 252b29ac5f43 fts: Added support for handling multiple namespaces. Timo Sirainen <tss@iki.fi> parents: 8495 diff changeset	336 str_append(str, "fl=uid&rows=1&q=last_uid:TRUE+");
8492 ee1ecdda8a6b fts-solr: Keep last_uid field updated containing the mailbox's last added UID. Timo Sirainen <tss@iki.fi> parents: 8484 diff changeset	337
8950 ec1ae90af21a fts: Fixes to how virtual mailboxes are searched. Timo Sirainen <tss@iki.fi> parents: 8946 diff changeset	338 box_name = fts_box_get_root(box, &ns);
ec1ae90af21a fts: Fixes to how virtual mailboxes are searched. Timo Sirainen <tss@iki.fi> parents: 8946 diff changeset	339
8499 252b29ac5f43 fts: Added support for handling multiple namespaces. Timo Sirainen <tss@iki.fi> parents: 8495 diff changeset	340 mailbox_get_status(box, STATUS_UIDVALIDITY, &status);
252b29ac5f43 fts: Added support for handling multiple namespaces. Timo Sirainen <tss@iki.fi> parents: 8495 diff changeset	341 str_printfa(str, "uidv:%u+box:", status.uidvalidity);
8950 ec1ae90af21a fts: Fixes to how virtual mailboxes are searched. Timo Sirainen <tss@iki.fi> parents: 8946 diff changeset	342 solr_quote_http(str, box_name);
ec1ae90af21a fts: Fixes to how virtual mailboxes are searched. Timo Sirainen <tss@iki.fi> parents: 8946 diff changeset	343 solr_add_ns_query_http(str, backend, ns);
8499 252b29ac5f43 fts: Added support for handling multiple namespaces. Timo Sirainen <tss@iki.fi> parents: 8495 diff changeset	344 str_append(str, "+user:");
8950 ec1ae90af21a fts: Fixes to how virtual mailboxes are searched. Timo Sirainen <tss@iki.fi> parents: 8946 diff changeset	345 solr_quote_http(str, ns->user->username);
8492 ee1ecdda8a6b fts-solr: Keep last_uid field updated containing the mailbox's last added UID. Timo Sirainen <tss@iki.fi> parents: 8484 diff changeset	346
ee1ecdda8a6b fts-solr: Keep last_uid field updated containing the mailbox's last added UID. Timo Sirainen <tss@iki.fi> parents: 8484 diff changeset	347 t_array_init(&uids, 1);
ee1ecdda8a6b fts-solr: Keep last_uid field updated containing the mailbox's last added UID. Timo Sirainen <tss@iki.fi> parents: 8484 diff changeset	348 if (solr_connection_select(solr_conn, str_c(str),
ee1ecdda8a6b fts-solr: Keep last_uid field updated containing the mailbox's last added UID. Timo Sirainen <tss@iki.fi> parents: 8484 diff changeset	349 NULL, NULL, &uids, NULL) < 0)
ee1ecdda8a6b fts-solr: Keep last_uid field updated containing the mailbox's last added UID. Timo Sirainen <tss@iki.fi> parents: 8484 diff changeset	350 return -1;
ee1ecdda8a6b fts-solr: Keep last_uid field updated containing the mailbox's last added UID. Timo Sirainen <tss@iki.fi> parents: 8484 diff changeset	351
ee1ecdda8a6b fts-solr: Keep last_uid field updated containing the mailbox's last added UID. Timo Sirainen <tss@iki.fi> parents: 8484 diff changeset	352 uidvals = array_get(&uids, &count);
ee1ecdda8a6b fts-solr: Keep last_uid field updated containing the mailbox's last added UID. Timo Sirainen <tss@iki.fi> parents: 8484 diff changeset	353 if (count == 0) {
ee1ecdda8a6b fts-solr: Keep last_uid field updated containing the mailbox's last added UID. Timo Sirainen <tss@iki.fi> parents: 8484 diff changeset	354 /* either nothing is indexed or we're converting from an
ee1ecdda8a6b fts-solr: Keep last_uid field updated containing the mailbox's last added UID. Timo Sirainen <tss@iki.fi> parents: 8484 diff changeset	355 older database format without the last_uid fields */
ee1ecdda8a6b fts-solr: Keep last_uid field updated containing the mailbox's last added UID. Timo Sirainen <tss@iki.fi> parents: 8484 diff changeset	356 return fts_backend_solr_get_last_uid_fallback(backend,
ee1ecdda8a6b fts-solr: Keep last_uid field updated containing the mailbox's last added UID. Timo Sirainen <tss@iki.fi> parents: 8484 diff changeset	357 last_uid_r);
ee1ecdda8a6b fts-solr: Keep last_uid field updated containing the mailbox's last added UID. Timo Sirainen <tss@iki.fi> parents: 8484 diff changeset	358 } else if (count == 1 && uidvals[0].seq1 == uidvals[0].seq2) {
ee1ecdda8a6b fts-solr: Keep last_uid field updated containing the mailbox's last added UID. Timo Sirainen <tss@iki.fi> parents: 8484 diff changeset	359 *last_uid_r = uidvals[0].seq1;
ee1ecdda8a6b fts-solr: Keep last_uid field updated containing the mailbox's last added UID. Timo Sirainen <tss@iki.fi> parents: 8484 diff changeset	360 } else {
ee1ecdda8a6b fts-solr: Keep last_uid field updated containing the mailbox's last added UID. Timo Sirainen <tss@iki.fi> parents: 8484 diff changeset	361 i_error("fts_solr: Last UID lookup returned multiple rows");
ee1ecdda8a6b fts-solr: Keep last_uid field updated containing the mailbox's last added UID. Timo Sirainen <tss@iki.fi> parents: 8484 diff changeset	362 return -1;
ee1ecdda8a6b fts-solr: Keep last_uid field updated containing the mailbox's last added UID. Timo Sirainen <tss@iki.fi> parents: 8484 diff changeset	363 }
ee1ecdda8a6b fts-solr: Keep last_uid field updated containing the mailbox's last added UID. Timo Sirainen <tss@iki.fi> parents: 8484 diff changeset	364 return 0;
ee1ecdda8a6b fts-solr: Keep last_uid field updated containing the mailbox's last added UID. Timo Sirainen <tss@iki.fi> parents: 8484 diff changeset	365 }
ee1ecdda8a6b fts-solr: Keep last_uid field updated containing the mailbox's last added UID. Timo Sirainen <tss@iki.fi> parents: 8484 diff changeset	366
8500 3efcdc45d111 Added alias_for setting for namespaces. Fixes namespace issues with fts. Timo Sirainen <tss@iki.fi> parents: 8499 diff changeset	367 static struct mail_namespace *
3efcdc45d111 Added alias_for setting for namespaces. Fixes namespace issues with fts. Timo Sirainen <tss@iki.fi> parents: 8499 diff changeset	368 solr_get_namespaces(struct fts_backend *_backend,
3efcdc45d111 Added alias_for setting for namespaces. Fixes namespace issues with fts. Timo Sirainen <tss@iki.fi> parents: 8499 diff changeset	369 struct mailbox box, const char ns_prefix)
8499 252b29ac5f43 fts: Added support for handling multiple namespaces. Timo Sirainen <tss@iki.fi> parents: 8495 diff changeset	370 {
252b29ac5f43 fts: Added support for handling multiple namespaces. Timo Sirainen <tss@iki.fi> parents: 8495 diff changeset	371 struct solr_fts_backend backend = (struct solr_fts_backend )_backend;
252b29ac5f43 fts: Added support for handling multiple namespaces. Timo Sirainen <tss@iki.fi> parents: 8495 diff changeset	372 struct mail_namespace *namespaces = box->storage->ns->user->namespaces;
252b29ac5f43 fts: Added support for handling multiple namespaces. Timo Sirainen <tss@iki.fi> parents: 8495 diff changeset	373
252b29ac5f43 fts: Added support for handling multiple namespaces. Timo Sirainen <tss@iki.fi> parents: 8495 diff changeset	374 if (ns_prefix == NULL)
8500 3efcdc45d111 Added alias_for setting for namespaces. Fixes namespace issues with fts. Timo Sirainen <tss@iki.fi> parents: 8499 diff changeset	375 return backend->default_ns;
3efcdc45d111 Added alias_for setting for namespaces. Fixes namespace issues with fts. Timo Sirainen <tss@iki.fi> parents: 8499 diff changeset	376 else
3efcdc45d111 Added alias_for setting for namespaces. Fixes namespace issues with fts. Timo Sirainen <tss@iki.fi> parents: 8499 diff changeset	377 return mail_namespace_find_prefix(namespaces, ns_prefix);
8499 252b29ac5f43 fts: Added support for handling multiple namespaces. Timo Sirainen <tss@iki.fi> parents: 8495 diff changeset	378 }
252b29ac5f43 fts: Added support for handling multiple namespaces. Timo Sirainen <tss@iki.fi> parents: 8495 diff changeset	379
8495 ffb37c392166 Finished making full text search indexes work fast with virtual mailboxes. Timo Sirainen <tss@iki.fi> parents: 8493 diff changeset	380 static bool
8499 252b29ac5f43 fts: Added support for handling multiple namespaces. Timo Sirainen <tss@iki.fi> parents: 8495 diff changeset	381 solr_virtual_get_last_uids(const char ns_prefix, const char mailbox,
252b29ac5f43 fts: Added support for handling multiple namespaces. Timo Sirainen <tss@iki.fi> parents: 8495 diff changeset	382 uint32_t uidvalidity, uint32_t uid, void context)
8495 ffb37c392166 Finished making full text search indexes work fast with virtual mailboxes. Timo Sirainen <tss@iki.fi> parents: 8493 diff changeset	383 {
ffb37c392166 Finished making full text search indexes work fast with virtual mailboxes. Timo Sirainen <tss@iki.fi> parents: 8493 diff changeset	384 struct fts_backend_solr_get_last_uids_context *ctx = context;
ffb37c392166 Finished making full text search indexes work fast with virtual mailboxes. Timo Sirainen <tss@iki.fi> parents: 8493 diff changeset	385 struct fts_backend_uid_map *map;
8500 3efcdc45d111 Added alias_for setting for namespaces. Fixes namespace issues with fts. Timo Sirainen <tss@iki.fi> parents: 8499 diff changeset	386 struct mail_namespace *ns;
8499 252b29ac5f43 fts: Added support for handling multiple namespaces. Timo Sirainen <tss@iki.fi> parents: 8495 diff changeset	387 const char *vname;
252b29ac5f43 fts: Added support for handling multiple namespaces. Timo Sirainen <tss@iki.fi> parents: 8495 diff changeset	388
8500 3efcdc45d111 Added alias_for setting for namespaces. Fixes namespace issues with fts. Timo Sirainen <tss@iki.fi> parents: 8499 diff changeset	389 ns = solr_get_namespaces(ctx->backend, ctx->box, ns_prefix);
3efcdc45d111 Added alias_for setting for namespaces. Fixes namespace issues with fts. Timo Sirainen <tss@iki.fi> parents: 8499 diff changeset	390 for (; ns != NULL; ns = ns->alias_chain_next) {
3efcdc45d111 Added alias_for setting for namespaces. Fixes namespace issues with fts. Timo Sirainen <tss@iki.fi> parents: 8499 diff changeset	391 vname = mail_namespace_get_vname(ns, ctx->vname, mailbox);
3efcdc45d111 Added alias_for setting for namespaces. Fixes namespace issues with fts. Timo Sirainen <tss@iki.fi> parents: 8499 diff changeset	392 map = array_append_space(ctx->last_uids);
3efcdc45d111 Added alias_for setting for namespaces. Fixes namespace issues with fts. Timo Sirainen <tss@iki.fi> parents: 8499 diff changeset	393 map->mailbox = p_strdup(ctx->pool, vname);
3efcdc45d111 Added alias_for setting for namespaces. Fixes namespace issues with fts. Timo Sirainen <tss@iki.fi> parents: 8499 diff changeset	394 map->uidvalidity = uidvalidity;
3efcdc45d111 Added alias_for setting for namespaces. Fixes namespace issues with fts. Timo Sirainen <tss@iki.fi> parents: 8499 diff changeset	395 map->uid = *uid;
3efcdc45d111 Added alias_for setting for namespaces. Fixes namespace issues with fts. Timo Sirainen <tss@iki.fi> parents: 8499 diff changeset	396 }
8495 ffb37c392166 Finished making full text search indexes work fast with virtual mailboxes. Timo Sirainen <tss@iki.fi> parents: 8493 diff changeset	397 return FALSE;
ffb37c392166 Finished making full text search indexes work fast with virtual mailboxes. Timo Sirainen <tss@iki.fi> parents: 8493 diff changeset	398 }
ffb37c392166 Finished making full text search indexes work fast with virtual mailboxes. Timo Sirainen <tss@iki.fi> parents: 8493 diff changeset	399
8499 252b29ac5f43 fts: Added support for handling multiple namespaces. Timo Sirainen <tss@iki.fi> parents: 8495 diff changeset	400 static void
252b29ac5f43 fts: Added support for handling multiple namespaces. Timo Sirainen <tss@iki.fi> parents: 8495 diff changeset	401 solr_add_pattern(string_t str, const struct mailbox_virtual_pattern pattern)
8495 ffb37c392166 Finished making full text search indexes work fast with virtual mailboxes. Timo Sirainen <tss@iki.fi> parents: 8493 diff changeset	402 {
8950 ec1ae90af21a fts: Fixes to how virtual mailboxes are searched. Timo Sirainen <tss@iki.fi> parents: 8946 diff changeset	403 struct mail_namespace *ns = pattern->ns;
8499 252b29ac5f43 fts: Added support for handling multiple namespaces. Timo Sirainen <tss@iki.fi> parents: 8495 diff changeset	404 const char name, p;
252b29ac5f43 fts: Added support for handling multiple namespaces. Timo Sirainen <tss@iki.fi> parents: 8495 diff changeset	405
252b29ac5f43 fts: Added support for handling multiple namespaces. Timo Sirainen <tss@iki.fi> parents: 8495 diff changeset	406 name = pattern->pattern;
252b29ac5f43 fts: Added support for handling multiple namespaces. Timo Sirainen <tss@iki.fi> parents: 8495 diff changeset	407 if (!mail_namespace_update_name(pattern->ns, &name))
252b29ac5f43 fts: Added support for handling multiple namespaces. Timo Sirainen <tss@iki.fi> parents: 8495 diff changeset	408 name = mail_namespace_fix_sep(pattern->ns, name);
8495 ffb37c392166 Finished making full text search indexes work fast with virtual mailboxes. Timo Sirainen <tss@iki.fi> parents: 8493 diff changeset	409
8950 ec1ae90af21a fts: Fixes to how virtual mailboxes are searched. Timo Sirainen <tss@iki.fi> parents: 8946 diff changeset	410 fts_box_name_get_root(&ns, &name);
ec1ae90af21a fts: Fixes to how virtual mailboxes are searched. Timo Sirainen <tss@iki.fi> parents: 8946 diff changeset	411
8945 43b68bd6023a fts-solr: Fixes with virtual mailboxes. Timo Sirainen <tss@iki.fi> parents: 8879 diff changeset	412 if (strcmp(name, "*") == 0) {
43b68bd6023a fts-solr: Fixes with virtual mailboxes. Timo Sirainen <tss@iki.fi> parents: 8879 diff changeset	413 str_append(str, "[* TO *]");
43b68bd6023a fts-solr: Fixes with virtual mailboxes. Timo Sirainen <tss@iki.fi> parents: 8879 diff changeset	414 return;
43b68bd6023a fts-solr: Fixes with virtual mailboxes. Timo Sirainen <tss@iki.fi> parents: 8879 diff changeset	415 }
43b68bd6023a fts-solr: Fixes with virtual mailboxes. Timo Sirainen <tss@iki.fi> parents: 8879 diff changeset	416
8495 ffb37c392166 Finished making full text search indexes work fast with virtual mailboxes. Timo Sirainen <tss@iki.fi> parents: 8493 diff changeset	417 /* first check if there are any wildcards in the pattern */
8499 252b29ac5f43 fts: Added support for handling multiple namespaces. Timo Sirainen <tss@iki.fi> parents: 8495 diff changeset	418 for (p = name; *p != '\0'; p++) {
8495 ffb37c392166 Finished making full text search indexes work fast with virtual mailboxes. Timo Sirainen <tss@iki.fi> parents: 8493 diff changeset	419 if (p == '%' \|\| p == '*')
ffb37c392166 Finished making full text search indexes work fast with virtual mailboxes. Timo Sirainen <tss@iki.fi> parents: 8493 diff changeset	420 break;
ffb37c392166 Finished making full text search indexes work fast with virtual mailboxes. Timo Sirainen <tss@iki.fi> parents: 8493 diff changeset	421 }
ffb37c392166 Finished making full text search indexes work fast with virtual mailboxes. Timo Sirainen <tss@iki.fi> parents: 8493 diff changeset	422 if (*p == '\0') {
ffb37c392166 Finished making full text search indexes work fast with virtual mailboxes. Timo Sirainen <tss@iki.fi> parents: 8493 diff changeset	423 /* full mailbox name */
8499 252b29ac5f43 fts: Added support for handling multiple namespaces. Timo Sirainen <tss@iki.fi> parents: 8495 diff changeset	424 solr_quote(str, name);
8495 ffb37c392166 Finished making full text search indexes work fast with virtual mailboxes. Timo Sirainen <tss@iki.fi> parents: 8493 diff changeset	425 return;
ffb37c392166 Finished making full text search indexes work fast with virtual mailboxes. Timo Sirainen <tss@iki.fi> parents: 8493 diff changeset	426 }
ffb37c392166 Finished making full text search indexes work fast with virtual mailboxes. Timo Sirainen <tss@iki.fi> parents: 8493 diff changeset	427
ffb37c392166 Finished making full text search indexes work fast with virtual mailboxes. Timo Sirainen <tss@iki.fi> parents: 8493 diff changeset	428 /* there are at least some wildcards. */
8499 252b29ac5f43 fts: Added support for handling multiple namespaces. Timo Sirainen <tss@iki.fi> parents: 8495 diff changeset	429 for (p = name; *p != '\0'; p++) {
8495 ffb37c392166 Finished making full text search indexes work fast with virtual mailboxes. Timo Sirainen <tss@iki.fi> parents: 8493 diff changeset	430 if (p == '%' \|\| p == '*') {
8499 252b29ac5f43 fts: Added support for handling multiple namespaces. Timo Sirainen <tss@iki.fi> parents: 8495 diff changeset	431 if (p == name \|\| (p[-1] != '%' && p[-1] != '*'))
8495 ffb37c392166 Finished making full text search indexes work fast with virtual mailboxes. Timo Sirainen <tss@iki.fi> parents: 8493 diff changeset	432 str_append_c(str, '*');
ffb37c392166 Finished making full text search indexes work fast with virtual mailboxes. Timo Sirainen <tss@iki.fi> parents: 8493 diff changeset	433 } else {
ffb37c392166 Finished making full text search indexes work fast with virtual mailboxes. Timo Sirainen <tss@iki.fi> parents: 8493 diff changeset	434 if (!i_isalnum(*p))
ffb37c392166 Finished making full text search indexes work fast with virtual mailboxes. Timo Sirainen <tss@iki.fi> parents: 8493 diff changeset	435 str_append_c(str, '\\');
ffb37c392166 Finished making full text search indexes work fast with virtual mailboxes. Timo Sirainen <tss@iki.fi> parents: 8493 diff changeset	436 str_append_c(str, *p);
ffb37c392166 Finished making full text search indexes work fast with virtual mailboxes. Timo Sirainen <tss@iki.fi> parents: 8493 diff changeset	437 }
ffb37c392166 Finished making full text search indexes work fast with virtual mailboxes. Timo Sirainen <tss@iki.fi> parents: 8493 diff changeset	438 }
ffb37c392166 Finished making full text search indexes work fast with virtual mailboxes. Timo Sirainen <tss@iki.fi> parents: 8493 diff changeset	439 }
ffb37c392166 Finished making full text search indexes work fast with virtual mailboxes. Timo Sirainen <tss@iki.fi> parents: 8493 diff changeset	440
ffb37c392166 Finished making full text search indexes work fast with virtual mailboxes. Timo Sirainen <tss@iki.fi> parents: 8493 diff changeset	441 static void
8499 252b29ac5f43 fts: Added support for handling multiple namespaces. Timo Sirainen <tss@iki.fi> parents: 8495 diff changeset	442 fts_backend_solr_filter_mailboxes(struct fts_backend *_backend,
8495 ffb37c392166 Finished making full text search indexes work fast with virtual mailboxes. Timo Sirainen <tss@iki.fi> parents: 8493 diff changeset	443 string_t str, struct mailbox box)
ffb37c392166 Finished making full text search indexes work fast with virtual mailboxes. Timo Sirainen <tss@iki.fi> parents: 8493 diff changeset	444 {
8499 252b29ac5f43 fts: Added support for handling multiple namespaces. Timo Sirainen <tss@iki.fi> parents: 8495 diff changeset	445 ARRAY_TYPE(mailbox_virtual_patterns) includes_arr, excludes_arr;
8500 3efcdc45d111 Added alias_for setting for namespaces. Fixes namespace issues with fts. Timo Sirainen <tss@iki.fi> parents: 8499 diff changeset	446 struct mail_namespace *ns;
8499 252b29ac5f43 fts: Added support for handling multiple namespaces. Timo Sirainen <tss@iki.fi> parents: 8495 diff changeset	447 const struct mailbox_virtual_pattern includes, excludes;
8519 f456043e10d3 fts-solr: Minor bugfix Timo Sirainen <tss@iki.fi> parents: 8505 diff changeset	448 unsigned int i, inc_count, exc_count;
8495 ffb37c392166 Finished making full text search indexes work fast with virtual mailboxes. Timo Sirainen <tss@iki.fi> parents: 8493 diff changeset	449 string_t *fq;
ffb37c392166 Finished making full text search indexes work fast with virtual mailboxes. Timo Sirainen <tss@iki.fi> parents: 8493 diff changeset	450
ffb37c392166 Finished making full text search indexes work fast with virtual mailboxes. Timo Sirainen <tss@iki.fi> parents: 8493 diff changeset	451 t_array_init(&includes_arr, 16);
ffb37c392166 Finished making full text search indexes work fast with virtual mailboxes. Timo Sirainen <tss@iki.fi> parents: 8493 diff changeset	452 t_array_init(&excludes_arr, 16);
ffb37c392166 Finished making full text search indexes work fast with virtual mailboxes. Timo Sirainen <tss@iki.fi> parents: 8493 diff changeset	453 mailbox_get_virtual_box_patterns(box, &includes_arr, &excludes_arr);
ffb37c392166 Finished making full text search indexes work fast with virtual mailboxes. Timo Sirainen <tss@iki.fi> parents: 8493 diff changeset	454 includes = array_get(&includes_arr, &inc_count);
ffb37c392166 Finished making full text search indexes work fast with virtual mailboxes. Timo Sirainen <tss@iki.fi> parents: 8493 diff changeset	455 excludes = array_get(&excludes_arr, &exc_count);
ffb37c392166 Finished making full text search indexes work fast with virtual mailboxes. Timo Sirainen <tss@iki.fi> parents: 8493 diff changeset	456 i_assert(inc_count > 0);
ffb37c392166 Finished making full text search indexes work fast with virtual mailboxes. Timo Sirainen <tss@iki.fi> parents: 8493 diff changeset	457
ffb37c392166 Finished making full text search indexes work fast with virtual mailboxes. Timo Sirainen <tss@iki.fi> parents: 8493 diff changeset	458 /* First see if there are any patterns that begin with a wildcard.
ffb37c392166 Finished making full text search indexes work fast with virtual mailboxes. Timo Sirainen <tss@iki.fi> parents: 8493 diff changeset	459 Solr doesn't allow them, so in that case we'll need to return
ffb37c392166 Finished making full text search indexes work fast with virtual mailboxes. Timo Sirainen <tss@iki.fi> parents: 8493 diff changeset	460 all mailboxes. */
ffb37c392166 Finished making full text search indexes work fast with virtual mailboxes. Timo Sirainen <tss@iki.fi> parents: 8493 diff changeset	461 for (i = 0; i < inc_count; i++) {
8499 252b29ac5f43 fts: Added support for handling multiple namespaces. Timo Sirainen <tss@iki.fi> parents: 8495 diff changeset	462 if (includes[i].pattern == '' \|\|
252b29ac5f43 fts: Added support for handling multiple namespaces. Timo Sirainen <tss@iki.fi> parents: 8495 diff changeset	463 *includes[i].pattern == '%')
8495 ffb37c392166 Finished making full text search indexes work fast with virtual mailboxes. Timo Sirainen <tss@iki.fi> parents: 8493 diff changeset	464 break;
ffb37c392166 Finished making full text search indexes work fast with virtual mailboxes. Timo Sirainen <tss@iki.fi> parents: 8493 diff changeset	465 }
ffb37c392166 Finished making full text search indexes work fast with virtual mailboxes. Timo Sirainen <tss@iki.fi> parents: 8493 diff changeset	466
ffb37c392166 Finished making full text search indexes work fast with virtual mailboxes. Timo Sirainen <tss@iki.fi> parents: 8493 diff changeset	467 fq = t_str_new(128);
8499 252b29ac5f43 fts: Added support for handling multiple namespaces. Timo Sirainen <tss@iki.fi> parents: 8495 diff changeset	468 if (i == inc_count && inc_count <= FTS_SOLR_MAX_BOX_INC_PATTERNS) {
8495 ffb37c392166 Finished making full text search indexes work fast with virtual mailboxes. Timo Sirainen <tss@iki.fi> parents: 8493 diff changeset	469 /* we can filter what mailboxes we want returned */
ffb37c392166 Finished making full text search indexes work fast with virtual mailboxes. Timo Sirainen <tss@iki.fi> parents: 8493 diff changeset	470 str_append_c(fq, '(');
ffb37c392166 Finished making full text search indexes work fast with virtual mailboxes. Timo Sirainen <tss@iki.fi> parents: 8493 diff changeset	471 for (i = 0; i < inc_count; i++) {
ffb37c392166 Finished making full text search indexes work fast with virtual mailboxes. Timo Sirainen <tss@iki.fi> parents: 8493 diff changeset	472 if (i != 0)
8879 e81dfb69bfc5 fts-solr: Don't double-escape "+" if we're filtering what mailboxes we want. Timo Sirainen <tss@iki.fi> parents: 8848 diff changeset	473 str_append(fq, " OR +");
8499 252b29ac5f43 fts: Added support for handling multiple namespaces. Timo Sirainen <tss@iki.fi> parents: 8495 diff changeset	474 str_append_c(fq, '(');
8879 e81dfb69bfc5 fts-solr: Don't double-escape "+" if we're filtering what mailboxes we want. Timo Sirainen <tss@iki.fi> parents: 8848 diff changeset	475 str_append(fq, "+box:");
8499 252b29ac5f43 fts: Added support for handling multiple namespaces. Timo Sirainen <tss@iki.fi> parents: 8495 diff changeset	476 solr_add_pattern(fq, &includes[i]);
8945 43b68bd6023a fts-solr: Fixes with virtual mailboxes. Timo Sirainen <tss@iki.fi> parents: 8879 diff changeset	477 solr_add_ns_query(fq, _backend, includes[i].ns, FALSE);
8499 252b29ac5f43 fts: Added support for handling multiple namespaces. Timo Sirainen <tss@iki.fi> parents: 8495 diff changeset	478 str_append_c(fq, ')');
8495 ffb37c392166 Finished making full text search indexes work fast with virtual mailboxes. Timo Sirainen <tss@iki.fi> parents: 8493 diff changeset	479 }
ffb37c392166 Finished making full text search indexes work fast with virtual mailboxes. Timo Sirainen <tss@iki.fi> parents: 8493 diff changeset	480 str_append_c(fq, ')');
ffb37c392166 Finished making full text search indexes work fast with virtual mailboxes. Timo Sirainen <tss@iki.fi> parents: 8493 diff changeset	481 }
8499 252b29ac5f43 fts: Added support for handling multiple namespaces. Timo Sirainen <tss@iki.fi> parents: 8495 diff changeset	482 exc_count = I_MIN(FTS_SOLR_MAX_BOX_EXC_PATTERNS, exc_count);
8495 ffb37c392166 Finished making full text search indexes work fast with virtual mailboxes. Timo Sirainen <tss@iki.fi> parents: 8493 diff changeset	483 for (i = 0; i < exc_count; i++) {
8519 f456043e10d3 fts-solr: Minor bugfix Timo Sirainen <tss@iki.fi> parents: 8505 diff changeset	484 if (str_len(fq) > 0)
8495 ffb37c392166 Finished making full text search indexes work fast with virtual mailboxes. Timo Sirainen <tss@iki.fi> parents: 8493 diff changeset	485 str_append_c(fq, ' ');
8945 43b68bd6023a fts-solr: Fixes with virtual mailboxes. Timo Sirainen <tss@iki.fi> parents: 8879 diff changeset	486 str_append(fq, "NOT (");
43b68bd6023a fts-solr: Fixes with virtual mailboxes. Timo Sirainen <tss@iki.fi> parents: 8879 diff changeset	487 str_append(fq, "box:");
8499 252b29ac5f43 fts: Added support for handling multiple namespaces. Timo Sirainen <tss@iki.fi> parents: 8495 diff changeset	488 solr_add_pattern(fq, &excludes[i]);
8500 3efcdc45d111 Added alias_for setting for namespaces. Fixes namespace issues with fts. Timo Sirainen <tss@iki.fi> parents: 8499 diff changeset	489
3efcdc45d111 Added alias_for setting for namespaces. Fixes namespace issues with fts. Timo Sirainen <tss@iki.fi> parents: 8499 diff changeset	490 for (ns = excludes[i].ns; ns->alias_for != NULL; )
3efcdc45d111 Added alias_for setting for namespaces. Fixes namespace issues with fts. Timo Sirainen <tss@iki.fi> parents: 8499 diff changeset	491 ns = ns->alias_for;
8945 43b68bd6023a fts-solr: Fixes with virtual mailboxes. Timo Sirainen <tss@iki.fi> parents: 8879 diff changeset	492 solr_add_ns_query(fq, _backend, ns, FALSE);
8499 252b29ac5f43 fts: Added support for handling multiple namespaces. Timo Sirainen <tss@iki.fi> parents: 8495 diff changeset	493 str_append_c(fq, ')');
8495 ffb37c392166 Finished making full text search indexes work fast with virtual mailboxes. Timo Sirainen <tss@iki.fi> parents: 8493 diff changeset	494 }
ffb37c392166 Finished making full text search indexes work fast with virtual mailboxes. Timo Sirainen <tss@iki.fi> parents: 8493 diff changeset	495 if (str_len(fq) > 0) {
ffb37c392166 Finished making full text search indexes work fast with virtual mailboxes. Timo Sirainen <tss@iki.fi> parents: 8493 diff changeset	496 str_append(str, "&fq=");
ffb37c392166 Finished making full text search indexes work fast with virtual mailboxes. Timo Sirainen <tss@iki.fi> parents: 8493 diff changeset	497 solr_connection_http_escape(solr_conn, str, str_c(fq));
ffb37c392166 Finished making full text search indexes work fast with virtual mailboxes. Timo Sirainen <tss@iki.fi> parents: 8493 diff changeset	498 }
ffb37c392166 Finished making full text search indexes work fast with virtual mailboxes. Timo Sirainen <tss@iki.fi> parents: 8493 diff changeset	499 }
ffb37c392166 Finished making full text search indexes work fast with virtual mailboxes. Timo Sirainen <tss@iki.fi> parents: 8493 diff changeset	500
ffb37c392166 Finished making full text search indexes work fast with virtual mailboxes. Timo Sirainen <tss@iki.fi> parents: 8493 diff changeset	501 static int
ffb37c392166 Finished making full text search indexes work fast with virtual mailboxes. Timo Sirainen <tss@iki.fi> parents: 8493 diff changeset	502 fts_backend_solr_get_all_last_uids(struct fts_backend *backend, pool_t pool,
ffb37c392166 Finished making full text search indexes work fast with virtual mailboxes. Timo Sirainen <tss@iki.fi> parents: 8493 diff changeset	503 ARRAY_TYPE(fts_backend_uid_map) *last_uids)
ffb37c392166 Finished making full text search indexes work fast with virtual mailboxes. Timo Sirainen <tss@iki.fi> parents: 8493 diff changeset	504 {
ffb37c392166 Finished making full text search indexes work fast with virtual mailboxes. Timo Sirainen <tss@iki.fi> parents: 8493 diff changeset	505 struct fts_backend_solr_get_last_uids_context ctx;
ffb37c392166 Finished making full text search indexes work fast with virtual mailboxes. Timo Sirainen <tss@iki.fi> parents: 8493 diff changeset	506 string_t *str;
ffb37c392166 Finished making full text search indexes work fast with virtual mailboxes. Timo Sirainen <tss@iki.fi> parents: 8493 diff changeset	507
ffb37c392166 Finished making full text search indexes work fast with virtual mailboxes. Timo Sirainen <tss@iki.fi> parents: 8493 diff changeset	508 memset(&ctx, 0, sizeof(ctx));
8499 252b29ac5f43 fts: Added support for handling multiple namespaces. Timo Sirainen <tss@iki.fi> parents: 8495 diff changeset	509 ctx.backend = backend;
8495 ffb37c392166 Finished making full text search indexes work fast with virtual mailboxes. Timo Sirainen <tss@iki.fi> parents: 8493 diff changeset	510 ctx.pool = pool;
ffb37c392166 Finished making full text search indexes work fast with virtual mailboxes. Timo Sirainen <tss@iki.fi> parents: 8493 diff changeset	511 ctx.last_uids = last_uids;
8499 252b29ac5f43 fts: Added support for handling multiple namespaces. Timo Sirainen <tss@iki.fi> parents: 8495 diff changeset	512 ctx.box = backend->box;
252b29ac5f43 fts: Added support for handling multiple namespaces. Timo Sirainen <tss@iki.fi> parents: 8495 diff changeset	513 ctx.vname = t_str_new(256);
8495 ffb37c392166 Finished making full text search indexes work fast with virtual mailboxes. Timo Sirainen <tss@iki.fi> parents: 8493 diff changeset	514
ffb37c392166 Finished making full text search indexes work fast with virtual mailboxes. Timo Sirainen <tss@iki.fi> parents: 8493 diff changeset	515 str = t_str_new(256);
8499 252b29ac5f43 fts: Added support for handling multiple namespaces. Timo Sirainen <tss@iki.fi> parents: 8495 diff changeset	516 str_printfa(str, "fl=uid,box,uidv,ns&rows=%u&q=last_uid:TRUE+user:",
8495 ffb37c392166 Finished making full text search indexes work fast with virtual mailboxes. Timo Sirainen <tss@iki.fi> parents: 8493 diff changeset	517 SOLR_MAX_ROWS);
8499 252b29ac5f43 fts: Added support for handling multiple namespaces. Timo Sirainen <tss@iki.fi> parents: 8495 diff changeset	518 solr_quote_http(str, backend->box->storage->ns->user->username);
252b29ac5f43 fts: Added support for handling multiple namespaces. Timo Sirainen <tss@iki.fi> parents: 8495 diff changeset	519 fts_backend_solr_filter_mailboxes(backend, str, backend->box);
8495 ffb37c392166 Finished making full text search indexes work fast with virtual mailboxes. Timo Sirainen <tss@iki.fi> parents: 8493 diff changeset	520
ffb37c392166 Finished making full text search indexes work fast with virtual mailboxes. Timo Sirainen <tss@iki.fi> parents: 8493 diff changeset	521 return solr_connection_select(solr_conn, str_c(str),
ffb37c392166 Finished making full text search indexes work fast with virtual mailboxes. Timo Sirainen <tss@iki.fi> parents: 8493 diff changeset	522 solr_virtual_get_last_uids, &ctx,
ffb37c392166 Finished making full text search indexes work fast with virtual mailboxes. Timo Sirainen <tss@iki.fi> parents: 8493 diff changeset	523 NULL, NULL);
ffb37c392166 Finished making full text search indexes work fast with virtual mailboxes. Timo Sirainen <tss@iki.fi> parents: 8493 diff changeset	524 }
ffb37c392166 Finished making full text search indexes work fast with virtual mailboxes. Timo Sirainen <tss@iki.fi> parents: 8493 diff changeset	525
7990 662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	526 static int
662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	527 fts_backend_solr_build_init(struct fts_backend backend, uint32_t last_uid_r,
662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	528 struct fts_backend_build_context **ctx_r)
662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	529 {
662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	530 struct solr_fts_backend_build_context *ctx;
662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	531 struct mailbox_status status;
662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	532
662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	533 *last_uid_r = (uint32_t)-1;
662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	534
662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	535 ctx = i_new(struct solr_fts_backend_build_context, 1);
662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	536 ctx->ctx.backend = backend;
8001 3fbfdda3e5d3 fts-solr: Send data to Solr in larger packets. Commit after expunges. Timo Sirainen <tss@iki.fi> parents: 8000 diff changeset	537 ctx->cmd = str_new(default_pool, SOLR_CMDBUF_SIZE);
7990 662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	538
662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	539 mailbox_get_status(backend->box, STATUS_UIDVALIDITY, &status);
662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	540 ctx->uid_validity = status.uidvalidity;
662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	541
662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	542 *ctx_r = &ctx->ctx;
662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	543 return 0;
662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	544 }
662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	545
8492 ee1ecdda8a6b fts-solr: Keep last_uid field updated containing the mailbox's last added UID. Timo Sirainen <tss@iki.fi> parents: 8484 diff changeset	546 static void
ee1ecdda8a6b fts-solr: Keep last_uid field updated containing the mailbox's last added UID. Timo Sirainen <tss@iki.fi> parents: 8484 diff changeset	547 fts_backend_solr_add_doc_prefix(struct solr_fts_backend_build_context *ctx,
ee1ecdda8a6b fts-solr: Keep last_uid field updated containing the mailbox's last added UID. Timo Sirainen <tss@iki.fi> parents: 8484 diff changeset	548 uint32_t uid)
ee1ecdda8a6b fts-solr: Keep last_uid field updated containing the mailbox's last added UID. Timo Sirainen <tss@iki.fi> parents: 8484 diff changeset	549 {
8499 252b29ac5f43 fts: Added support for handling multiple namespaces. Timo Sirainen <tss@iki.fi> parents: 8495 diff changeset	550 struct solr_fts_backend *backend =
252b29ac5f43 fts: Added support for handling multiple namespaces. Timo Sirainen <tss@iki.fi> parents: 8495 diff changeset	551 (struct solr_fts_backend *)ctx->ctx.backend;
8492 ee1ecdda8a6b fts-solr: Keep last_uid field updated containing the mailbox's last added UID. Timo Sirainen <tss@iki.fi> parents: 8484 diff changeset	552 struct mailbox *box = ctx->ctx.backend->box;
8950 ec1ae90af21a fts: Fixes to how virtual mailboxes are searched. Timo Sirainen <tss@iki.fi> parents: 8946 diff changeset	553 struct mail_namespace *ns;
ec1ae90af21a fts: Fixes to how virtual mailboxes are searched. Timo Sirainen <tss@iki.fi> parents: 8946 diff changeset	554 const char *box_name;
8492 ee1ecdda8a6b fts-solr: Keep last_uid field updated containing the mailbox's last added UID. Timo Sirainen <tss@iki.fi> parents: 8484 diff changeset	555
ee1ecdda8a6b fts-solr: Keep last_uid field updated containing the mailbox's last added UID. Timo Sirainen <tss@iki.fi> parents: 8484 diff changeset	556 str_printfa(ctx->cmd, "<doc>"
ee1ecdda8a6b fts-solr: Keep last_uid field updated containing the mailbox's last added UID. Timo Sirainen <tss@iki.fi> parents: 8484 diff changeset	557 "<field name=\"uid\">%u</field>"
ee1ecdda8a6b fts-solr: Keep last_uid field updated containing the mailbox's last added UID. Timo Sirainen <tss@iki.fi> parents: 8484 diff changeset	558 "<field name=\"uidv\">%u</field>",
ee1ecdda8a6b fts-solr: Keep last_uid field updated containing the mailbox's last added UID. Timo Sirainen <tss@iki.fi> parents: 8484 diff changeset	559 uid, ctx->uid_validity);
ee1ecdda8a6b fts-solr: Keep last_uid field updated containing the mailbox's last added UID. Timo Sirainen <tss@iki.fi> parents: 8484 diff changeset	560
8950 ec1ae90af21a fts: Fixes to how virtual mailboxes are searched. Timo Sirainen <tss@iki.fi> parents: 8946 diff changeset	561 box_name = fts_box_get_root(box, &ns);
8500 3efcdc45d111 Added alias_for setting for namespaces. Fixes namespace issues with fts. Timo Sirainen <tss@iki.fi> parents: 8499 diff changeset	562
3efcdc45d111 Added alias_for setting for namespaces. Fixes namespace issues with fts. Timo Sirainen <tss@iki.fi> parents: 8499 diff changeset	563 if (ns != backend->default_ns) {
8499 252b29ac5f43 fts: Added support for handling multiple namespaces. Timo Sirainen <tss@iki.fi> parents: 8495 diff changeset	564 str_append(ctx->cmd, "<field name=\"ns\">");
8500 3efcdc45d111 Added alias_for setting for namespaces. Fixes namespace issues with fts. Timo Sirainen <tss@iki.fi> parents: 8499 diff changeset	565 xml_encode(ctx->cmd, ns->prefix);
8499 252b29ac5f43 fts: Added support for handling multiple namespaces. Timo Sirainen <tss@iki.fi> parents: 8495 diff changeset	566 str_append(ctx->cmd, "</field>");
252b29ac5f43 fts: Added support for handling multiple namespaces. Timo Sirainen <tss@iki.fi> parents: 8495 diff changeset	567 }
8492 ee1ecdda8a6b fts-solr: Keep last_uid field updated containing the mailbox's last added UID. Timo Sirainen <tss@iki.fi> parents: 8484 diff changeset	568 str_append(ctx->cmd, "<field name=\"box\">");
8950 ec1ae90af21a fts: Fixes to how virtual mailboxes are searched. Timo Sirainen <tss@iki.fi> parents: 8946 diff changeset	569 xml_encode(ctx->cmd, box_name);
8492 ee1ecdda8a6b fts-solr: Keep last_uid field updated containing the mailbox's last added UID. Timo Sirainen <tss@iki.fi> parents: 8484 diff changeset	570 str_append(ctx->cmd, "</field><field name=\"user\">");
8500 3efcdc45d111 Added alias_for setting for namespaces. Fixes namespace issues with fts. Timo Sirainen <tss@iki.fi> parents: 8499 diff changeset	571 xml_encode(ctx->cmd, ns->user->username);
8492 ee1ecdda8a6b fts-solr: Keep last_uid field updated containing the mailbox's last added UID. Timo Sirainen <tss@iki.fi> parents: 8484 diff changeset	572 str_append(ctx->cmd, "</field>");
ee1ecdda8a6b fts-solr: Keep last_uid field updated containing the mailbox's last added UID. Timo Sirainen <tss@iki.fi> parents: 8484 diff changeset	573 }
ee1ecdda8a6b fts-solr: Keep last_uid field updated containing the mailbox's last added UID. Timo Sirainen <tss@iki.fi> parents: 8484 diff changeset	574
8499 252b29ac5f43 fts: Added support for handling multiple namespaces. Timo Sirainen <tss@iki.fi> parents: 8495 diff changeset	575 static void xml_encode_id(string_t str, struct fts_backend _backend,
8950 ec1ae90af21a fts: Fixes to how virtual mailboxes are searched. Timo Sirainen <tss@iki.fi> parents: 8946 diff changeset	576 uint32_t uid, uint32_t uid_validity)
8499 252b29ac5f43 fts: Added support for handling multiple namespaces. Timo Sirainen <tss@iki.fi> parents: 8495 diff changeset	577 {
252b29ac5f43 fts: Added support for handling multiple namespaces. Timo Sirainen <tss@iki.fi> parents: 8495 diff changeset	578 struct solr_fts_backend backend = (struct solr_fts_backend )_backend;
252b29ac5f43 fts: Added support for handling multiple namespaces. Timo Sirainen <tss@iki.fi> parents: 8495 diff changeset	579
252b29ac5f43 fts: Added support for handling multiple namespaces. Timo Sirainen <tss@iki.fi> parents: 8495 diff changeset	580 if (uid != 0)
252b29ac5f43 fts: Added support for handling multiple namespaces. Timo Sirainen <tss@iki.fi> parents: 8495 diff changeset	581 str_printfa(str, "%u/", uid);
252b29ac5f43 fts: Added support for handling multiple namespaces. Timo Sirainen <tss@iki.fi> parents: 8495 diff changeset	582 else
252b29ac5f43 fts: Added support for handling multiple namespaces. Timo Sirainen <tss@iki.fi> parents: 8495 diff changeset	583 str_append(str, "L/");
252b29ac5f43 fts: Added support for handling multiple namespaces. Timo Sirainen <tss@iki.fi> parents: 8495 diff changeset	584 if (backend->id_namespace != NULL) {
252b29ac5f43 fts: Added support for handling multiple namespaces. Timo Sirainen <tss@iki.fi> parents: 8495 diff changeset	585 xml_encode(str, backend->id_namespace);
252b29ac5f43 fts: Added support for handling multiple namespaces. Timo Sirainen <tss@iki.fi> parents: 8495 diff changeset	586 str_append_c(str, '/');
252b29ac5f43 fts: Added support for handling multiple namespaces. Timo Sirainen <tss@iki.fi> parents: 8495 diff changeset	587 }
252b29ac5f43 fts: Added support for handling multiple namespaces. Timo Sirainen <tss@iki.fi> parents: 8495 diff changeset	588 str_printfa(str, "%u/", uid_validity);
252b29ac5f43 fts: Added support for handling multiple namespaces. Timo Sirainen <tss@iki.fi> parents: 8495 diff changeset	589 xml_encode(str, backend->id_username);
252b29ac5f43 fts: Added support for handling multiple namespaces. Timo Sirainen <tss@iki.fi> parents: 8495 diff changeset	590 str_append_c(str, '/');
8950 ec1ae90af21a fts: Fixes to how virtual mailboxes are searched. Timo Sirainen <tss@iki.fi> parents: 8946 diff changeset	591 xml_encode(str, backend->id_box_name);
8499 252b29ac5f43 fts: Added support for handling multiple namespaces. Timo Sirainen <tss@iki.fi> parents: 8495 diff changeset	592 }
252b29ac5f43 fts: Added support for handling multiple namespaces. Timo Sirainen <tss@iki.fi> parents: 8495 diff changeset	593
7990 662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	594 static int
662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	595 fts_backend_solr_build_more(struct fts_backend_build_context *_ctx,
662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	596 uint32_t uid, const unsigned char *data,
662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	597 size_t size, bool headers)
662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	598 {
662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	599 struct solr_fts_backend_build_context *ctx =
662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	600 (struct solr_fts_backend_build_context *)_ctx;
662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	601 string_t *cmd = ctx->cmd;
662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	602
662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	603 /* body comes first, then headers */
662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	604 if (ctx->prev_uid != uid) {
662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	605 /* uid changed */
8495 ffb37c392166 Finished making full text search indexes work fast with virtual mailboxes. Timo Sirainen <tss@iki.fi> parents: 8493 diff changeset	606 if (ctx->post == NULL) {
ffb37c392166 Finished making full text search indexes work fast with virtual mailboxes. Timo Sirainen <tss@iki.fi> parents: 8493 diff changeset	607 ctx->post = solr_connection_post_begin(solr_conn);
7990 662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	608 str_append(cmd, "<add>");
8495 ffb37c392166 Finished making full text search indexes work fast with virtual mailboxes. Timo Sirainen <tss@iki.fi> parents: 8493 diff changeset	609 } else {
7990 662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	610 str_append(cmd, "</field></doc>");
8495 ffb37c392166 Finished making full text search indexes work fast with virtual mailboxes. Timo Sirainen <tss@iki.fi> parents: 8493 diff changeset	611 }
7990 662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	612 ctx->prev_uid = uid;
662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	613
8492 ee1ecdda8a6b fts-solr: Keep last_uid field updated containing the mailbox's last added UID. Timo Sirainen <tss@iki.fi> parents: 8484 diff changeset	614 fts_backend_solr_add_doc_prefix(ctx, uid);
8499 252b29ac5f43 fts: Added support for handling multiple namespaces. Timo Sirainen <tss@iki.fi> parents: 8495 diff changeset	615 str_printfa(cmd, "<field name=\"id\">");
8950 ec1ae90af21a fts: Fixes to how virtual mailboxes are searched. Timo Sirainen <tss@iki.fi> parents: 8946 diff changeset	616 xml_encode_id(cmd, _ctx->backend, uid, ctx->uid_validity);
7990 662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	617 str_append(cmd, "</field>");
662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	618
662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	619 ctx->headers = headers;
662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	620 if (headers) {
662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	621 str_append(cmd, "<field name=\"hdr\">");
662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	622 } else {
662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	623 str_append(cmd, "<field name=\"body\">");
662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	624 }
662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	625 } else if (headers && !ctx->headers) {
662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	626 str_append(cmd, "</field><field name=\"hdr\">");
662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	627 } else {
662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	628 i_assert(!(!headers && ctx->headers));
662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	629 }
662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	630
8000 61be6a4722c5 fts-solr: Encode entities when indexing data. Timo Sirainen <tss@iki.fi> parents: 7999 diff changeset	631 xml_encode_data(cmd, data, size);
8001 3fbfdda3e5d3 fts-solr: Send data to Solr in larger packets. Commit after expunges. Timo Sirainen <tss@iki.fi> parents: 8000 diff changeset	632 if (str_len(cmd) > SOLR_CMDBUF_SIZE-128) {
3fbfdda3e5d3 fts-solr: Send data to Solr in larger packets. Commit after expunges. Timo Sirainen <tss@iki.fi> parents: 8000 diff changeset	633 solr_connection_post_more(ctx->post, str_data(cmd),
3fbfdda3e5d3 fts-solr: Send data to Solr in larger packets. Commit after expunges. Timo Sirainen <tss@iki.fi> parents: 8000 diff changeset	634 str_len(cmd));
3fbfdda3e5d3 fts-solr: Send data to Solr in larger packets. Commit after expunges. Timo Sirainen <tss@iki.fi> parents: 8000 diff changeset	635 str_truncate(cmd, 0);
3fbfdda3e5d3 fts-solr: Send data to Solr in larger packets. Commit after expunges. Timo Sirainen <tss@iki.fi> parents: 8000 diff changeset	636 }
7990 662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	637 return 0;
662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	638 }
662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	639
662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	640 static int
8492 ee1ecdda8a6b fts-solr: Keep last_uid field updated containing the mailbox's last added UID. Timo Sirainen <tss@iki.fi> parents: 8484 diff changeset	641 fts_backed_solr_build_commit(struct solr_fts_backend_build_context *ctx)
ee1ecdda8a6b fts-solr: Keep last_uid field updated containing the mailbox's last added UID. Timo Sirainen <tss@iki.fi> parents: 8484 diff changeset	642 {
ee1ecdda8a6b fts-solr: Keep last_uid field updated containing the mailbox's last added UID. Timo Sirainen <tss@iki.fi> parents: 8484 diff changeset	643 int ret;
ee1ecdda8a6b fts-solr: Keep last_uid field updated containing the mailbox's last added UID. Timo Sirainen <tss@iki.fi> parents: 8484 diff changeset	644
8495 ffb37c392166 Finished making full text search indexes work fast with virtual mailboxes. Timo Sirainen <tss@iki.fi> parents: 8493 diff changeset	645 if (ctx->post == NULL)
8492 ee1ecdda8a6b fts-solr: Keep last_uid field updated containing the mailbox's last added UID. Timo Sirainen <tss@iki.fi> parents: 8484 diff changeset	646 return 0;
ee1ecdda8a6b fts-solr: Keep last_uid field updated containing the mailbox's last added UID. Timo Sirainen <tss@iki.fi> parents: 8484 diff changeset	647
ee1ecdda8a6b fts-solr: Keep last_uid field updated containing the mailbox's last added UID. Timo Sirainen <tss@iki.fi> parents: 8484 diff changeset	648 str_append(ctx->cmd, "</field></doc>");
ee1ecdda8a6b fts-solr: Keep last_uid field updated containing the mailbox's last added UID. Timo Sirainen <tss@iki.fi> parents: 8484 diff changeset	649
ee1ecdda8a6b fts-solr: Keep last_uid field updated containing the mailbox's last added UID. Timo Sirainen <tss@iki.fi> parents: 8484 diff changeset	650 /* Update the mailbox's last_uid field, replacing the existing
ee1ecdda8a6b fts-solr: Keep last_uid field updated containing the mailbox's last added UID. Timo Sirainen <tss@iki.fi> parents: 8484 diff changeset	651 document. Note that since there is no locking, it's possible that
ee1ecdda8a6b fts-solr: Keep last_uid field updated containing the mailbox's last added UID. Timo Sirainen <tss@iki.fi> parents: 8484 diff changeset	652 if another session is indexing at the same time, the last_uid value
ee1ecdda8a6b fts-solr: Keep last_uid field updated containing the mailbox's last added UID. Timo Sirainen <tss@iki.fi> parents: 8484 diff changeset	653 may shrink. This doesn't really matter, we'll simply do more work
ee1ecdda8a6b fts-solr: Keep last_uid field updated containing the mailbox's last added UID. Timo Sirainen <tss@iki.fi> parents: 8484 diff changeset	654 in future by reindexing some messages. */
ee1ecdda8a6b fts-solr: Keep last_uid field updated containing the mailbox's last added UID. Timo Sirainen <tss@iki.fi> parents: 8484 diff changeset	655 fts_backend_solr_add_doc_prefix(ctx, ctx->prev_uid);
ee1ecdda8a6b fts-solr: Keep last_uid field updated containing the mailbox's last added UID. Timo Sirainen <tss@iki.fi> parents: 8484 diff changeset	656 str_printfa(ctx->cmd, "<field name=\"last_uid\">TRUE</field>"
8499 252b29ac5f43 fts: Added support for handling multiple namespaces. Timo Sirainen <tss@iki.fi> parents: 8495 diff changeset	657 "<field name=\"id\">");
8950 ec1ae90af21a fts: Fixes to how virtual mailboxes are searched. Timo Sirainen <tss@iki.fi> parents: 8946 diff changeset	658 xml_encode_id(ctx->cmd, ctx->ctx.backend, 0, ctx->uid_validity);
8492 ee1ecdda8a6b fts-solr: Keep last_uid field updated containing the mailbox's last added UID. Timo Sirainen <tss@iki.fi> parents: 8484 diff changeset	659 str_append(ctx->cmd, "</field></doc></add>");
ee1ecdda8a6b fts-solr: Keep last_uid field updated containing the mailbox's last added UID. Timo Sirainen <tss@iki.fi> parents: 8484 diff changeset	660
ee1ecdda8a6b fts-solr: Keep last_uid field updated containing the mailbox's last added UID. Timo Sirainen <tss@iki.fi> parents: 8484 diff changeset	661 solr_connection_post_more(ctx->post, str_data(ctx->cmd),
ee1ecdda8a6b fts-solr: Keep last_uid field updated containing the mailbox's last added UID. Timo Sirainen <tss@iki.fi> parents: 8484 diff changeset	662 str_len(ctx->cmd));
ee1ecdda8a6b fts-solr: Keep last_uid field updated containing the mailbox's last added UID. Timo Sirainen <tss@iki.fi> parents: 8484 diff changeset	663 ret = solr_connection_post_end(ctx->post);
ee1ecdda8a6b fts-solr: Keep last_uid field updated containing the mailbox's last added UID. Timo Sirainen <tss@iki.fi> parents: 8484 diff changeset	664 /* commit and wait until the documents we just indexed are
ee1ecdda8a6b fts-solr: Keep last_uid field updated containing the mailbox's last added UID. Timo Sirainen <tss@iki.fi> parents: 8484 diff changeset	665 visible to the following search */
ee1ecdda8a6b fts-solr: Keep last_uid field updated containing the mailbox's last added UID. Timo Sirainen <tss@iki.fi> parents: 8484 diff changeset	666 if (solr_connection_post(solr_conn, "<commit waitFlush=\"false\" "
ee1ecdda8a6b fts-solr: Keep last_uid field updated containing the mailbox's last added UID. Timo Sirainen <tss@iki.fi> parents: 8484 diff changeset	667 "waitSearcher=\"true\"/>") < 0)
ee1ecdda8a6b fts-solr: Keep last_uid field updated containing the mailbox's last added UID. Timo Sirainen <tss@iki.fi> parents: 8484 diff changeset	668 ret = -1;
ee1ecdda8a6b fts-solr: Keep last_uid field updated containing the mailbox's last added UID. Timo Sirainen <tss@iki.fi> parents: 8484 diff changeset	669 return ret;
ee1ecdda8a6b fts-solr: Keep last_uid field updated containing the mailbox's last added UID. Timo Sirainen <tss@iki.fi> parents: 8484 diff changeset	670 }
ee1ecdda8a6b fts-solr: Keep last_uid field updated containing the mailbox's last added UID. Timo Sirainen <tss@iki.fi> parents: 8484 diff changeset	671
ee1ecdda8a6b fts-solr: Keep last_uid field updated containing the mailbox's last added UID. Timo Sirainen <tss@iki.fi> parents: 8484 diff changeset	672 static int
7990 662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	673 fts_backend_solr_build_deinit(struct fts_backend_build_context *_ctx)
662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	674 {
662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	675 struct solr_fts_backend_build_context *ctx =
662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	676 (struct solr_fts_backend_build_context *)_ctx;
8492 ee1ecdda8a6b fts-solr: Keep last_uid field updated containing the mailbox's last added UID. Timo Sirainen <tss@iki.fi> parents: 8484 diff changeset	677 int ret;
7990 662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	678
8492 ee1ecdda8a6b fts-solr: Keep last_uid field updated containing the mailbox's last added UID. Timo Sirainen <tss@iki.fi> parents: 8484 diff changeset	679 ret = fts_backed_solr_build_commit(ctx);
7990 662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	680 str_free(&ctx->cmd);
662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	681 i_free(ctx);
662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	682 return ret;
662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	683 }
662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	684
662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	685 static void
8493 149c6ccc3df4 fts-solr: Use '!' as escape character in usernames. Escape '/' and '!'. Timo Sirainen <tss@iki.fi> parents: 8492 diff changeset	686 fts_backend_solr_expunge(struct fts_backend backend, struct mail mail)
7990 662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	687 {
662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	688 struct mailbox_status status;
662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	689
662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	690 mailbox_get_status(mail->box, STATUS_UIDVALIDITY, &status);
662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	691
662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	692 T_BEGIN {
662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	693 string_t *cmd;
662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	694
662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	695 cmd = t_str_new(256);
8499 252b29ac5f43 fts: Added support for handling multiple namespaces. Timo Sirainen <tss@iki.fi> parents: 8495 diff changeset	696 str_append(cmd, "<delete><id>");
8950 ec1ae90af21a fts: Fixes to how virtual mailboxes are searched. Timo Sirainen <tss@iki.fi> parents: 8946 diff changeset	697 xml_encode_id(cmd, backend, mail->uid, status.uidvalidity);
7990 662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	698 str_append(cmd, "</id></delete>");
662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	699
662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	700 (void)solr_connection_post(solr_conn, str_c(cmd));
662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	701 } T_END;
662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	702 }
662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	703
662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	704 static void
662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	705 fts_backend_solr_expunge_finish(struct fts_backend *backend ATTR_UNUSED,
662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	706 struct mailbox *box ATTR_UNUSED,
662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	707 bool committed ATTR_UNUSED)
662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	708 {
8001 3fbfdda3e5d3 fts-solr: Send data to Solr in larger packets. Commit after expunges. Timo Sirainen <tss@iki.fi> parents: 8000 diff changeset	709 solr_connection_post(solr_conn,
3fbfdda3e5d3 fts-solr: Send data to Solr in larger packets. Commit after expunges. Timo Sirainen <tss@iki.fi> parents: 8000 diff changeset	710 "<commit waitFlush=\"false\" waitSearcher=\"false\"/>");
7990 662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	711 }
662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	712
662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	713 static int fts_backend_solr_lock(struct fts_backend *backend ATTR_UNUSED)
662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	714 {
662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	715 return 1;
662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	716 }
662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	717
662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	718 static void fts_backend_solr_unlock(struct fts_backend *backend ATTR_UNUSED)
662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	719 {
662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	720 }
662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	721
8499 252b29ac5f43 fts: Added support for handling multiple namespaces. Timo Sirainen <tss@iki.fi> parents: 8495 diff changeset	722 static bool solr_virtual_uid_map(const char ns_prefix, const char mailbox,
252b29ac5f43 fts: Added support for handling multiple namespaces. Timo Sirainen <tss@iki.fi> parents: 8495 diff changeset	723 uint32_t uidvalidity, uint32_t *uid,
252b29ac5f43 fts: Added support for handling multiple namespaces. Timo Sirainen <tss@iki.fi> parents: 8495 diff changeset	724 void *context)
8484 db78eab2ba5d Initial support for getting full text search working nicely with virtual mailboxes. Timo Sirainen <tss@iki.fi> parents: 8109 diff changeset	725 {
8499 252b29ac5f43 fts: Added support for handling multiple namespaces. Timo Sirainen <tss@iki.fi> parents: 8495 diff changeset	726 struct solr_virtual_uid_map_context *ctx = context;
8500 3efcdc45d111 Added alias_for setting for namespaces. Fixes namespace issues with fts. Timo Sirainen <tss@iki.fi> parents: 8499 diff changeset	727 struct mail_namespace *ns;
8499 252b29ac5f43 fts: Added support for handling multiple namespaces. Timo Sirainen <tss@iki.fi> parents: 8495 diff changeset	728 const char *vname;
8950 ec1ae90af21a fts: Fixes to how virtual mailboxes are searched. Timo Sirainen <tss@iki.fi> parents: 8946 diff changeset	729 bool convert_inbox;
8484 db78eab2ba5d Initial support for getting full text search working nicely with virtual mailboxes. Timo Sirainen <tss@iki.fi> parents: 8109 diff changeset	730
8500 3efcdc45d111 Added alias_for setting for namespaces. Fixes namespace issues with fts. Timo Sirainen <tss@iki.fi> parents: 8499 diff changeset	731 ns = solr_get_namespaces(ctx->backend, ctx->box, ns_prefix);
8950 ec1ae90af21a fts: Fixes to how virtual mailboxes are searched. Timo Sirainen <tss@iki.fi> parents: 8946 diff changeset	732 convert_inbox = (ns->flags & NAMESPACE_FLAG_INBOX) != 0 &&
ec1ae90af21a fts: Fixes to how virtual mailboxes are searched. Timo Sirainen <tss@iki.fi> parents: 8946 diff changeset	733 strcmp(mailbox, "INBOX") == 0;
8500 3efcdc45d111 Added alias_for setting for namespaces. Fixes namespace issues with fts. Timo Sirainen <tss@iki.fi> parents: 8499 diff changeset	734 for (; ns != NULL; ns = ns->alias_chain_next) {
8950 ec1ae90af21a fts: Fixes to how virtual mailboxes are searched. Timo Sirainen <tss@iki.fi> parents: 8946 diff changeset	735 vname = convert_inbox ? ns->prefix :
ec1ae90af21a fts: Fixes to how virtual mailboxes are searched. Timo Sirainen <tss@iki.fi> parents: 8946 diff changeset	736 mail_namespace_get_vname(ns, ctx->vname, mailbox);
8500 3efcdc45d111 Added alias_for setting for namespaces. Fixes namespace issues with fts. Timo Sirainen <tss@iki.fi> parents: 8499 diff changeset	737 if (mailbox_get_virtual_uid(ctx->box, vname, uidvalidity,
3efcdc45d111 Added alias_for setting for namespaces. Fixes namespace issues with fts. Timo Sirainen <tss@iki.fi> parents: 8499 diff changeset	738 *uid, uid))
3efcdc45d111 Added alias_for setting for namespaces. Fixes namespace issues with fts. Timo Sirainen <tss@iki.fi> parents: 8499 diff changeset	739 return TRUE;
3efcdc45d111 Added alias_for setting for namespaces. Fixes namespace issues with fts. Timo Sirainen <tss@iki.fi> parents: 8499 diff changeset	740 }
3efcdc45d111 Added alias_for setting for namespaces. Fixes namespace issues with fts. Timo Sirainen <tss@iki.fi> parents: 8499 diff changeset	741 return FALSE;
8484 db78eab2ba5d Initial support for getting full text search working nicely with virtual mailboxes. Timo Sirainen <tss@iki.fi> parents: 8109 diff changeset	742 }
db78eab2ba5d Initial support for getting full text search working nicely with virtual mailboxes. Timo Sirainen <tss@iki.fi> parents: 8109 diff changeset	743
8004 6418d4c0ff16 fts: Added a new lookup API where the backend can look up all the fields Timo Sirainen <tss@iki.fi> parents: 8002 diff changeset	744 static int fts_backend_solr_lookup(struct fts_backend_lookup_context *ctx,
6418d4c0ff16 fts: Added a new lookup API where the backend can look up all the fields Timo Sirainen <tss@iki.fi> parents: 8002 diff changeset	745 ARRAY_TYPE(seq_range) *definite_uids,
8006 0a3b5ddb0869 Added support for sorting by X-SCORE. Currently it's only set by fts-solr. Timo Sirainen <tss@iki.fi> parents: 8004 diff changeset	746 ARRAY_TYPE(seq_range) *maybe_uids,
0a3b5ddb0869 Added support for sorting by X-SCORE. Currently it's only set by fts-solr. Timo Sirainen <tss@iki.fi> parents: 8004 diff changeset	747 ARRAY_TYPE(fts_score_map) *scores)
7990 662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	748 {
8004 6418d4c0ff16 fts: Added a new lookup API where the backend can look up all the fields Timo Sirainen <tss@iki.fi> parents: 8002 diff changeset	749 struct mailbox *box = ctx->backend->box;
8950 ec1ae90af21a fts: Fixes to how virtual mailboxes are searched. Timo Sirainen <tss@iki.fi> parents: 8946 diff changeset	750 struct mail_namespace *ns;
8499 252b29ac5f43 fts: Added support for handling multiple namespaces. Timo Sirainen <tss@iki.fi> parents: 8495 diff changeset	751 struct solr_virtual_uid_map_context uid_map_ctx;
8004 6418d4c0ff16 fts: Added a new lookup API where the backend can look up all the fields Timo Sirainen <tss@iki.fi> parents: 8002 diff changeset	752 const struct fts_backend_lookup_field *fields;
8950 ec1ae90af21a fts: Fixes to how virtual mailboxes are searched. Timo Sirainen <tss@iki.fi> parents: 8946 diff changeset	753 const char *box_name;
8004 6418d4c0ff16 fts: Added a new lookup API where the backend can look up all the fields Timo Sirainen <tss@iki.fi> parents: 8002 diff changeset	754 unsigned int i, count;
7990 662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	755 struct mailbox_status status;
662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	756 string_t *str;
8484 db78eab2ba5d Initial support for getting full text search working nicely with virtual mailboxes. Timo Sirainen <tss@iki.fi> parents: 8109 diff changeset	757 bool virtual;
7990 662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	758
8484 db78eab2ba5d Initial support for getting full text search working nicely with virtual mailboxes. Timo Sirainen <tss@iki.fi> parents: 8109 diff changeset	759 virtual = strcmp(box->storage->name, "virtual") == 0;
8004 6418d4c0ff16 fts: Added a new lookup API where the backend can look up all the fields Timo Sirainen <tss@iki.fi> parents: 8002 diff changeset	760 mailbox_get_status(box, STATUS_UIDVALIDITY, &status);
7990 662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	761
662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	762 str = t_str_new(256);
8484 db78eab2ba5d Initial support for getting full text search working nicely with virtual mailboxes. Timo Sirainen <tss@iki.fi> parents: 8109 diff changeset	763 if (!virtual) {
8499 252b29ac5f43 fts: Added support for handling multiple namespaces. Timo Sirainen <tss@iki.fi> parents: 8495 diff changeset	764 str_printfa(str, "fl=uid,score&rows=%u&sort=uid+asc&q=",
8484 db78eab2ba5d Initial support for getting full text search working nicely with virtual mailboxes. Timo Sirainen <tss@iki.fi> parents: 8109 diff changeset	765 status.uidnext);
db78eab2ba5d Initial support for getting full text search working nicely with virtual mailboxes. Timo Sirainen <tss@iki.fi> parents: 8109 diff changeset	766 } else {
8499 252b29ac5f43 fts: Added support for handling multiple namespaces. Timo Sirainen <tss@iki.fi> parents: 8495 diff changeset	767 str_printfa(str, "fl=uid,score,box,uidv,ns&rows=%u"
252b29ac5f43 fts: Added support for handling multiple namespaces. Timo Sirainen <tss@iki.fi> parents: 8495 diff changeset	768 "&sort=box+asc,uid+asc&q=",
8484 db78eab2ba5d Initial support for getting full text search working nicely with virtual mailboxes. Timo Sirainen <tss@iki.fi> parents: 8109 diff changeset	769 SOLR_MAX_ROWS);
db78eab2ba5d Initial support for getting full text search working nicely with virtual mailboxes. Timo Sirainen <tss@iki.fi> parents: 8109 diff changeset	770 }
8004 6418d4c0ff16 fts: Added a new lookup API where the backend can look up all the fields Timo Sirainen <tss@iki.fi> parents: 8002 diff changeset	771
6418d4c0ff16 fts: Added a new lookup API where the backend can look up all the fields Timo Sirainen <tss@iki.fi> parents: 8002 diff changeset	772 /* build a lucene search query from the fields */
6418d4c0ff16 fts: Added a new lookup API where the backend can look up all the fields Timo Sirainen <tss@iki.fi> parents: 8002 diff changeset	773 fields = array_get(&ctx->fields, &count);
6418d4c0ff16 fts: Added a new lookup API where the backend can look up all the fields Timo Sirainen <tss@iki.fi> parents: 8002 diff changeset	774 for (i = 0; i < count; i++) {
6418d4c0ff16 fts: Added a new lookup API where the backend can look up all the fields Timo Sirainen <tss@iki.fi> parents: 8002 diff changeset	775 if (i > 0)
8499 252b29ac5f43 fts: Added support for handling multiple namespaces. Timo Sirainen <tss@iki.fi> parents: 8495 diff changeset	776 str_append_c(str, '+');
8004 6418d4c0ff16 fts: Added a new lookup API where the backend can look up all the fields Timo Sirainen <tss@iki.fi> parents: 8002 diff changeset	777
6418d4c0ff16 fts: Added a new lookup API where the backend can look up all the fields Timo Sirainen <tss@iki.fi> parents: 8002 diff changeset	778 if ((fields[i].flags & FTS_LOOKUP_FLAG_INVERT) != 0)
6418d4c0ff16 fts: Added a new lookup API where the backend can look up all the fields Timo Sirainen <tss@iki.fi> parents: 8002 diff changeset	779 str_append_c(str, '-');
6418d4c0ff16 fts: Added a new lookup API where the backend can look up all the fields Timo Sirainen <tss@iki.fi> parents: 8002 diff changeset	780
6418d4c0ff16 fts: Added a new lookup API where the backend can look up all the fields Timo Sirainen <tss@iki.fi> parents: 8002 diff changeset	781 if ((fields[i].flags & FTS_LOOKUP_FLAG_HEADER) == 0) {
6418d4c0ff16 fts: Added a new lookup API where the backend can look up all the fields Timo Sirainen <tss@iki.fi> parents: 8002 diff changeset	782 /* body only */
6418d4c0ff16 fts: Added a new lookup API where the backend can look up all the fields Timo Sirainen <tss@iki.fi> parents: 8002 diff changeset	783 i_assert((fields[i].flags & FTS_LOOKUP_FLAG_BODY) != 0);
6418d4c0ff16 fts: Added a new lookup API where the backend can look up all the fields Timo Sirainen <tss@iki.fi> parents: 8002 diff changeset	784 str_append(str, "body:");
9141 d21bd1a1710a solr: Don't use "any" copyfield, it doubles the index size. Timo Sirainen <tss@iki.fi> parents: 8950 diff changeset	785 solr_quote_http(str, fields[i].key);
8004 6418d4c0ff16 fts: Added a new lookup API where the backend can look up all the fields Timo Sirainen <tss@iki.fi> parents: 8002 diff changeset	786 } else if ((fields[i].flags & FTS_LOOKUP_FLAG_BODY) == 0) {
6418d4c0ff16 fts: Added a new lookup API where the backend can look up all the fields Timo Sirainen <tss@iki.fi> parents: 8002 diff changeset	787 /* header only */
6418d4c0ff16 fts: Added a new lookup API where the backend can look up all the fields Timo Sirainen <tss@iki.fi> parents: 8002 diff changeset	788 str_append(str, "hdr:");
9141 d21bd1a1710a solr: Don't use "any" copyfield, it doubles the index size. Timo Sirainen <tss@iki.fi> parents: 8950 diff changeset	789 solr_quote_http(str, fields[i].key);
8004 6418d4c0ff16 fts: Added a new lookup API where the backend can look up all the fields Timo Sirainen <tss@iki.fi> parents: 8002 diff changeset	790 } else {
6418d4c0ff16 fts: Added a new lookup API where the backend can look up all the fields Timo Sirainen <tss@iki.fi> parents: 8002 diff changeset	791 /* both */
9141 d21bd1a1710a solr: Don't use "any" copyfield, it doubles the index size. Timo Sirainen <tss@iki.fi> parents: 8950 diff changeset	792 str_append(str, "(body:");
d21bd1a1710a solr: Don't use "any" copyfield, it doubles the index size. Timo Sirainen <tss@iki.fi> parents: 8950 diff changeset	793 solr_quote_http(str, fields[i].key);
d21bd1a1710a solr: Don't use "any" copyfield, it doubles the index size. Timo Sirainen <tss@iki.fi> parents: 8950 diff changeset	794 str_append(str, "+OR+hdr:");
d21bd1a1710a solr: Don't use "any" copyfield, it doubles the index size. Timo Sirainen <tss@iki.fi> parents: 8950 diff changeset	795 solr_quote_http(str, fields[i].key);
d21bd1a1710a solr: Don't use "any" copyfield, it doubles the index size. Timo Sirainen <tss@iki.fi> parents: 8950 diff changeset	796 str_append_c(str, ')');
8004 6418d4c0ff16 fts: Added a new lookup API where the backend can look up all the fields Timo Sirainen <tss@iki.fi> parents: 8002 diff changeset	797 }
7990 662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	798 }
662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	799
8004 6418d4c0ff16 fts: Added a new lookup API where the backend can look up all the fields Timo Sirainen <tss@iki.fi> parents: 8002 diff changeset	800 /* use a separate filter query for selecting the mailbox. it shouldn't
6418d4c0ff16 fts: Added a new lookup API where the backend can look up all the fields Timo Sirainen <tss@iki.fi> parents: 8002 diff changeset	801 affect the score and there could be some caching benefits too. */
8505 5cc1c4f3d38b fts-solr: In filter queries use "+" prefixes to require all the parameters to match. Timo Sirainen <tss@iki.fi> parents: 8500 diff changeset	802 str_append(str, "&fq=%2Buser:");
8499 252b29ac5f43 fts: Added support for handling multiple namespaces. Timo Sirainen <tss@iki.fi> parents: 8495 diff changeset	803 solr_quote_http(str, box->storage->ns->user->username);
8495 ffb37c392166 Finished making full text search indexes work fast with virtual mailboxes. Timo Sirainen <tss@iki.fi> parents: 8493 diff changeset	804 if (virtual)
8499 252b29ac5f43 fts: Added support for handling multiple namespaces. Timo Sirainen <tss@iki.fi> parents: 8495 diff changeset	805 fts_backend_solr_filter_mailboxes(ctx->backend, str, box);
8495 ffb37c392166 Finished making full text search indexes work fast with virtual mailboxes. Timo Sirainen <tss@iki.fi> parents: 8493 diff changeset	806 else {
8950 ec1ae90af21a fts: Fixes to how virtual mailboxes are searched. Timo Sirainen <tss@iki.fi> parents: 8946 diff changeset	807 box_name = fts_box_get_root(box, &ns);
8505 5cc1c4f3d38b fts-solr: In filter queries use "+" prefixes to require all the parameters to match. Timo Sirainen <tss@iki.fi> parents: 8500 diff changeset	808 str_printfa(str, "+%%2Buidv:%u+%%2Bbox:", status.uidvalidity);
8950 ec1ae90af21a fts: Fixes to how virtual mailboxes are searched. Timo Sirainen <tss@iki.fi> parents: 8946 diff changeset	809 solr_quote_http(str, box_name);
ec1ae90af21a fts: Fixes to how virtual mailboxes are searched. Timo Sirainen <tss@iki.fi> parents: 8946 diff changeset	810 solr_add_ns_query_http(str, ctx->backend, ns);
8484 db78eab2ba5d Initial support for getting full text search working nicely with virtual mailboxes. Timo Sirainen <tss@iki.fi> parents: 8109 diff changeset	811 }
db78eab2ba5d Initial support for getting full text search working nicely with virtual mailboxes. Timo Sirainen <tss@iki.fi> parents: 8109 diff changeset	812
7990 662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	813 array_clear(maybe_uids);
8484 db78eab2ba5d Initial support for getting full text search working nicely with virtual mailboxes. Timo Sirainen <tss@iki.fi> parents: 8109 diff changeset	814 if (!virtual) {
db78eab2ba5d Initial support for getting full text search working nicely with virtual mailboxes. Timo Sirainen <tss@iki.fi> parents: 8109 diff changeset	815 return solr_connection_select(solr_conn, str_c(str), NULL, NULL,
db78eab2ba5d Initial support for getting full text search working nicely with virtual mailboxes. Timo Sirainen <tss@iki.fi> parents: 8109 diff changeset	816 definite_uids, scores);
db78eab2ba5d Initial support for getting full text search working nicely with virtual mailboxes. Timo Sirainen <tss@iki.fi> parents: 8109 diff changeset	817 } else {
8499 252b29ac5f43 fts: Added support for handling multiple namespaces. Timo Sirainen <tss@iki.fi> parents: 8495 diff changeset	818 memset(&uid_map_ctx, 0, sizeof(uid_map_ctx));
252b29ac5f43 fts: Added support for handling multiple namespaces. Timo Sirainen <tss@iki.fi> parents: 8495 diff changeset	819 uid_map_ctx.backend = ctx->backend;
252b29ac5f43 fts: Added support for handling multiple namespaces. Timo Sirainen <tss@iki.fi> parents: 8495 diff changeset	820 uid_map_ctx.box = box;
252b29ac5f43 fts: Added support for handling multiple namespaces. Timo Sirainen <tss@iki.fi> parents: 8495 diff changeset	821 uid_map_ctx.vname = t_str_new(256);
8484 db78eab2ba5d Initial support for getting full text search working nicely with virtual mailboxes. Timo Sirainen <tss@iki.fi> parents: 8109 diff changeset	822 return solr_connection_select(solr_conn, str_c(str),
8499 252b29ac5f43 fts: Added support for handling multiple namespaces. Timo Sirainen <tss@iki.fi> parents: 8495 diff changeset	823 solr_virtual_uid_map,
252b29ac5f43 fts: Added support for handling multiple namespaces. Timo Sirainen <tss@iki.fi> parents: 8495 diff changeset	824 &uid_map_ctx,
8484 db78eab2ba5d Initial support for getting full text search working nicely with virtual mailboxes. Timo Sirainen <tss@iki.fi> parents: 8109 diff changeset	825 definite_uids, scores);
db78eab2ba5d Initial support for getting full text search working nicely with virtual mailboxes. Timo Sirainen <tss@iki.fi> parents: 8109 diff changeset	826 }
7990 662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	827 }
662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	828
662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	829 struct fts_backend fts_backend_solr = {
662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	830 MEMBER(name) "solr",
9292 96b764649cbb fts: Fixed crashing with Squat. Timo Sirainen <tss@iki.fi> parents: 9141 diff changeset	831 MEMBER(flags) FTS_BACKEND_FLAG_VIRTUAL_LOOKUPS,
7990 662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	832
662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	833 {
662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	834 fts_backend_solr_init,
662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	835 fts_backend_solr_deinit,
662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	836 fts_backend_solr_get_last_uid,
8495 ffb37c392166 Finished making full text search indexes work fast with virtual mailboxes. Timo Sirainen <tss@iki.fi> parents: 8493 diff changeset	837 fts_backend_solr_get_all_last_uids,
7990 662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	838 fts_backend_solr_build_init,
662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	839 fts_backend_solr_build_more,
662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	840 fts_backend_solr_build_deinit,
662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	841 fts_backend_solr_expunge,
662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	842 fts_backend_solr_expunge_finish,
662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	843 fts_backend_solr_lock,
662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	844 fts_backend_solr_unlock,
8004 6418d4c0ff16 fts: Added a new lookup API where the backend can look up all the fields Timo Sirainen <tss@iki.fi> parents: 8002 diff changeset	845 NULL,
6418d4c0ff16 fts: Added a new lookup API where the backend can look up all the fields Timo Sirainen <tss@iki.fi> parents: 8002 diff changeset	846 NULL,
6418d4c0ff16 fts: Added a new lookup API where the backend can look up all the fields Timo Sirainen <tss@iki.fi> parents: 8002 diff changeset	847 fts_backend_solr_lookup
7990 662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	848 }
662172573fe1 Initial code to support Apache Solr (Lucene indexing server). Timo Sirainen <tss@iki.fi> parents: diff changeset	849 };

Mercurial > dovecot > original-hg > dovecot-1.2

annotate src/plugins/fts-solr/fts-backend-solr.c @ 9604:cf0da2cd31fb HEAD