Mercurial > dovecot > original-hg > dovecot-1.2

annotate src/plugins/fts-squat/squat-trie.c @ 7197:bb9304dee6d4 HEAD

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
uidlist renumbering: changed iteration code to iterate children first. This will be needed for handling expunges.
author Timo Sirainen <tss@iki.fi>
date Sun, 27 Jan 2008 11:49:35 +0200
parents d9b87e3ce6c8
children f5f77a3ae203
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
7086
7ed926ed7aa4 Updated copyright notices to include year 2008.
Timo Sirainen <tss@iki.fi>
parents: 7043
diff changeset
1 /* Copyright (c) 2007-2008 Dovecot authors, see the included COPYING file */
4855
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
2
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
3 #include "lib.h"
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
4 #include "array.h"
6898
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
5 #include "str.h"
6944
66b3e894041b Support mmap_disable=yes and some error handling improvements.
Timo Sirainen <tss@iki.fi>
parents: 6940
diff changeset
6 #include "read-full.h"
4855
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
7 #include "istream.h"
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
8 #include "ostream.h"
6911
a215deb3de8d Fixed non-ASCII searches.
Timo Sirainen <tss@iki.fi>
parents: 6905
diff changeset
9 #include "unichar.h"
6944
66b3e894041b Support mmap_disable=yes and some error handling improvements.
Timo Sirainen <tss@iki.fi>
parents: 6940
diff changeset
10 #include "file-cache.h"
6898
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
11 #include "seq-range-array.h"
4855
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
12 #include "squat-uidlist.h"
5068
65a4e1e61555 Some bugfixes. Moved header struct to a separate file.
Timo Sirainen <tss@iki.fi>
parents: 5010
diff changeset
13 #include "squat-trie-private.h"
4855
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
14
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
15 #include <stdio.h>
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
16 #include <stdlib.h>
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
17 #include <unistd.h>
6898
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
18 #include <sys/mman.h>
4855
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
19
6905
8c779f3e16be Fixes to FTS handling.
Timo Sirainen <tss@iki.fi>
parents: 6902
diff changeset
20 #define DEFAULT_NORMALIZE_MAP_CHARS \
8c779f3e16be Fixes to FTS handling.
Timo Sirainen <tss@iki.fi>
parents: 6902
diff changeset
21 "EOTIRSACDNLMVUGPHBFWYXKJQZ0123456789@.-+#$%_&"
8c779f3e16be Fixes to FTS handling.
Timo Sirainen <tss@iki.fi>
parents: 6902
diff changeset
22 #define DEFAULT_PARTIAL_LEN 4
8c779f3e16be Fixes to FTS handling.
Timo Sirainen <tss@iki.fi>
parents: 6902
diff changeset
23 #define DEFAULT_FULL_LEN 4
8c779f3e16be Fixes to FTS handling.
Timo Sirainen <tss@iki.fi>
parents: 6902
diff changeset
24
6898
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
25 #define MAX_FAST_LEVEL 3
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
26 #define SEQUENTIAL_COUNT 46
4878
88a91d9a867b Fixes. Should be pretty much working now.
Timo Sirainen <tss@iki.fi>
parents: 4855
diff changeset
27
6944
66b3e894041b Support mmap_disable=yes and some error handling improvements.
Timo Sirainen <tss@iki.fi>
parents: 6940
diff changeset
28 #define TRIE_BYTES_LEFT(n) \
66b3e894041b Support mmap_disable=yes and some error handling improvements.
Timo Sirainen <tss@iki.fi>
parents: 6940
diff changeset
29 ((n) * SQUAT_PACK_MAX_SIZE)
66b3e894041b Support mmap_disable=yes and some error handling improvements.
Timo Sirainen <tss@iki.fi>
parents: 6940
diff changeset
30 #define TRIE_READAHEAD_SIZE \
66b3e894041b Support mmap_disable=yes and some error handling improvements.
Timo Sirainen <tss@iki.fi>
parents: 6940
diff changeset
31 I_MAX(4096, 1 + 256 + TRIE_BYTES_LEFT(256))
66b3e894041b Support mmap_disable=yes and some error handling improvements.
Timo Sirainen <tss@iki.fi>
parents: 6940
diff changeset
32
4878
88a91d9a867b Fixes. Should be pretty much working now.
Timo Sirainen <tss@iki.fi>
parents: 4855
diff changeset
33 struct squat_trie_build_context {
88a91d9a867b Fixes. Should be pretty much working now.
Timo Sirainen <tss@iki.fi>
parents: 4855
diff changeset
34 struct squat_trie *trie;
4855
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
35 struct ostream *output;
6931
97702c9c4111 Locking and error handling fixes.
Timo Sirainen <tss@iki.fi>
parents: 6914
diff changeset
36 struct squat_uidlist_build_context *uidlist_build_ctx;
97702c9c4111 Locking and error handling fixes.
Timo Sirainen <tss@iki.fi>
parents: 6914
diff changeset
37
97702c9c4111 Locking and error handling fixes.
Timo Sirainen <tss@iki.fi>
parents: 6914
diff changeset
38 struct file_lock *file_lock;
4855
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
39
6898
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
40 uint32_t first_uid;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
41 unsigned int compress_nodes:1;
4878
88a91d9a867b Fixes. Should be pretty much working now.
Timo Sirainen <tss@iki.fi>
parents: 4855
diff changeset
42 };
88a91d9a867b Fixes. Should be pretty much working now.
Timo Sirainen <tss@iki.fi>
parents: 4855
diff changeset
43
6898
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
44 struct squat_trie_iterate_node {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
45 struct squat_node *node;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
46 unsigned int idx;
4855
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
47 };
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
48
6898
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
49 struct squat_trie_iterate_context {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
50 struct squat_trie *trie;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
51 struct squat_trie_iterate_node cur;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
52 ARRAY_DEFINE(parents, struct squat_trie_iterate_node);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
53 bool failed;
4855
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
54 };
6898
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
55
6931
97702c9c4111 Locking and error handling fixes.
Timo Sirainen <tss@iki.fi>
parents: 6914
diff changeset
56 static int squat_trie_map(struct squat_trie *trie, bool building);
4855
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
57
6898
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
58 void squat_trie_delete(struct squat_trie *trie)
4855
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
59 {
6898
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
60 if (unlink(trie->path) < 0 && errno != ENOENT)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
61 i_error("unlink(%s) failed: %m", trie->path);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
62 squat_uidlist_delete(trie->uidlist);
4855
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
63 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
64
6898
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
65 static void squat_trie_set_corrupted(struct squat_trie *trie)
4855
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
66 {
6898
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
67 trie->corrupted = TRUE;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
68 i_error("Corrupted file %s", trie->path);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
69 squat_trie_delete(trie);
4855
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
70 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
71
6898
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
72 static void squat_trie_normalize_map_build(struct squat_trie *trie)
4855
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
73 {
6898
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
74 static unsigned char valid_chars[] =
6905
8c779f3e16be Fixes to FTS handling.
Timo Sirainen <tss@iki.fi>
parents: 6902
diff changeset
75 DEFAULT_NORMALIZE_MAP_CHARS;
6898
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
76 unsigned int i, j;
4855
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
77
6905
8c779f3e16be Fixes to FTS handling.
Timo Sirainen <tss@iki.fi>
parents: 6902
diff changeset
78 memset(trie->default_normalize_map, 0,
8c779f3e16be Fixes to FTS handling.
Timo Sirainen <tss@iki.fi>
parents: 6902
diff changeset
79 sizeof(trie->default_normalize_map));
4901
5e604d6f391e Only ASCII texts were actually indexed. Now UTF-8 input is properly
Timo Sirainen <tss@iki.fi>
parents: 4898
diff changeset
80
6898
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
81 #if 1
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
82 for (i = 0, j = 1; i < sizeof(valid_chars)-1; i++) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
83 unsigned char chr = valid_chars[i];
4855
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
84
6898
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
85 if (chr >= 'A' && chr <= 'Z')
6905
8c779f3e16be Fixes to FTS handling.
Timo Sirainen <tss@iki.fi>
parents: 6902
diff changeset
86 trie->default_normalize_map[chr-'A'+'a'] = j;
8c779f3e16be Fixes to FTS handling.
Timo Sirainen <tss@iki.fi>
parents: 6902
diff changeset
87 trie->default_normalize_map[chr] = j++;
6898
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
88 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
89 i_assert(j <= SEQUENTIAL_COUNT);
4855
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
90
6898
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
91 for (i = 128; i < 256; i++)
6905
8c779f3e16be Fixes to FTS handling.
Timo Sirainen <tss@iki.fi>
parents: 6902
diff changeset
92 trie->default_normalize_map[i] = j++;
6898
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
93 #else
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
94 for (i = 0; i < sizeof(valid_chars)-1; i++) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
95 unsigned char chr = valid_chars[i];
4890
cb3b4153136c 64bit and big endian fixes
Timo Sirainen <tss@iki.fi>
parents: 4887
diff changeset
96
6898
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
97 if (chr >= 'A' && chr <= 'Z')
6905
8c779f3e16be Fixes to FTS handling.
Timo Sirainen <tss@iki.fi>
parents: 6902
diff changeset
98 trie->default_normalize_map[chr-'A'+'a'] = chr;
8c779f3e16be Fixes to FTS handling.
Timo Sirainen <tss@iki.fi>
parents: 6902
diff changeset
99 trie->default_normalize_map[chr] = chr;
4890
cb3b4153136c 64bit and big endian fixes
Timo Sirainen <tss@iki.fi>
parents: 4887
diff changeset
100 }
6898
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
101 for (i = 128; i < 256; i++)
6905
8c779f3e16be Fixes to FTS handling.
Timo Sirainen <tss@iki.fi>
parents: 6902
diff changeset
102 trie->default_normalize_map[i] = i_toupper(i);
4890
cb3b4153136c 64bit and big endian fixes
Timo Sirainen <tss@iki.fi>
parents: 4887
diff changeset
103 #endif
cb3b4153136c 64bit and big endian fixes
Timo Sirainen <tss@iki.fi>
parents: 4887
diff changeset
104 }
cb3b4153136c 64bit and big endian fixes
Timo Sirainen <tss@iki.fi>
parents: 4887
diff changeset
105
6898
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
106 static void node_free(struct squat_trie *trie, struct squat_node *node)
4855
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
107 {
6898
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
108 struct squat_node *children;
4890
cb3b4153136c 64bit and big endian fixes
Timo Sirainen <tss@iki.fi>
parents: 4887
diff changeset
109 unsigned int i;
4855
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
110
6898
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
111 if (NODE_IS_DYNAMIC_LEAF(node))
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
112 i_free(node->children.leaf_string);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
113 else if (!node->children_not_mapped && node->child_count > 0) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
114 children = NODE_CHILDREN_NODES(node);
4883
d8adbe93c969 Added support for mmap_disable=yes and several other fixes.
Timo Sirainen <tss@iki.fi>
parents: 4879
diff changeset
115
6898
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
116 trie->node_alloc_size -=
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
117 NODE_CHILDREN_ALLOC_SIZE(node->child_count);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
118 for (i = 0; i < node->child_count; i++)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
119 node_free(trie, &children[i]);
4883
d8adbe93c969 Added support for mmap_disable=yes and several other fixes.
Timo Sirainen <tss@iki.fi>
parents: 4879
diff changeset
120
6898
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
121 i_free(node->children.data);
4898
07a038b57946 Optimization.
Timo Sirainen <tss@iki.fi>
parents: 4897
diff changeset
122 }
07a038b57946 Optimization.
Timo Sirainen <tss@iki.fi>
parents: 4897
diff changeset
123 }
07a038b57946 Optimization.
Timo Sirainen <tss@iki.fi>
parents: 4897
diff changeset
124
4878
88a91d9a867b Fixes. Should be pretty much working now.
Timo Sirainen <tss@iki.fi>
parents: 4855
diff changeset
125 struct squat_trie *
6898
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
126 squat_trie_init(const char *path, uint32_t uidvalidity,
4879
Timo Sirainen <tss@iki.fi>
parents: 4878
diff changeset
127 enum file_lock_method lock_method, bool mmap_disable)
4855
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
128 {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
129 struct squat_trie *trie;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
130
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
131 trie = i_new(struct squat_trie, 1);
6898
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
132 trie->path = i_strdup(path);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
133 trie->uidlist = squat_uidlist_init(trie);
4855
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
134 trie->fd = -1;
4878
88a91d9a867b Fixes. Should be pretty much working now.
Timo Sirainen <tss@iki.fi>
parents: 4855
diff changeset
135 trie->lock_method = lock_method;
6898
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
136 trie->uidvalidity = uidvalidity;
6944
66b3e894041b Support mmap_disable=yes and some error handling improvements.
Timo Sirainen <tss@iki.fi>
parents: 6940
diff changeset
137 trie->mmap_disable = mmap_disable;
6898
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
138 squat_trie_normalize_map_build(trie);
4855
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
139 return trie;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
140 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
141
6898
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
142 static void squat_trie_close(struct squat_trie *trie)
4878
88a91d9a867b Fixes. Should be pretty much working now.
Timo Sirainen <tss@iki.fi>
parents: 4855
diff changeset
143 {
6931
97702c9c4111 Locking and error handling fixes.
Timo Sirainen <tss@iki.fi>
parents: 6914
diff changeset
144 trie->corrupted = FALSE;
97702c9c4111 Locking and error handling fixes.
Timo Sirainen <tss@iki.fi>
parents: 6914
diff changeset
145 node_free(trie, &trie->root);
97702c9c4111 Locking and error handling fixes.
Timo Sirainen <tss@iki.fi>
parents: 6914
diff changeset
146 memset(&trie->root, 0, sizeof(trie->root));
97702c9c4111 Locking and error handling fixes.
Timo Sirainen <tss@iki.fi>
parents: 6914
diff changeset
147 memset(&trie->hdr, 0, sizeof(trie->hdr));
97702c9c4111 Locking and error handling fixes.
Timo Sirainen <tss@iki.fi>
parents: 6914
diff changeset
148
6944
66b3e894041b Support mmap_disable=yes and some error handling improvements.
Timo Sirainen <tss@iki.fi>
parents: 6940
diff changeset
149 trie->data = NULL;
66b3e894041b Support mmap_disable=yes and some error handling improvements.
Timo Sirainen <tss@iki.fi>
parents: 6940
diff changeset
150 trie->data_size = 0;
66b3e894041b Support mmap_disable=yes and some error handling improvements.
Timo Sirainen <tss@iki.fi>
parents: 6940
diff changeset
151
66b3e894041b Support mmap_disable=yes and some error handling improvements.
Timo Sirainen <tss@iki.fi>
parents: 6940
diff changeset
152 if (trie->file_cache != NULL)
66b3e894041b Support mmap_disable=yes and some error handling improvements.
Timo Sirainen <tss@iki.fi>
parents: 6940
diff changeset
153 file_cache_free(&trie->file_cache);
6898
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
154 if (trie->mmap_size != 0) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
155 if (munmap(trie->mmap_base, trie->mmap_size) < 0)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
156 i_error("munmap(%s) failed: %m", trie->path);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
157 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
158 if (trie->fd != -1) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
159 if (close(trie->fd) < 0)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
160 i_error("close(%s) failed: %m", trie->path);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
161 trie->fd = -1;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
162 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
163 trie->locked_file_size = 0;
6931
97702c9c4111 Locking and error handling fixes.
Timo Sirainen <tss@iki.fi>
parents: 6914
diff changeset
164 }
97702c9c4111 Locking and error handling fixes.
Timo Sirainen <tss@iki.fi>
parents: 6914
diff changeset
165
97702c9c4111 Locking and error handling fixes.
Timo Sirainen <tss@iki.fi>
parents: 6914
diff changeset
166 void squat_trie_deinit(struct squat_trie **_trie)
97702c9c4111 Locking and error handling fixes.
Timo Sirainen <tss@iki.fi>
parents: 6914
diff changeset
167 {
97702c9c4111 Locking and error handling fixes.
Timo Sirainen <tss@iki.fi>
parents: 6914
diff changeset
168 struct squat_trie *trie = *_trie;
97702c9c4111 Locking and error handling fixes.
Timo Sirainen <tss@iki.fi>
parents: 6914
diff changeset
169
97702c9c4111 Locking and error handling fixes.
Timo Sirainen <tss@iki.fi>
parents: 6914
diff changeset
170 *_trie = NULL;
97702c9c4111 Locking and error handling fixes.
Timo Sirainen <tss@iki.fi>
parents: 6914
diff changeset
171 squat_trie_close(trie);
97702c9c4111 Locking and error handling fixes.
Timo Sirainen <tss@iki.fi>
parents: 6914
diff changeset
172 squat_uidlist_deinit(trie->uidlist);
97702c9c4111 Locking and error handling fixes.
Timo Sirainen <tss@iki.fi>
parents: 6914
diff changeset
173 i_free(trie->path);
97702c9c4111 Locking and error handling fixes.
Timo Sirainen <tss@iki.fi>
parents: 6914
diff changeset
174 i_free(trie);
6898
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
175 }
4883
d8adbe93c969 Added support for mmap_disable=yes and several other fixes.
Timo Sirainen <tss@iki.fi>
parents: 4879
diff changeset
176
6898
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
177 static void squat_trie_header_init(struct squat_trie *trie)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
178 {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
179 memset(&trie->hdr, 0, sizeof(trie->hdr));
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
180 trie->hdr.version = SQUAT_TRIE_VERSION;
7159
2de2058a5cdc Don't use ioloop_time. Fixes squat-test which doesn't create ioloop.
Timo Sirainen <tss@iki.fi>
parents: 7158
diff changeset
181 trie->hdr.indexid = time(NULL);
6898
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
182 trie->hdr.uidvalidity = trie->uidvalidity;
6905
8c779f3e16be Fixes to FTS handling.
Timo Sirainen <tss@iki.fi>
parents: 6902
diff changeset
183 trie->hdr.partial_len = DEFAULT_PARTIAL_LEN;
8c779f3e16be Fixes to FTS handling.
Timo Sirainen <tss@iki.fi>
parents: 6902
diff changeset
184 trie->hdr.full_len = DEFAULT_FULL_LEN;
8c779f3e16be Fixes to FTS handling.
Timo Sirainen <tss@iki.fi>
parents: 6902
diff changeset
185
8c779f3e16be Fixes to FTS handling.
Timo Sirainen <tss@iki.fi>
parents: 6902
diff changeset
186 i_assert(sizeof(trie->hdr.normalize_map) ==
8c779f3e16be Fixes to FTS handling.
Timo Sirainen <tss@iki.fi>
parents: 6902
diff changeset
187 sizeof(trie->default_normalize_map));
8c779f3e16be Fixes to FTS handling.
Timo Sirainen <tss@iki.fi>
parents: 6902
diff changeset
188 memcpy(trie->hdr.normalize_map, trie->default_normalize_map,
8c779f3e16be Fixes to FTS handling.
Timo Sirainen <tss@iki.fi>
parents: 6902
diff changeset
189 sizeof(trie->hdr.normalize_map));
6898
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
190 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
191
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
192 static int squat_trie_open_fd(struct squat_trie *trie)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
193 {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
194 trie->fd = open(trie->path, O_RDWR);
4883
d8adbe93c969 Added support for mmap_disable=yes and several other fixes.
Timo Sirainen <tss@iki.fi>
parents: 4879
diff changeset
195 if (trie->fd == -1) {
6898
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
196 if (errno == ENOENT) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
197 squat_trie_header_init(trie);
4883
d8adbe93c969 Added support for mmap_disable=yes and several other fixes.
Timo Sirainen <tss@iki.fi>
parents: 4879
diff changeset
198 return 0;
d8adbe93c969 Added support for mmap_disable=yes and several other fixes.
Timo Sirainen <tss@iki.fi>
parents: 4879
diff changeset
199 }
6898
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
200 i_error("open(%s) failed: %m", trie->path);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
201 return -1;
4883
d8adbe93c969 Added support for mmap_disable=yes and several other fixes.
Timo Sirainen <tss@iki.fi>
parents: 4879
diff changeset
202 }
6898
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
203 return 0;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
204 }
4883
d8adbe93c969 Added support for mmap_disable=yes and several other fixes.
Timo Sirainen <tss@iki.fi>
parents: 4879
diff changeset
205
6898
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
206 static int squat_trie_open(struct squat_trie *trie)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
207 {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
208 squat_trie_close(trie);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
209
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
210 if (squat_trie_open_fd(trie) < 0)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
211 return -1;
6931
97702c9c4111 Locking and error handling fixes.
Timo Sirainen <tss@iki.fi>
parents: 6914
diff changeset
212 return squat_trie_map(trie, FALSE);
4878
88a91d9a867b Fixes. Should be pretty much working now.
Timo Sirainen <tss@iki.fi>
parents: 4855
diff changeset
213 }
88a91d9a867b Fixes. Should be pretty much working now.
Timo Sirainen <tss@iki.fi>
parents: 4855
diff changeset
214
4879
Timo Sirainen <tss@iki.fi>
parents: 4878
diff changeset
215 static int squat_trie_is_file_stale(struct squat_trie *trie)
Timo Sirainen <tss@iki.fi>
parents: 4878
diff changeset
216 {
6898
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
217 struct stat st, st2;
4879
Timo Sirainen <tss@iki.fi>
parents: 4878
diff changeset
218
6898
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
219 if (stat(trie->path, &st) < 0) {
4883
d8adbe93c969 Added support for mmap_disable=yes and several other fixes.
Timo Sirainen <tss@iki.fi>
parents: 4879
diff changeset
220 if (errno == ENOENT)
d8adbe93c969 Added support for mmap_disable=yes and several other fixes.
Timo Sirainen <tss@iki.fi>
parents: 4879
diff changeset
221 return 1;
d8adbe93c969 Added support for mmap_disable=yes and several other fixes.
Timo Sirainen <tss@iki.fi>
parents: 4879
diff changeset
222
6898
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
223 i_error("stat(%s) failed: %m", trie->path);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
224 return -1;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
225 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
226 if (fstat(trie->fd, &st2) < 0) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
227 i_error("fstat(%s) failed: %m", trie->path);
4879
Timo Sirainen <tss@iki.fi>
parents: 4878
diff changeset
228 return -1;
Timo Sirainen <tss@iki.fi>
parents: 4878
diff changeset
229 }
6898
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
230 trie->locked_file_size = st2.st_size;
4879
Timo Sirainen <tss@iki.fi>
parents: 4878
diff changeset
231
6898
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
232 return st.st_ino == st2.st_ino &&
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
233 CMP_DEV_T(st.st_dev, st2.st_dev) ? 0 : 1;
4879
Timo Sirainen <tss@iki.fi>
parents: 4878
diff changeset
234 }
Timo Sirainen <tss@iki.fi>
parents: 4878
diff changeset
235
6898
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
236 void squat_trie_refresh(struct squat_trie *trie)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
237 {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
238 if (squat_trie_is_file_stale(trie) > 0)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
239 (void)squat_trie_open(trie);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
240 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
241
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
242 static int squat_trie_lock(struct squat_trie *trie, int lock_type,
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
243 struct file_lock **file_lock_r)
4886
ae2114f11a0e Memory leak fixes. Also when building a large mailbox flush once in a while
Timo Sirainen <tss@iki.fi>
parents: 4883
diff changeset
244 {
ae2114f11a0e Memory leak fixes. Also when building a large mailbox flush once in a while
Timo Sirainen <tss@iki.fi>
parents: 4883
diff changeset
245 int ret;
ae2114f11a0e Memory leak fixes. Also when building a large mailbox flush once in a while
Timo Sirainen <tss@iki.fi>
parents: 4883
diff changeset
246
6898
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
247 while (trie->fd != -1) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
248 ret = file_wait_lock(trie->fd, trie->path, lock_type,
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
249 trie->lock_method, SQUAT_TRIE_LOCK_TIMEOUT,
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
250 file_lock_r);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
251 if (ret == 0) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
252 i_error("file_wait_lock(%s) failed: %m", trie->path);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
253 return 0;
4879
Timo Sirainen <tss@iki.fi>
parents: 4878
diff changeset
254 }
6898
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
255 if (ret < 0)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
256 return -1;
4879
Timo Sirainen <tss@iki.fi>
parents: 4878
diff changeset
257
Timo Sirainen <tss@iki.fi>
parents: 4878
diff changeset
258 /* if the trie has been compressed, we need to reopen the
Timo Sirainen <tss@iki.fi>
parents: 4878
diff changeset
259 file and try to lock again */
Timo Sirainen <tss@iki.fi>
parents: 4878
diff changeset
260 ret = squat_trie_is_file_stale(trie);
Timo Sirainen <tss@iki.fi>
parents: 4878
diff changeset
261 if (ret == 0)
6898
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
262 return 1;
4879
Timo Sirainen <tss@iki.fi>
parents: 4878
diff changeset
263
6898
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
264 file_unlock(file_lock_r);
4879
Timo Sirainen <tss@iki.fi>
parents: 4878
diff changeset
265 if (ret < 0)
Timo Sirainen <tss@iki.fi>
parents: 4878
diff changeset
266 return -1;
Timo Sirainen <tss@iki.fi>
parents: 4878
diff changeset
267
6898
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
268 squat_trie_close(trie);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
269 if (squat_trie_open_fd(trie) < 0)
4855
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
270 return -1;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
271 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
272 return 0;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
273 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
274
6898
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
275 static void
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
276 node_make_squential(struct squat_trie *trie, struct squat_node *node, int level)
4855
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
277 {
6898
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
278 const unsigned int alloc_size =
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
279 NODE_CHILDREN_ALLOC_SIZE(SEQUENTIAL_COUNT);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
280 struct squat_node *children;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
281 unsigned char *chars;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
282 unsigned int i;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
283
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
284 i_assert(node->child_count == 0);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
285
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
286 trie->node_alloc_size += alloc_size;
4855
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
287
6898
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
288 node->want_sequential = FALSE;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
289 node->have_sequential = TRUE;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
290
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
291 node->child_count = SEQUENTIAL_COUNT;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
292 node->children.data = i_malloc(alloc_size);
4855
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
293
6898
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
294 chars = NODE_CHILDREN_CHARS(node);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
295 for (i = 0; i < SEQUENTIAL_COUNT; i++)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
296 chars[i] = i;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
297
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
298 if (level < MAX_FAST_LEVEL) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
299 children = NODE_CHILDREN_NODES(node);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
300 for (i = 0; i < SEQUENTIAL_COUNT; i++)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
301 children[i].want_sequential = TRUE;
4855
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
302 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
303 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
304
6898
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
305 static unsigned int
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
306 node_add_child(struct squat_trie *trie, struct squat_node *node,
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
307 unsigned char chr, int level)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
308 {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
309 unsigned int old_child_count = node->child_count;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
310 struct squat_node *children, *old_children;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
311 unsigned char *chars;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
312 size_t old_size, new_size;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
313
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
314 i_assert(node->leaf_string_length == 0);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
315
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
316 if (node->want_sequential) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
317 node_make_squential(trie, node, level);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
318
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
319 if (chr < SEQUENTIAL_COUNT)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
320 return chr;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
321 old_child_count = SEQUENTIAL_COUNT;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
322 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
323
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
324 node->child_count++;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
325 new_size = NODE_CHILDREN_ALLOC_SIZE(node->child_count);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
326
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
327 if (old_child_count == 0) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
328 /* first child */
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
329 node->children.data = i_malloc(new_size);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
330 trie->node_alloc_size += new_size;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
331 children = NODE_CHILDREN_NODES(node);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
332 } else {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
333 old_size = NODE_CHILDREN_ALLOC_SIZE(old_child_count);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
334 if (old_size != new_size) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
335 trie->node_alloc_size += new_size - old_size;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
336 node->children.data = i_realloc(node->children.data,
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
337 old_size, new_size);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
338 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
339
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
340 children = NODE_CHILDREN_NODES(node);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
341 old_children = (void *)(NODE_CHILDREN_CHARS(node) +
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
342 MEM_ALIGN(old_child_count));
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
343 if (children != old_children) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
344 memmove(children, old_children,
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
345 old_child_count * sizeof(struct squat_node));
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
346 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
347 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
348
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
349 chars = NODE_CHILDREN_CHARS(node);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
350 chars[node->child_count - 1] = chr;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
351 return node->child_count - 1;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
352 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
353
4855
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
354 static int
6944
66b3e894041b Support mmap_disable=yes and some error handling improvements.
Timo Sirainen <tss@iki.fi>
parents: 6940
diff changeset
355 trie_file_cache_read(struct squat_trie *trie, size_t offset, size_t size)
66b3e894041b Support mmap_disable=yes and some error handling improvements.
Timo Sirainen <tss@iki.fi>
parents: 6940
diff changeset
356 {
66b3e894041b Support mmap_disable=yes and some error handling improvements.
Timo Sirainen <tss@iki.fi>
parents: 6940
diff changeset
357 if (trie->file_cache == NULL)
66b3e894041b Support mmap_disable=yes and some error handling improvements.
Timo Sirainen <tss@iki.fi>
parents: 6940
diff changeset
358 return 0;
66b3e894041b Support mmap_disable=yes and some error handling improvements.
Timo Sirainen <tss@iki.fi>
parents: 6940
diff changeset
359
66b3e894041b Support mmap_disable=yes and some error handling improvements.
Timo Sirainen <tss@iki.fi>
parents: 6940
diff changeset
360 if (file_cache_read(trie->file_cache, offset, size) < 0) {
66b3e894041b Support mmap_disable=yes and some error handling improvements.
Timo Sirainen <tss@iki.fi>
parents: 6940
diff changeset
361 i_error("read(%s) failed: %m", trie->path);
66b3e894041b Support mmap_disable=yes and some error handling improvements.
Timo Sirainen <tss@iki.fi>
parents: 6940
diff changeset
362 return -1;
66b3e894041b Support mmap_disable=yes and some error handling improvements.
Timo Sirainen <tss@iki.fi>
parents: 6940
diff changeset
363 }
66b3e894041b Support mmap_disable=yes and some error handling improvements.
Timo Sirainen <tss@iki.fi>
parents: 6940
diff changeset
364 trie->data = file_cache_get_map(trie->file_cache, &trie->data_size);
66b3e894041b Support mmap_disable=yes and some error handling improvements.
Timo Sirainen <tss@iki.fi>
parents: 6940
diff changeset
365 return 0;
66b3e894041b Support mmap_disable=yes and some error handling improvements.
Timo Sirainen <tss@iki.fi>
parents: 6940
diff changeset
366 }
66b3e894041b Support mmap_disable=yes and some error handling improvements.
Timo Sirainen <tss@iki.fi>
parents: 6940
diff changeset
367
66b3e894041b Support mmap_disable=yes and some error handling improvements.
Timo Sirainen <tss@iki.fi>
parents: 6940
diff changeset
368 static int
6898
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
369 node_read_children(struct squat_trie *trie, struct squat_node *node, int level)
4855
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
370 {
6898
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
371 const uint8_t *data, *end;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
372 const unsigned char *child_chars;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
373 struct squat_node *child, *children = NULL;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
374 uoff_t node_offset;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
375 unsigned int i, child_idx, child_count;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
376 uoff_t base_offset;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
377 uint32_t num;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
378
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
379 i_assert(node->children_not_mapped);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
380 i_assert(!node->have_sequential);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
381 i_assert(trie->unmapped_child_count > 0);
6944
66b3e894041b Support mmap_disable=yes and some error handling improvements.
Timo Sirainen <tss@iki.fi>
parents: 6940
diff changeset
382 i_assert(trie->data_size <= trie->locked_file_size);
6898
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
383
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
384 trie->unmapped_child_count--;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
385 node_offset = node->children.offset;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
386 node->children_not_mapped = FALSE;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
387 node->children.data = NULL;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
388
6944
66b3e894041b Support mmap_disable=yes and some error handling improvements.
Timo Sirainen <tss@iki.fi>
parents: 6940
diff changeset
389 if (trie_file_cache_read(trie, node_offset, TRIE_READAHEAD_SIZE) < 0)
66b3e894041b Support mmap_disable=yes and some error handling improvements.
Timo Sirainen <tss@iki.fi>
parents: 6940
diff changeset
390 return -1;
66b3e894041b Support mmap_disable=yes and some error handling improvements.
Timo Sirainen <tss@iki.fi>
parents: 6940
diff changeset
391 if (unlikely(node_offset >= trie->data_size)) {
6898
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
392 squat_trie_set_corrupted(trie);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
393 return -1;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
394 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
395
6944
66b3e894041b Support mmap_disable=yes and some error handling improvements.
Timo Sirainen <tss@iki.fi>
parents: 6940
diff changeset
396 data = CONST_PTR_OFFSET(trie->data, node_offset);
66b3e894041b Support mmap_disable=yes and some error handling improvements.
Timo Sirainen <tss@iki.fi>
parents: 6940
diff changeset
397 end = CONST_PTR_OFFSET(trie->data, trie->data_size);
6898
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
398 child_count = *data++;
6944
66b3e894041b Support mmap_disable=yes and some error handling improvements.
Timo Sirainen <tss@iki.fi>
parents: 6940
diff changeset
399 if (unlikely(node_offset + child_count >= trie->data_size)) {
6898
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
400 squat_trie_set_corrupted(trie);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
401 return -1;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
402 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
403
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
404 if (child_count == 0)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
405 return 0;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
406
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
407 child_chars = data;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
408 data += child_count;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
409
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
410 /* get child offsets */
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
411 base_offset = node_offset;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
412 for (i = 0; i < child_count; i++) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
413 /* we always start with !have_sequential, so at i=0 this
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
414 check always goes to add the first child */
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
415 if (node->have_sequential && child_chars[i] < SEQUENTIAL_COUNT)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
416 child_idx = child_chars[i];
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
417 else {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
418 child_idx = node_add_child(trie, node, child_chars[i],
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
419 level);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
420 children = NODE_CHILDREN_NODES(node);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
421 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
422 child = &children[child_idx];
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
423
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
424 /* 1) child offset */
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
425 num = squat_unpack_num(&data, end);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
426 if (num == 0) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
427 /* no children */
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
428 } else {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
429 if ((num & 1) != 0) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
430 base_offset += num >> 1;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
431 } else {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
432 base_offset -= num >> 1;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
433 }
6944
66b3e894041b Support mmap_disable=yes and some error handling improvements.
Timo Sirainen <tss@iki.fi>
parents: 6940
diff changeset
434 if (base_offset >= trie->locked_file_size) {
6898
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
435 squat_trie_set_corrupted(trie);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
436 return -1;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
437 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
438 trie->unmapped_child_count++;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
439 child->children_not_mapped = TRUE;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
440 child->children.offset = base_offset;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
441 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
442
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
443 /* 2) uidlist */
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
444 child->uid_list_idx = squat_unpack_num(&data, end);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
445 if (child->uid_list_idx == 0) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
446 /* we don't write nodes with empty uidlists */
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
447 squat_trie_set_corrupted(trie);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
448 return -1;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
449 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
450 if (!UIDLIST_IS_SINGLETON(child->uid_list_idx)) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
451 /* 3) next uid */
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
452 child->next_uid = squat_unpack_num(&data, end) + 1;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
453 } else {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
454 uint32_t idx = child->uid_list_idx;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
455
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
456 child->next_uid = 1 +
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
457 squat_uidlist_singleton_last_uid(idx);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
458 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
459
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
460 /* 4) unused uids + leaf string flag */
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
461 num = squat_unpack_num(&data, end);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
462 child->unused_uids = num >> 1;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
463 if ((num & 1) != 0) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
464 /* leaf string */
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
465 unsigned int len;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
466 unsigned char *dest;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
467
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
468 /* 5) leaf string length */
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
469 len = child->leaf_string_length =
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
470 squat_unpack_num(&data, end) + 1;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
471 if (!NODE_IS_DYNAMIC_LEAF(child))
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
472 dest = child->children.static_leaf_string;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
473 else {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
474 dest = child->children.leaf_string =
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
475 i_malloc(len);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
476 }
6944
66b3e894041b Support mmap_disable=yes and some error handling improvements.
Timo Sirainen <tss@iki.fi>
parents: 6940
diff changeset
477
66b3e894041b Support mmap_disable=yes and some error handling improvements.
Timo Sirainen <tss@iki.fi>
parents: 6940
diff changeset
478 if (trie->file_cache != NULL) {
66b3e894041b Support mmap_disable=yes and some error handling improvements.
Timo Sirainen <tss@iki.fi>
parents: 6940
diff changeset
479 /* the string may be long -
66b3e894041b Support mmap_disable=yes and some error handling improvements.
Timo Sirainen <tss@iki.fi>
parents: 6940
diff changeset
480 recalculate the end pos */
66b3e894041b Support mmap_disable=yes and some error handling improvements.
Timo Sirainen <tss@iki.fi>
parents: 6940
diff changeset
481 size_t offset, size;
66b3e894041b Support mmap_disable=yes and some error handling improvements.
Timo Sirainen <tss@iki.fi>
parents: 6940
diff changeset
482
66b3e894041b Support mmap_disable=yes and some error handling improvements.
Timo Sirainen <tss@iki.fi>
parents: 6940
diff changeset
483 offset = (const char *)data -
66b3e894041b Support mmap_disable=yes and some error handling improvements.
Timo Sirainen <tss@iki.fi>
parents: 6940
diff changeset
484 (const char *)trie->data;
66b3e894041b Support mmap_disable=yes and some error handling improvements.
Timo Sirainen <tss@iki.fi>
parents: 6940
diff changeset
485 size = len + TRIE_BYTES_LEFT(child_count - i);
66b3e894041b Support mmap_disable=yes and some error handling improvements.
Timo Sirainen <tss@iki.fi>
parents: 6940
diff changeset
486
66b3e894041b Support mmap_disable=yes and some error handling improvements.
Timo Sirainen <tss@iki.fi>
parents: 6940
diff changeset
487 if (trie_file_cache_read(trie, offset,
66b3e894041b Support mmap_disable=yes and some error handling improvements.
Timo Sirainen <tss@iki.fi>
parents: 6940
diff changeset
488 size) < 0)
66b3e894041b Support mmap_disable=yes and some error handling improvements.
Timo Sirainen <tss@iki.fi>
parents: 6940
diff changeset
489 return -1;
66b3e894041b Support mmap_disable=yes and some error handling improvements.
Timo Sirainen <tss@iki.fi>
parents: 6940
diff changeset
490 data = CONST_PTR_OFFSET(trie->data, offset);
66b3e894041b Support mmap_disable=yes and some error handling improvements.
Timo Sirainen <tss@iki.fi>
parents: 6940
diff changeset
491 end = CONST_PTR_OFFSET(trie->data,
66b3e894041b Support mmap_disable=yes and some error handling improvements.
Timo Sirainen <tss@iki.fi>
parents: 6940
diff changeset
492 trie->data_size);
66b3e894041b Support mmap_disable=yes and some error handling improvements.
Timo Sirainen <tss@iki.fi>
parents: 6940
diff changeset
493 }
66b3e894041b Support mmap_disable=yes and some error handling improvements.
Timo Sirainen <tss@iki.fi>
parents: 6940
diff changeset
494
7043
7708a8c166d6 Compiler warning fixes on 32bit systems.
Timo Sirainen <tss@iki.fi>
parents: 6945
diff changeset
495 if ((size_t)(end - data) < len) {
6898
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
496 squat_trie_set_corrupted(trie);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
497 return -1;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
498 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
499 memcpy(dest, data, len);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
500 data += len;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
501 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
502 }
6944
66b3e894041b Support mmap_disable=yes and some error handling improvements.
Timo Sirainen <tss@iki.fi>
parents: 6940
diff changeset
503 if (unlikely(data == end)) {
66b3e894041b Support mmap_disable=yes and some error handling improvements.
Timo Sirainen <tss@iki.fi>
parents: 6940
diff changeset
504 /* we should never get this far */
66b3e894041b Support mmap_disable=yes and some error handling improvements.
Timo Sirainen <tss@iki.fi>
parents: 6940
diff changeset
505 squat_trie_set_corrupted(trie);
66b3e894041b Support mmap_disable=yes and some error handling improvements.
Timo Sirainen <tss@iki.fi>
parents: 6940
diff changeset
506 return -1;
66b3e894041b Support mmap_disable=yes and some error handling improvements.
Timo Sirainen <tss@iki.fi>
parents: 6940
diff changeset
507 }
6898
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
508 return 0;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
509 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
510
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
511 static void
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
512 node_write_children(struct squat_trie_build_context *ctx,
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
513 struct squat_node *node, const uoff_t *node_offsets)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
514 {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
515 struct squat_node *children;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
516 const unsigned char *chars;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
517 uint8_t child_count, buf[SQUAT_PACK_MAX_SIZE * 5], *bufp;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
518 uoff_t base_offset;
4855
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
519 unsigned int i;
6898
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
520
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
521 chars = NODE_CHILDREN_CHARS(node);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
522 children = NODE_CHILDREN_NODES(node);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
523
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
524 base_offset = ctx->output->offset;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
525 child_count = node->child_count;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
526 o_stream_send(ctx->output, &child_count, 1);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
527 o_stream_send(ctx->output, chars, child_count);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
528
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
529 for (i = 0; i < child_count; i++) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
530 bufp = buf;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
531 /* 1) child offset */
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
532 if (node_offsets[i] == 0)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
533 *bufp++ = 0;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
534 else if (node_offsets[i] >= base_offset) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
535 squat_pack_num(&bufp,
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
536 ((node_offsets[i] - base_offset) << 1) | 1);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
537 base_offset = node_offsets[i];
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
538 } else {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
539 squat_pack_num(&bufp,
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
540 (base_offset - node_offsets[i]) << 1);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
541 base_offset = node_offsets[i];
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
542 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
543
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
544 /* 2) uidlist */
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
545 squat_pack_num(&bufp, children[i].uid_list_idx);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
546 if (!UIDLIST_IS_SINGLETON(children[i].uid_list_idx)) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
547 /* 3) next uid */
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
548 squat_pack_num(&bufp, children[i].next_uid - 1);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
549 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
550
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
551 if (children[i].leaf_string_length == 0) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
552 /* 4a) unused uids */
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
553 squat_pack_num(&bufp, children[i].unused_uids << 1);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
554 o_stream_send(ctx->output, buf, bufp - buf);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
555 } else {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
556 i_assert(node_offsets[i] == 0);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
557 /* 4b) unused uids + flag */
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
558 squat_pack_num(&bufp, (children[i].unused_uids << 1) | 1);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
559 /* 5) leaf string length */
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
560 squat_pack_num(&bufp, children[i].leaf_string_length - 1);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
561 o_stream_send(ctx->output, buf, bufp - buf);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
562 o_stream_send(ctx->output,
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
563 NODE_LEAF_STRING(&children[i]),
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
564 children[i].leaf_string_length);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
565 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
566 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
567 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
568
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
569 static inline void
6931
97702c9c4111 Locking and error handling fixes.
Timo Sirainen <tss@iki.fi>
parents: 6914
diff changeset
570 node_add_uid(struct squat_trie_build_context *ctx, uint32_t uid,
97702c9c4111 Locking and error handling fixes.
Timo Sirainen <tss@iki.fi>
parents: 6914
diff changeset
571 struct squat_node *node)
6898
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
572 {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
573 if (uid < node->next_uid) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
574 /* duplicate */
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
575 return;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
576 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
577 node->unused_uids += uid - node->next_uid;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
578 node->next_uid = uid + 1;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
579
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
580 node->uid_list_idx =
6931
97702c9c4111 Locking and error handling fixes.
Timo Sirainen <tss@iki.fi>
parents: 6914
diff changeset
581 squat_uidlist_build_add_uid(ctx->uidlist_build_ctx,
6898
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
582 node->uid_list_idx, uid);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
583 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
584
6931
97702c9c4111 Locking and error handling fixes.
Timo Sirainen <tss@iki.fi>
parents: 6914
diff changeset
585 static void
97702c9c4111 Locking and error handling fixes.
Timo Sirainen <tss@iki.fi>
parents: 6914
diff changeset
586 node_split_string(struct squat_trie_build_context *ctx, struct squat_node *node)
6898
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
587 {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
588 struct squat_node *child;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
589 unsigned char *str;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
590 unsigned int uid, idx, str_len = node->leaf_string_length;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
591
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
592 i_assert(str_len > 0);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
593
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
594 /* make a copy of the leaf string and convert to normal node by
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
595 removing it. */
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
596 str = t_malloc(str_len);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
597 if (!NODE_IS_DYNAMIC_LEAF(node))
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
598 memcpy(str, node->children.static_leaf_string, str_len);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
599 else {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
600 memcpy(str, node->children.leaf_string, str_len);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
601 i_free(node->children.leaf_string);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
602 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
603 node->leaf_string_length = 0;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
604
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
605 /* create a new child node for the rest of the string */
6931
97702c9c4111 Locking and error handling fixes.
Timo Sirainen <tss@iki.fi>
parents: 6914
diff changeset
606 idx = node_add_child(ctx->trie, node, str[0], MAX_FAST_LEVEL);
6898
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
607 child = NODE_CHILDREN_NODES(node) + idx;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
608
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
609 /* update uidlist to contain all of parent's UIDs */
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
610 child->next_uid = node->next_uid - node->unused_uids;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
611 for (uid = 0; uid < child->next_uid; uid++) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
612 child->uid_list_idx =
6931
97702c9c4111 Locking and error handling fixes.
Timo Sirainen <tss@iki.fi>
parents: 6914
diff changeset
613 squat_uidlist_build_add_uid(ctx->uidlist_build_ctx,
6898
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
614 child->uid_list_idx, uid);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
615 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
616
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
617 i_assert(!child->have_sequential && child->children.data == NULL);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
618 if (str_len > 1) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
619 /* make the child a leaf string */
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
620 str_len--;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
621 child->leaf_string_length = str_len;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
622 if (!NODE_IS_DYNAMIC_LEAF(child)) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
623 memcpy(child->children.static_leaf_string,
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
624 str + 1, str_len);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
625 } else {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
626 child->children.leaf_string = i_malloc(str_len);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
627 memcpy(child->children.leaf_string, str + 1, str_len);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
628 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
629 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
630 }
4855
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
631
6898
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
632 static bool
6931
97702c9c4111 Locking and error handling fixes.
Timo Sirainen <tss@iki.fi>
parents: 6914
diff changeset
633 node_leaf_string_add_or_split(struct squat_trie_build_context *ctx,
97702c9c4111 Locking and error handling fixes.
Timo Sirainen <tss@iki.fi>
parents: 6914
diff changeset
634 struct squat_node *node,
6898
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
635 const unsigned char *data, unsigned int data_len)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
636 {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
637 const unsigned char *str = NODE_LEAF_STRING(node);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
638 const unsigned int str_len = node->leaf_string_length;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
639 unsigned int i;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
640
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
641 if (data_len != str_len) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
642 /* different lengths, can't match */
6940
414c9d631a81 Replaced t_push/t_pop calls with T_FRAME*() macros.
Timo Sirainen <tss@iki.fi>
parents: 6931
diff changeset
643 T_FRAME(
414c9d631a81 Replaced t_push/t_pop calls with T_FRAME*() macros.
Timo Sirainen <tss@iki.fi>
parents: 6931
diff changeset
644 node_split_string(ctx, node);
414c9d631a81 Replaced t_push/t_pop calls with T_FRAME*() macros.
Timo Sirainen <tss@iki.fi>
parents: 6931
diff changeset
645 );
6898
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
646 return FALSE;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
647 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
648
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
649 for (i = 0; i < data_len; i++) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
650 if (data[i] != str[i]) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
651 /* non-match */
6940
414c9d631a81 Replaced t_push/t_pop calls with T_FRAME*() macros.
Timo Sirainen <tss@iki.fi>
parents: 6931
diff changeset
652 T_FRAME(
414c9d631a81 Replaced t_push/t_pop calls with T_FRAME*() macros.
Timo Sirainen <tss@iki.fi>
parents: 6931
diff changeset
653 node_split_string(ctx, node);
414c9d631a81 Replaced t_push/t_pop calls with T_FRAME*() macros.
Timo Sirainen <tss@iki.fi>
parents: 6931
diff changeset
654 );
6898
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
655 return FALSE;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
656 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
657 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
658 return TRUE;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
659 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
660
6931
97702c9c4111 Locking and error handling fixes.
Timo Sirainen <tss@iki.fi>
parents: 6914
diff changeset
661 static int squat_build_add(struct squat_trie_build_context *ctx, uint32_t uid,
6898
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
662 const unsigned char *data, unsigned int size)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
663 {
6931
97702c9c4111 Locking and error handling fixes.
Timo Sirainen <tss@iki.fi>
parents: 6914
diff changeset
664 struct squat_trie *trie = ctx->trie;
6898
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
665 struct squat_node *node = &trie->root;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
666 const unsigned char *end = data + size;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
667 unsigned char *chars;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
668 unsigned int idx;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
669 int level = 0;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
670
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
671 for (;;) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
672 if (node->children_not_mapped) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
673 if (unlikely(node_read_children(trie, node, level) < 0))
4855
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
674 return -1;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
675 }
6898
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
676
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
677 if (node->leaf_string_length != 0) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
678 /* the whole string must match or we need to split
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
679 the node */
6931
97702c9c4111 Locking and error handling fixes.
Timo Sirainen <tss@iki.fi>
parents: 6914
diff changeset
680 if (node_leaf_string_add_or_split(ctx, node, data,
6898
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
681 end - data)) {
6931
97702c9c4111 Locking and error handling fixes.
Timo Sirainen <tss@iki.fi>
parents: 6914
diff changeset
682 node_add_uid(ctx, uid, node);
6898
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
683 return 0;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
684 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
685 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
686
6931
97702c9c4111 Locking and error handling fixes.
Timo Sirainen <tss@iki.fi>
parents: 6914
diff changeset
687 node_add_uid(ctx, uid, node);
6898
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
688
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
689 if (unlikely(uid < node->unused_uids)) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
690 squat_trie_set_corrupted(trie);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
691 return -1;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
692 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
693 /* child node's UIDs are relative to ours. so for example if
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
694 we're adding UID 4 and this node now has [2,4] UIDs,
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
695 unused_uids=3 and so the child node will be adding
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
696 UID 4-3 = 1. */
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
697 uid -= node->unused_uids;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
698
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
699 if (data == end)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
700 return 0;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
701 level++;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
702
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
703 if (node->have_sequential) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
704 if (*data < SEQUENTIAL_COUNT) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
705 idx = *data;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
706 goto found;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
707 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
708 idx = SEQUENTIAL_COUNT;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
709 } else {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
710 idx = 0;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
711 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
712 chars = NODE_CHILDREN_CHARS(node);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
713 for (; idx < node->child_count; idx++) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
714 if (chars[idx] == *data)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
715 goto found;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
716 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
717 break;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
718 found:
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
719 data++;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
720 node = NODE_CHILDREN_NODES(node) + idx;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
721 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
722
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
723 /* create new children */
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
724 i_assert(node->leaf_string_length == 0);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
725
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
726 for (;;) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
727 idx = node_add_child(trie, node, *data,
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
728 size - (end - data) + 1);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
729 node = NODE_CHILDREN_NODES(node) + idx;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
730
6931
97702c9c4111 Locking and error handling fixes.
Timo Sirainen <tss@iki.fi>
parents: 6914
diff changeset
731 node_add_uid(ctx, uid, node);
6898
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
732 uid = 0;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
733
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
734 if (++data == end)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
735 break;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
736
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
737 if (!node->have_sequential) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
738 /* convert the node into a leaf string */
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
739 unsigned int len = end - data;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
740
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
741 i_assert(node->children.data == NULL);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
742 node->leaf_string_length = len;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
743 if (!NODE_IS_DYNAMIC_LEAF(node)) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
744 memcpy(node->children.static_leaf_string,
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
745 data, len);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
746 } else {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
747 node->children.leaf_string = i_malloc(len);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
748 memcpy(node->children.leaf_string, data, len);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
749 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
750 break;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
751 }
4855
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
752 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
753 return 0;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
754 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
755
6898
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
756 static int
6931
97702c9c4111 Locking and error handling fixes.
Timo Sirainen <tss@iki.fi>
parents: 6914
diff changeset
757 squat_build_word_bytes(struct squat_trie_build_context *ctx, uint32_t uid,
6911
a215deb3de8d Fixed non-ASCII searches.
Timo Sirainen <tss@iki.fi>
parents: 6905
diff changeset
758 const unsigned char *data, unsigned int size)
4855
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
759 {
6931
97702c9c4111 Locking and error handling fixes.
Timo Sirainen <tss@iki.fi>
parents: 6914
diff changeset
760 struct squat_trie *trie = ctx->trie;
6898
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
761 unsigned int i;
4855
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
762
6911
a215deb3de8d Fixed non-ASCII searches.
Timo Sirainen <tss@iki.fi>
parents: 6905
diff changeset
763 if (trie->hdr.full_len <= trie->hdr.partial_len)
a215deb3de8d Fixed non-ASCII searches.
Timo Sirainen <tss@iki.fi>
parents: 6905
diff changeset
764 i = 0;
a215deb3de8d Fixed non-ASCII searches.
Timo Sirainen <tss@iki.fi>
parents: 6905
diff changeset
765 else {
a215deb3de8d Fixed non-ASCII searches.
Timo Sirainen <tss@iki.fi>
parents: 6905
diff changeset
766 /* the first word is longer than others */
6931
97702c9c4111 Locking and error handling fixes.
Timo Sirainen <tss@iki.fi>
parents: 6914
diff changeset
767 if (squat_build_add(ctx, uid, data,
6911
a215deb3de8d Fixed non-ASCII searches.
Timo Sirainen <tss@iki.fi>
parents: 6905
diff changeset
768 I_MIN(size, trie->hdr.full_len)) < 0)
a215deb3de8d Fixed non-ASCII searches.
Timo Sirainen <tss@iki.fi>
parents: 6905
diff changeset
769 return -1;
a215deb3de8d Fixed non-ASCII searches.
Timo Sirainen <tss@iki.fi>
parents: 6905
diff changeset
770 i = 1;
a215deb3de8d Fixed non-ASCII searches.
Timo Sirainen <tss@iki.fi>
parents: 6905
diff changeset
771 }
a215deb3de8d Fixed non-ASCII searches.
Timo Sirainen <tss@iki.fi>
parents: 6905
diff changeset
772
a215deb3de8d Fixed non-ASCII searches.
Timo Sirainen <tss@iki.fi>
parents: 6905
diff changeset
773 for (; i < size; i++) {
6931
97702c9c4111 Locking and error handling fixes.
Timo Sirainen <tss@iki.fi>
parents: 6914
diff changeset
774 if (squat_build_add(ctx, uid, data + i,
6911
a215deb3de8d Fixed non-ASCII searches.
Timo Sirainen <tss@iki.fi>
parents: 6905
diff changeset
775 I_MIN(trie->hdr.partial_len, size-i)) < 0)
4883
d8adbe93c969 Added support for mmap_disable=yes and several other fixes.
Timo Sirainen <tss@iki.fi>
parents: 4879
diff changeset
776 return -1;
d8adbe93c969 Added support for mmap_disable=yes and several other fixes.
Timo Sirainen <tss@iki.fi>
parents: 4879
diff changeset
777 }
6911
a215deb3de8d Fixed non-ASCII searches.
Timo Sirainen <tss@iki.fi>
parents: 6905
diff changeset
778 return 0;
a215deb3de8d Fixed non-ASCII searches.
Timo Sirainen <tss@iki.fi>
parents: 6905
diff changeset
779 }
a215deb3de8d Fixed non-ASCII searches.
Timo Sirainen <tss@iki.fi>
parents: 6905
diff changeset
780
a215deb3de8d Fixed non-ASCII searches.
Timo Sirainen <tss@iki.fi>
parents: 6905
diff changeset
781 static int
6931
97702c9c4111 Locking and error handling fixes.
Timo Sirainen <tss@iki.fi>
parents: 6914
diff changeset
782 squat_build_word(struct squat_trie_build_context *ctx, uint32_t uid,
6911
a215deb3de8d Fixed non-ASCII searches.
Timo Sirainen <tss@iki.fi>
parents: 6905
diff changeset
783 const unsigned char *data, const uint8_t *char_lengths,
a215deb3de8d Fixed non-ASCII searches.
Timo Sirainen <tss@iki.fi>
parents: 6905
diff changeset
784 unsigned int size)
a215deb3de8d Fixed non-ASCII searches.
Timo Sirainen <tss@iki.fi>
parents: 6905
diff changeset
785 {
6931
97702c9c4111 Locking and error handling fixes.
Timo Sirainen <tss@iki.fi>
parents: 6914
diff changeset
786 struct squat_trie *trie = ctx->trie;
6911
a215deb3de8d Fixed non-ASCII searches.
Timo Sirainen <tss@iki.fi>
parents: 6905
diff changeset
787 unsigned int i, j, bytelen;
a215deb3de8d Fixed non-ASCII searches.
Timo Sirainen <tss@iki.fi>
parents: 6905
diff changeset
788
a215deb3de8d Fixed non-ASCII searches.
Timo Sirainen <tss@iki.fi>
parents: 6905
diff changeset
789 if (char_lengths == NULL) {
a215deb3de8d Fixed non-ASCII searches.
Timo Sirainen <tss@iki.fi>
parents: 6905
diff changeset
790 /* optimization path: all characters are bytes */
6931
97702c9c4111 Locking and error handling fixes.
Timo Sirainen <tss@iki.fi>
parents: 6914
diff changeset
791 return squat_build_word_bytes(ctx, uid, data, size);
6911
a215deb3de8d Fixed non-ASCII searches.
Timo Sirainen <tss@iki.fi>
parents: 6905
diff changeset
792 }
a215deb3de8d Fixed non-ASCII searches.
Timo Sirainen <tss@iki.fi>
parents: 6905
diff changeset
793
a215deb3de8d Fixed non-ASCII searches.
Timo Sirainen <tss@iki.fi>
parents: 6905
diff changeset
794 if (trie->hdr.full_len <= trie->hdr.partial_len)
a215deb3de8d Fixed non-ASCII searches.
Timo Sirainen <tss@iki.fi>
parents: 6905
diff changeset
795 i = 0;
a215deb3de8d Fixed non-ASCII searches.
Timo Sirainen <tss@iki.fi>
parents: 6905
diff changeset
796 else {
a215deb3de8d Fixed non-ASCII searches.
Timo Sirainen <tss@iki.fi>
parents: 6905
diff changeset
797 /* the first word is longer than others */
a215deb3de8d Fixed non-ASCII searches.
Timo Sirainen <tss@iki.fi>
parents: 6905
diff changeset
798 bytelen = 0;
a215deb3de8d Fixed non-ASCII searches.
Timo Sirainen <tss@iki.fi>
parents: 6905
diff changeset
799 for (j = 0; j < trie->hdr.full_len && bytelen < size; j++)
a215deb3de8d Fixed non-ASCII searches.
Timo Sirainen <tss@iki.fi>
parents: 6905
diff changeset
800 bytelen += char_lengths[bytelen];
a215deb3de8d Fixed non-ASCII searches.
Timo Sirainen <tss@iki.fi>
parents: 6905
diff changeset
801 i_assert(bytelen <= size);
a215deb3de8d Fixed non-ASCII searches.
Timo Sirainen <tss@iki.fi>
parents: 6905
diff changeset
802
6931
97702c9c4111 Locking and error handling fixes.
Timo Sirainen <tss@iki.fi>
parents: 6914
diff changeset
803 if (squat_build_add(ctx, uid, data, bytelen) < 0)
6911
a215deb3de8d Fixed non-ASCII searches.
Timo Sirainen <tss@iki.fi>
parents: 6905
diff changeset
804 return -1;
a215deb3de8d Fixed non-ASCII searches.
Timo Sirainen <tss@iki.fi>
parents: 6905
diff changeset
805 i = char_lengths[0];
a215deb3de8d Fixed non-ASCII searches.
Timo Sirainen <tss@iki.fi>
parents: 6905
diff changeset
806 }
a215deb3de8d Fixed non-ASCII searches.
Timo Sirainen <tss@iki.fi>
parents: 6905
diff changeset
807
a215deb3de8d Fixed non-ASCII searches.
Timo Sirainen <tss@iki.fi>
parents: 6905
diff changeset
808 for (; i < size; i += char_lengths[i]) {
a215deb3de8d Fixed non-ASCII searches.
Timo Sirainen <tss@iki.fi>
parents: 6905
diff changeset
809 bytelen = 0;
a215deb3de8d Fixed non-ASCII searches.
Timo Sirainen <tss@iki.fi>
parents: 6905
diff changeset
810 for (j = 0; j < trie->hdr.partial_len && i+bytelen < size; j++)
a215deb3de8d Fixed non-ASCII searches.
Timo Sirainen <tss@iki.fi>
parents: 6905
diff changeset
811 bytelen += char_lengths[i + bytelen];
a215deb3de8d Fixed non-ASCII searches.
Timo Sirainen <tss@iki.fi>
parents: 6905
diff changeset
812 i_assert(i + bytelen <= size);
a215deb3de8d Fixed non-ASCII searches.
Timo Sirainen <tss@iki.fi>
parents: 6905
diff changeset
813
6931
97702c9c4111 Locking and error handling fixes.
Timo Sirainen <tss@iki.fi>
parents: 6914
diff changeset
814 if (squat_build_add(ctx, uid, data + i, bytelen) < 0)
6911
a215deb3de8d Fixed non-ASCII searches.
Timo Sirainen <tss@iki.fi>
parents: 6905
diff changeset
815 return -1;
a215deb3de8d Fixed non-ASCII searches.
Timo Sirainen <tss@iki.fi>
parents: 6905
diff changeset
816 }
a215deb3de8d Fixed non-ASCII searches.
Timo Sirainen <tss@iki.fi>
parents: 6905
diff changeset
817 return 0;
4855
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
818 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
819
6898
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
820 static unsigned char *
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
821 squat_data_normalize(struct squat_trie *trie, const unsigned char *data,
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
822 unsigned int size)
4855
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
823 {
7187
d9b87e3ce6c8 Treat replacement characters as non-indexed chars.
Timo Sirainen <tss@iki.fi>
parents: 7161
diff changeset
824 static const unsigned char replacement_utf8[] = { 0xef, 0xbf, 0xbd };
6898
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
825 unsigned char *dest;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
826 unsigned int i;
4855
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
827
6898
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
828 dest = t_malloc(size);
7187
d9b87e3ce6c8 Treat replacement characters as non-indexed chars.
Timo Sirainen <tss@iki.fi>
parents: 7161
diff changeset
829 for (i = 0; i < size; i++) {
d9b87e3ce6c8 Treat replacement characters as non-indexed chars.
Timo Sirainen <tss@iki.fi>
parents: 7161
diff changeset
830 if (data[i] == replacement_utf8[0] && i + 2 < size &&
d9b87e3ce6c8 Treat replacement characters as non-indexed chars.
Timo Sirainen <tss@iki.fi>
parents: 7161
diff changeset
831 data[i+1] == replacement_utf8[1] &&
d9b87e3ce6c8 Treat replacement characters as non-indexed chars.
Timo Sirainen <tss@iki.fi>
parents: 7161
diff changeset
832 data[i+2] == replacement_utf8[2]) {
d9b87e3ce6c8 Treat replacement characters as non-indexed chars.
Timo Sirainen <tss@iki.fi>
parents: 7161
diff changeset
833 /* Don't index replacement character */
d9b87e3ce6c8 Treat replacement characters as non-indexed chars.
Timo Sirainen <tss@iki.fi>
parents: 7161
diff changeset
834 dest[i++] = 0;
d9b87e3ce6c8 Treat replacement characters as non-indexed chars.
Timo Sirainen <tss@iki.fi>
parents: 7161
diff changeset
835 dest[i++] = 0;
d9b87e3ce6c8 Treat replacement characters as non-indexed chars.
Timo Sirainen <tss@iki.fi>
parents: 7161
diff changeset
836 dest[i] = 0;
d9b87e3ce6c8 Treat replacement characters as non-indexed chars.
Timo Sirainen <tss@iki.fi>
parents: 7161
diff changeset
837 } else {
d9b87e3ce6c8 Treat replacement characters as non-indexed chars.
Timo Sirainen <tss@iki.fi>
parents: 7161
diff changeset
838 dest[i] = trie->hdr.normalize_map[data[i]];
d9b87e3ce6c8 Treat replacement characters as non-indexed chars.
Timo Sirainen <tss@iki.fi>
parents: 7161
diff changeset
839 }
d9b87e3ce6c8 Treat replacement characters as non-indexed chars.
Timo Sirainen <tss@iki.fi>
parents: 7161
diff changeset
840 }
6898
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
841 return dest;
4855
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
842 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
843
6940
414c9d631a81 Replaced t_push/t_pop calls with T_FRAME*() macros.
Timo Sirainen <tss@iki.fi>
parents: 6931
diff changeset
844 static int
414c9d631a81 Replaced t_push/t_pop calls with T_FRAME*() macros.
Timo Sirainen <tss@iki.fi>
parents: 6931
diff changeset
845 squat_trie_build_more_real(struct squat_trie_build_context *ctx,
414c9d631a81 Replaced t_push/t_pop calls with T_FRAME*() macros.
Timo Sirainen <tss@iki.fi>
parents: 6931
diff changeset
846 uint32_t uid, enum squat_index_type type,
414c9d631a81 Replaced t_push/t_pop calls with T_FRAME*() macros.
Timo Sirainen <tss@iki.fi>
parents: 6931
diff changeset
847 const unsigned char *input, unsigned int size)
4855
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
848 {
4878
88a91d9a867b Fixes. Should be pretty much working now.
Timo Sirainen <tss@iki.fi>
parents: 4855
diff changeset
849 struct squat_trie *trie = ctx->trie;
6911
a215deb3de8d Fixed non-ASCII searches.
Timo Sirainen <tss@iki.fi>
parents: 6905
diff changeset
850 const unsigned char *data;
a215deb3de8d Fixed non-ASCII searches.
Timo Sirainen <tss@iki.fi>
parents: 6905
diff changeset
851 uint8_t *char_lengths;
6898
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
852 unsigned int i, start = 0;
6911
a215deb3de8d Fixed non-ASCII searches.
Timo Sirainen <tss@iki.fi>
parents: 6905
diff changeset
853 bool multibyte_chars = FALSE;
6898
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
854 int ret = 0;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
855
6914
9c3f0e180751 Index message's all headers (including MIME headers) after the body.
Timo Sirainen <tss@iki.fi>
parents: 6912
diff changeset
856 uid = uid * 2 + (type == SQUAT_INDEX_TYPE_HEADER ? 1 : 0);
4855
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
857
6911
a215deb3de8d Fixed non-ASCII searches.
Timo Sirainen <tss@iki.fi>
parents: 6905
diff changeset
858 char_lengths = t_malloc(size);
a215deb3de8d Fixed non-ASCII searches.
Timo Sirainen <tss@iki.fi>
parents: 6905
diff changeset
859 data = squat_data_normalize(trie, input, size);
6898
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
860 for (i = 0; i < size; i++) {
6911
a215deb3de8d Fixed non-ASCII searches.
Timo Sirainen <tss@iki.fi>
parents: 6905
diff changeset
861 char_lengths[i] = uni_utf8_char_bytes(input[i]);
a215deb3de8d Fixed non-ASCII searches.
Timo Sirainen <tss@iki.fi>
parents: 6905
diff changeset
862 if (char_lengths[i] != 1)
a215deb3de8d Fixed non-ASCII searches.
Timo Sirainen <tss@iki.fi>
parents: 6905
diff changeset
863 multibyte_chars = TRUE;
6898
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
864 if (data[i] != '\0')
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
865 continue;
4886
ae2114f11a0e Memory leak fixes. Also when building a large mailbox flush once in a while
Timo Sirainen <tss@iki.fi>
parents: 4883
diff changeset
866
6898
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
867 while (start < i && data[start] == '\0')
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
868 start++;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
869 if (i != start) {
6931
97702c9c4111 Locking and error handling fixes.
Timo Sirainen <tss@iki.fi>
parents: 6914
diff changeset
870 if (squat_build_word(ctx, uid, data + start,
6911
a215deb3de8d Fixed non-ASCII searches.
Timo Sirainen <tss@iki.fi>
parents: 6905
diff changeset
871 !multibyte_chars ? NULL :
a215deb3de8d Fixed non-ASCII searches.
Timo Sirainen <tss@iki.fi>
parents: 6905
diff changeset
872 char_lengths + start,
6898
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
873 i - start) < 0) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
874 ret = -1;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
875 start = i;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
876 break;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
877 }
4883
d8adbe93c969 Added support for mmap_disable=yes and several other fixes.
Timo Sirainen <tss@iki.fi>
parents: 4879
diff changeset
878 }
6898
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
879 start = i + 1;
4855
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
880 }
6898
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
881 while (start < i && data[start] == '\0')
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
882 start++;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
883 if (i != start) {
6931
97702c9c4111 Locking and error handling fixes.
Timo Sirainen <tss@iki.fi>
parents: 6914
diff changeset
884 if (squat_build_word(ctx, uid, data + start,
6911
a215deb3de8d Fixed non-ASCII searches.
Timo Sirainen <tss@iki.fi>
parents: 6905
diff changeset
885 !multibyte_chars ? NULL :
a215deb3de8d Fixed non-ASCII searches.
Timo Sirainen <tss@iki.fi>
parents: 6905
diff changeset
886 char_lengths + start, i - start) < 0)
6898
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
887 ret = -1;
4855
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
888 }
6940
414c9d631a81 Replaced t_push/t_pop calls with T_FRAME*() macros.
Timo Sirainen <tss@iki.fi>
parents: 6931
diff changeset
889 return ret;
414c9d631a81 Replaced t_push/t_pop calls with T_FRAME*() macros.
Timo Sirainen <tss@iki.fi>
parents: 6931
diff changeset
890 }
414c9d631a81 Replaced t_push/t_pop calls with T_FRAME*() macros.
Timo Sirainen <tss@iki.fi>
parents: 6931
diff changeset
891
414c9d631a81 Replaced t_push/t_pop calls with T_FRAME*() macros.
Timo Sirainen <tss@iki.fi>
parents: 6931
diff changeset
892 int squat_trie_build_more(struct squat_trie_build_context *ctx,
414c9d631a81 Replaced t_push/t_pop calls with T_FRAME*() macros.
Timo Sirainen <tss@iki.fi>
parents: 6931
diff changeset
893 uint32_t uid, enum squat_index_type type,
414c9d631a81 Replaced t_push/t_pop calls with T_FRAME*() macros.
Timo Sirainen <tss@iki.fi>
parents: 6931
diff changeset
894 const unsigned char *input, unsigned int size)
414c9d631a81 Replaced t_push/t_pop calls with T_FRAME*() macros.
Timo Sirainen <tss@iki.fi>
parents: 6931
diff changeset
895 {
414c9d631a81 Replaced t_push/t_pop calls with T_FRAME*() macros.
Timo Sirainen <tss@iki.fi>
parents: 6931
diff changeset
896 int ret;
414c9d631a81 Replaced t_push/t_pop calls with T_FRAME*() macros.
Timo Sirainen <tss@iki.fi>
parents: 6931
diff changeset
897
414c9d631a81 Replaced t_push/t_pop calls with T_FRAME*() macros.
Timo Sirainen <tss@iki.fi>
parents: 6931
diff changeset
898 T_FRAME(
414c9d631a81 Replaced t_push/t_pop calls with T_FRAME*() macros.
Timo Sirainen <tss@iki.fi>
parents: 6931
diff changeset
899 ret = squat_trie_build_more_real(ctx, uid, type, input, size);
414c9d631a81 Replaced t_push/t_pop calls with T_FRAME*() macros.
Timo Sirainen <tss@iki.fi>
parents: 6931
diff changeset
900 );
6898
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
901 return ret;
4855
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
902 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
903
6898
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
904 static void node_drop_unused_children(struct squat_node *node)
4855
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
905 {
6898
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
906 unsigned char *chars;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
907 struct squat_node *children_src, *children_dest;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
908 unsigned int i, j, orig_child_count = node->child_count;
4855
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
909
6898
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
910 chars = NODE_CHILDREN_CHARS(node);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
911 children_src = NODE_CHILDREN_NODES(node);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
912
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
913 /* move chars */
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
914 for (i = j = 0; i < orig_child_count; i++) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
915 if (children_src[i].next_uid != 0)
4855
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
916 chars[j++] = chars[i];
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
917 }
6898
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
918 node->child_count = j;
4890
cb3b4153136c 64bit and big endian fixes
Timo Sirainen <tss@iki.fi>
parents: 4887
diff changeset
919
6898
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
920 /* move children. note that children_dest may point to different
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
921 location than children_src, although they both point to the
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
922 same node. */
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
923 children_dest = NODE_CHILDREN_NODES(node);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
924 for (i = j = 0; i < orig_child_count; i++) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
925 if (children_src[i].next_uid != 0)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
926 children_dest[j++] = children_src[i];
4890
cb3b4153136c 64bit and big endian fixes
Timo Sirainen <tss@iki.fi>
parents: 4887
diff changeset
927 }
cb3b4153136c 64bit and big endian fixes
Timo Sirainen <tss@iki.fi>
parents: 4887
diff changeset
928 }
cb3b4153136c 64bit and big endian fixes
Timo Sirainen <tss@iki.fi>
parents: 4887
diff changeset
929
4855
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
930 static int
6898
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
931 squat_write_node(struct squat_trie_build_context *ctx, struct squat_node *node,
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
932 uoff_t *node_offset_r, int level)
4855
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
933 {
6898
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
934 struct squat_trie *trie = ctx->trie;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
935 struct squat_node *children;
4855
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
936 unsigned int i;
6898
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
937 uoff_t *node_offsets;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
938 uint8_t child_count;
6940
414c9d631a81 Replaced t_push/t_pop calls with T_FRAME*() macros.
Timo Sirainen <tss@iki.fi>
parents: 6931
diff changeset
939 int ret;
4901
5e604d6f391e Only ASCII texts were actually indexed. Now UTF-8 input is properly
Timo Sirainen <tss@iki.fi>
parents: 4898
diff changeset
940
6898
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
941 i_assert(node->next_uid != 0);
4855
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
942
6898
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
943 if (node->children_not_mapped && ctx->compress_nodes) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
944 if (node_read_children(trie, node, MAX_FAST_LEVEL) < 0)
4855
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
945 return -1;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
946 }
6898
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
947
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
948 node->have_sequential = FALSE;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
949 node_drop_unused_children(node);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
950
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
951 child_count = node->child_count;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
952 if (child_count == 0) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
953 i_assert(!node->children_not_mapped ||
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
954 node->leaf_string_length == 0);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
955 *node_offset_r = !node->children_not_mapped ? 0 :
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
956 node->children.offset;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
957 return 0;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
958 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
959 i_assert(!node->children_not_mapped);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
960
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
961 trie->hdr.node_count++;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
962
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
963 children = NODE_CHILDREN_NODES(node);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
964 node_offsets = t_new(uoff_t, child_count);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
965 for (i = 0; i < child_count; i++) {
6940
414c9d631a81 Replaced t_push/t_pop calls with T_FRAME*() macros.
Timo Sirainen <tss@iki.fi>
parents: 6931
diff changeset
966 T_FRAME(
414c9d631a81 Replaced t_push/t_pop calls with T_FRAME*() macros.
Timo Sirainen <tss@iki.fi>
parents: 6931
diff changeset
967 ret = squat_write_node(ctx, &children[i],
414c9d631a81 Replaced t_push/t_pop calls with T_FRAME*() macros.
Timo Sirainen <tss@iki.fi>
parents: 6931
diff changeset
968 &node_offsets[i], level + 1);
414c9d631a81 Replaced t_push/t_pop calls with T_FRAME*() macros.
Timo Sirainen <tss@iki.fi>
parents: 6931
diff changeset
969 );
414c9d631a81 Replaced t_push/t_pop calls with T_FRAME*() macros.
Timo Sirainen <tss@iki.fi>
parents: 6931
diff changeset
970 if (ret < 0)
6898
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
971 return -1;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
972 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
973
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
974 *node_offset_r = ctx->output->offset;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
975 node_write_children(ctx, node, node_offsets);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
976 return 0;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
977 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
978
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
979 static int squat_write_nodes(struct squat_trie_build_context *ctx)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
980 {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
981 struct squat_trie *trie = ctx->trie;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
982 uoff_t node_offset;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
983 int ret;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
984
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
985 if (ctx->trie->root.next_uid == 0)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
986 return 0;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
987
6940
414c9d631a81 Replaced t_push/t_pop calls with T_FRAME*() macros.
Timo Sirainen <tss@iki.fi>
parents: 6931
diff changeset
988 T_FRAME(
414c9d631a81 Replaced t_push/t_pop calls with T_FRAME*() macros.
Timo Sirainen <tss@iki.fi>
parents: 6931
diff changeset
989 ret = squat_write_node(ctx, &ctx->trie->root, &node_offset, 0);
414c9d631a81 Replaced t_push/t_pop calls with T_FRAME*() macros.
Timo Sirainen <tss@iki.fi>
parents: 6931
diff changeset
990 );
6898
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
991 if (ret < 0)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
992 return -1;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
993
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
994 trie->hdr.root_offset = node_offset;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
995 trie->hdr.root_unused_uids = trie->root.unused_uids;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
996 trie->hdr.root_next_uid = trie->root.next_uid;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
997 trie->hdr.root_uidlist_idx = trie->root.uid_list_idx;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
998 return 0;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
999 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1000
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1001 static struct squat_trie_iterate_context *
7197
bb9304dee6d4 uidlist renumbering: changed iteration code to iterate children first. This
Timo Sirainen <tss@iki.fi>
parents: 7187
diff changeset
1002 squat_trie_iterate_init(struct squat_trie *trie)
6898
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1003 {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1004 struct squat_trie_iterate_context *ctx;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1005
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1006 ctx = i_new(struct squat_trie_iterate_context, 1);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1007 ctx->trie = trie;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1008 ctx->cur.node = &trie->root;
6905
8c779f3e16be Fixes to FTS handling.
Timo Sirainen <tss@iki.fi>
parents: 6902
diff changeset
1009 i_array_init(&ctx->parents, trie->hdr.partial_len*2);
6898
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1010 return ctx;
4855
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
1011 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
1012
4878
88a91d9a867b Fixes. Should be pretty much working now.
Timo Sirainen <tss@iki.fi>
parents: 4855
diff changeset
1013 static int
7197
bb9304dee6d4 uidlist renumbering: changed iteration code to iterate children first. This
Timo Sirainen <tss@iki.fi>
parents: 7187
diff changeset
1014 squat_trie_iterate_deinit(struct squat_trie_iterate_context *ctx)
6898
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1015 {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1016 int ret = ctx->failed ? -1 : 0;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1017
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1018 array_free(&ctx->parents);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1019 i_free(ctx);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1020 return ret;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1021 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1022
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1023 static struct squat_node *
7197
bb9304dee6d4 uidlist renumbering: changed iteration code to iterate children first. This
Timo Sirainen <tss@iki.fi>
parents: 7187
diff changeset
1024 squat_trie_iterate_first(struct squat_trie_iterate_context *ctx)
4855
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
1025 {
7197
bb9304dee6d4 uidlist renumbering: changed iteration code to iterate children first. This
Timo Sirainen <tss@iki.fi>
parents: 7187
diff changeset
1026 if (ctx->cur.node->children_not_mapped) {
bb9304dee6d4 uidlist renumbering: changed iteration code to iterate children first. This
Timo Sirainen <tss@iki.fi>
parents: 7187
diff changeset
1027 if (node_read_children(ctx->trie, ctx->cur.node, 1) < 0) {
6898
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1028 ctx->failed = TRUE;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1029 return NULL;
4855
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
1030 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
1031 }
7197
bb9304dee6d4 uidlist renumbering: changed iteration code to iterate children first. This
Timo Sirainen <tss@iki.fi>
parents: 7187
diff changeset
1032 return ctx->cur.node;
6898
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1033 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1034
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1035 static struct squat_node *
7197
bb9304dee6d4 uidlist renumbering: changed iteration code to iterate children first. This
Timo Sirainen <tss@iki.fi>
parents: 7187
diff changeset
1036 squat_trie_iterate_next(struct squat_trie_iterate_context *ctx)
6898
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1037 {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1038 struct squat_trie_iterate_node *iter_nodes;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1039 struct squat_node *children;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1040 unsigned int count;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1041
7197
bb9304dee6d4 uidlist renumbering: changed iteration code to iterate children first. This
Timo Sirainen <tss@iki.fi>
parents: 7187
diff changeset
1042 while (ctx->cur.idx == ctx->cur.node->child_count) {
6898
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1043 iter_nodes = array_get_modifiable(&ctx->parents, &count);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1044 if (count == 0)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1045 return NULL;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1046 ctx->cur = iter_nodes[count-1];
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1047 array_delete(&ctx->parents, count-1, 1);
7197
bb9304dee6d4 uidlist renumbering: changed iteration code to iterate children first. This
Timo Sirainen <tss@iki.fi>
parents: 7187
diff changeset
1048 }
6898
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1049
7197
bb9304dee6d4 uidlist renumbering: changed iteration code to iterate children first. This
Timo Sirainen <tss@iki.fi>
parents: 7187
diff changeset
1050 ctx->cur.idx++;
bb9304dee6d4 uidlist renumbering: changed iteration code to iterate children first. This
Timo Sirainen <tss@iki.fi>
parents: 7187
diff changeset
1051 array_append(&ctx->parents, &ctx->cur, 1);
bb9304dee6d4 uidlist renumbering: changed iteration code to iterate children first. This
Timo Sirainen <tss@iki.fi>
parents: 7187
diff changeset
1052 children = NODE_CHILDREN_NODES(ctx->cur.node);
bb9304dee6d4 uidlist renumbering: changed iteration code to iterate children first. This
Timo Sirainen <tss@iki.fi>
parents: 7187
diff changeset
1053 ctx->cur.node = &children[ctx->cur.idx-1];
bb9304dee6d4 uidlist renumbering: changed iteration code to iterate children first. This
Timo Sirainen <tss@iki.fi>
parents: 7187
diff changeset
1054 ctx->cur.idx = 0;
bb9304dee6d4 uidlist renumbering: changed iteration code to iterate children first. This
Timo Sirainen <tss@iki.fi>
parents: 7187
diff changeset
1055 return ctx->cur.node;
6898
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1056 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1057
6931
97702c9c4111 Locking and error handling fixes.
Timo Sirainen <tss@iki.fi>
parents: 6914
diff changeset
1058 static int
97702c9c4111 Locking and error handling fixes.
Timo Sirainen <tss@iki.fi>
parents: 6914
diff changeset
1059 squat_trie_renumber_uidlists(struct squat_trie_build_context *ctx,
97702c9c4111 Locking and error handling fixes.
Timo Sirainen <tss@iki.fi>
parents: 6914
diff changeset
1060 bool compress)
6898
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1061 {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1062 struct squat_trie_iterate_context *iter;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1063 struct squat_node *node;
6931
97702c9c4111 Locking and error handling fixes.
Timo Sirainen <tss@iki.fi>
parents: 6914
diff changeset
1064 struct squat_uidlist_rebuild_context *rebuild_ctx;
6898
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1065 ARRAY_TYPE(uint32_t) uids;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1066 uint32_t new_uid_list_idx, max_count=0;
7159
2de2058a5cdc Don't use ioloop_time. Fixes squat-test which doesn't create ioloop.
Timo Sirainen <tss@iki.fi>
parents: 7158
diff changeset
1067 time_t now;
6898
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1068 int ret = 0;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1069
6931
97702c9c4111 Locking and error handling fixes.
Timo Sirainen <tss@iki.fi>
parents: 6914
diff changeset
1070 /* FIXME: update indexid */
97702c9c4111 Locking and error handling fixes.
Timo Sirainen <tss@iki.fi>
parents: 6914
diff changeset
1071 if ((ret = squat_uidlist_rebuild_init(ctx->uidlist_build_ctx,
97702c9c4111 Locking and error handling fixes.
Timo Sirainen <tss@iki.fi>
parents: 6914
diff changeset
1072 compress, &rebuild_ctx)) <= 0)
6898
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1073 return ret;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1074
7159
2de2058a5cdc Don't use ioloop_time. Fixes squat-test which doesn't create ioloop.
Timo Sirainen <tss@iki.fi>
parents: 7158
diff changeset
1075 now = time(NULL);
7043
7708a8c166d6 Compiler warning fixes on 32bit systems.
Timo Sirainen <tss@iki.fi>
parents: 6945
diff changeset
1076 ctx->trie->hdr.indexid =
7159
2de2058a5cdc Don't use ioloop_time. Fixes squat-test which doesn't create ioloop.
Timo Sirainen <tss@iki.fi>
parents: 7158
diff changeset
1077 I_MAX((unsigned int)now, ctx->trie->hdr.indexid + 1);
6931
97702c9c4111 Locking and error handling fixes.
Timo Sirainen <tss@iki.fi>
parents: 6914
diff changeset
1078
6898
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1079 i_array_init(&uids, 1024);
7197
bb9304dee6d4 uidlist renumbering: changed iteration code to iterate children first. This
Timo Sirainen <tss@iki.fi>
parents: 7187
diff changeset
1080 iter = squat_trie_iterate_init(ctx->trie);
bb9304dee6d4 uidlist renumbering: changed iteration code to iterate children first. This
Timo Sirainen <tss@iki.fi>
parents: 7187
diff changeset
1081 node = squat_trie_iterate_first(iter);
6898
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1082 new_uid_list_idx = 0x100;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1083 while (node != NULL) {
7197
bb9304dee6d4 uidlist renumbering: changed iteration code to iterate children first. This
Timo Sirainen <tss@iki.fi>
parents: 7187
diff changeset
1084 if (!UIDLIST_IS_SINGLETON(node->uid_list_idx)) {
bb9304dee6d4 uidlist renumbering: changed iteration code to iterate children first. This
Timo Sirainen <tss@iki.fi>
parents: 7187
diff changeset
1085 array_clear(&uids);
bb9304dee6d4 uidlist renumbering: changed iteration code to iterate children first. This
Timo Sirainen <tss@iki.fi>
parents: 7187
diff changeset
1086 if (squat_uidlist_get(ctx->trie->uidlist,
bb9304dee6d4 uidlist renumbering: changed iteration code to iterate children first. This
Timo Sirainen <tss@iki.fi>
parents: 7187
diff changeset
1087 node->uid_list_idx, &uids) < 0) {
bb9304dee6d4 uidlist renumbering: changed iteration code to iterate children first. This
Timo Sirainen <tss@iki.fi>
parents: 7187
diff changeset
1088 ret = -1;
bb9304dee6d4 uidlist renumbering: changed iteration code to iterate children first. This
Timo Sirainen <tss@iki.fi>
parents: 7187
diff changeset
1089 break;
bb9304dee6d4 uidlist renumbering: changed iteration code to iterate children first. This
Timo Sirainen <tss@iki.fi>
parents: 7187
diff changeset
1090 }
bb9304dee6d4 uidlist renumbering: changed iteration code to iterate children first. This
Timo Sirainen <tss@iki.fi>
parents: 7187
diff changeset
1091 max_count = I_MAX(max_count, array_count(&uids));
bb9304dee6d4 uidlist renumbering: changed iteration code to iterate children first. This
Timo Sirainen <tss@iki.fi>
parents: 7187
diff changeset
1092 squat_uidlist_rebuild_next(rebuild_ctx, &uids);
bb9304dee6d4 uidlist renumbering: changed iteration code to iterate children first. This
Timo Sirainen <tss@iki.fi>
parents: 7187
diff changeset
1093 node->uid_list_idx = new_uid_list_idx << 1;
bb9304dee6d4 uidlist renumbering: changed iteration code to iterate children first. This
Timo Sirainen <tss@iki.fi>
parents: 7187
diff changeset
1094 new_uid_list_idx++;
6898
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1095 }
7197
bb9304dee6d4 uidlist renumbering: changed iteration code to iterate children first. This
Timo Sirainen <tss@iki.fi>
parents: 7187
diff changeset
1096 node = squat_trie_iterate_next(iter);
6898
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1097 }
7197
bb9304dee6d4 uidlist renumbering: changed iteration code to iterate children first. This
Timo Sirainen <tss@iki.fi>
parents: 7187
diff changeset
1098 squat_trie_iterate_deinit(iter);
6898
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1099 array_free(&uids);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1100
6931
97702c9c4111 Locking and error handling fixes.
Timo Sirainen <tss@iki.fi>
parents: 6914
diff changeset
1101 /* lock the trie before we rename uidlist */
97702c9c4111 Locking and error handling fixes.
Timo Sirainen <tss@iki.fi>
parents: 6914
diff changeset
1102 if (squat_trie_lock(ctx->trie, F_WRLCK, &ctx->file_lock) <= 0)
97702c9c4111 Locking and error handling fixes.
Timo Sirainen <tss@iki.fi>
parents: 6914
diff changeset
1103 ret = -1;
97702c9c4111 Locking and error handling fixes.
Timo Sirainen <tss@iki.fi>
parents: 6914
diff changeset
1104 return squat_uidlist_rebuild_finish(rebuild_ctx, ret < 0);
6898
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1105 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1106
6905
8c779f3e16be Fixes to FTS handling.
Timo Sirainen <tss@iki.fi>
parents: 6902
diff changeset
1107 static bool squat_trie_check_header(struct squat_trie *trie)
8c779f3e16be Fixes to FTS handling.
Timo Sirainen <tss@iki.fi>
parents: 6902
diff changeset
1108 {
8c779f3e16be Fixes to FTS handling.
Timo Sirainen <tss@iki.fi>
parents: 6902
diff changeset
1109 if (trie->hdr.version != SQUAT_TRIE_VERSION ||
8c779f3e16be Fixes to FTS handling.
Timo Sirainen <tss@iki.fi>
parents: 6902
diff changeset
1110 trie->hdr.uidvalidity != trie->uidvalidity)
8c779f3e16be Fixes to FTS handling.
Timo Sirainen <tss@iki.fi>
parents: 6902
diff changeset
1111 return FALSE;
8c779f3e16be Fixes to FTS handling.
Timo Sirainen <tss@iki.fi>
parents: 6902
diff changeset
1112
8c779f3e16be Fixes to FTS handling.
Timo Sirainen <tss@iki.fi>
parents: 6902
diff changeset
1113 if (trie->hdr.partial_len > trie->hdr.full_len) {
8c779f3e16be Fixes to FTS handling.
Timo Sirainen <tss@iki.fi>
parents: 6902
diff changeset
1114 i_error("Corrupted %s: partial len > full len", trie->path);
8c779f3e16be Fixes to FTS handling.
Timo Sirainen <tss@iki.fi>
parents: 6902
diff changeset
1115 return FALSE;
8c779f3e16be Fixes to FTS handling.
Timo Sirainen <tss@iki.fi>
parents: 6902
diff changeset
1116 }
8c779f3e16be Fixes to FTS handling.
Timo Sirainen <tss@iki.fi>
parents: 6902
diff changeset
1117 if (trie->hdr.full_len == 0) {
8c779f3e16be Fixes to FTS handling.
Timo Sirainen <tss@iki.fi>
parents: 6902
diff changeset
1118 i_error("Corrupted %s: full len=0", trie->path);
8c779f3e16be Fixes to FTS handling.
Timo Sirainen <tss@iki.fi>
parents: 6902
diff changeset
1119 return FALSE;
8c779f3e16be Fixes to FTS handling.
Timo Sirainen <tss@iki.fi>
parents: 6902
diff changeset
1120 }
8c779f3e16be Fixes to FTS handling.
Timo Sirainen <tss@iki.fi>
parents: 6902
diff changeset
1121 return TRUE;
8c779f3e16be Fixes to FTS handling.
Timo Sirainen <tss@iki.fi>
parents: 6902
diff changeset
1122 }
8c779f3e16be Fixes to FTS handling.
Timo Sirainen <tss@iki.fi>
parents: 6902
diff changeset
1123
6898
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1124 static int squat_trie_map_header(struct squat_trie *trie)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1125 {
6944
66b3e894041b Support mmap_disable=yes and some error handling improvements.
Timo Sirainen <tss@iki.fi>
parents: 6940
diff changeset
1126 int ret;
66b3e894041b Support mmap_disable=yes and some error handling improvements.
Timo Sirainen <tss@iki.fi>
parents: 6940
diff changeset
1127
6898
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1128 if (trie->locked_file_size == 0) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1129 /* newly created file */
7158
f41bd3065ec8 Init header for newly created files.
Timo Sirainen <tss@iki.fi>
parents: 7086
diff changeset
1130 squat_trie_header_init(trie);
6944
66b3e894041b Support mmap_disable=yes and some error handling improvements.
Timo Sirainen <tss@iki.fi>
parents: 6940
diff changeset
1131 return 1;
6898
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1132 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1133 i_assert(trie->fd != -1);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1134
6944
66b3e894041b Support mmap_disable=yes and some error handling improvements.
Timo Sirainen <tss@iki.fi>
parents: 6940
diff changeset
1135 if (trie->mmap_disable) {
66b3e894041b Support mmap_disable=yes and some error handling improvements.
Timo Sirainen <tss@iki.fi>
parents: 6940
diff changeset
1136 ret = pread_full(trie->fd, &trie->hdr, sizeof(trie->hdr), 0);
66b3e894041b Support mmap_disable=yes and some error handling improvements.
Timo Sirainen <tss@iki.fi>
parents: 6940
diff changeset
1137 if (ret <= 0) {
66b3e894041b Support mmap_disable=yes and some error handling improvements.
Timo Sirainen <tss@iki.fi>
parents: 6940
diff changeset
1138 if (ret < 0) {
66b3e894041b Support mmap_disable=yes and some error handling improvements.
Timo Sirainen <tss@iki.fi>
parents: 6940
diff changeset
1139 i_error("pread(%s) failed: %m", trie->path);
66b3e894041b Support mmap_disable=yes and some error handling improvements.
Timo Sirainen <tss@iki.fi>
parents: 6940
diff changeset
1140 return -1;
66b3e894041b Support mmap_disable=yes and some error handling improvements.
Timo Sirainen <tss@iki.fi>
parents: 6940
diff changeset
1141 }
66b3e894041b Support mmap_disable=yes and some error handling improvements.
Timo Sirainen <tss@iki.fi>
parents: 6940
diff changeset
1142 i_error("Corrupted %s: File too small", trie->path);
66b3e894041b Support mmap_disable=yes and some error handling improvements.
Timo Sirainen <tss@iki.fi>
parents: 6940
diff changeset
1143 return 0;
66b3e894041b Support mmap_disable=yes and some error handling improvements.
Timo Sirainen <tss@iki.fi>
parents: 6940
diff changeset
1144 }
66b3e894041b Support mmap_disable=yes and some error handling improvements.
Timo Sirainen <tss@iki.fi>
parents: 6940
diff changeset
1145 trie->data = NULL;
66b3e894041b Support mmap_disable=yes and some error handling improvements.
Timo Sirainen <tss@iki.fi>
parents: 6940
diff changeset
1146 trie->data_size = 0;
66b3e894041b Support mmap_disable=yes and some error handling improvements.
Timo Sirainen <tss@iki.fi>
parents: 6940
diff changeset
1147 } else {
66b3e894041b Support mmap_disable=yes and some error handling improvements.
Timo Sirainen <tss@iki.fi>
parents: 6940
diff changeset
1148 if (trie->locked_file_size < sizeof(trie->hdr)) {
66b3e894041b Support mmap_disable=yes and some error handling improvements.
Timo Sirainen <tss@iki.fi>
parents: 6940
diff changeset
1149 i_error("Corrupted %s: File too small", trie->path);
66b3e894041b Support mmap_disable=yes and some error handling improvements.
Timo Sirainen <tss@iki.fi>
parents: 6940
diff changeset
1150 return 0;
66b3e894041b Support mmap_disable=yes and some error handling improvements.
Timo Sirainen <tss@iki.fi>
parents: 6940
diff changeset
1151 }
66b3e894041b Support mmap_disable=yes and some error handling improvements.
Timo Sirainen <tss@iki.fi>
parents: 6940
diff changeset
1152 if (trie->mmap_size != 0) {
66b3e894041b Support mmap_disable=yes and some error handling improvements.
Timo Sirainen <tss@iki.fi>
parents: 6940
diff changeset
1153 if (munmap(trie->mmap_base, trie->mmap_size) < 0)
66b3e894041b Support mmap_disable=yes and some error handling improvements.
Timo Sirainen <tss@iki.fi>
parents: 6940
diff changeset
1154 i_error("munmap(%s) failed: %m", trie->path);
66b3e894041b Support mmap_disable=yes and some error handling improvements.
Timo Sirainen <tss@iki.fi>
parents: 6940
diff changeset
1155 }
66b3e894041b Support mmap_disable=yes and some error handling improvements.
Timo Sirainen <tss@iki.fi>
parents: 6940
diff changeset
1156
66b3e894041b Support mmap_disable=yes and some error handling improvements.
Timo Sirainen <tss@iki.fi>
parents: 6940
diff changeset
1157 trie->mmap_size = trie->locked_file_size;
66b3e894041b Support mmap_disable=yes and some error handling improvements.
Timo Sirainen <tss@iki.fi>
parents: 6940
diff changeset
1158 trie->mmap_base = mmap(NULL, trie->mmap_size,
66b3e894041b Support mmap_disable=yes and some error handling improvements.
Timo Sirainen <tss@iki.fi>
parents: 6940
diff changeset
1159 PROT_READ | PROT_WRITE,
66b3e894041b Support mmap_disable=yes and some error handling improvements.
Timo Sirainen <tss@iki.fi>
parents: 6940
diff changeset
1160 MAP_SHARED, trie->fd, 0);
66b3e894041b Support mmap_disable=yes and some error handling improvements.
Timo Sirainen <tss@iki.fi>
parents: 6940
diff changeset
1161 if (trie->mmap_base == MAP_FAILED) {
66b3e894041b Support mmap_disable=yes and some error handling improvements.
Timo Sirainen <tss@iki.fi>
parents: 6940
diff changeset
1162 trie->data = trie->mmap_base = NULL;
66b3e894041b Support mmap_disable=yes and some error handling improvements.
Timo Sirainen <tss@iki.fi>
parents: 6940
diff changeset
1163 trie->data_size = trie->mmap_size = 0;
66b3e894041b Support mmap_disable=yes and some error handling improvements.
Timo Sirainen <tss@iki.fi>
parents: 6940
diff changeset
1164 i_error("mmap(%s) failed: %m", trie->path);
66b3e894041b Support mmap_disable=yes and some error handling improvements.
Timo Sirainen <tss@iki.fi>
parents: 6940
diff changeset
1165 return -1;
66b3e894041b Support mmap_disable=yes and some error handling improvements.
Timo Sirainen <tss@iki.fi>
parents: 6940
diff changeset
1166 }
66b3e894041b Support mmap_disable=yes and some error handling improvements.
Timo Sirainen <tss@iki.fi>
parents: 6940
diff changeset
1167 memcpy(&trie->hdr, trie->mmap_base, sizeof(trie->hdr));
66b3e894041b Support mmap_disable=yes and some error handling improvements.
Timo Sirainen <tss@iki.fi>
parents: 6940
diff changeset
1168 trie->data = trie->mmap_base;
66b3e894041b Support mmap_disable=yes and some error handling improvements.
Timo Sirainen <tss@iki.fi>
parents: 6940
diff changeset
1169 trie->data_size = trie->mmap_size;
6898
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1170 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1171
6944
66b3e894041b Support mmap_disable=yes and some error handling improvements.
Timo Sirainen <tss@iki.fi>
parents: 6940
diff changeset
1172 return squat_trie_check_header(trie) ? 1 : 0;
6898
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1173 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1174
6931
97702c9c4111 Locking and error handling fixes.
Timo Sirainen <tss@iki.fi>
parents: 6914
diff changeset
1175 static int squat_trie_map(struct squat_trie *trie, bool building)
6898
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1176 {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1177 struct file_lock *file_lock = NULL;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1178 bool changed;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1179 int ret;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1180
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1181 if (trie->fd != -1) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1182 if (squat_trie_lock(trie, F_RDLCK, &file_lock) <= 0)
4855
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
1183 return -1;
6944
66b3e894041b Support mmap_disable=yes and some error handling improvements.
Timo Sirainen <tss@iki.fi>
parents: 6940
diff changeset
1184 if (trie->mmap_disable && trie->file_cache == NULL)
66b3e894041b Support mmap_disable=yes and some error handling improvements.
Timo Sirainen <tss@iki.fi>
parents: 6940
diff changeset
1185 trie->file_cache = file_cache_new(trie->fd);
6898
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1186 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1187
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1188 ret = squat_trie_map_header(trie);
6944
66b3e894041b Support mmap_disable=yes and some error handling improvements.
Timo Sirainen <tss@iki.fi>
parents: 6940
diff changeset
1189 if (ret == 0) {
66b3e894041b Support mmap_disable=yes and some error handling improvements.
Timo Sirainen <tss@iki.fi>
parents: 6940
diff changeset
1190 file_lock_free(&file_lock);
66b3e894041b Support mmap_disable=yes and some error handling improvements.
Timo Sirainen <tss@iki.fi>
parents: 6940
diff changeset
1191 squat_trie_delete(trie);
66b3e894041b Support mmap_disable=yes and some error handling improvements.
Timo Sirainen <tss@iki.fi>
parents: 6940
diff changeset
1192 squat_trie_close(trie);
66b3e894041b Support mmap_disable=yes and some error handling improvements.
Timo Sirainen <tss@iki.fi>
parents: 6940
diff changeset
1193 squat_trie_header_init(trie);
66b3e894041b Support mmap_disable=yes and some error handling improvements.
Timo Sirainen <tss@iki.fi>
parents: 6940
diff changeset
1194 }
6898
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1195 changed = trie->root.children.offset != trie->hdr.root_offset;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1196
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1197 if (changed || trie->hdr.root_offset == 0) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1198 memset(&trie->root, 0, sizeof(trie->root));
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1199 trie->root.want_sequential = TRUE;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1200 trie->root.unused_uids = trie->hdr.root_unused_uids;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1201 trie->root.next_uid = trie->hdr.root_next_uid;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1202 trie->root.uid_list_idx = trie->hdr.root_uidlist_idx;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1203 trie->root.children.offset = trie->hdr.root_offset;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1204
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1205 if (trie->hdr.root_offset == 0) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1206 trie->unmapped_child_count = 0;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1207 trie->root.children_not_mapped = FALSE;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1208 } else {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1209 trie->unmapped_child_count = 1;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1210 trie->root.children_not_mapped = TRUE;
4855
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
1211 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
1212 }
6898
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1213
6944
66b3e894041b Support mmap_disable=yes and some error handling improvements.
Timo Sirainen <tss@iki.fi>
parents: 6940
diff changeset
1214 if (ret >= 0 && !building) {
6931
97702c9c4111 Locking and error handling fixes.
Timo Sirainen <tss@iki.fi>
parents: 6914
diff changeset
1215 /* do this while we're still locked */
97702c9c4111 Locking and error handling fixes.
Timo Sirainen <tss@iki.fi>
parents: 6914
diff changeset
1216 ret = squat_uidlist_refresh(trie->uidlist);
97702c9c4111 Locking and error handling fixes.
Timo Sirainen <tss@iki.fi>
parents: 6914
diff changeset
1217 }
97702c9c4111 Locking and error handling fixes.
Timo Sirainen <tss@iki.fi>
parents: 6914
diff changeset
1218
6898
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1219 if (file_lock != NULL)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1220 file_unlock(&file_lock);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1221 if (ret < 0)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1222 return -1;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1223
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1224 return trie->hdr.root_offset == 0 || !changed ? 0 :
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1225 node_read_children(trie, &trie->root, 1);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1226 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1227
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1228 int squat_trie_build_init(struct squat_trie *trie, uint32_t *last_uid_r,
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1229 struct squat_trie_build_context **ctx_r)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1230 {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1231 struct squat_trie_build_context *ctx;
6931
97702c9c4111 Locking and error handling fixes.
Timo Sirainen <tss@iki.fi>
parents: 6914
diff changeset
1232 struct squat_uidlist_build_context *uidlist_build_ctx;
6898
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1233
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1234 if (trie->fd == -1) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1235 trie->fd = open(trie->path, O_RDWR | O_CREAT, 0600);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1236 if (trie->fd == -1) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1237 i_error("creat(%s) failed: %m", trie->path);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1238 return -1;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1239 }
4855
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
1240 }
6898
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1241
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1242 /* uidlist locks building */
6931
97702c9c4111 Locking and error handling fixes.
Timo Sirainen <tss@iki.fi>
parents: 6914
diff changeset
1243 if (squat_uidlist_build_init(trie->uidlist, &uidlist_build_ctx) < 0)
6898
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1244 return -1;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1245
6931
97702c9c4111 Locking and error handling fixes.
Timo Sirainen <tss@iki.fi>
parents: 6914
diff changeset
1246 if (squat_trie_map(trie, TRUE) < 0) {
97702c9c4111 Locking and error handling fixes.
Timo Sirainen <tss@iki.fi>
parents: 6914
diff changeset
1247 squat_uidlist_build_deinit(&uidlist_build_ctx);
6898
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1248 return -1;
6931
97702c9c4111 Locking and error handling fixes.
Timo Sirainen <tss@iki.fi>
parents: 6914
diff changeset
1249 }
6898
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1250
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1251 ctx = i_new(struct squat_trie_build_context, 1);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1252 ctx->trie = trie;
6931
97702c9c4111 Locking and error handling fixes.
Timo Sirainen <tss@iki.fi>
parents: 6914
diff changeset
1253 ctx->uidlist_build_ctx = uidlist_build_ctx;
6898
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1254 ctx->first_uid = trie->root.next_uid;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1255
6905
8c779f3e16be Fixes to FTS handling.
Timo Sirainen <tss@iki.fi>
parents: 6902
diff changeset
1256 *last_uid_r = I_MAX((trie->root.next_uid+1)/2, 1) - 1;
6898
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1257 *ctx_r = ctx;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1258 return 0;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1259 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1260
6931
97702c9c4111 Locking and error handling fixes.
Timo Sirainen <tss@iki.fi>
parents: 6914
diff changeset
1261 static int squat_trie_write_lock(struct squat_trie_build_context *ctx)
97702c9c4111 Locking and error handling fixes.
Timo Sirainen <tss@iki.fi>
parents: 6914
diff changeset
1262 {
97702c9c4111 Locking and error handling fixes.
Timo Sirainen <tss@iki.fi>
parents: 6914
diff changeset
1263 if (ctx->file_lock != NULL)
97702c9c4111 Locking and error handling fixes.
Timo Sirainen <tss@iki.fi>
parents: 6914
diff changeset
1264 return 0;
97702c9c4111 Locking and error handling fixes.
Timo Sirainen <tss@iki.fi>
parents: 6914
diff changeset
1265
97702c9c4111 Locking and error handling fixes.
Timo Sirainen <tss@iki.fi>
parents: 6914
diff changeset
1266 if (squat_trie_lock(ctx->trie, F_WRLCK, &ctx->file_lock) <= 0)
97702c9c4111 Locking and error handling fixes.
Timo Sirainen <tss@iki.fi>
parents: 6914
diff changeset
1267 return -1;
97702c9c4111 Locking and error handling fixes.
Timo Sirainen <tss@iki.fi>
parents: 6914
diff changeset
1268 return 0;
97702c9c4111 Locking and error handling fixes.
Timo Sirainen <tss@iki.fi>
parents: 6914
diff changeset
1269 }
97702c9c4111 Locking and error handling fixes.
Timo Sirainen <tss@iki.fi>
parents: 6914
diff changeset
1270
6898
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1271 static int squat_trie_write(struct squat_trie_build_context *ctx)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1272 {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1273 struct squat_trie *trie = ctx->trie;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1274 struct ostream *output;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1275 const char *path;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1276 int fd = -1, ret = 0;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1277
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1278 if ((trie->hdr.used_file_size > sizeof(trie->hdr) &&
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1279 trie->unmapped_child_count < trie->hdr.node_count/4) || 1) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1280 /* we might as well recreate the file */
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1281 ctx->compress_nodes = TRUE;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1282
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1283 path = t_strconcat(trie->path, ".tmp", NULL);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1284 fd = open(path, O_RDWR | O_CREAT, 0600);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1285 if (fd == -1) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1286 i_error("creat(%s) failed: %m", path);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1287 return -1;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1288 }
6931
97702c9c4111 Locking and error handling fixes.
Timo Sirainen <tss@iki.fi>
parents: 6914
diff changeset
1289 ret = file_wait_lock(fd, path, F_WRLCK, trie->lock_method,
97702c9c4111 Locking and error handling fixes.
Timo Sirainen <tss@iki.fi>
parents: 6914
diff changeset
1290 SQUAT_TRIE_LOCK_TIMEOUT, &ctx->file_lock);
97702c9c4111 Locking and error handling fixes.
Timo Sirainen <tss@iki.fi>
parents: 6914
diff changeset
1291 if (ret <= 0) {
97702c9c4111 Locking and error handling fixes.
Timo Sirainen <tss@iki.fi>
parents: 6914
diff changeset
1292 if (ret == 0)
97702c9c4111 Locking and error handling fixes.
Timo Sirainen <tss@iki.fi>
parents: 6914
diff changeset
1293 i_error("file_wait_lock(%s) failed: %m", path);
97702c9c4111 Locking and error handling fixes.
Timo Sirainen <tss@iki.fi>
parents: 6914
diff changeset
1294 (void)close(fd);
97702c9c4111 Locking and error handling fixes.
Timo Sirainen <tss@iki.fi>
parents: 6914
diff changeset
1295 return -1;
97702c9c4111 Locking and error handling fixes.
Timo Sirainen <tss@iki.fi>
parents: 6914
diff changeset
1296 }
97702c9c4111 Locking and error handling fixes.
Timo Sirainen <tss@iki.fi>
parents: 6914
diff changeset
1297
6898
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1298 output = o_stream_create_fd(fd, 0, FALSE);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1299 o_stream_cork(output);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1300 o_stream_send(output, &trie->hdr, sizeof(trie->hdr));
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1301 } else {
6931
97702c9c4111 Locking and error handling fixes.
Timo Sirainen <tss@iki.fi>
parents: 6914
diff changeset
1302 /* we need to lock only while header is being written */
6898
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1303 path = trie->path;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1304 ctx->compress_nodes =
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1305 trie->hdr.used_file_size == sizeof(trie->hdr);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1306 output = o_stream_create_fd(trie->fd, 0, FALSE);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1307 o_stream_cork(output);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1308
6931
97702c9c4111 Locking and error handling fixes.
Timo Sirainen <tss@iki.fi>
parents: 6914
diff changeset
1309 if (trie->hdr.used_file_size != 0)
97702c9c4111 Locking and error handling fixes.
Timo Sirainen <tss@iki.fi>
parents: 6914
diff changeset
1310 o_stream_seek(output, trie->hdr.used_file_size);
97702c9c4111 Locking and error handling fixes.
Timo Sirainen <tss@iki.fi>
parents: 6914
diff changeset
1311 else {
97702c9c4111 Locking and error handling fixes.
Timo Sirainen <tss@iki.fi>
parents: 6914
diff changeset
1312 if (squat_trie_write_lock(ctx) < 0) {
97702c9c4111 Locking and error handling fixes.
Timo Sirainen <tss@iki.fi>
parents: 6914
diff changeset
1313 o_stream_unref(&output);
97702c9c4111 Locking and error handling fixes.
Timo Sirainen <tss@iki.fi>
parents: 6914
diff changeset
1314 return -1;
97702c9c4111 Locking and error handling fixes.
Timo Sirainen <tss@iki.fi>
parents: 6914
diff changeset
1315 }
6898
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1316 o_stream_send(output, &trie->hdr, sizeof(trie->hdr));
6931
97702c9c4111 Locking and error handling fixes.
Timo Sirainen <tss@iki.fi>
parents: 6914
diff changeset
1317 }
6898
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1318 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1319
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1320 ctx->output = output;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1321 ret = squat_write_nodes(ctx);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1322 ctx->output = NULL;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1323
6944
66b3e894041b Support mmap_disable=yes and some error handling improvements.
Timo Sirainen <tss@iki.fi>
parents: 6940
diff changeset
1324 /* write 1 byte guard at the end of file, so that we can verify broken
66b3e894041b Support mmap_disable=yes and some error handling improvements.
Timo Sirainen <tss@iki.fi>
parents: 6940
diff changeset
1325 squat_unpack_num() input by checking if data==end */
66b3e894041b Support mmap_disable=yes and some error handling improvements.
Timo Sirainen <tss@iki.fi>
parents: 6940
diff changeset
1326 o_stream_send(output, "", 1);
66b3e894041b Support mmap_disable=yes and some error handling improvements.
Timo Sirainen <tss@iki.fi>
parents: 6940
diff changeset
1327
6905
8c779f3e16be Fixes to FTS handling.
Timo Sirainen <tss@iki.fi>
parents: 6902
diff changeset
1328 if (trie->corrupted)
8c779f3e16be Fixes to FTS handling.
Timo Sirainen <tss@iki.fi>
parents: 6902
diff changeset
1329 ret = -1;
6931
97702c9c4111 Locking and error handling fixes.
Timo Sirainen <tss@iki.fi>
parents: 6914
diff changeset
1330 if (ret == 0)
97702c9c4111 Locking and error handling fixes.
Timo Sirainen <tss@iki.fi>
parents: 6914
diff changeset
1331 ret = squat_trie_write_lock(ctx);
6898
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1332 if (ret == 0) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1333 trie->hdr.used_file_size = output->offset;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1334 o_stream_seek(output, 0);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1335 o_stream_send(output, &trie->hdr, sizeof(trie->hdr));
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1336 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1337 if (output->last_failed_errno != 0) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1338 errno = output->last_failed_errno;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1339 i_error("write() to %s failed: %m", path);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1340 ret = -1;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1341 }
6931
97702c9c4111 Locking and error handling fixes.
Timo Sirainen <tss@iki.fi>
parents: 6914
diff changeset
1342 o_stream_destroy(&output);
97702c9c4111 Locking and error handling fixes.
Timo Sirainen <tss@iki.fi>
parents: 6914
diff changeset
1343
97702c9c4111 Locking and error handling fixes.
Timo Sirainen <tss@iki.fi>
parents: 6914
diff changeset
1344 if (fd == -1) {
97702c9c4111 Locking and error handling fixes.
Timo Sirainen <tss@iki.fi>
parents: 6914
diff changeset
1345 /* appended to the existing file */
97702c9c4111 Locking and error handling fixes.
Timo Sirainen <tss@iki.fi>
parents: 6914
diff changeset
1346 return ret;
97702c9c4111 Locking and error handling fixes.
Timo Sirainen <tss@iki.fi>
parents: 6914
diff changeset
1347 }
6898
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1348
6931
97702c9c4111 Locking and error handling fixes.
Timo Sirainen <tss@iki.fi>
parents: 6914
diff changeset
1349 /* recreating the trie file */
97702c9c4111 Locking and error handling fixes.
Timo Sirainen <tss@iki.fi>
parents: 6914
diff changeset
1350 if (ret < 0) {
97702c9c4111 Locking and error handling fixes.
Timo Sirainen <tss@iki.fi>
parents: 6914
diff changeset
1351 if (close(fd) < 0)
97702c9c4111 Locking and error handling fixes.
Timo Sirainen <tss@iki.fi>
parents: 6914
diff changeset
1352 i_error("close(%s) failed: %m", path);
97702c9c4111 Locking and error handling fixes.
Timo Sirainen <tss@iki.fi>
parents: 6914
diff changeset
1353 fd = -1;
97702c9c4111 Locking and error handling fixes.
Timo Sirainen <tss@iki.fi>
parents: 6914
diff changeset
1354 } else if (rename(path, trie->path) < 0) {
97702c9c4111 Locking and error handling fixes.
Timo Sirainen <tss@iki.fi>
parents: 6914
diff changeset
1355 i_error("rename(%s, %s) failed: %m", path, trie->path);
97702c9c4111 Locking and error handling fixes.
Timo Sirainen <tss@iki.fi>
parents: 6914
diff changeset
1356 ret = -1;
97702c9c4111 Locking and error handling fixes.
Timo Sirainen <tss@iki.fi>
parents: 6914
diff changeset
1357 }
97702c9c4111 Locking and error handling fixes.
Timo Sirainen <tss@iki.fi>
parents: 6914
diff changeset
1358
97702c9c4111 Locking and error handling fixes.
Timo Sirainen <tss@iki.fi>
parents: 6914
diff changeset
1359 if (ret < 0) {
97702c9c4111 Locking and error handling fixes.
Timo Sirainen <tss@iki.fi>
parents: 6914
diff changeset
1360 if (unlink(path) < 0 && errno != ENOENT)
97702c9c4111 Locking and error handling fixes.
Timo Sirainen <tss@iki.fi>
parents: 6914
diff changeset
1361 i_error("unlink(%s) failed: %m", path);
97702c9c4111 Locking and error handling fixes.
Timo Sirainen <tss@iki.fi>
parents: 6914
diff changeset
1362 } else {
97702c9c4111 Locking and error handling fixes.
Timo Sirainen <tss@iki.fi>
parents: 6914
diff changeset
1363 if (trie->fd != -1) {
6898
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1364 if (close(trie->fd) < 0)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1365 i_error("close(%s) failed: %m", trie->path);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1366 }
6931
97702c9c4111 Locking and error handling fixes.
Timo Sirainen <tss@iki.fi>
parents: 6914
diff changeset
1367 trie->fd = fd;
6898
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1368 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1369 return ret;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1370 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1371
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1372 int squat_trie_build_deinit(struct squat_trie_build_context **_ctx)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1373 {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1374 struct squat_trie_build_context *ctx = *_ctx;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1375 bool compress;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1376 int ret;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1377
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1378 *_ctx = NULL;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1379
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1380 compress = (ctx->trie->root.next_uid - ctx->first_uid) > 10;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1381
6931
97702c9c4111 Locking and error handling fixes.
Timo Sirainen <tss@iki.fi>
parents: 6914
diff changeset
1382 /* keep trie locked while header is being written and when files are
97702c9c4111 Locking and error handling fixes.
Timo Sirainen <tss@iki.fi>
parents: 6914
diff changeset
1383 being renamed, so that while trie is read locked, uidlist can't
97702c9c4111 Locking and error handling fixes.
Timo Sirainen <tss@iki.fi>
parents: 6914
diff changeset
1384 change under. */
97702c9c4111 Locking and error handling fixes.
Timo Sirainen <tss@iki.fi>
parents: 6914
diff changeset
1385 squat_uidlist_build_flush(ctx->uidlist_build_ctx);
97702c9c4111 Locking and error handling fixes.
Timo Sirainen <tss@iki.fi>
parents: 6914
diff changeset
1386 ret = squat_trie_renumber_uidlists(ctx, compress);
97702c9c4111 Locking and error handling fixes.
Timo Sirainen <tss@iki.fi>
parents: 6914
diff changeset
1387 if (ret == 0)
6898
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1388 ret = squat_trie_write(ctx);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1389
6931
97702c9c4111 Locking and error handling fixes.
Timo Sirainen <tss@iki.fi>
parents: 6914
diff changeset
1390 if (ret == 0)
97702c9c4111 Locking and error handling fixes.
Timo Sirainen <tss@iki.fi>
parents: 6914
diff changeset
1391 ret = squat_uidlist_build_finish(ctx->uidlist_build_ctx);
97702c9c4111 Locking and error handling fixes.
Timo Sirainen <tss@iki.fi>
parents: 6914
diff changeset
1392 if (ctx->file_lock != NULL)
97702c9c4111 Locking and error handling fixes.
Timo Sirainen <tss@iki.fi>
parents: 6914
diff changeset
1393 file_unlock(&ctx->file_lock);
97702c9c4111 Locking and error handling fixes.
Timo Sirainen <tss@iki.fi>
parents: 6914
diff changeset
1394 squat_uidlist_build_deinit(&ctx->uidlist_build_ctx);
97702c9c4111 Locking and error handling fixes.
Timo Sirainen <tss@iki.fi>
parents: 6914
diff changeset
1395
6898
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1396 i_free(ctx);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1397 return ret;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1398 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1399
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1400 int squat_trie_get_last_uid(struct squat_trie *trie, uint32_t *last_uid_r)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1401 {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1402 if (trie->fd == -1) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1403 if (squat_trie_open(trie) < 0)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1404 return -1;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1405 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1406
6905
8c779f3e16be Fixes to FTS handling.
Timo Sirainen <tss@iki.fi>
parents: 6902
diff changeset
1407 *last_uid_r = I_MAX((trie->root.next_uid+1)/2, 1) - 1;
4855
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
1408 return 0;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
1409 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
1410
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
1411 static int
6898
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1412 squat_trie_lookup_data(struct squat_trie *trie, const unsigned char *data,
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1413 unsigned int size, ARRAY_TYPE(seq_range) *uids)
4855
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
1414 {
6898
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1415 struct squat_node *node = &trie->root;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1416 unsigned char *chars;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1417 unsigned int idx;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1418 int level = 0;
4855
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
1419
6898
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1420 array_clear(uids);
4855
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
1421
6898
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1422 for (;;) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1423 if (node->children_not_mapped) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1424 if (node_read_children(trie, node, level) < 0)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1425 return -1;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1426 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1427 if (node->leaf_string_length != 0) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1428 unsigned int str_len = node->leaf_string_length;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1429 const unsigned char *str;
5414
Timo Sirainen <tss@iki.fi>
parents: 5336
diff changeset
1430
6898
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1431 if (str_len > sizeof(node->children.static_leaf_string))
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1432 str = node->children.leaf_string;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1433 else
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1434 str = node->children.static_leaf_string;
4855
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
1435
6898
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1436 if (size > str_len || memcmp(data, str, size) != 0)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1437 return 0;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1438
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1439 /* match */
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1440 break;
4855
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
1441 }
4898
07a038b57946 Optimization.
Timo Sirainen <tss@iki.fi>
parents: 4897
diff changeset
1442
6898
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1443 if (size == 0)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1444 break;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1445 level++;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1446
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1447 if (node->have_sequential) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1448 if (*data < SEQUENTIAL_COUNT) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1449 idx = *data;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1450 goto found;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1451 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1452 idx = SEQUENTIAL_COUNT;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1453 } else {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1454 idx = 0;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1455 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1456 chars = NODE_CHILDREN_CHARS(node);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1457 for (; idx < node->child_count; idx++) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1458 if (chars[idx] == *data)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1459 goto found;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1460 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1461 return 0;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1462 found:
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1463 /* follow to children */
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1464 if (level == 1) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1465 /* root level, add all UIDs */
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1466 if (squat_uidlist_get_seqrange(trie->uidlist,
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1467 node->uid_list_idx,
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1468 uids) < 0)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1469 return -1;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1470 } else {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1471 if (squat_uidlist_filter(trie->uidlist,
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1472 node->uid_list_idx, uids) < 0)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1473 return -1;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1474 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1475 data++;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1476 size--;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1477 node = NODE_CHILDREN_NODES(node) + idx;
4855
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
1478 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
1479
6898
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1480 if (squat_uidlist_filter(trie->uidlist, node->uid_list_idx, uids) < 0)
4878
88a91d9a867b Fixes. Should be pretty much working now.
Timo Sirainen <tss@iki.fi>
parents: 4855
diff changeset
1481 return -1;
6898
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1482 return 1;
4855
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
1483 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
1484
4878
88a91d9a867b Fixes. Should be pretty much working now.
Timo Sirainen <tss@iki.fi>
parents: 4855
diff changeset
1485 static void
6898
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1486 squat_trie_filter_type(enum squat_index_type type,
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1487 const ARRAY_TYPE(seq_range) *src,
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1488 ARRAY_TYPE(seq_range) *dest)
4855
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
1489 {
6898
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1490 const struct seq_range *src_range;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1491 struct seq_range new_range;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1492 unsigned int i, count, mask;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1493 uint32_t next_seq, uid;
4855
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
1494
6898
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1495 array_clear(dest);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1496 src_range = array_get(src, &count);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1497 if (count == 0)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1498 return;
4855
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
1499
6898
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1500 if ((type & SQUAT_INDEX_TYPE_HEADER) != 0 &&
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1501 (type & SQUAT_INDEX_TYPE_BODY) != 0) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1502 /* everything is fine, just fix the UIDs */
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1503 new_range.seq1 = src_range[0].seq1 / 2;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1504 new_range.seq2 = src_range[0].seq2 / 2;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1505 for (i = 1; i < count; i++) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1506 next_seq = src_range[i].seq1 / 2;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1507 if (next_seq == new_range.seq2 + 1) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1508 /* we can continue the previous range */
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1509 } else {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1510 array_append(dest, &new_range, 1);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1511 new_range.seq1 = src_range[i].seq1 / 2;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1512 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1513 new_range.seq2 = src_range[i].seq2 / 2;
4878
88a91d9a867b Fixes. Should be pretty much working now.
Timo Sirainen <tss@iki.fi>
parents: 4855
diff changeset
1514 }
6898
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1515 array_append(dest, &new_range, 1);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1516 return;
4886
ae2114f11a0e Memory leak fixes. Also when building a large mailbox flush once in a while
Timo Sirainen <tss@iki.fi>
parents: 4883
diff changeset
1517 }
ae2114f11a0e Memory leak fixes. Also when building a large mailbox flush once in a while
Timo Sirainen <tss@iki.fi>
parents: 4883
diff changeset
1518
6898
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1519 /* we'll have to drop either header or body UIDs */
6914
9c3f0e180751 Index message's all headers (including MIME headers) after the body.
Timo Sirainen <tss@iki.fi>
parents: 6912
diff changeset
1520 mask = (type & SQUAT_INDEX_TYPE_HEADER) != 0 ? 1 : 0;
6898
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1521 for (i = 0; i < count; i++) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1522 for (uid = src_range[i].seq1; uid <= src_range[i].seq2; uid++) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1523 if ((uid & 1) == mask)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1524 seq_range_array_add(dest, 0, uid/2);
4855
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
1525 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
1526 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
1527 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
1528
6902
cff5428c3c4d Fixed handling non-indexed characters in search key.
Timo Sirainen <tss@iki.fi>
parents: 6898
diff changeset
1529 struct squat_trie_lookup_context {
cff5428c3c4d Fixed handling non-indexed characters in search key.
Timo Sirainen <tss@iki.fi>
parents: 6898
diff changeset
1530 struct squat_trie *trie;
cff5428c3c4d Fixed handling non-indexed characters in search key.
Timo Sirainen <tss@iki.fi>
parents: 6898
diff changeset
1531 enum squat_index_type type;
cff5428c3c4d Fixed handling non-indexed characters in search key.
Timo Sirainen <tss@iki.fi>
parents: 6898
diff changeset
1532
cff5428c3c4d Fixed handling non-indexed characters in search key.
Timo Sirainen <tss@iki.fi>
parents: 6898
diff changeset
1533 ARRAY_TYPE(seq_range) *definite_uids, *maybe_uids;
cff5428c3c4d Fixed handling non-indexed characters in search key.
Timo Sirainen <tss@iki.fi>
parents: 6898
diff changeset
1534 ARRAY_TYPE(seq_range) tmp_uids, tmp_uids2;
cff5428c3c4d Fixed handling non-indexed characters in search key.
Timo Sirainen <tss@iki.fi>
parents: 6898
diff changeset
1535 bool first;
cff5428c3c4d Fixed handling non-indexed characters in search key.
Timo Sirainen <tss@iki.fi>
parents: 6898
diff changeset
1536 };
cff5428c3c4d Fixed handling non-indexed characters in search key.
Timo Sirainen <tss@iki.fi>
parents: 6898
diff changeset
1537
cff5428c3c4d Fixed handling non-indexed characters in search key.
Timo Sirainen <tss@iki.fi>
parents: 6898
diff changeset
1538 static int
cff5428c3c4d Fixed handling non-indexed characters in search key.
Timo Sirainen <tss@iki.fi>
parents: 6898
diff changeset
1539 squat_trie_lookup_partial(struct squat_trie_lookup_context *ctx,
6911
a215deb3de8d Fixed non-ASCII searches.
Timo Sirainen <tss@iki.fi>
parents: 6905
diff changeset
1540 const unsigned char *data, uint8_t *char_lengths,
a215deb3de8d Fixed non-ASCII searches.
Timo Sirainen <tss@iki.fi>
parents: 6905
diff changeset
1541 unsigned int size)
6902
cff5428c3c4d Fixed handling non-indexed characters in search key.
Timo Sirainen <tss@iki.fi>
parents: 6898
diff changeset
1542 {
6911
a215deb3de8d Fixed non-ASCII searches.
Timo Sirainen <tss@iki.fi>
parents: 6905
diff changeset
1543 const unsigned int partial_len = ctx->trie->hdr.partial_len;
a215deb3de8d Fixed non-ASCII searches.
Timo Sirainen <tss@iki.fi>
parents: 6905
diff changeset
1544 unsigned int char_idx, max_chars, i, j, bytelen;
6902
cff5428c3c4d Fixed handling non-indexed characters in search key.
Timo Sirainen <tss@iki.fi>
parents: 6898
diff changeset
1545 int ret;
cff5428c3c4d Fixed handling non-indexed characters in search key.
Timo Sirainen <tss@iki.fi>
parents: 6898
diff changeset
1546
6911
a215deb3de8d Fixed non-ASCII searches.
Timo Sirainen <tss@iki.fi>
parents: 6905
diff changeset
1547 max_chars = uni_utf8_strlen_n(data, size);
a215deb3de8d Fixed non-ASCII searches.
Timo Sirainen <tss@iki.fi>
parents: 6905
diff changeset
1548 if (max_chars > ctx->trie->hdr.partial_len)
a215deb3de8d Fixed non-ASCII searches.
Timo Sirainen <tss@iki.fi>
parents: 6905
diff changeset
1549 max_chars = partial_len;
6902
cff5428c3c4d Fixed handling non-indexed characters in search key.
Timo Sirainen <tss@iki.fi>
parents: 6898
diff changeset
1550
6911
a215deb3de8d Fixed non-ASCII searches.
Timo Sirainen <tss@iki.fi>
parents: 6905
diff changeset
1551 for (i = 0, char_idx = 0; char_idx < max_chars; char_idx++) {
a215deb3de8d Fixed non-ASCII searches.
Timo Sirainen <tss@iki.fi>
parents: 6905
diff changeset
1552 bytelen = 0;
a215deb3de8d Fixed non-ASCII searches.
Timo Sirainen <tss@iki.fi>
parents: 6905
diff changeset
1553 for (j = 0; j < partial_len && i+bytelen < size; j++)
a215deb3de8d Fixed non-ASCII searches.
Timo Sirainen <tss@iki.fi>
parents: 6905
diff changeset
1554 bytelen += char_lengths[i + bytelen];
a215deb3de8d Fixed non-ASCII searches.
Timo Sirainen <tss@iki.fi>
parents: 6905
diff changeset
1555
a215deb3de8d Fixed non-ASCII searches.
Timo Sirainen <tss@iki.fi>
parents: 6905
diff changeset
1556 ret = squat_trie_lookup_data(ctx->trie, data + i, bytelen,
6902
cff5428c3c4d Fixed handling non-indexed characters in search key.
Timo Sirainen <tss@iki.fi>
parents: 6898
diff changeset
1557 &ctx->tmp_uids);
cff5428c3c4d Fixed handling non-indexed characters in search key.
Timo Sirainen <tss@iki.fi>
parents: 6898
diff changeset
1558 if (ret <= 0) {
cff5428c3c4d Fixed handling non-indexed characters in search key.
Timo Sirainen <tss@iki.fi>
parents: 6898
diff changeset
1559 array_clear(ctx->maybe_uids);
cff5428c3c4d Fixed handling non-indexed characters in search key.
Timo Sirainen <tss@iki.fi>
parents: 6898
diff changeset
1560 return ret;
cff5428c3c4d Fixed handling non-indexed characters in search key.
Timo Sirainen <tss@iki.fi>
parents: 6898
diff changeset
1561 }
cff5428c3c4d Fixed handling non-indexed characters in search key.
Timo Sirainen <tss@iki.fi>
parents: 6898
diff changeset
1562
cff5428c3c4d Fixed handling non-indexed characters in search key.
Timo Sirainen <tss@iki.fi>
parents: 6898
diff changeset
1563 if (ctx->first) {
cff5428c3c4d Fixed handling non-indexed characters in search key.
Timo Sirainen <tss@iki.fi>
parents: 6898
diff changeset
1564 squat_trie_filter_type(ctx->type, &ctx->tmp_uids,
cff5428c3c4d Fixed handling non-indexed characters in search key.
Timo Sirainen <tss@iki.fi>
parents: 6898
diff changeset
1565 ctx->maybe_uids);
cff5428c3c4d Fixed handling non-indexed characters in search key.
Timo Sirainen <tss@iki.fi>
parents: 6898
diff changeset
1566 ctx->first = FALSE;
cff5428c3c4d Fixed handling non-indexed characters in search key.
Timo Sirainen <tss@iki.fi>
parents: 6898
diff changeset
1567 } else {
cff5428c3c4d Fixed handling non-indexed characters in search key.
Timo Sirainen <tss@iki.fi>
parents: 6898
diff changeset
1568 squat_trie_filter_type(ctx->type, &ctx->tmp_uids,
cff5428c3c4d Fixed handling non-indexed characters in search key.
Timo Sirainen <tss@iki.fi>
parents: 6898
diff changeset
1569 &ctx->tmp_uids2);
cff5428c3c4d Fixed handling non-indexed characters in search key.
Timo Sirainen <tss@iki.fi>
parents: 6898
diff changeset
1570 seq_range_array_remove_invert_range(ctx->maybe_uids,
cff5428c3c4d Fixed handling non-indexed characters in search key.
Timo Sirainen <tss@iki.fi>
parents: 6898
diff changeset
1571 &ctx->tmp_uids2);
cff5428c3c4d Fixed handling non-indexed characters in search key.
Timo Sirainen <tss@iki.fi>
parents: 6898
diff changeset
1572 }
6911
a215deb3de8d Fixed non-ASCII searches.
Timo Sirainen <tss@iki.fi>
parents: 6905
diff changeset
1573
a215deb3de8d Fixed non-ASCII searches.
Timo Sirainen <tss@iki.fi>
parents: 6905
diff changeset
1574 i += char_lengths[i];
a215deb3de8d Fixed non-ASCII searches.
Timo Sirainen <tss@iki.fi>
parents: 6905
diff changeset
1575 }
6902
cff5428c3c4d Fixed handling non-indexed characters in search key.
Timo Sirainen <tss@iki.fi>
parents: 6898
diff changeset
1576 return 1;
cff5428c3c4d Fixed handling non-indexed characters in search key.
Timo Sirainen <tss@iki.fi>
parents: 6898
diff changeset
1577 }
cff5428c3c4d Fixed handling non-indexed characters in search key.
Timo Sirainen <tss@iki.fi>
parents: 6898
diff changeset
1578
6905
8c779f3e16be Fixes to FTS handling.
Timo Sirainen <tss@iki.fi>
parents: 6902
diff changeset
1579 static void squat_trie_add_unknown(struct squat_trie *trie,
8c779f3e16be Fixes to FTS handling.
Timo Sirainen <tss@iki.fi>
parents: 6902
diff changeset
1580 ARRAY_TYPE(seq_range) *maybe_uids)
8c779f3e16be Fixes to FTS handling.
Timo Sirainen <tss@iki.fi>
parents: 6902
diff changeset
1581 {
8c779f3e16be Fixes to FTS handling.
Timo Sirainen <tss@iki.fi>
parents: 6902
diff changeset
1582 struct seq_range *range, new_range;
8c779f3e16be Fixes to FTS handling.
Timo Sirainen <tss@iki.fi>
parents: 6902
diff changeset
1583 unsigned int count;
8c779f3e16be Fixes to FTS handling.
Timo Sirainen <tss@iki.fi>
parents: 6902
diff changeset
1584 uint32_t last_uid;
8c779f3e16be Fixes to FTS handling.
Timo Sirainen <tss@iki.fi>
parents: 6902
diff changeset
1585
8c779f3e16be Fixes to FTS handling.
Timo Sirainen <tss@iki.fi>
parents: 6902
diff changeset
1586 last_uid = I_MAX((trie->root.next_uid+1)/2, 1) - 1;
8c779f3e16be Fixes to FTS handling.
Timo Sirainen <tss@iki.fi>
parents: 6902
diff changeset
1587
8c779f3e16be Fixes to FTS handling.
Timo Sirainen <tss@iki.fi>
parents: 6902
diff changeset
1588 range = array_get_modifiable(maybe_uids, &count);
8c779f3e16be Fixes to FTS handling.
Timo Sirainen <tss@iki.fi>
parents: 6902
diff changeset
1589 if (count > 0 && range[count-1].seq2 == last_uid) {
8c779f3e16be Fixes to FTS handling.
Timo Sirainen <tss@iki.fi>
parents: 6902
diff changeset
1590 /* increase the range */
8c779f3e16be Fixes to FTS handling.
Timo Sirainen <tss@iki.fi>
parents: 6902
diff changeset
1591 range[count-1].seq2 = (uint32_t)-1;
8c779f3e16be Fixes to FTS handling.
Timo Sirainen <tss@iki.fi>
parents: 6902
diff changeset
1592 } else {
8c779f3e16be Fixes to FTS handling.
Timo Sirainen <tss@iki.fi>
parents: 6902
diff changeset
1593 new_range.seq1 = last_uid + 1;
8c779f3e16be Fixes to FTS handling.
Timo Sirainen <tss@iki.fi>
parents: 6902
diff changeset
1594 new_range.seq2 = (uint32_t)-1;
8c779f3e16be Fixes to FTS handling.
Timo Sirainen <tss@iki.fi>
parents: 6902
diff changeset
1595 array_append(maybe_uids, &new_range, 1);
8c779f3e16be Fixes to FTS handling.
Timo Sirainen <tss@iki.fi>
parents: 6902
diff changeset
1596 }
8c779f3e16be Fixes to FTS handling.
Timo Sirainen <tss@iki.fi>
parents: 6902
diff changeset
1597 }
8c779f3e16be Fixes to FTS handling.
Timo Sirainen <tss@iki.fi>
parents: 6902
diff changeset
1598
6940
414c9d631a81 Replaced t_push/t_pop calls with T_FRAME*() macros.
Timo Sirainen <tss@iki.fi>
parents: 6931
diff changeset
1599 static int
414c9d631a81 Replaced t_push/t_pop calls with T_FRAME*() macros.
Timo Sirainen <tss@iki.fi>
parents: 6931
diff changeset
1600 squat_trie_lookup_real(struct squat_trie *trie, const char *str,
414c9d631a81 Replaced t_push/t_pop calls with T_FRAME*() macros.
Timo Sirainen <tss@iki.fi>
parents: 6931
diff changeset
1601 enum squat_index_type type,
414c9d631a81 Replaced t_push/t_pop calls with T_FRAME*() macros.
Timo Sirainen <tss@iki.fi>
parents: 6931
diff changeset
1602 ARRAY_TYPE(seq_range) *definite_uids,
414c9d631a81 Replaced t_push/t_pop calls with T_FRAME*() macros.
Timo Sirainen <tss@iki.fi>
parents: 6931
diff changeset
1603 ARRAY_TYPE(seq_range) *maybe_uids)
4855
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
1604 {
6902
cff5428c3c4d Fixed handling non-indexed characters in search key.
Timo Sirainen <tss@iki.fi>
parents: 6898
diff changeset
1605 struct squat_trie_lookup_context ctx;
cff5428c3c4d Fixed handling non-indexed characters in search key.
Timo Sirainen <tss@iki.fi>
parents: 6898
diff changeset
1606 unsigned char *data;
6911
a215deb3de8d Fixed non-ASCII searches.
Timo Sirainen <tss@iki.fi>
parents: 6905
diff changeset
1607 uint8_t *char_lengths;
a215deb3de8d Fixed non-ASCII searches.
Timo Sirainen <tss@iki.fi>
parents: 6905
diff changeset
1608 unsigned int i, start, bytes, str_bytelen, str_charlen;
6898
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1609 int ret = 0;
4855
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
1610
7161
b2333f0f23af squat_trie_lookup(): Make sure we clear result arrays first.
Timo Sirainen <tss@iki.fi>
parents: 7159
diff changeset
1611 array_clear(definite_uids);
b2333f0f23af squat_trie_lookup(): Make sure we clear result arrays first.
Timo Sirainen <tss@iki.fi>
parents: 7159
diff changeset
1612 array_clear(maybe_uids);
b2333f0f23af squat_trie_lookup(): Make sure we clear result arrays first.
Timo Sirainen <tss@iki.fi>
parents: 7159
diff changeset
1613
6902
cff5428c3c4d Fixed handling non-indexed characters in search key.
Timo Sirainen <tss@iki.fi>
parents: 6898
diff changeset
1614 memset(&ctx, 0, sizeof(ctx));
cff5428c3c4d Fixed handling non-indexed characters in search key.
Timo Sirainen <tss@iki.fi>
parents: 6898
diff changeset
1615 ctx.trie = trie;
cff5428c3c4d Fixed handling non-indexed characters in search key.
Timo Sirainen <tss@iki.fi>
parents: 6898
diff changeset
1616 ctx.type = type;
cff5428c3c4d Fixed handling non-indexed characters in search key.
Timo Sirainen <tss@iki.fi>
parents: 6898
diff changeset
1617 ctx.definite_uids = definite_uids;
cff5428c3c4d Fixed handling non-indexed characters in search key.
Timo Sirainen <tss@iki.fi>
parents: 6898
diff changeset
1618 ctx.maybe_uids = maybe_uids;
6945
5915aea5f070 UID ranges can be large, don't allocate them from data stack.
Timo Sirainen <tss@iki.fi>
parents: 6944
diff changeset
1619 i_array_init(&ctx.tmp_uids, 128);
5915aea5f070 UID ranges can be large, don't allocate them from data stack.
Timo Sirainen <tss@iki.fi>
parents: 6944
diff changeset
1620 i_array_init(&ctx.tmp_uids2, 128);
6902
cff5428c3c4d Fixed handling non-indexed characters in search key.
Timo Sirainen <tss@iki.fi>
parents: 6898
diff changeset
1621 ctx.first = TRUE;
cff5428c3c4d Fixed handling non-indexed characters in search key.
Timo Sirainen <tss@iki.fi>
parents: 6898
diff changeset
1622
6911
a215deb3de8d Fixed non-ASCII searches.
Timo Sirainen <tss@iki.fi>
parents: 6905
diff changeset
1623 str_bytelen = strlen(str);
a215deb3de8d Fixed non-ASCII searches.
Timo Sirainen <tss@iki.fi>
parents: 6905
diff changeset
1624 char_lengths = t_malloc0(str_bytelen);
6912
b326327b1f10 Fix for searching <4 char keys.
Timo Sirainen <tss@iki.fi>
parents: 6911
diff changeset
1625 for (i = 0, str_charlen = 0; i < str_bytelen; str_charlen++) {
6911
a215deb3de8d Fixed non-ASCII searches.
Timo Sirainen <tss@iki.fi>
parents: 6905
diff changeset
1626 bytes = uni_utf8_char_bytes(str[i]);
a215deb3de8d Fixed non-ASCII searches.
Timo Sirainen <tss@iki.fi>
parents: 6905
diff changeset
1627 char_lengths[i] = bytes;
a215deb3de8d Fixed non-ASCII searches.
Timo Sirainen <tss@iki.fi>
parents: 6905
diff changeset
1628 i += bytes;
a215deb3de8d Fixed non-ASCII searches.
Timo Sirainen <tss@iki.fi>
parents: 6905
diff changeset
1629 }
a215deb3de8d Fixed non-ASCII searches.
Timo Sirainen <tss@iki.fi>
parents: 6905
diff changeset
1630 data = squat_data_normalize(trie, (const unsigned char *)str,
a215deb3de8d Fixed non-ASCII searches.
Timo Sirainen <tss@iki.fi>
parents: 6905
diff changeset
1631 str_bytelen);
6902
cff5428c3c4d Fixed handling non-indexed characters in search key.
Timo Sirainen <tss@iki.fi>
parents: 6898
diff changeset
1632
6911
a215deb3de8d Fixed non-ASCII searches.
Timo Sirainen <tss@iki.fi>
parents: 6905
diff changeset
1633 for (i = start = 0; i < str_bytelen && ret >= 0; i += char_lengths[i]) {
6902
cff5428c3c4d Fixed handling non-indexed characters in search key.
Timo Sirainen <tss@iki.fi>
parents: 6898
diff changeset
1634 if (data[i] != '\0')
cff5428c3c4d Fixed handling non-indexed characters in search key.
Timo Sirainen <tss@iki.fi>
parents: 6898
diff changeset
1635 continue;
cff5428c3c4d Fixed handling non-indexed characters in search key.
Timo Sirainen <tss@iki.fi>
parents: 6898
diff changeset
1636
cff5428c3c4d Fixed handling non-indexed characters in search key.
Timo Sirainen <tss@iki.fi>
parents: 6898
diff changeset
1637 /* string has nonindexed characters.
cff5428c3c4d Fixed handling non-indexed characters in search key.
Timo Sirainen <tss@iki.fi>
parents: 6898
diff changeset
1638 search it in parts. */
cff5428c3c4d Fixed handling non-indexed characters in search key.
Timo Sirainen <tss@iki.fi>
parents: 6898
diff changeset
1639 if (i != start) {
cff5428c3c4d Fixed handling non-indexed characters in search key.
Timo Sirainen <tss@iki.fi>
parents: 6898
diff changeset
1640 ret = squat_trie_lookup_partial(&ctx, data + start,
6911
a215deb3de8d Fixed non-ASCII searches.
Timo Sirainen <tss@iki.fi>
parents: 6905
diff changeset
1641 char_lengths,
6902
cff5428c3c4d Fixed handling non-indexed characters in search key.
Timo Sirainen <tss@iki.fi>
parents: 6898
diff changeset
1642 i - start);
cff5428c3c4d Fixed handling non-indexed characters in search key.
Timo Sirainen <tss@iki.fi>
parents: 6898
diff changeset
1643 }
6911
a215deb3de8d Fixed non-ASCII searches.
Timo Sirainen <tss@iki.fi>
parents: 6905
diff changeset
1644 start = i + char_lengths[i];
6902
cff5428c3c4d Fixed handling non-indexed characters in search key.
Timo Sirainen <tss@iki.fi>
parents: 6898
diff changeset
1645 }
cff5428c3c4d Fixed handling non-indexed characters in search key.
Timo Sirainen <tss@iki.fi>
parents: 6898
diff changeset
1646
cff5428c3c4d Fixed handling non-indexed characters in search key.
Timo Sirainen <tss@iki.fi>
parents: 6898
diff changeset
1647 if (start != 0) {
cff5428c3c4d Fixed handling non-indexed characters in search key.
Timo Sirainen <tss@iki.fi>
parents: 6898
diff changeset
1648 /* string has nonindexed characters. finish the search. */
cff5428c3c4d Fixed handling non-indexed characters in search key.
Timo Sirainen <tss@iki.fi>
parents: 6898
diff changeset
1649 array_clear(definite_uids);
cff5428c3c4d Fixed handling non-indexed characters in search key.
Timo Sirainen <tss@iki.fi>
parents: 6898
diff changeset
1650 if (i != start && ret >= 0) {
cff5428c3c4d Fixed handling non-indexed characters in search key.
Timo Sirainen <tss@iki.fi>
parents: 6898
diff changeset
1651 ret = squat_trie_lookup_partial(&ctx, data + start,
6911
a215deb3de8d Fixed non-ASCII searches.
Timo Sirainen <tss@iki.fi>
parents: 6905
diff changeset
1652 char_lengths,
6902
cff5428c3c4d Fixed handling non-indexed characters in search key.
Timo Sirainen <tss@iki.fi>
parents: 6898
diff changeset
1653 i - start);
cff5428c3c4d Fixed handling non-indexed characters in search key.
Timo Sirainen <tss@iki.fi>
parents: 6898
diff changeset
1654 }
6945
5915aea5f070 UID ranges can be large, don't allocate them from data stack.
Timo Sirainen <tss@iki.fi>
parents: 6944
diff changeset
1655 } else {
5915aea5f070 UID ranges can be large, don't allocate them from data stack.
Timo Sirainen <tss@iki.fi>
parents: 6944
diff changeset
1656 if (str_charlen <= trie->hdr.partial_len ||
5915aea5f070 UID ranges can be large, don't allocate them from data stack.
Timo Sirainen <tss@iki.fi>
parents: 6944
diff changeset
1657 trie->hdr.full_len > trie->hdr.partial_len) {
5915aea5f070 UID ranges can be large, don't allocate them from data stack.
Timo Sirainen <tss@iki.fi>
parents: 6944
diff changeset
1658 ret = squat_trie_lookup_data(trie, data, str_bytelen,
5915aea5f070 UID ranges can be large, don't allocate them from data stack.
Timo Sirainen <tss@iki.fi>
parents: 6944
diff changeset
1659 &ctx.tmp_uids);
5915aea5f070 UID ranges can be large, don't allocate them from data stack.
Timo Sirainen <tss@iki.fi>
parents: 6944
diff changeset
1660 if (ret > 0) {
5915aea5f070 UID ranges can be large, don't allocate them from data stack.
Timo Sirainen <tss@iki.fi>
parents: 6944
diff changeset
1661 squat_trie_filter_type(type, &ctx.tmp_uids,
5915aea5f070 UID ranges can be large, don't allocate them from data stack.
Timo Sirainen <tss@iki.fi>
parents: 6944
diff changeset
1662 definite_uids);
5915aea5f070 UID ranges can be large, don't allocate them from data stack.
Timo Sirainen <tss@iki.fi>
parents: 6944
diff changeset
1663 }
5915aea5f070 UID ranges can be large, don't allocate them from data stack.
Timo Sirainen <tss@iki.fi>
parents: 6944
diff changeset
1664 } else {
5915aea5f070 UID ranges can be large, don't allocate them from data stack.
Timo Sirainen <tss@iki.fi>
parents: 6944
diff changeset
1665 array_clear(definite_uids);
6902
cff5428c3c4d Fixed handling non-indexed characters in search key.
Timo Sirainen <tss@iki.fi>
parents: 6898
diff changeset
1666 }
4878
88a91d9a867b Fixes. Should be pretty much working now.
Timo Sirainen <tss@iki.fi>
parents: 4855
diff changeset
1667
6945
5915aea5f070 UID ranges can be large, don't allocate them from data stack.
Timo Sirainen <tss@iki.fi>
parents: 6944
diff changeset
1668 if (str_charlen <= trie->hdr.partial_len ||
5915aea5f070 UID ranges can be large, don't allocate them from data stack.
Timo Sirainen <tss@iki.fi>
parents: 6944
diff changeset
1669 trie->hdr.partial_len == 0) {
5915aea5f070 UID ranges can be large, don't allocate them from data stack.
Timo Sirainen <tss@iki.fi>
parents: 6944
diff changeset
1670 /* we have the result */
5915aea5f070 UID ranges can be large, don't allocate them from data stack.
Timo Sirainen <tss@iki.fi>
parents: 6944
diff changeset
1671 array_clear(maybe_uids);
5915aea5f070 UID ranges can be large, don't allocate them from data stack.
Timo Sirainen <tss@iki.fi>
parents: 6944
diff changeset
1672 } else {
5915aea5f070 UID ranges can be large, don't allocate them from data stack.
Timo Sirainen <tss@iki.fi>
parents: 6944
diff changeset
1673 ret = squat_trie_lookup_partial(&ctx, data + start,
5915aea5f070 UID ranges can be large, don't allocate them from data stack.
Timo Sirainen <tss@iki.fi>
parents: 6944
diff changeset
1674 char_lengths,
5915aea5f070 UID ranges can be large, don't allocate them from data stack.
Timo Sirainen <tss@iki.fi>
parents: 6944
diff changeset
1675 i - start);
5915aea5f070 UID ranges can be large, don't allocate them from data stack.
Timo Sirainen <tss@iki.fi>
parents: 6944
diff changeset
1676 }
4878
88a91d9a867b Fixes. Should be pretty much working now.
Timo Sirainen <tss@iki.fi>
parents: 4855
diff changeset
1677 }
6905
8c779f3e16be Fixes to FTS handling.
Timo Sirainen <tss@iki.fi>
parents: 6902
diff changeset
1678 squat_trie_add_unknown(trie, maybe_uids);
6945
5915aea5f070 UID ranges can be large, don't allocate them from data stack.
Timo Sirainen <tss@iki.fi>
parents: 6944
diff changeset
1679 array_free(&ctx.tmp_uids);
5915aea5f070 UID ranges can be large, don't allocate them from data stack.
Timo Sirainen <tss@iki.fi>
parents: 6944
diff changeset
1680 array_free(&ctx.tmp_uids2);
6905
8c779f3e16be Fixes to FTS handling.
Timo Sirainen <tss@iki.fi>
parents: 6902
diff changeset
1681 return ret < 0 ? -1 : 0;
4855
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
1682 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
1683
6940
414c9d631a81 Replaced t_push/t_pop calls with T_FRAME*() macros.
Timo Sirainen <tss@iki.fi>
parents: 6931
diff changeset
1684 int squat_trie_lookup(struct squat_trie *trie, const char *str,
414c9d631a81 Replaced t_push/t_pop calls with T_FRAME*() macros.
Timo Sirainen <tss@iki.fi>
parents: 6931
diff changeset
1685 enum squat_index_type type,
414c9d631a81 Replaced t_push/t_pop calls with T_FRAME*() macros.
Timo Sirainen <tss@iki.fi>
parents: 6931
diff changeset
1686 ARRAY_TYPE(seq_range) *definite_uids,
414c9d631a81 Replaced t_push/t_pop calls with T_FRAME*() macros.
Timo Sirainen <tss@iki.fi>
parents: 6931
diff changeset
1687 ARRAY_TYPE(seq_range) *maybe_uids)
414c9d631a81 Replaced t_push/t_pop calls with T_FRAME*() macros.
Timo Sirainen <tss@iki.fi>
parents: 6931
diff changeset
1688 {
414c9d631a81 Replaced t_push/t_pop calls with T_FRAME*() macros.
Timo Sirainen <tss@iki.fi>
parents: 6931
diff changeset
1689 int ret;
414c9d631a81 Replaced t_push/t_pop calls with T_FRAME*() macros.
Timo Sirainen <tss@iki.fi>
parents: 6931
diff changeset
1690
414c9d631a81 Replaced t_push/t_pop calls with T_FRAME*() macros.
Timo Sirainen <tss@iki.fi>
parents: 6931
diff changeset
1691 T_FRAME(
414c9d631a81 Replaced t_push/t_pop calls with T_FRAME*() macros.
Timo Sirainen <tss@iki.fi>
parents: 6931
diff changeset
1692 ret = squat_trie_lookup_real(trie, str, type,
414c9d631a81 Replaced t_push/t_pop calls with T_FRAME*() macros.
Timo Sirainen <tss@iki.fi>
parents: 6931
diff changeset
1693 definite_uids, maybe_uids);
414c9d631a81 Replaced t_push/t_pop calls with T_FRAME*() macros.
Timo Sirainen <tss@iki.fi>
parents: 6931
diff changeset
1694 );
414c9d631a81 Replaced t_push/t_pop calls with T_FRAME*() macros.
Timo Sirainen <tss@iki.fi>
parents: 6931
diff changeset
1695 return ret;
414c9d631a81 Replaced t_push/t_pop calls with T_FRAME*() macros.
Timo Sirainen <tss@iki.fi>
parents: 6931
diff changeset
1696 }
414c9d631a81 Replaced t_push/t_pop calls with T_FRAME*() macros.
Timo Sirainen <tss@iki.fi>
parents: 6931
diff changeset
1697
6418
46d9ee79f292 Removed _ prefix from all public APIs.
Timo Sirainen <tss@iki.fi>
parents: 6414
diff changeset
1698 struct squat_uidlist *squat_trie_get_uidlist(struct squat_trie *trie)
4855
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
1699 {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
1700 return trie->uidlist;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas
Timo Sirainen <tss@iki.fi>
parents:
diff changeset
1701 }
6898
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1702
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1703 size_t squat_trie_mem_used(struct squat_trie *trie, unsigned int *count_r)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1704 {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1705 *count_r = trie->hdr.node_count;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1706 return trie->node_alloc_size;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling.
Timo Sirainen <tss@iki.fi>
parents: 6429
diff changeset
1707 }