dovecot/original-hg/dovecot-1.2: src/plugins/fts-squat/squat-trie.c annotate

annotate src/plugins/fts-squat/squat-trie.c @ 7086:7ed926ed7aa4 HEAD

Updated copyright notices to include year 2008.

author	Timo Sirainen <tss@iki.fi>
date	Tue, 01 Jan 2008 22:05:21 +0200
parents	7708a8c166d6
children	f41bd3065ec8

rev	line source
7086 7ed926ed7aa4 Updated copyright notices to include year 2008. Timo Sirainen <tss@iki.fi> parents: 7043 diff changeset	1 /* Copyright (c) 2007-2008 Dovecot authors, see the included COPYING file */
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	2
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	3 #include "lib.h"
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	4 #include "array.h"
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	5 #include "str.h"
6944 66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	6 #include "read-full.h"
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	7 #include "istream.h"
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	8 #include "ostream.h"
6911 a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	9 #include "unichar.h"
6944 66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	10 #include "file-cache.h"
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	11 #include "seq-range-array.h"
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	12 #include "squat-uidlist.h"
5068 65a4e1e61555 Some bugfixes. Moved header struct to a separate file. Timo Sirainen <tss@iki.fi> parents: 5010 diff changeset	13 #include "squat-trie-private.h"
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	14
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	15 #include <stdio.h>
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	16 #include <stdlib.h>
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	17 #include <unistd.h>
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	18 #include <sys/mman.h>
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	19
6905 8c779f3e16be Fixes to FTS handling. Timo Sirainen <tss@iki.fi> parents: 6902 diff changeset	20 #define DEFAULT_NORMALIZE_MAP_CHARS \
8c779f3e16be Fixes to FTS handling. Timo Sirainen <tss@iki.fi> parents: 6902 diff changeset	21 "EOTIRSACDNLMVUGPHBFWYXKJQZ0123456789@.-+#$%_&"
8c779f3e16be Fixes to FTS handling. Timo Sirainen <tss@iki.fi> parents: 6902 diff changeset	22 #define DEFAULT_PARTIAL_LEN 4
8c779f3e16be Fixes to FTS handling. Timo Sirainen <tss@iki.fi> parents: 6902 diff changeset	23 #define DEFAULT_FULL_LEN 4
8c779f3e16be Fixes to FTS handling. Timo Sirainen <tss@iki.fi> parents: 6902 diff changeset	24
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	25 #define MAX_FAST_LEVEL 3
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	26 #define SEQUENTIAL_COUNT 46
4878 88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	27
6944 66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	28 #define TRIE_BYTES_LEFT(n) \
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	29 ((n) * SQUAT_PACK_MAX_SIZE)
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	30 #define TRIE_READAHEAD_SIZE \
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	31 I_MAX(4096, 1 + 256 + TRIE_BYTES_LEFT(256))
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	32
4878 88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	33 struct squat_trie_build_context {
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	34 struct squat_trie *trie;
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	35 struct ostream *output;
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	36 struct squat_uidlist_build_context *uidlist_build_ctx;
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	37
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	38 struct file_lock *file_lock;
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	39
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	40 uint32_t first_uid;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	41 unsigned int compress_nodes:1;
4878 88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	42 };
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	43
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	44 struct squat_trie_iterate_node {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	45 struct squat_node *node;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	46 unsigned int idx;
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	47 };
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	48
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	49 struct squat_trie_iterate_context {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	50 struct squat_trie *trie;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	51 struct squat_trie_iterate_node cur;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	52 ARRAY_DEFINE(parents, struct squat_trie_iterate_node);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	53 bool failed;
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	54 };
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	55
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	56 static int squat_trie_map(struct squat_trie *trie, bool building);
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	57
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	58 void squat_trie_delete(struct squat_trie *trie)
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	59 {
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	60 if (unlink(trie->path) < 0 && errno != ENOENT)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	61 i_error("unlink(%s) failed: %m", trie->path);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	62 squat_uidlist_delete(trie->uidlist);
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	63 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	64
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	65 static void squat_trie_set_corrupted(struct squat_trie *trie)
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	66 {
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	67 trie->corrupted = TRUE;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	68 i_error("Corrupted file %s", trie->path);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	69 squat_trie_delete(trie);
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	70 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	71
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	72 static void squat_trie_normalize_map_build(struct squat_trie *trie)
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	73 {
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	74 static unsigned char valid_chars[] =
6905 8c779f3e16be Fixes to FTS handling. Timo Sirainen <tss@iki.fi> parents: 6902 diff changeset	75 DEFAULT_NORMALIZE_MAP_CHARS;
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	76 unsigned int i, j;
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	77
6905 8c779f3e16be Fixes to FTS handling. Timo Sirainen <tss@iki.fi> parents: 6902 diff changeset	78 memset(trie->default_normalize_map, 0,
8c779f3e16be Fixes to FTS handling. Timo Sirainen <tss@iki.fi> parents: 6902 diff changeset	79 sizeof(trie->default_normalize_map));
4901 5e604d6f391e Only ASCII texts were actually indexed. Now UTF-8 input is properly Timo Sirainen <tss@iki.fi> parents: 4898 diff changeset	80
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	81 #if 1
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	82 for (i = 0, j = 1; i < sizeof(valid_chars)-1; i++) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	83 unsigned char chr = valid_chars[i];
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	84
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	85 if (chr >= 'A' && chr <= 'Z')
6905 8c779f3e16be Fixes to FTS handling. Timo Sirainen <tss@iki.fi> parents: 6902 diff changeset	86 trie->default_normalize_map[chr-'A'+'a'] = j;
8c779f3e16be Fixes to FTS handling. Timo Sirainen <tss@iki.fi> parents: 6902 diff changeset	87 trie->default_normalize_map[chr] = j++;
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	88 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	89 i_assert(j <= SEQUENTIAL_COUNT);
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	90
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	91 for (i = 128; i < 256; i++)
6905 8c779f3e16be Fixes to FTS handling. Timo Sirainen <tss@iki.fi> parents: 6902 diff changeset	92 trie->default_normalize_map[i] = j++;
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	93 #else
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	94 for (i = 0; i < sizeof(valid_chars)-1; i++) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	95 unsigned char chr = valid_chars[i];
4890 cb3b4153136c 64bit and big endian fixes Timo Sirainen <tss@iki.fi> parents: 4887 diff changeset	96
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	97 if (chr >= 'A' && chr <= 'Z')
6905 8c779f3e16be Fixes to FTS handling. Timo Sirainen <tss@iki.fi> parents: 6902 diff changeset	98 trie->default_normalize_map[chr-'A'+'a'] = chr;
8c779f3e16be Fixes to FTS handling. Timo Sirainen <tss@iki.fi> parents: 6902 diff changeset	99 trie->default_normalize_map[chr] = chr;
4890 cb3b4153136c 64bit and big endian fixes Timo Sirainen <tss@iki.fi> parents: 4887 diff changeset	100 }
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	101 for (i = 128; i < 256; i++)
6905 8c779f3e16be Fixes to FTS handling. Timo Sirainen <tss@iki.fi> parents: 6902 diff changeset	102 trie->default_normalize_map[i] = i_toupper(i);
4890 cb3b4153136c 64bit and big endian fixes Timo Sirainen <tss@iki.fi> parents: 4887 diff changeset	103 #endif
cb3b4153136c 64bit and big endian fixes Timo Sirainen <tss@iki.fi> parents: 4887 diff changeset	104 }
cb3b4153136c 64bit and big endian fixes Timo Sirainen <tss@iki.fi> parents: 4887 diff changeset	105
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	106 static void node_free(struct squat_trie trie, struct squat_node node)
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	107 {
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	108 struct squat_node *children;
4890 cb3b4153136c 64bit and big endian fixes Timo Sirainen <tss@iki.fi> parents: 4887 diff changeset	109 unsigned int i;
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	110
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	111 if (NODE_IS_DYNAMIC_LEAF(node))
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	112 i_free(node->children.leaf_string);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	113 else if (!node->children_not_mapped && node->child_count > 0) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	114 children = NODE_CHILDREN_NODES(node);
4883 d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	115
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	116 trie->node_alloc_size -=
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	117 NODE_CHILDREN_ALLOC_SIZE(node->child_count);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	118 for (i = 0; i < node->child_count; i++)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	119 node_free(trie, &children[i]);
4883 d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	120
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	121 i_free(node->children.data);
4898 07a038b57946 Optimization. Timo Sirainen <tss@iki.fi> parents: 4897 diff changeset	122 }
07a038b57946 Optimization. Timo Sirainen <tss@iki.fi> parents: 4897 diff changeset	123 }
07a038b57946 Optimization. Timo Sirainen <tss@iki.fi> parents: 4897 diff changeset	124
4878 88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	125 struct squat_trie *
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	126 squat_trie_init(const char *path, uint32_t uidvalidity,
4879 b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	127 enum file_lock_method lock_method, bool mmap_disable)
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	128 {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	129 struct squat_trie *trie;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	130
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	131 trie = i_new(struct squat_trie, 1);
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	132 trie->path = i_strdup(path);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	133 trie->uidlist = squat_uidlist_init(trie);
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	134 trie->fd = -1;
4878 88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	135 trie->lock_method = lock_method;
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	136 trie->uidvalidity = uidvalidity;
6944 66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	137 trie->mmap_disable = mmap_disable;
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	138 squat_trie_normalize_map_build(trie);
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	139 return trie;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	140 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	141
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	142 static void squat_trie_close(struct squat_trie *trie)
4878 88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	143 {
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	144 trie->corrupted = FALSE;
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	145 node_free(trie, &trie->root);
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	146 memset(&trie->root, 0, sizeof(trie->root));
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	147 memset(&trie->hdr, 0, sizeof(trie->hdr));
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	148
6944 66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	149 trie->data = NULL;
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	150 trie->data_size = 0;
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	151
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	152 if (trie->file_cache != NULL)
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	153 file_cache_free(&trie->file_cache);
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	154 if (trie->mmap_size != 0) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	155 if (munmap(trie->mmap_base, trie->mmap_size) < 0)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	156 i_error("munmap(%s) failed: %m", trie->path);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	157 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	158 if (trie->fd != -1) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	159 if (close(trie->fd) < 0)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	160 i_error("close(%s) failed: %m", trie->path);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	161 trie->fd = -1;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	162 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	163 trie->locked_file_size = 0;
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	164 }
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	165
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	166 void squat_trie_deinit(struct squat_trie **_trie)
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	167 {
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	168 struct squat_trie trie = _trie;
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	169
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	170 *_trie = NULL;
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	171 squat_trie_close(trie);
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	172 squat_uidlist_deinit(trie->uidlist);
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	173 i_free(trie->path);
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	174 i_free(trie);
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	175 }
4883 d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	176
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	177 static void squat_trie_header_init(struct squat_trie *trie)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	178 {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	179 memset(&trie->hdr, 0, sizeof(trie->hdr));
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	180 trie->hdr.version = SQUAT_TRIE_VERSION;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	181 trie->hdr.indexid = ioloop_time;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	182 trie->hdr.uidvalidity = trie->uidvalidity;
6905 8c779f3e16be Fixes to FTS handling. Timo Sirainen <tss@iki.fi> parents: 6902 diff changeset	183 trie->hdr.partial_len = DEFAULT_PARTIAL_LEN;
8c779f3e16be Fixes to FTS handling. Timo Sirainen <tss@iki.fi> parents: 6902 diff changeset	184 trie->hdr.full_len = DEFAULT_FULL_LEN;
8c779f3e16be Fixes to FTS handling. Timo Sirainen <tss@iki.fi> parents: 6902 diff changeset	185
8c779f3e16be Fixes to FTS handling. Timo Sirainen <tss@iki.fi> parents: 6902 diff changeset	186 i_assert(sizeof(trie->hdr.normalize_map) ==
8c779f3e16be Fixes to FTS handling. Timo Sirainen <tss@iki.fi> parents: 6902 diff changeset	187 sizeof(trie->default_normalize_map));
8c779f3e16be Fixes to FTS handling. Timo Sirainen <tss@iki.fi> parents: 6902 diff changeset	188 memcpy(trie->hdr.normalize_map, trie->default_normalize_map,
8c779f3e16be Fixes to FTS handling. Timo Sirainen <tss@iki.fi> parents: 6902 diff changeset	189 sizeof(trie->hdr.normalize_map));
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	190 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	191
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	192 static int squat_trie_open_fd(struct squat_trie *trie)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	193 {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	194 trie->fd = open(trie->path, O_RDWR);
4883 d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	195 if (trie->fd == -1) {
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	196 if (errno == ENOENT) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	197 squat_trie_header_init(trie);
4883 d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	198 return 0;
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	199 }
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	200 i_error("open(%s) failed: %m", trie->path);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	201 return -1;
4883 d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	202 }
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	203 return 0;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	204 }
4883 d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	205
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	206 static int squat_trie_open(struct squat_trie *trie)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	207 {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	208 squat_trie_close(trie);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	209
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	210 if (squat_trie_open_fd(trie) < 0)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	211 return -1;
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	212 return squat_trie_map(trie, FALSE);
4878 88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	213 }
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	214
4879 b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	215 static int squat_trie_is_file_stale(struct squat_trie *trie)
b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	216 {
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	217 struct stat st, st2;
4879 b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	218
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	219 if (stat(trie->path, &st) < 0) {
4883 d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	220 if (errno == ENOENT)
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	221 return 1;
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	222
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	223 i_error("stat(%s) failed: %m", trie->path);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	224 return -1;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	225 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	226 if (fstat(trie->fd, &st2) < 0) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	227 i_error("fstat(%s) failed: %m", trie->path);
4879 b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	228 return -1;
b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	229 }
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	230 trie->locked_file_size = st2.st_size;
4879 b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	231
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	232 return st.st_ino == st2.st_ino &&
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	233 CMP_DEV_T(st.st_dev, st2.st_dev) ? 0 : 1;
4879 b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	234 }
b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	235
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	236 void squat_trie_refresh(struct squat_trie *trie)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	237 {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	238 if (squat_trie_is_file_stale(trie) > 0)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	239 (void)squat_trie_open(trie);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	240 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	241
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	242 static int squat_trie_lock(struct squat_trie *trie, int lock_type,
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	243 struct file_lock **file_lock_r)
4886 ae2114f11a0e Memory leak fixes. Also when building a large mailbox flush once in a while Timo Sirainen <tss@iki.fi> parents: 4883 diff changeset	244 {
ae2114f11a0e Memory leak fixes. Also when building a large mailbox flush once in a while Timo Sirainen <tss@iki.fi> parents: 4883 diff changeset	245 int ret;
ae2114f11a0e Memory leak fixes. Also when building a large mailbox flush once in a while Timo Sirainen <tss@iki.fi> parents: 4883 diff changeset	246
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	247 while (trie->fd != -1) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	248 ret = file_wait_lock(trie->fd, trie->path, lock_type,
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	249 trie->lock_method, SQUAT_TRIE_LOCK_TIMEOUT,
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	250 file_lock_r);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	251 if (ret == 0) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	252 i_error("file_wait_lock(%s) failed: %m", trie->path);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	253 return 0;
4879 b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	254 }
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	255 if (ret < 0)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	256 return -1;
4879 b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	257
b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	258 /* if the trie has been compressed, we need to reopen the
b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	259 file and try to lock again */
b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	260 ret = squat_trie_is_file_stale(trie);
b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	261 if (ret == 0)
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	262 return 1;
4879 b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	263
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	264 file_unlock(file_lock_r);
4879 b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	265 if (ret < 0)
b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	266 return -1;
b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	267
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	268 squat_trie_close(trie);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	269 if (squat_trie_open_fd(trie) < 0)
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	270 return -1;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	271 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	272 return 0;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	273 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	274
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	275 static void
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	276 node_make_squential(struct squat_trie trie, struct squat_node node, int level)
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	277 {
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	278 const unsigned int alloc_size =
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	279 NODE_CHILDREN_ALLOC_SIZE(SEQUENTIAL_COUNT);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	280 struct squat_node *children;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	281 unsigned char *chars;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	282 unsigned int i;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	283
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	284 i_assert(node->child_count == 0);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	285
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	286 trie->node_alloc_size += alloc_size;
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	287
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	288 node->want_sequential = FALSE;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	289 node->have_sequential = TRUE;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	290
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	291 node->child_count = SEQUENTIAL_COUNT;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	292 node->children.data = i_malloc(alloc_size);
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	293
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	294 chars = NODE_CHILDREN_CHARS(node);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	295 for (i = 0; i < SEQUENTIAL_COUNT; i++)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	296 chars[i] = i;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	297
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	298 if (level < MAX_FAST_LEVEL) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	299 children = NODE_CHILDREN_NODES(node);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	300 for (i = 0; i < SEQUENTIAL_COUNT; i++)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	301 children[i].want_sequential = TRUE;
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	302 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	303 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	304
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	305 static unsigned int
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	306 node_add_child(struct squat_trie trie, struct squat_node node,
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	307 unsigned char chr, int level)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	308 {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	309 unsigned int old_child_count = node->child_count;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	310 struct squat_node children, old_children;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	311 unsigned char *chars;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	312 size_t old_size, new_size;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	313
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	314 i_assert(node->leaf_string_length == 0);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	315
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	316 if (node->want_sequential) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	317 node_make_squential(trie, node, level);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	318
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	319 if (chr < SEQUENTIAL_COUNT)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	320 return chr;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	321 old_child_count = SEQUENTIAL_COUNT;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	322 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	323
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	324 node->child_count++;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	325 new_size = NODE_CHILDREN_ALLOC_SIZE(node->child_count);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	326
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	327 if (old_child_count == 0) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	328 /* first child */
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	329 node->children.data = i_malloc(new_size);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	330 trie->node_alloc_size += new_size;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	331 children = NODE_CHILDREN_NODES(node);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	332 } else {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	333 old_size = NODE_CHILDREN_ALLOC_SIZE(old_child_count);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	334 if (old_size != new_size) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	335 trie->node_alloc_size += new_size - old_size;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	336 node->children.data = i_realloc(node->children.data,
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	337 old_size, new_size);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	338 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	339
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	340 children = NODE_CHILDREN_NODES(node);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	341 old_children = (void *)(NODE_CHILDREN_CHARS(node) +
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	342 MEM_ALIGN(old_child_count));
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	343 if (children != old_children) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	344 memmove(children, old_children,
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	345 old_child_count * sizeof(struct squat_node));
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	346 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	347 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	348
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	349 chars = NODE_CHILDREN_CHARS(node);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	350 chars[node->child_count - 1] = chr;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	351 return node->child_count - 1;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	352 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	353
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	354 static int
6944 66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	355 trie_file_cache_read(struct squat_trie *trie, size_t offset, size_t size)
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	356 {
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	357 if (trie->file_cache == NULL)
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	358 return 0;
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	359
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	360 if (file_cache_read(trie->file_cache, offset, size) < 0) {
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	361 i_error("read(%s) failed: %m", trie->path);
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	362 return -1;
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	363 }
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	364 trie->data = file_cache_get_map(trie->file_cache, &trie->data_size);
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	365 return 0;
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	366 }
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	367
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	368 static int
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	369 node_read_children(struct squat_trie trie, struct squat_node node, int level)
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	370 {
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	371 const uint8_t data, end;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	372 const unsigned char *child_chars;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	373 struct squat_node child, children = NULL;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	374 uoff_t node_offset;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	375 unsigned int i, child_idx, child_count;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	376 uoff_t base_offset;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	377 uint32_t num;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	378
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	379 i_assert(node->children_not_mapped);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	380 i_assert(!node->have_sequential);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	381 i_assert(trie->unmapped_child_count > 0);
6944 66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	382 i_assert(trie->data_size <= trie->locked_file_size);
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	383
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	384 trie->unmapped_child_count--;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	385 node_offset = node->children.offset;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	386 node->children_not_mapped = FALSE;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	387 node->children.data = NULL;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	388
6944 66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	389 if (trie_file_cache_read(trie, node_offset, TRIE_READAHEAD_SIZE) < 0)
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	390 return -1;
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	391 if (unlikely(node_offset >= trie->data_size)) {
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	392 squat_trie_set_corrupted(trie);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	393 return -1;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	394 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	395
6944 66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	396 data = CONST_PTR_OFFSET(trie->data, node_offset);
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	397 end = CONST_PTR_OFFSET(trie->data, trie->data_size);
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	398 child_count = *data++;
6944 66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	399 if (unlikely(node_offset + child_count >= trie->data_size)) {
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	400 squat_trie_set_corrupted(trie);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	401 return -1;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	402 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	403
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	404 if (child_count == 0)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	405 return 0;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	406
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	407 child_chars = data;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	408 data += child_count;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	409
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	410 /* get child offsets */
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	411 base_offset = node_offset;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	412 for (i = 0; i < child_count; i++) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	413 /* we always start with !have_sequential, so at i=0 this
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	414 check always goes to add the first child */
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	415 if (node->have_sequential && child_chars[i] < SEQUENTIAL_COUNT)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	416 child_idx = child_chars[i];
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	417 else {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	418 child_idx = node_add_child(trie, node, child_chars[i],
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	419 level);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	420 children = NODE_CHILDREN_NODES(node);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	421 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	422 child = &children[child_idx];
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	423
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	424 /* 1) child offset */
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	425 num = squat_unpack_num(&data, end);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	426 if (num == 0) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	427 /* no children */
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	428 } else {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	429 if ((num & 1) != 0) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	430 base_offset += num >> 1;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	431 } else {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	432 base_offset -= num >> 1;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	433 }
6944 66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	434 if (base_offset >= trie->locked_file_size) {
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	435 squat_trie_set_corrupted(trie);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	436 return -1;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	437 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	438 trie->unmapped_child_count++;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	439 child->children_not_mapped = TRUE;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	440 child->children.offset = base_offset;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	441 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	442
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	443 /* 2) uidlist */
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	444 child->uid_list_idx = squat_unpack_num(&data, end);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	445 if (child->uid_list_idx == 0) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	446 /* we don't write nodes with empty uidlists */
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	447 squat_trie_set_corrupted(trie);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	448 return -1;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	449 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	450 if (!UIDLIST_IS_SINGLETON(child->uid_list_idx)) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	451 /* 3) next uid */
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	452 child->next_uid = squat_unpack_num(&data, end) + 1;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	453 } else {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	454 uint32_t idx = child->uid_list_idx;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	455
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	456 child->next_uid = 1 +
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	457 squat_uidlist_singleton_last_uid(idx);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	458 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	459
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	460 /* 4) unused uids + leaf string flag */
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	461 num = squat_unpack_num(&data, end);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	462 child->unused_uids = num >> 1;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	463 if ((num & 1) != 0) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	464 /* leaf string */
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	465 unsigned int len;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	466 unsigned char *dest;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	467
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	468 /* 5) leaf string length */
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	469 len = child->leaf_string_length =
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	470 squat_unpack_num(&data, end) + 1;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	471 if (!NODE_IS_DYNAMIC_LEAF(child))
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	472 dest = child->children.static_leaf_string;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	473 else {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	474 dest = child->children.leaf_string =
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	475 i_malloc(len);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	476 }
6944 66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	477
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	478 if (trie->file_cache != NULL) {
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	479 /* the string may be long -
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	480 recalculate the end pos */
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	481 size_t offset, size;
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	482
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	483 offset = (const char *)data -
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	484 (const char *)trie->data;
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	485 size = len + TRIE_BYTES_LEFT(child_count - i);
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	486
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	487 if (trie_file_cache_read(trie, offset,
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	488 size) < 0)
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	489 return -1;
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	490 data = CONST_PTR_OFFSET(trie->data, offset);
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	491 end = CONST_PTR_OFFSET(trie->data,
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	492 trie->data_size);
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	493 }
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	494
7043 7708a8c166d6 Compiler warning fixes on 32bit systems. Timo Sirainen <tss@iki.fi> parents: 6945 diff changeset	495 if ((size_t)(end - data) < len) {
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	496 squat_trie_set_corrupted(trie);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	497 return -1;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	498 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	499 memcpy(dest, data, len);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	500 data += len;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	501 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	502 }
6944 66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	503 if (unlikely(data == end)) {
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	504 /* we should never get this far */
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	505 squat_trie_set_corrupted(trie);
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	506 return -1;
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	507 }
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	508 return 0;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	509 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	510
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	511 static void
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	512 node_write_children(struct squat_trie_build_context *ctx,
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	513 struct squat_node node, const uoff_t node_offsets)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	514 {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	515 struct squat_node *children;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	516 const unsigned char *chars;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	517 uint8_t child_count, buf[SQUAT_PACK_MAX_SIZE * 5], *bufp;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	518 uoff_t base_offset;
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	519 unsigned int i;
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	520
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	521 chars = NODE_CHILDREN_CHARS(node);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	522 children = NODE_CHILDREN_NODES(node);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	523
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	524 base_offset = ctx->output->offset;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	525 child_count = node->child_count;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	526 o_stream_send(ctx->output, &child_count, 1);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	527 o_stream_send(ctx->output, chars, child_count);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	528
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	529 for (i = 0; i < child_count; i++) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	530 bufp = buf;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	531 /* 1) child offset */
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	532 if (node_offsets[i] == 0)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	533 *bufp++ = 0;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	534 else if (node_offsets[i] >= base_offset) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	535 squat_pack_num(&bufp,
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	536 ((node_offsets[i] - base_offset) << 1) \| 1);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	537 base_offset = node_offsets[i];
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	538 } else {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	539 squat_pack_num(&bufp,
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	540 (base_offset - node_offsets[i]) << 1);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	541 base_offset = node_offsets[i];
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	542 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	543
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	544 /* 2) uidlist */
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	545 squat_pack_num(&bufp, children[i].uid_list_idx);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	546 if (!UIDLIST_IS_SINGLETON(children[i].uid_list_idx)) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	547 /* 3) next uid */
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	548 squat_pack_num(&bufp, children[i].next_uid - 1);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	549 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	550
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	551 if (children[i].leaf_string_length == 0) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	552 /* 4a) unused uids */
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	553 squat_pack_num(&bufp, children[i].unused_uids << 1);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	554 o_stream_send(ctx->output, buf, bufp - buf);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	555 } else {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	556 i_assert(node_offsets[i] == 0);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	557 /* 4b) unused uids + flag */
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	558 squat_pack_num(&bufp, (children[i].unused_uids << 1) \| 1);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	559 /* 5) leaf string length */
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	560 squat_pack_num(&bufp, children[i].leaf_string_length - 1);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	561 o_stream_send(ctx->output, buf, bufp - buf);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	562 o_stream_send(ctx->output,
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	563 NODE_LEAF_STRING(&children[i]),
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	564 children[i].leaf_string_length);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	565 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	566 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	567 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	568
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	569 static inline void
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	570 node_add_uid(struct squat_trie_build_context *ctx, uint32_t uid,
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	571 struct squat_node *node)
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	572 {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	573 if (uid < node->next_uid) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	574 /* duplicate */
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	575 return;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	576 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	577 node->unused_uids += uid - node->next_uid;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	578 node->next_uid = uid + 1;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	579
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	580 node->uid_list_idx =
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	581 squat_uidlist_build_add_uid(ctx->uidlist_build_ctx,
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	582 node->uid_list_idx, uid);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	583 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	584
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	585 static void
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	586 node_split_string(struct squat_trie_build_context ctx, struct squat_node node)
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	587 {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	588 struct squat_node *child;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	589 unsigned char *str;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	590 unsigned int uid, idx, str_len = node->leaf_string_length;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	591
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	592 i_assert(str_len > 0);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	593
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	594 /* make a copy of the leaf string and convert to normal node by
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	595 removing it. */
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	596 str = t_malloc(str_len);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	597 if (!NODE_IS_DYNAMIC_LEAF(node))
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	598 memcpy(str, node->children.static_leaf_string, str_len);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	599 else {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	600 memcpy(str, node->children.leaf_string, str_len);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	601 i_free(node->children.leaf_string);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	602 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	603 node->leaf_string_length = 0;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	604
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	605 /* create a new child node for the rest of the string */
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	606 idx = node_add_child(ctx->trie, node, str[0], MAX_FAST_LEVEL);
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	607 child = NODE_CHILDREN_NODES(node) + idx;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	608
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	609 /* update uidlist to contain all of parent's UIDs */
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	610 child->next_uid = node->next_uid - node->unused_uids;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	611 for (uid = 0; uid < child->next_uid; uid++) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	612 child->uid_list_idx =
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	613 squat_uidlist_build_add_uid(ctx->uidlist_build_ctx,
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	614 child->uid_list_idx, uid);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	615 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	616
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	617 i_assert(!child->have_sequential && child->children.data == NULL);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	618 if (str_len > 1) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	619 /* make the child a leaf string */
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	620 str_len--;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	621 child->leaf_string_length = str_len;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	622 if (!NODE_IS_DYNAMIC_LEAF(child)) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	623 memcpy(child->children.static_leaf_string,
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	624 str + 1, str_len);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	625 } else {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	626 child->children.leaf_string = i_malloc(str_len);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	627 memcpy(child->children.leaf_string, str + 1, str_len);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	628 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	629 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	630 }
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	631
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	632 static bool
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	633 node_leaf_string_add_or_split(struct squat_trie_build_context *ctx,
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	634 struct squat_node *node,
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	635 const unsigned char *data, unsigned int data_len)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	636 {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	637 const unsigned char *str = NODE_LEAF_STRING(node);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	638 const unsigned int str_len = node->leaf_string_length;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	639 unsigned int i;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	640
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	641 if (data_len != str_len) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	642 /* different lengths, can't match */
6940 414c9d631a81 Replaced t_push/t_pop calls with T_FRAME() macros. Timo Sirainen <tss@iki.fi>* parents: 6931 diff changeset	643 T_FRAME(
414c9d631a81 Replaced t_push/t_pop calls with T_FRAME() macros. Timo Sirainen <tss@iki.fi>* parents: 6931 diff changeset	644 node_split_string(ctx, node);
414c9d631a81 Replaced t_push/t_pop calls with T_FRAME() macros. Timo Sirainen <tss@iki.fi>* parents: 6931 diff changeset	645 );
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	646 return FALSE;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	647 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	648
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	649 for (i = 0; i < data_len; i++) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	650 if (data[i] != str[i]) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	651 /* non-match */
6940 414c9d631a81 Replaced t_push/t_pop calls with T_FRAME() macros. Timo Sirainen <tss@iki.fi>* parents: 6931 diff changeset	652 T_FRAME(
414c9d631a81 Replaced t_push/t_pop calls with T_FRAME() macros. Timo Sirainen <tss@iki.fi>* parents: 6931 diff changeset	653 node_split_string(ctx, node);
414c9d631a81 Replaced t_push/t_pop calls with T_FRAME() macros. Timo Sirainen <tss@iki.fi>* parents: 6931 diff changeset	654 );
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	655 return FALSE;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	656 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	657 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	658 return TRUE;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	659 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	660
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	661 static int squat_build_add(struct squat_trie_build_context *ctx, uint32_t uid,
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	662 const unsigned char *data, unsigned int size)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	663 {
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	664 struct squat_trie *trie = ctx->trie;
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	665 struct squat_node *node = &trie->root;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	666 const unsigned char *end = data + size;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	667 unsigned char *chars;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	668 unsigned int idx;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	669 int level = 0;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	670
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	671 for (;;) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	672 if (node->children_not_mapped) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	673 if (unlikely(node_read_children(trie, node, level) < 0))
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	674 return -1;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	675 }
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	676
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	677 if (node->leaf_string_length != 0) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	678 /* the whole string must match or we need to split
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	679 the node */
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	680 if (node_leaf_string_add_or_split(ctx, node, data,
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	681 end - data)) {
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	682 node_add_uid(ctx, uid, node);
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	683 return 0;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	684 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	685 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	686
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	687 node_add_uid(ctx, uid, node);
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	688
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	689 if (unlikely(uid < node->unused_uids)) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	690 squat_trie_set_corrupted(trie);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	691 return -1;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	692 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	693 /* child node's UIDs are relative to ours. so for example if
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	694 we're adding UID 4 and this node now has [2,4] UIDs,
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	695 unused_uids=3 and so the child node will be adding
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	696 UID 4-3 = 1. */
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	697 uid -= node->unused_uids;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	698
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	699 if (data == end)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	700 return 0;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	701 level++;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	702
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	703 if (node->have_sequential) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	704 if (*data < SEQUENTIAL_COUNT) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	705 idx = *data;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	706 goto found;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	707 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	708 idx = SEQUENTIAL_COUNT;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	709 } else {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	710 idx = 0;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	711 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	712 chars = NODE_CHILDREN_CHARS(node);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	713 for (; idx < node->child_count; idx++) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	714 if (chars[idx] == *data)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	715 goto found;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	716 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	717 break;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	718 found:
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	719 data++;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	720 node = NODE_CHILDREN_NODES(node) + idx;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	721 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	722
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	723 /* create new children */
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	724 i_assert(node->leaf_string_length == 0);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	725
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	726 for (;;) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	727 idx = node_add_child(trie, node, *data,
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	728 size - (end - data) + 1);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	729 node = NODE_CHILDREN_NODES(node) + idx;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	730
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	731 node_add_uid(ctx, uid, node);
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	732 uid = 0;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	733
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	734 if (++data == end)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	735 break;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	736
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	737 if (!node->have_sequential) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	738 /* convert the node into a leaf string */
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	739 unsigned int len = end - data;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	740
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	741 i_assert(node->children.data == NULL);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	742 node->leaf_string_length = len;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	743 if (!NODE_IS_DYNAMIC_LEAF(node)) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	744 memcpy(node->children.static_leaf_string,
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	745 data, len);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	746 } else {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	747 node->children.leaf_string = i_malloc(len);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	748 memcpy(node->children.leaf_string, data, len);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	749 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	750 break;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	751 }
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	752 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	753 return 0;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	754 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	755
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	756 static int
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	757 squat_build_word_bytes(struct squat_trie_build_context *ctx, uint32_t uid,
6911 a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	758 const unsigned char *data, unsigned int size)
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	759 {
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	760 struct squat_trie *trie = ctx->trie;
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	761 unsigned int i;
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	762
6911 a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	763 if (trie->hdr.full_len <= trie->hdr.partial_len)
a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	764 i = 0;
a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	765 else {
a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	766 /* the first word is longer than others */
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	767 if (squat_build_add(ctx, uid, data,
6911 a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	768 I_MIN(size, trie->hdr.full_len)) < 0)
a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	769 return -1;
a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	770 i = 1;
a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	771 }
a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	772
a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	773 for (; i < size; i++) {
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	774 if (squat_build_add(ctx, uid, data + i,
6911 a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	775 I_MIN(trie->hdr.partial_len, size-i)) < 0)
4883 d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	776 return -1;
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	777 }
6911 a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	778 return 0;
a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	779 }
a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	780
a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	781 static int
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	782 squat_build_word(struct squat_trie_build_context *ctx, uint32_t uid,
6911 a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	783 const unsigned char data, const uint8_t char_lengths,
a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	784 unsigned int size)
a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	785 {
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	786 struct squat_trie *trie = ctx->trie;
6911 a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	787 unsigned int i, j, bytelen;
a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	788
a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	789 if (char_lengths == NULL) {
a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	790 /* optimization path: all characters are bytes */
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	791 return squat_build_word_bytes(ctx, uid, data, size);
6911 a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	792 }
a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	793
a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	794 if (trie->hdr.full_len <= trie->hdr.partial_len)
a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	795 i = 0;
a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	796 else {
a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	797 /* the first word is longer than others */
a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	798 bytelen = 0;
a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	799 for (j = 0; j < trie->hdr.full_len && bytelen < size; j++)
a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	800 bytelen += char_lengths[bytelen];
a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	801 i_assert(bytelen <= size);
a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	802
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	803 if (squat_build_add(ctx, uid, data, bytelen) < 0)
6911 a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	804 return -1;
a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	805 i = char_lengths[0];
a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	806 }
a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	807
a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	808 for (; i < size; i += char_lengths[i]) {
a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	809 bytelen = 0;
a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	810 for (j = 0; j < trie->hdr.partial_len && i+bytelen < size; j++)
a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	811 bytelen += char_lengths[i + bytelen];
a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	812 i_assert(i + bytelen <= size);
a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	813
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	814 if (squat_build_add(ctx, uid, data + i, bytelen) < 0)
6911 a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	815 return -1;
a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	816 }
a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	817 return 0;
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	818 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	819
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	820 static unsigned char *
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	821 squat_data_normalize(struct squat_trie trie, const unsigned char data,
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	822 unsigned int size)
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	823 {
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	824 unsigned char *dest;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	825 unsigned int i;
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	826
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	827 dest = t_malloc(size);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	828 for (i = 0; i < size; i++)
6905 8c779f3e16be Fixes to FTS handling. Timo Sirainen <tss@iki.fi> parents: 6902 diff changeset	829 dest[i] = trie->hdr.normalize_map[data[i]];
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	830 return dest;
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	831 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	832
6940 414c9d631a81 Replaced t_push/t_pop calls with T_FRAME() macros. Timo Sirainen <tss@iki.fi>* parents: 6931 diff changeset	833 static int
414c9d631a81 Replaced t_push/t_pop calls with T_FRAME() macros. Timo Sirainen <tss@iki.fi>* parents: 6931 diff changeset	834 squat_trie_build_more_real(struct squat_trie_build_context *ctx,
414c9d631a81 Replaced t_push/t_pop calls with T_FRAME() macros. Timo Sirainen <tss@iki.fi>* parents: 6931 diff changeset	835 uint32_t uid, enum squat_index_type type,
414c9d631a81 Replaced t_push/t_pop calls with T_FRAME() macros. Timo Sirainen <tss@iki.fi>* parents: 6931 diff changeset	836 const unsigned char *input, unsigned int size)
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	837 {
4878 88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	838 struct squat_trie *trie = ctx->trie;
6911 a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	839 const unsigned char *data;
a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	840 uint8_t *char_lengths;
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	841 unsigned int i, start = 0;
6911 a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	842 bool multibyte_chars = FALSE;
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	843 int ret = 0;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	844
6914 9c3f0e180751 Index message's all headers (including MIME headers) after the body. Timo Sirainen <tss@iki.fi> parents: 6912 diff changeset	845 uid = uid * 2 + (type == SQUAT_INDEX_TYPE_HEADER ? 1 : 0);
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	846
6911 a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	847 char_lengths = t_malloc(size);
a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	848 data = squat_data_normalize(trie, input, size);
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	849 for (i = 0; i < size; i++) {
6911 a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	850 char_lengths[i] = uni_utf8_char_bytes(input[i]);
a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	851 if (char_lengths[i] != 1)
a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	852 multibyte_chars = TRUE;
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	853 if (data[i] != '\0')
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	854 continue;
4886 ae2114f11a0e Memory leak fixes. Also when building a large mailbox flush once in a while Timo Sirainen <tss@iki.fi> parents: 4883 diff changeset	855
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	856 while (start < i && data[start] == '\0')
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	857 start++;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	858 if (i != start) {
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	859 if (squat_build_word(ctx, uid, data + start,
6911 a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	860 !multibyte_chars ? NULL :
a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	861 char_lengths + start,
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	862 i - start) < 0) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	863 ret = -1;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	864 start = i;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	865 break;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	866 }
4883 d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	867 }
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	868 start = i + 1;
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	869 }
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	870 while (start < i && data[start] == '\0')
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	871 start++;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	872 if (i != start) {
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	873 if (squat_build_word(ctx, uid, data + start,
6911 a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	874 !multibyte_chars ? NULL :
a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	875 char_lengths + start, i - start) < 0)
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	876 ret = -1;
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	877 }
6940 414c9d631a81 Replaced t_push/t_pop calls with T_FRAME() macros. Timo Sirainen <tss@iki.fi>* parents: 6931 diff changeset	878 return ret;
414c9d631a81 Replaced t_push/t_pop calls with T_FRAME() macros. Timo Sirainen <tss@iki.fi>* parents: 6931 diff changeset	879 }
414c9d631a81 Replaced t_push/t_pop calls with T_FRAME() macros. Timo Sirainen <tss@iki.fi>* parents: 6931 diff changeset	880
414c9d631a81 Replaced t_push/t_pop calls with T_FRAME() macros. Timo Sirainen <tss@iki.fi>* parents: 6931 diff changeset	881 int squat_trie_build_more(struct squat_trie_build_context *ctx,
414c9d631a81 Replaced t_push/t_pop calls with T_FRAME() macros. Timo Sirainen <tss@iki.fi>* parents: 6931 diff changeset	882 uint32_t uid, enum squat_index_type type,
414c9d631a81 Replaced t_push/t_pop calls with T_FRAME() macros. Timo Sirainen <tss@iki.fi>* parents: 6931 diff changeset	883 const unsigned char *input, unsigned int size)
414c9d631a81 Replaced t_push/t_pop calls with T_FRAME() macros. Timo Sirainen <tss@iki.fi>* parents: 6931 diff changeset	884 {
414c9d631a81 Replaced t_push/t_pop calls with T_FRAME() macros. Timo Sirainen <tss@iki.fi>* parents: 6931 diff changeset	885 int ret;
414c9d631a81 Replaced t_push/t_pop calls with T_FRAME() macros. Timo Sirainen <tss@iki.fi>* parents: 6931 diff changeset	886
414c9d631a81 Replaced t_push/t_pop calls with T_FRAME() macros. Timo Sirainen <tss@iki.fi>* parents: 6931 diff changeset	887 T_FRAME(
414c9d631a81 Replaced t_push/t_pop calls with T_FRAME() macros. Timo Sirainen <tss@iki.fi>* parents: 6931 diff changeset	888 ret = squat_trie_build_more_real(ctx, uid, type, input, size);
414c9d631a81 Replaced t_push/t_pop calls with T_FRAME() macros. Timo Sirainen <tss@iki.fi>* parents: 6931 diff changeset	889 );
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	890 return ret;
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	891 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	892
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	893 static void node_drop_unused_children(struct squat_node *node)
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	894 {
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	895 unsigned char *chars;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	896 struct squat_node children_src, children_dest;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	897 unsigned int i, j, orig_child_count = node->child_count;
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	898
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	899 chars = NODE_CHILDREN_CHARS(node);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	900 children_src = NODE_CHILDREN_NODES(node);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	901
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	902 /* move chars */
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	903 for (i = j = 0; i < orig_child_count; i++) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	904 if (children_src[i].next_uid != 0)
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	905 chars[j++] = chars[i];
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	906 }
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	907 node->child_count = j;
4890 cb3b4153136c 64bit and big endian fixes Timo Sirainen <tss@iki.fi> parents: 4887 diff changeset	908
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	909 /* move children. note that children_dest may point to different
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	910 location than children_src, although they both point to the
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	911 same node. */
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	912 children_dest = NODE_CHILDREN_NODES(node);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	913 for (i = j = 0; i < orig_child_count; i++) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	914 if (children_src[i].next_uid != 0)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	915 children_dest[j++] = children_src[i];
4890 cb3b4153136c 64bit and big endian fixes Timo Sirainen <tss@iki.fi> parents: 4887 diff changeset	916 }
cb3b4153136c 64bit and big endian fixes Timo Sirainen <tss@iki.fi> parents: 4887 diff changeset	917 }
cb3b4153136c 64bit and big endian fixes Timo Sirainen <tss@iki.fi> parents: 4887 diff changeset	918
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	919 static int
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	920 squat_write_node(struct squat_trie_build_context ctx, struct squat_node node,
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	921 uoff_t *node_offset_r, int level)
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	922 {
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	923 struct squat_trie *trie = ctx->trie;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	924 struct squat_node *children;
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	925 unsigned int i;
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	926 uoff_t *node_offsets;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	927 uint8_t child_count;
6940 414c9d631a81 Replaced t_push/t_pop calls with T_FRAME() macros. Timo Sirainen <tss@iki.fi>* parents: 6931 diff changeset	928 int ret;
4901 5e604d6f391e Only ASCII texts were actually indexed. Now UTF-8 input is properly Timo Sirainen <tss@iki.fi> parents: 4898 diff changeset	929
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	930 i_assert(node->next_uid != 0);
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	931
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	932 if (node->children_not_mapped && ctx->compress_nodes) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	933 if (node_read_children(trie, node, MAX_FAST_LEVEL) < 0)
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	934 return -1;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	935 }
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	936
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	937 node->have_sequential = FALSE;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	938 node_drop_unused_children(node);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	939
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	940 child_count = node->child_count;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	941 if (child_count == 0) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	942 i_assert(!node->children_not_mapped \|\|
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	943 node->leaf_string_length == 0);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	944 *node_offset_r = !node->children_not_mapped ? 0 :
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	945 node->children.offset;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	946 return 0;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	947 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	948 i_assert(!node->children_not_mapped);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	949
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	950 trie->hdr.node_count++;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	951
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	952 children = NODE_CHILDREN_NODES(node);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	953 node_offsets = t_new(uoff_t, child_count);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	954 for (i = 0; i < child_count; i++) {
6940 414c9d631a81 Replaced t_push/t_pop calls with T_FRAME() macros. Timo Sirainen <tss@iki.fi>* parents: 6931 diff changeset	955 T_FRAME(
414c9d631a81 Replaced t_push/t_pop calls with T_FRAME() macros. Timo Sirainen <tss@iki.fi>* parents: 6931 diff changeset	956 ret = squat_write_node(ctx, &children[i],
414c9d631a81 Replaced t_push/t_pop calls with T_FRAME() macros. Timo Sirainen <tss@iki.fi>* parents: 6931 diff changeset	957 &node_offsets[i], level + 1);
414c9d631a81 Replaced t_push/t_pop calls with T_FRAME() macros. Timo Sirainen <tss@iki.fi>* parents: 6931 diff changeset	958 );
414c9d631a81 Replaced t_push/t_pop calls with T_FRAME() macros. Timo Sirainen <tss@iki.fi>* parents: 6931 diff changeset	959 if (ret < 0)
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	960 return -1;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	961 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	962
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	963 *node_offset_r = ctx->output->offset;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	964 node_write_children(ctx, node, node_offsets);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	965 return 0;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	966 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	967
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	968 static int squat_write_nodes(struct squat_trie_build_context *ctx)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	969 {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	970 struct squat_trie *trie = ctx->trie;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	971 uoff_t node_offset;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	972 int ret;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	973
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	974 if (ctx->trie->root.next_uid == 0)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	975 return 0;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	976
6940 414c9d631a81 Replaced t_push/t_pop calls with T_FRAME() macros. Timo Sirainen <tss@iki.fi>* parents: 6931 diff changeset	977 T_FRAME(
414c9d631a81 Replaced t_push/t_pop calls with T_FRAME() macros. Timo Sirainen <tss@iki.fi>* parents: 6931 diff changeset	978 ret = squat_write_node(ctx, &ctx->trie->root, &node_offset, 0);
414c9d631a81 Replaced t_push/t_pop calls with T_FRAME() macros. Timo Sirainen <tss@iki.fi>* parents: 6931 diff changeset	979 );
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	980 if (ret < 0)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	981 return -1;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	982
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	983 trie->hdr.root_offset = node_offset;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	984 trie->hdr.root_unused_uids = trie->root.unused_uids;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	985 trie->hdr.root_next_uid = trie->root.next_uid;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	986 trie->hdr.root_uidlist_idx = trie->root.uid_list_idx;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	987 return 0;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	988 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	989
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	990 static struct squat_trie_iterate_context *
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	991 squat_trie_iterate_uidlist_init(struct squat_trie *trie)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	992 {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	993 struct squat_trie_iterate_context *ctx;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	994
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	995 ctx = i_new(struct squat_trie_iterate_context, 1);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	996 ctx->trie = trie;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	997 ctx->cur.node = &trie->root;
6905 8c779f3e16be Fixes to FTS handling. Timo Sirainen <tss@iki.fi> parents: 6902 diff changeset	998 i_array_init(&ctx->parents, trie->hdr.partial_len*2);
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	999 return ctx;
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1000 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1001
4878 88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1002 static int
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1003 squat_trie_iterate_uidlist_deinit(struct squat_trie_iterate_context *ctx)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1004 {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1005 int ret = ctx->failed ? -1 : 0;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1006
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1007 array_free(&ctx->parents);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1008 i_free(ctx);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1009 return ret;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1010 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1011
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1012 static struct squat_node *
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1013 squat_trie_iterate_uidlist_first(struct squat_trie_iterate_context *ctx)
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1014 {
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1015 struct squat_node *node = ctx->cur.node;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1016 int level;
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1017
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1018 if (UIDLIST_IS_SINGLETON(node->uid_list_idx)) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1019 /* no uidlists */
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1020 i_assert(node == &ctx->trie->root);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1021 return NULL;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1022 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1023
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1024 if (node->children_not_mapped) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1025 level = array_count(&ctx->parents);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1026
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1027 if (node_read_children(ctx->trie, node, level) < 0) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1028 ctx->failed = TRUE;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1029 return NULL;
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1030 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1031 }
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1032 return node;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1033 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1034
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1035 static struct squat_node *
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1036 squat_trie_iterate_uidlist_next(struct squat_trie_iterate_context *ctx)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1037 {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1038 struct squat_trie_iterate_node *iter_nodes;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1039 struct squat_node *node = ctx->cur.node;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1040 struct squat_node *children;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1041 unsigned int count;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1042
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1043 /* return our children first */
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1044 children = NODE_CHILDREN_NODES(node);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1045 for (; ctx->cur.idx < node->child_count; ctx->cur.idx++) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1046 if (!UIDLIST_IS_SINGLETON(children[ctx->cur.idx].uid_list_idx))
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1047 return &children[ctx->cur.idx++];
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1048 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1049
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1050 ctx->cur.idx = 0;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1051 for (;;) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1052 /* next start iterating our childrens' children */
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1053 while (ctx->cur.idx < node->child_count) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1054 uint32_t list_idx =
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1055 children[ctx->cur.idx++].uid_list_idx;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1056
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1057 if (UIDLIST_IS_SINGLETON(list_idx))
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1058 continue;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1059
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1060 array_append(&ctx->parents, &ctx->cur, 1);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1061 ctx->cur.node = &children[ctx->cur.idx-1];
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1062 ctx->cur.idx = 0;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1063 if (squat_trie_iterate_uidlist_first(ctx) == NULL)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1064 return NULL;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1065 return squat_trie_iterate_uidlist_next(ctx);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1066 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1067
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1068 /* no more children. go to next sibling's children */
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1069 iter_nodes = array_get_modifiable(&ctx->parents, &count);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1070 if (count == 0)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1071 return NULL;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1072
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1073 ctx->cur = iter_nodes[count-1];
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1074 array_delete(&ctx->parents, count-1, 1);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1075
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1076 node = ctx->cur.node;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1077 children = NODE_CHILDREN_NODES(node);
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1078 }
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1079 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1080
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	1081 static int
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	1082 squat_trie_renumber_uidlists(struct squat_trie_build_context *ctx,
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	1083 bool compress)
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1084 {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1085 struct squat_trie_iterate_context *iter;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1086 struct squat_node *node;
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	1087 struct squat_uidlist_rebuild_context *rebuild_ctx;
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1088 ARRAY_TYPE(uint32_t) uids;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1089 uint32_t new_uid_list_idx, max_count=0;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1090 int ret = 0;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1091
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	1092 /* FIXME: update indexid */
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	1093 if ((ret = squat_uidlist_rebuild_init(ctx->uidlist_build_ctx,
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	1094 compress, &rebuild_ctx)) <= 0)
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1095 return ret;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1096
7043 7708a8c166d6 Compiler warning fixes on 32bit systems. Timo Sirainen <tss@iki.fi> parents: 6945 diff changeset	1097 ctx->trie->hdr.indexid =
7708a8c166d6 Compiler warning fixes on 32bit systems. Timo Sirainen <tss@iki.fi> parents: 6945 diff changeset	1098 I_MAX((unsigned int)ioloop_time, ctx->trie->hdr.indexid + 1);
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	1099
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1100 i_array_init(&uids, 1024);
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	1101 iter = squat_trie_iterate_uidlist_init(ctx->trie);
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1102 node = squat_trie_iterate_uidlist_first(iter);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1103 new_uid_list_idx = 0x100;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1104 while (node != NULL) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1105 array_clear(&uids);
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	1106 if (squat_uidlist_get(ctx->trie->uidlist, node->uid_list_idx,
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	1107 &uids) < 0) {
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1108 ret = -1;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1109 break;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1110 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1111 max_count = I_MAX(max_count, array_count(&uids));
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	1112 squat_uidlist_rebuild_next(rebuild_ctx, &uids);
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1113 node->uid_list_idx = new_uid_list_idx << 1;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1114 new_uid_list_idx++;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1115
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1116 node = squat_trie_iterate_uidlist_next(iter);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1117 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1118 squat_trie_iterate_uidlist_deinit(iter);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1119 array_free(&uids);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1120
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	1121 /* lock the trie before we rename uidlist */
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	1122 if (squat_trie_lock(ctx->trie, F_WRLCK, &ctx->file_lock) <= 0)
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	1123 ret = -1;
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	1124 return squat_uidlist_rebuild_finish(rebuild_ctx, ret < 0);
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1125 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1126
6905 8c779f3e16be Fixes to FTS handling. Timo Sirainen <tss@iki.fi> parents: 6902 diff changeset	1127 static bool squat_trie_check_header(struct squat_trie *trie)
8c779f3e16be Fixes to FTS handling. Timo Sirainen <tss@iki.fi> parents: 6902 diff changeset	1128 {
8c779f3e16be Fixes to FTS handling. Timo Sirainen <tss@iki.fi> parents: 6902 diff changeset	1129 if (trie->hdr.version != SQUAT_TRIE_VERSION \|\|
8c779f3e16be Fixes to FTS handling. Timo Sirainen <tss@iki.fi> parents: 6902 diff changeset	1130 trie->hdr.uidvalidity != trie->uidvalidity)
8c779f3e16be Fixes to FTS handling. Timo Sirainen <tss@iki.fi> parents: 6902 diff changeset	1131 return FALSE;
8c779f3e16be Fixes to FTS handling. Timo Sirainen <tss@iki.fi> parents: 6902 diff changeset	1132
8c779f3e16be Fixes to FTS handling. Timo Sirainen <tss@iki.fi> parents: 6902 diff changeset	1133 if (trie->hdr.partial_len > trie->hdr.full_len) {
8c779f3e16be Fixes to FTS handling. Timo Sirainen <tss@iki.fi> parents: 6902 diff changeset	1134 i_error("Corrupted %s: partial len > full len", trie->path);
8c779f3e16be Fixes to FTS handling. Timo Sirainen <tss@iki.fi> parents: 6902 diff changeset	1135 return FALSE;
8c779f3e16be Fixes to FTS handling. Timo Sirainen <tss@iki.fi> parents: 6902 diff changeset	1136 }
8c779f3e16be Fixes to FTS handling. Timo Sirainen <tss@iki.fi> parents: 6902 diff changeset	1137 if (trie->hdr.full_len == 0) {
8c779f3e16be Fixes to FTS handling. Timo Sirainen <tss@iki.fi> parents: 6902 diff changeset	1138 i_error("Corrupted %s: full len=0", trie->path);
8c779f3e16be Fixes to FTS handling. Timo Sirainen <tss@iki.fi> parents: 6902 diff changeset	1139 return FALSE;
8c779f3e16be Fixes to FTS handling. Timo Sirainen <tss@iki.fi> parents: 6902 diff changeset	1140 }
8c779f3e16be Fixes to FTS handling. Timo Sirainen <tss@iki.fi> parents: 6902 diff changeset	1141 return TRUE;
8c779f3e16be Fixes to FTS handling. Timo Sirainen <tss@iki.fi> parents: 6902 diff changeset	1142 }
8c779f3e16be Fixes to FTS handling. Timo Sirainen <tss@iki.fi> parents: 6902 diff changeset	1143
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1144 static int squat_trie_map_header(struct squat_trie *trie)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1145 {
6944 66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	1146 int ret;
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	1147
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1148 if (trie->locked_file_size == 0) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1149 /* newly created file */
6944 66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	1150 return 1;
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1151 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1152 i_assert(trie->fd != -1);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1153
6944 66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	1154 if (trie->mmap_disable) {
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	1155 ret = pread_full(trie->fd, &trie->hdr, sizeof(trie->hdr), 0);
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	1156 if (ret <= 0) {
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	1157 if (ret < 0) {
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	1158 i_error("pread(%s) failed: %m", trie->path);
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	1159 return -1;
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	1160 }
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	1161 i_error("Corrupted %s: File too small", trie->path);
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	1162 return 0;
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	1163 }
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	1164 trie->data = NULL;
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	1165 trie->data_size = 0;
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	1166 } else {
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	1167 if (trie->locked_file_size < sizeof(trie->hdr)) {
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	1168 i_error("Corrupted %s: File too small", trie->path);
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	1169 return 0;
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	1170 }
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	1171 if (trie->mmap_size != 0) {
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	1172 if (munmap(trie->mmap_base, trie->mmap_size) < 0)
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	1173 i_error("munmap(%s) failed: %m", trie->path);
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	1174 }
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	1175
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	1176 trie->mmap_size = trie->locked_file_size;
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	1177 trie->mmap_base = mmap(NULL, trie->mmap_size,
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	1178 PROT_READ \| PROT_WRITE,
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	1179 MAP_SHARED, trie->fd, 0);
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	1180 if (trie->mmap_base == MAP_FAILED) {
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	1181 trie->data = trie->mmap_base = NULL;
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	1182 trie->data_size = trie->mmap_size = 0;
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	1183 i_error("mmap(%s) failed: %m", trie->path);
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	1184 return -1;
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	1185 }
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	1186 memcpy(&trie->hdr, trie->mmap_base, sizeof(trie->hdr));
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	1187 trie->data = trie->mmap_base;
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	1188 trie->data_size = trie->mmap_size;
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1189 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1190
6944 66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	1191 return squat_trie_check_header(trie) ? 1 : 0;
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1192 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1193
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	1194 static int squat_trie_map(struct squat_trie *trie, bool building)
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1195 {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1196 struct file_lock *file_lock = NULL;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1197 bool changed;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1198 int ret;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1199
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1200 if (trie->fd != -1) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1201 if (squat_trie_lock(trie, F_RDLCK, &file_lock) <= 0)
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1202 return -1;
6944 66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	1203 if (trie->mmap_disable && trie->file_cache == NULL)
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	1204 trie->file_cache = file_cache_new(trie->fd);
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1205 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1206
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1207 ret = squat_trie_map_header(trie);
6944 66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	1208 if (ret == 0) {
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	1209 file_lock_free(&file_lock);
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	1210 squat_trie_delete(trie);
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	1211 squat_trie_close(trie);
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	1212 squat_trie_header_init(trie);
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	1213 }
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1214 changed = trie->root.children.offset != trie->hdr.root_offset;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1215
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1216 if (changed \|\| trie->hdr.root_offset == 0) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1217 memset(&trie->root, 0, sizeof(trie->root));
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1218 trie->root.want_sequential = TRUE;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1219 trie->root.unused_uids = trie->hdr.root_unused_uids;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1220 trie->root.next_uid = trie->hdr.root_next_uid;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1221 trie->root.uid_list_idx = trie->hdr.root_uidlist_idx;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1222 trie->root.children.offset = trie->hdr.root_offset;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1223
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1224 if (trie->hdr.root_offset == 0) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1225 trie->unmapped_child_count = 0;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1226 trie->root.children_not_mapped = FALSE;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1227 } else {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1228 trie->unmapped_child_count = 1;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1229 trie->root.children_not_mapped = TRUE;
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1230 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1231 }
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1232
6944 66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	1233 if (ret >= 0 && !building) {
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	1234 /* do this while we're still locked */
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	1235 ret = squat_uidlist_refresh(trie->uidlist);
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	1236 }
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	1237
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1238 if (file_lock != NULL)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1239 file_unlock(&file_lock);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1240 if (ret < 0)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1241 return -1;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1242
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1243 return trie->hdr.root_offset == 0 \|\| !changed ? 0 :
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1244 node_read_children(trie, &trie->root, 1);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1245 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1246
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1247 int squat_trie_build_init(struct squat_trie trie, uint32_t last_uid_r,
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1248 struct squat_trie_build_context **ctx_r)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1249 {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1250 struct squat_trie_build_context *ctx;
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	1251 struct squat_uidlist_build_context *uidlist_build_ctx;
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1252
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1253 if (trie->fd == -1) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1254 trie->fd = open(trie->path, O_RDWR \| O_CREAT, 0600);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1255 if (trie->fd == -1) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1256 i_error("creat(%s) failed: %m", trie->path);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1257 return -1;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1258 }
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1259 }
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1260
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1261 /* uidlist locks building */
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	1262 if (squat_uidlist_build_init(trie->uidlist, &uidlist_build_ctx) < 0)
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1263 return -1;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1264
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	1265 if (squat_trie_map(trie, TRUE) < 0) {
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	1266 squat_uidlist_build_deinit(&uidlist_build_ctx);
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1267 return -1;
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	1268 }
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1269
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1270 ctx = i_new(struct squat_trie_build_context, 1);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1271 ctx->trie = trie;
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	1272 ctx->uidlist_build_ctx = uidlist_build_ctx;
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1273 ctx->first_uid = trie->root.next_uid;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1274
6905 8c779f3e16be Fixes to FTS handling. Timo Sirainen <tss@iki.fi> parents: 6902 diff changeset	1275 *last_uid_r = I_MAX((trie->root.next_uid+1)/2, 1) - 1;
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1276 *ctx_r = ctx;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1277 return 0;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1278 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1279
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	1280 static int squat_trie_write_lock(struct squat_trie_build_context *ctx)
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	1281 {
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	1282 if (ctx->file_lock != NULL)
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	1283 return 0;
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	1284
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	1285 if (squat_trie_lock(ctx->trie, F_WRLCK, &ctx->file_lock) <= 0)
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	1286 return -1;
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	1287 return 0;
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	1288 }
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	1289
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1290 static int squat_trie_write(struct squat_trie_build_context *ctx)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1291 {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1292 struct squat_trie *trie = ctx->trie;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1293 struct ostream *output;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1294 const char *path;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1295 int fd = -1, ret = 0;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1296
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1297 if ((trie->hdr.used_file_size > sizeof(trie->hdr) &&
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1298 trie->unmapped_child_count < trie->hdr.node_count/4) \|\| 1) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1299 /* we might as well recreate the file */
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1300 ctx->compress_nodes = TRUE;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1301
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1302 path = t_strconcat(trie->path, ".tmp", NULL);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1303 fd = open(path, O_RDWR \| O_CREAT, 0600);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1304 if (fd == -1) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1305 i_error("creat(%s) failed: %m", path);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1306 return -1;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1307 }
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	1308 ret = file_wait_lock(fd, path, F_WRLCK, trie->lock_method,
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	1309 SQUAT_TRIE_LOCK_TIMEOUT, &ctx->file_lock);
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	1310 if (ret <= 0) {
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	1311 if (ret == 0)
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	1312 i_error("file_wait_lock(%s) failed: %m", path);
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	1313 (void)close(fd);
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	1314 return -1;
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	1315 }
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	1316
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1317 output = o_stream_create_fd(fd, 0, FALSE);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1318 o_stream_cork(output);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1319 o_stream_send(output, &trie->hdr, sizeof(trie->hdr));
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1320 } else {
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	1321 /* we need to lock only while header is being written */
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1322 path = trie->path;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1323 ctx->compress_nodes =
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1324 trie->hdr.used_file_size == sizeof(trie->hdr);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1325 output = o_stream_create_fd(trie->fd, 0, FALSE);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1326 o_stream_cork(output);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1327
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	1328 if (trie->hdr.used_file_size != 0)
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	1329 o_stream_seek(output, trie->hdr.used_file_size);
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	1330 else {
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	1331 if (squat_trie_write_lock(ctx) < 0) {
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	1332 o_stream_unref(&output);
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	1333 return -1;
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	1334 }
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1335 o_stream_send(output, &trie->hdr, sizeof(trie->hdr));
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	1336 }
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1337 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1338
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1339 ctx->output = output;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1340 ret = squat_write_nodes(ctx);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1341 ctx->output = NULL;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1342
6944 66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	1343 /* write 1 byte guard at the end of file, so that we can verify broken
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	1344 squat_unpack_num() input by checking if data==end */
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	1345 o_stream_send(output, "", 1);
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	1346
6905 8c779f3e16be Fixes to FTS handling. Timo Sirainen <tss@iki.fi> parents: 6902 diff changeset	1347 if (trie->corrupted)
8c779f3e16be Fixes to FTS handling. Timo Sirainen <tss@iki.fi> parents: 6902 diff changeset	1348 ret = -1;
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	1349 if (ret == 0)
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	1350 ret = squat_trie_write_lock(ctx);
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1351 if (ret == 0) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1352 trie->hdr.used_file_size = output->offset;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1353 o_stream_seek(output, 0);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1354 o_stream_send(output, &trie->hdr, sizeof(trie->hdr));
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1355 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1356 if (output->last_failed_errno != 0) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1357 errno = output->last_failed_errno;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1358 i_error("write() to %s failed: %m", path);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1359 ret = -1;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1360 }
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	1361 o_stream_destroy(&output);
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	1362
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	1363 if (fd == -1) {
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	1364 /* appended to the existing file */
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	1365 return ret;
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	1366 }
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1367
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	1368 /* recreating the trie file */
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	1369 if (ret < 0) {
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	1370 if (close(fd) < 0)
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	1371 i_error("close(%s) failed: %m", path);
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	1372 fd = -1;
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	1373 } else if (rename(path, trie->path) < 0) {
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	1374 i_error("rename(%s, %s) failed: %m", path, trie->path);
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	1375 ret = -1;
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	1376 }
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	1377
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	1378 if (ret < 0) {
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	1379 if (unlink(path) < 0 && errno != ENOENT)
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	1380 i_error("unlink(%s) failed: %m", path);
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	1381 } else {
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	1382 if (trie->fd != -1) {
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1383 if (close(trie->fd) < 0)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1384 i_error("close(%s) failed: %m", trie->path);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1385 }
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	1386 trie->fd = fd;
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1387 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1388 return ret;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1389 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1390
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1391 int squat_trie_build_deinit(struct squat_trie_build_context **_ctx)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1392 {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1393 struct squat_trie_build_context ctx = _ctx;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1394 bool compress;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1395 int ret;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1396
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1397 *_ctx = NULL;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1398
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1399 compress = (ctx->trie->root.next_uid - ctx->first_uid) > 10;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1400
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	1401 /* keep trie locked while header is being written and when files are
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	1402 being renamed, so that while trie is read locked, uidlist can't
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	1403 change under. */
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	1404 squat_uidlist_build_flush(ctx->uidlist_build_ctx);
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	1405 ret = squat_trie_renumber_uidlists(ctx, compress);
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	1406 if (ret == 0)
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1407 ret = squat_trie_write(ctx);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1408
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	1409 if (ret == 0)
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	1410 ret = squat_uidlist_build_finish(ctx->uidlist_build_ctx);
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	1411 if (ctx->file_lock != NULL)
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	1412 file_unlock(&ctx->file_lock);
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	1413 squat_uidlist_build_deinit(&ctx->uidlist_build_ctx);
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	1414
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1415 i_free(ctx);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1416 return ret;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1417 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1418
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1419 int squat_trie_get_last_uid(struct squat_trie trie, uint32_t last_uid_r)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1420 {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1421 if (trie->fd == -1) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1422 if (squat_trie_open(trie) < 0)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1423 return -1;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1424 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1425
6905 8c779f3e16be Fixes to FTS handling. Timo Sirainen <tss@iki.fi> parents: 6902 diff changeset	1426 *last_uid_r = I_MAX((trie->root.next_uid+1)/2, 1) - 1;
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1427 return 0;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1428 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1429
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1430 static int
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1431 squat_trie_lookup_data(struct squat_trie trie, const unsigned char data,
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1432 unsigned int size, ARRAY_TYPE(seq_range) *uids)
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1433 {
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1434 struct squat_node *node = &trie->root;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1435 unsigned char *chars;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1436 unsigned int idx;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1437 int level = 0;
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1438
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1439 array_clear(uids);
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1440
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1441 for (;;) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1442 if (node->children_not_mapped) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1443 if (node_read_children(trie, node, level) < 0)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1444 return -1;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1445 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1446 if (node->leaf_string_length != 0) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1447 unsigned int str_len = node->leaf_string_length;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1448 const unsigned char *str;
5414 3ec03a4c0f26 fixes Timo Sirainen <tss@iki.fi> parents: 5336 diff changeset	1449
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1450 if (str_len > sizeof(node->children.static_leaf_string))
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1451 str = node->children.leaf_string;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1452 else
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1453 str = node->children.static_leaf_string;
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1454
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1455 if (size > str_len \|\| memcmp(data, str, size) != 0)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1456 return 0;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1457
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1458 /* match */
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1459 break;
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1460 }
4898 07a038b57946 Optimization. Timo Sirainen <tss@iki.fi> parents: 4897 diff changeset	1461
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1462 if (size == 0)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1463 break;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1464 level++;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1465
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1466 if (node->have_sequential) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1467 if (*data < SEQUENTIAL_COUNT) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1468 idx = *data;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1469 goto found;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1470 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1471 idx = SEQUENTIAL_COUNT;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1472 } else {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1473 idx = 0;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1474 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1475 chars = NODE_CHILDREN_CHARS(node);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1476 for (; idx < node->child_count; idx++) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1477 if (chars[idx] == *data)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1478 goto found;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1479 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1480 return 0;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1481 found:
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1482 /* follow to children */
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1483 if (level == 1) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1484 /* root level, add all UIDs */
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1485 if (squat_uidlist_get_seqrange(trie->uidlist,
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1486 node->uid_list_idx,
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1487 uids) < 0)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1488 return -1;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1489 } else {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1490 if (squat_uidlist_filter(trie->uidlist,
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1491 node->uid_list_idx, uids) < 0)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1492 return -1;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1493 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1494 data++;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1495 size--;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1496 node = NODE_CHILDREN_NODES(node) + idx;
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1497 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1498
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1499 if (squat_uidlist_filter(trie->uidlist, node->uid_list_idx, uids) < 0)
4878 88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1500 return -1;
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1501 return 1;
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1502 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1503
4878 88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1504 static void
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1505 squat_trie_filter_type(enum squat_index_type type,
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1506 const ARRAY_TYPE(seq_range) *src,
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1507 ARRAY_TYPE(seq_range) *dest)
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1508 {
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1509 const struct seq_range *src_range;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1510 struct seq_range new_range;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1511 unsigned int i, count, mask;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1512 uint32_t next_seq, uid;
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1513
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1514 array_clear(dest);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1515 src_range = array_get(src, &count);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1516 if (count == 0)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1517 return;
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1518
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1519 if ((type & SQUAT_INDEX_TYPE_HEADER) != 0 &&
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1520 (type & SQUAT_INDEX_TYPE_BODY) != 0) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1521 /* everything is fine, just fix the UIDs */
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1522 new_range.seq1 = src_range[0].seq1 / 2;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1523 new_range.seq2 = src_range[0].seq2 / 2;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1524 for (i = 1; i < count; i++) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1525 next_seq = src_range[i].seq1 / 2;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1526 if (next_seq == new_range.seq2 + 1) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1527 /* we can continue the previous range */
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1528 } else {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1529 array_append(dest, &new_range, 1);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1530 new_range.seq1 = src_range[i].seq1 / 2;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1531 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1532 new_range.seq2 = src_range[i].seq2 / 2;
4878 88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1533 }
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1534 array_append(dest, &new_range, 1);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1535 return;
4886 ae2114f11a0e Memory leak fixes. Also when building a large mailbox flush once in a while Timo Sirainen <tss@iki.fi> parents: 4883 diff changeset	1536 }
ae2114f11a0e Memory leak fixes. Also when building a large mailbox flush once in a while Timo Sirainen <tss@iki.fi> parents: 4883 diff changeset	1537
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1538 /* we'll have to drop either header or body UIDs */
6914 9c3f0e180751 Index message's all headers (including MIME headers) after the body. Timo Sirainen <tss@iki.fi> parents: 6912 diff changeset	1539 mask = (type & SQUAT_INDEX_TYPE_HEADER) != 0 ? 1 : 0;
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1540 for (i = 0; i < count; i++) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1541 for (uid = src_range[i].seq1; uid <= src_range[i].seq2; uid++) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1542 if ((uid & 1) == mask)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1543 seq_range_array_add(dest, 0, uid/2);
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1544 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1545 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1546 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1547
6902 cff5428c3c4d Fixed handling non-indexed characters in search key. Timo Sirainen <tss@iki.fi> parents: 6898 diff changeset	1548 struct squat_trie_lookup_context {
cff5428c3c4d Fixed handling non-indexed characters in search key. Timo Sirainen <tss@iki.fi> parents: 6898 diff changeset	1549 struct squat_trie *trie;
cff5428c3c4d Fixed handling non-indexed characters in search key. Timo Sirainen <tss@iki.fi> parents: 6898 diff changeset	1550 enum squat_index_type type;
cff5428c3c4d Fixed handling non-indexed characters in search key. Timo Sirainen <tss@iki.fi> parents: 6898 diff changeset	1551
cff5428c3c4d Fixed handling non-indexed characters in search key. Timo Sirainen <tss@iki.fi> parents: 6898 diff changeset	1552 ARRAY_TYPE(seq_range) definite_uids, maybe_uids;
cff5428c3c4d Fixed handling non-indexed characters in search key. Timo Sirainen <tss@iki.fi> parents: 6898 diff changeset	1553 ARRAY_TYPE(seq_range) tmp_uids, tmp_uids2;
cff5428c3c4d Fixed handling non-indexed characters in search key. Timo Sirainen <tss@iki.fi> parents: 6898 diff changeset	1554 bool first;
cff5428c3c4d Fixed handling non-indexed characters in search key. Timo Sirainen <tss@iki.fi> parents: 6898 diff changeset	1555 };
cff5428c3c4d Fixed handling non-indexed characters in search key. Timo Sirainen <tss@iki.fi> parents: 6898 diff changeset	1556
cff5428c3c4d Fixed handling non-indexed characters in search key. Timo Sirainen <tss@iki.fi> parents: 6898 diff changeset	1557 static int
cff5428c3c4d Fixed handling non-indexed characters in search key. Timo Sirainen <tss@iki.fi> parents: 6898 diff changeset	1558 squat_trie_lookup_partial(struct squat_trie_lookup_context *ctx,
6911 a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	1559 const unsigned char data, uint8_t char_lengths,
a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	1560 unsigned int size)
6902 cff5428c3c4d Fixed handling non-indexed characters in search key. Timo Sirainen <tss@iki.fi> parents: 6898 diff changeset	1561 {
6911 a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	1562 const unsigned int partial_len = ctx->trie->hdr.partial_len;
a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	1563 unsigned int char_idx, max_chars, i, j, bytelen;
6902 cff5428c3c4d Fixed handling non-indexed characters in search key. Timo Sirainen <tss@iki.fi> parents: 6898 diff changeset	1564 int ret;
cff5428c3c4d Fixed handling non-indexed characters in search key. Timo Sirainen <tss@iki.fi> parents: 6898 diff changeset	1565
6911 a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	1566 max_chars = uni_utf8_strlen_n(data, size);
a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	1567 if (max_chars > ctx->trie->hdr.partial_len)
a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	1568 max_chars = partial_len;
6902 cff5428c3c4d Fixed handling non-indexed characters in search key. Timo Sirainen <tss@iki.fi> parents: 6898 diff changeset	1569
6911 a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	1570 for (i = 0, char_idx = 0; char_idx < max_chars; char_idx++) {
a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	1571 bytelen = 0;
a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	1572 for (j = 0; j < partial_len && i+bytelen < size; j++)
a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	1573 bytelen += char_lengths[i + bytelen];
a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	1574
a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	1575 ret = squat_trie_lookup_data(ctx->trie, data + i, bytelen,
6902 cff5428c3c4d Fixed handling non-indexed characters in search key. Timo Sirainen <tss@iki.fi> parents: 6898 diff changeset	1576 &ctx->tmp_uids);
cff5428c3c4d Fixed handling non-indexed characters in search key. Timo Sirainen <tss@iki.fi> parents: 6898 diff changeset	1577 if (ret <= 0) {
cff5428c3c4d Fixed handling non-indexed characters in search key. Timo Sirainen <tss@iki.fi> parents: 6898 diff changeset	1578 array_clear(ctx->maybe_uids);
cff5428c3c4d Fixed handling non-indexed characters in search key. Timo Sirainen <tss@iki.fi> parents: 6898 diff changeset	1579 return ret;
cff5428c3c4d Fixed handling non-indexed characters in search key. Timo Sirainen <tss@iki.fi> parents: 6898 diff changeset	1580 }
cff5428c3c4d Fixed handling non-indexed characters in search key. Timo Sirainen <tss@iki.fi> parents: 6898 diff changeset	1581
cff5428c3c4d Fixed handling non-indexed characters in search key. Timo Sirainen <tss@iki.fi> parents: 6898 diff changeset	1582 if (ctx->first) {
cff5428c3c4d Fixed handling non-indexed characters in search key. Timo Sirainen <tss@iki.fi> parents: 6898 diff changeset	1583 squat_trie_filter_type(ctx->type, &ctx->tmp_uids,
cff5428c3c4d Fixed handling non-indexed characters in search key. Timo Sirainen <tss@iki.fi> parents: 6898 diff changeset	1584 ctx->maybe_uids);
cff5428c3c4d Fixed handling non-indexed characters in search key. Timo Sirainen <tss@iki.fi> parents: 6898 diff changeset	1585 ctx->first = FALSE;
cff5428c3c4d Fixed handling non-indexed characters in search key. Timo Sirainen <tss@iki.fi> parents: 6898 diff changeset	1586 } else {
cff5428c3c4d Fixed handling non-indexed characters in search key. Timo Sirainen <tss@iki.fi> parents: 6898 diff changeset	1587 squat_trie_filter_type(ctx->type, &ctx->tmp_uids,
cff5428c3c4d Fixed handling non-indexed characters in search key. Timo Sirainen <tss@iki.fi> parents: 6898 diff changeset	1588 &ctx->tmp_uids2);
cff5428c3c4d Fixed handling non-indexed characters in search key. Timo Sirainen <tss@iki.fi> parents: 6898 diff changeset	1589 seq_range_array_remove_invert_range(ctx->maybe_uids,
cff5428c3c4d Fixed handling non-indexed characters in search key. Timo Sirainen <tss@iki.fi> parents: 6898 diff changeset	1590 &ctx->tmp_uids2);
cff5428c3c4d Fixed handling non-indexed characters in search key. Timo Sirainen <tss@iki.fi> parents: 6898 diff changeset	1591 }
6911 a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	1592
a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	1593 i += char_lengths[i];
a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	1594 }
6902 cff5428c3c4d Fixed handling non-indexed characters in search key. Timo Sirainen <tss@iki.fi> parents: 6898 diff changeset	1595 return 1;
cff5428c3c4d Fixed handling non-indexed characters in search key. Timo Sirainen <tss@iki.fi> parents: 6898 diff changeset	1596 }
cff5428c3c4d Fixed handling non-indexed characters in search key. Timo Sirainen <tss@iki.fi> parents: 6898 diff changeset	1597
6905 8c779f3e16be Fixes to FTS handling. Timo Sirainen <tss@iki.fi> parents: 6902 diff changeset	1598 static void squat_trie_add_unknown(struct squat_trie *trie,
8c779f3e16be Fixes to FTS handling. Timo Sirainen <tss@iki.fi> parents: 6902 diff changeset	1599 ARRAY_TYPE(seq_range) *maybe_uids)
8c779f3e16be Fixes to FTS handling. Timo Sirainen <tss@iki.fi> parents: 6902 diff changeset	1600 {
8c779f3e16be Fixes to FTS handling. Timo Sirainen <tss@iki.fi> parents: 6902 diff changeset	1601 struct seq_range *range, new_range;
8c779f3e16be Fixes to FTS handling. Timo Sirainen <tss@iki.fi> parents: 6902 diff changeset	1602 unsigned int count;
8c779f3e16be Fixes to FTS handling. Timo Sirainen <tss@iki.fi> parents: 6902 diff changeset	1603 uint32_t last_uid;
8c779f3e16be Fixes to FTS handling. Timo Sirainen <tss@iki.fi> parents: 6902 diff changeset	1604
8c779f3e16be Fixes to FTS handling. Timo Sirainen <tss@iki.fi> parents: 6902 diff changeset	1605 last_uid = I_MAX((trie->root.next_uid+1)/2, 1) - 1;
8c779f3e16be Fixes to FTS handling. Timo Sirainen <tss@iki.fi> parents: 6902 diff changeset	1606
8c779f3e16be Fixes to FTS handling. Timo Sirainen <tss@iki.fi> parents: 6902 diff changeset	1607 range = array_get_modifiable(maybe_uids, &count);
8c779f3e16be Fixes to FTS handling. Timo Sirainen <tss@iki.fi> parents: 6902 diff changeset	1608 if (count > 0 && range[count-1].seq2 == last_uid) {
8c779f3e16be Fixes to FTS handling. Timo Sirainen <tss@iki.fi> parents: 6902 diff changeset	1609 /* increase the range */
8c779f3e16be Fixes to FTS handling. Timo Sirainen <tss@iki.fi> parents: 6902 diff changeset	1610 range[count-1].seq2 = (uint32_t)-1;
8c779f3e16be Fixes to FTS handling. Timo Sirainen <tss@iki.fi> parents: 6902 diff changeset	1611 } else {
8c779f3e16be Fixes to FTS handling. Timo Sirainen <tss@iki.fi> parents: 6902 diff changeset	1612 new_range.seq1 = last_uid + 1;
8c779f3e16be Fixes to FTS handling. Timo Sirainen <tss@iki.fi> parents: 6902 diff changeset	1613 new_range.seq2 = (uint32_t)-1;
8c779f3e16be Fixes to FTS handling. Timo Sirainen <tss@iki.fi> parents: 6902 diff changeset	1614 array_append(maybe_uids, &new_range, 1);
8c779f3e16be Fixes to FTS handling. Timo Sirainen <tss@iki.fi> parents: 6902 diff changeset	1615 }
8c779f3e16be Fixes to FTS handling. Timo Sirainen <tss@iki.fi> parents: 6902 diff changeset	1616 }
8c779f3e16be Fixes to FTS handling. Timo Sirainen <tss@iki.fi> parents: 6902 diff changeset	1617
6940 414c9d631a81 Replaced t_push/t_pop calls with T_FRAME() macros. Timo Sirainen <tss@iki.fi>* parents: 6931 diff changeset	1618 static int
414c9d631a81 Replaced t_push/t_pop calls with T_FRAME() macros. Timo Sirainen <tss@iki.fi>* parents: 6931 diff changeset	1619 squat_trie_lookup_real(struct squat_trie trie, const char str,
414c9d631a81 Replaced t_push/t_pop calls with T_FRAME() macros. Timo Sirainen <tss@iki.fi>* parents: 6931 diff changeset	1620 enum squat_index_type type,
414c9d631a81 Replaced t_push/t_pop calls with T_FRAME() macros. Timo Sirainen <tss@iki.fi>* parents: 6931 diff changeset	1621 ARRAY_TYPE(seq_range) *definite_uids,
414c9d631a81 Replaced t_push/t_pop calls with T_FRAME() macros. Timo Sirainen <tss@iki.fi>* parents: 6931 diff changeset	1622 ARRAY_TYPE(seq_range) *maybe_uids)
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1623 {
6902 cff5428c3c4d Fixed handling non-indexed characters in search key. Timo Sirainen <tss@iki.fi> parents: 6898 diff changeset	1624 struct squat_trie_lookup_context ctx;
cff5428c3c4d Fixed handling non-indexed characters in search key. Timo Sirainen <tss@iki.fi> parents: 6898 diff changeset	1625 unsigned char *data;
6911 a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	1626 uint8_t *char_lengths;
a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	1627 unsigned int i, start, bytes, str_bytelen, str_charlen;
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1628 int ret = 0;
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1629
6902 cff5428c3c4d Fixed handling non-indexed characters in search key. Timo Sirainen <tss@iki.fi> parents: 6898 diff changeset	1630 memset(&ctx, 0, sizeof(ctx));
cff5428c3c4d Fixed handling non-indexed characters in search key. Timo Sirainen <tss@iki.fi> parents: 6898 diff changeset	1631 ctx.trie = trie;
cff5428c3c4d Fixed handling non-indexed characters in search key. Timo Sirainen <tss@iki.fi> parents: 6898 diff changeset	1632 ctx.type = type;
cff5428c3c4d Fixed handling non-indexed characters in search key. Timo Sirainen <tss@iki.fi> parents: 6898 diff changeset	1633 ctx.definite_uids = definite_uids;
cff5428c3c4d Fixed handling non-indexed characters in search key. Timo Sirainen <tss@iki.fi> parents: 6898 diff changeset	1634 ctx.maybe_uids = maybe_uids;
6945 5915aea5f070 UID ranges can be large, don't allocate them from data stack. Timo Sirainen <tss@iki.fi> parents: 6944 diff changeset	1635 i_array_init(&ctx.tmp_uids, 128);
5915aea5f070 UID ranges can be large, don't allocate them from data stack. Timo Sirainen <tss@iki.fi> parents: 6944 diff changeset	1636 i_array_init(&ctx.tmp_uids2, 128);
6902 cff5428c3c4d Fixed handling non-indexed characters in search key. Timo Sirainen <tss@iki.fi> parents: 6898 diff changeset	1637 ctx.first = TRUE;
cff5428c3c4d Fixed handling non-indexed characters in search key. Timo Sirainen <tss@iki.fi> parents: 6898 diff changeset	1638
6911 a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	1639 str_bytelen = strlen(str);
a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	1640 char_lengths = t_malloc0(str_bytelen);
6912 b326327b1f10 Fix for searching <4 char keys. Timo Sirainen <tss@iki.fi> parents: 6911 diff changeset	1641 for (i = 0, str_charlen = 0; i < str_bytelen; str_charlen++) {
6911 a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	1642 bytes = uni_utf8_char_bytes(str[i]);
a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	1643 char_lengths[i] = bytes;
a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	1644 i += bytes;
a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	1645 }
a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	1646 data = squat_data_normalize(trie, (const unsigned char *)str,
a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	1647 str_bytelen);
6902 cff5428c3c4d Fixed handling non-indexed characters in search key. Timo Sirainen <tss@iki.fi> parents: 6898 diff changeset	1648
6911 a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	1649 for (i = start = 0; i < str_bytelen && ret >= 0; i += char_lengths[i]) {
6902 cff5428c3c4d Fixed handling non-indexed characters in search key. Timo Sirainen <tss@iki.fi> parents: 6898 diff changeset	1650 if (data[i] != '\0')
cff5428c3c4d Fixed handling non-indexed characters in search key. Timo Sirainen <tss@iki.fi> parents: 6898 diff changeset	1651 continue;
cff5428c3c4d Fixed handling non-indexed characters in search key. Timo Sirainen <tss@iki.fi> parents: 6898 diff changeset	1652
cff5428c3c4d Fixed handling non-indexed characters in search key. Timo Sirainen <tss@iki.fi> parents: 6898 diff changeset	1653 /* string has nonindexed characters.
cff5428c3c4d Fixed handling non-indexed characters in search key. Timo Sirainen <tss@iki.fi> parents: 6898 diff changeset	1654 search it in parts. */
cff5428c3c4d Fixed handling non-indexed characters in search key. Timo Sirainen <tss@iki.fi> parents: 6898 diff changeset	1655 if (i != start) {
cff5428c3c4d Fixed handling non-indexed characters in search key. Timo Sirainen <tss@iki.fi> parents: 6898 diff changeset	1656 ret = squat_trie_lookup_partial(&ctx, data + start,
6911 a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	1657 char_lengths,
6902 cff5428c3c4d Fixed handling non-indexed characters in search key. Timo Sirainen <tss@iki.fi> parents: 6898 diff changeset	1658 i - start);
cff5428c3c4d Fixed handling non-indexed characters in search key. Timo Sirainen <tss@iki.fi> parents: 6898 diff changeset	1659 }
6911 a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	1660 start = i + char_lengths[i];
6902 cff5428c3c4d Fixed handling non-indexed characters in search key. Timo Sirainen <tss@iki.fi> parents: 6898 diff changeset	1661 }
cff5428c3c4d Fixed handling non-indexed characters in search key. Timo Sirainen <tss@iki.fi> parents: 6898 diff changeset	1662
cff5428c3c4d Fixed handling non-indexed characters in search key. Timo Sirainen <tss@iki.fi> parents: 6898 diff changeset	1663 if (start != 0) {
cff5428c3c4d Fixed handling non-indexed characters in search key. Timo Sirainen <tss@iki.fi> parents: 6898 diff changeset	1664 /* string has nonindexed characters. finish the search. */
cff5428c3c4d Fixed handling non-indexed characters in search key. Timo Sirainen <tss@iki.fi> parents: 6898 diff changeset	1665 array_clear(definite_uids);
cff5428c3c4d Fixed handling non-indexed characters in search key. Timo Sirainen <tss@iki.fi> parents: 6898 diff changeset	1666 if (i != start && ret >= 0) {
cff5428c3c4d Fixed handling non-indexed characters in search key. Timo Sirainen <tss@iki.fi> parents: 6898 diff changeset	1667 ret = squat_trie_lookup_partial(&ctx, data + start,
6911 a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	1668 char_lengths,
6902 cff5428c3c4d Fixed handling non-indexed characters in search key. Timo Sirainen <tss@iki.fi> parents: 6898 diff changeset	1669 i - start);
cff5428c3c4d Fixed handling non-indexed characters in search key. Timo Sirainen <tss@iki.fi> parents: 6898 diff changeset	1670 }
6945 5915aea5f070 UID ranges can be large, don't allocate them from data stack. Timo Sirainen <tss@iki.fi> parents: 6944 diff changeset	1671 } else {
5915aea5f070 UID ranges can be large, don't allocate them from data stack. Timo Sirainen <tss@iki.fi> parents: 6944 diff changeset	1672 if (str_charlen <= trie->hdr.partial_len \|\|
5915aea5f070 UID ranges can be large, don't allocate them from data stack. Timo Sirainen <tss@iki.fi> parents: 6944 diff changeset	1673 trie->hdr.full_len > trie->hdr.partial_len) {
5915aea5f070 UID ranges can be large, don't allocate them from data stack. Timo Sirainen <tss@iki.fi> parents: 6944 diff changeset	1674 ret = squat_trie_lookup_data(trie, data, str_bytelen,
5915aea5f070 UID ranges can be large, don't allocate them from data stack. Timo Sirainen <tss@iki.fi> parents: 6944 diff changeset	1675 &ctx.tmp_uids);
5915aea5f070 UID ranges can be large, don't allocate them from data stack. Timo Sirainen <tss@iki.fi> parents: 6944 diff changeset	1676 if (ret > 0) {
5915aea5f070 UID ranges can be large, don't allocate them from data stack. Timo Sirainen <tss@iki.fi> parents: 6944 diff changeset	1677 squat_trie_filter_type(type, &ctx.tmp_uids,
5915aea5f070 UID ranges can be large, don't allocate them from data stack. Timo Sirainen <tss@iki.fi> parents: 6944 diff changeset	1678 definite_uids);
5915aea5f070 UID ranges can be large, don't allocate them from data stack. Timo Sirainen <tss@iki.fi> parents: 6944 diff changeset	1679 }
5915aea5f070 UID ranges can be large, don't allocate them from data stack. Timo Sirainen <tss@iki.fi> parents: 6944 diff changeset	1680 } else {
5915aea5f070 UID ranges can be large, don't allocate them from data stack. Timo Sirainen <tss@iki.fi> parents: 6944 diff changeset	1681 array_clear(definite_uids);
6902 cff5428c3c4d Fixed handling non-indexed characters in search key. Timo Sirainen <tss@iki.fi> parents: 6898 diff changeset	1682 }
4878 88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1683
6945 5915aea5f070 UID ranges can be large, don't allocate them from data stack. Timo Sirainen <tss@iki.fi> parents: 6944 diff changeset	1684 if (str_charlen <= trie->hdr.partial_len \|\|
5915aea5f070 UID ranges can be large, don't allocate them from data stack. Timo Sirainen <tss@iki.fi> parents: 6944 diff changeset	1685 trie->hdr.partial_len == 0) {
5915aea5f070 UID ranges can be large, don't allocate them from data stack. Timo Sirainen <tss@iki.fi> parents: 6944 diff changeset	1686 /* we have the result */
5915aea5f070 UID ranges can be large, don't allocate them from data stack. Timo Sirainen <tss@iki.fi> parents: 6944 diff changeset	1687 array_clear(maybe_uids);
5915aea5f070 UID ranges can be large, don't allocate them from data stack. Timo Sirainen <tss@iki.fi> parents: 6944 diff changeset	1688 } else {
5915aea5f070 UID ranges can be large, don't allocate them from data stack. Timo Sirainen <tss@iki.fi> parents: 6944 diff changeset	1689 ret = squat_trie_lookup_partial(&ctx, data + start,
5915aea5f070 UID ranges can be large, don't allocate them from data stack. Timo Sirainen <tss@iki.fi> parents: 6944 diff changeset	1690 char_lengths,
5915aea5f070 UID ranges can be large, don't allocate them from data stack. Timo Sirainen <tss@iki.fi> parents: 6944 diff changeset	1691 i - start);
5915aea5f070 UID ranges can be large, don't allocate them from data stack. Timo Sirainen <tss@iki.fi> parents: 6944 diff changeset	1692 }
4878 88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1693 }
6905 8c779f3e16be Fixes to FTS handling. Timo Sirainen <tss@iki.fi> parents: 6902 diff changeset	1694 squat_trie_add_unknown(trie, maybe_uids);
6945 5915aea5f070 UID ranges can be large, don't allocate them from data stack. Timo Sirainen <tss@iki.fi> parents: 6944 diff changeset	1695 array_free(&ctx.tmp_uids);
5915aea5f070 UID ranges can be large, don't allocate them from data stack. Timo Sirainen <tss@iki.fi> parents: 6944 diff changeset	1696 array_free(&ctx.tmp_uids2);
6905 8c779f3e16be Fixes to FTS handling. Timo Sirainen <tss@iki.fi> parents: 6902 diff changeset	1697 return ret < 0 ? -1 : 0;
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1698 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1699
6940 414c9d631a81 Replaced t_push/t_pop calls with T_FRAME() macros. Timo Sirainen <tss@iki.fi>* parents: 6931 diff changeset	1700 int squat_trie_lookup(struct squat_trie trie, const char str,
414c9d631a81 Replaced t_push/t_pop calls with T_FRAME() macros. Timo Sirainen <tss@iki.fi>* parents: 6931 diff changeset	1701 enum squat_index_type type,
414c9d631a81 Replaced t_push/t_pop calls with T_FRAME() macros. Timo Sirainen <tss@iki.fi>* parents: 6931 diff changeset	1702 ARRAY_TYPE(seq_range) *definite_uids,
414c9d631a81 Replaced t_push/t_pop calls with T_FRAME() macros. Timo Sirainen <tss@iki.fi>* parents: 6931 diff changeset	1703 ARRAY_TYPE(seq_range) *maybe_uids)
414c9d631a81 Replaced t_push/t_pop calls with T_FRAME() macros. Timo Sirainen <tss@iki.fi>* parents: 6931 diff changeset	1704 {
414c9d631a81 Replaced t_push/t_pop calls with T_FRAME() macros. Timo Sirainen <tss@iki.fi>* parents: 6931 diff changeset	1705 int ret;
414c9d631a81 Replaced t_push/t_pop calls with T_FRAME() macros. Timo Sirainen <tss@iki.fi>* parents: 6931 diff changeset	1706
414c9d631a81 Replaced t_push/t_pop calls with T_FRAME() macros. Timo Sirainen <tss@iki.fi>* parents: 6931 diff changeset	1707 T_FRAME(
414c9d631a81 Replaced t_push/t_pop calls with T_FRAME() macros. Timo Sirainen <tss@iki.fi>* parents: 6931 diff changeset	1708 ret = squat_trie_lookup_real(trie, str, type,
414c9d631a81 Replaced t_push/t_pop calls with T_FRAME() macros. Timo Sirainen <tss@iki.fi>* parents: 6931 diff changeset	1709 definite_uids, maybe_uids);
414c9d631a81 Replaced t_push/t_pop calls with T_FRAME() macros. Timo Sirainen <tss@iki.fi>* parents: 6931 diff changeset	1710 );
414c9d631a81 Replaced t_push/t_pop calls with T_FRAME() macros. Timo Sirainen <tss@iki.fi>* parents: 6931 diff changeset	1711 return ret;
414c9d631a81 Replaced t_push/t_pop calls with T_FRAME() macros. Timo Sirainen <tss@iki.fi>* parents: 6931 diff changeset	1712 }
414c9d631a81 Replaced t_push/t_pop calls with T_FRAME() macros. Timo Sirainen <tss@iki.fi>* parents: 6931 diff changeset	1713
6418 46d9ee79f292 Removed _ prefix from all public APIs. Timo Sirainen <tss@iki.fi> parents: 6414 diff changeset	1714 struct squat_uidlist squat_trie_get_uidlist(struct squat_trie trie)
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1715 {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1716 return trie->uidlist;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1717 }
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1718
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1719 size_t squat_trie_mem_used(struct squat_trie trie, unsigned int count_r)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1720 {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1721 *count_r = trie->hdr.node_count;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1722 return trie->node_alloc_size;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1723 }

Mercurial > dovecot > original-hg > dovecot-1.2

annotate src/plugins/fts-squat/squat-trie.c @ 7086:7ed926ed7aa4 HEAD