dovecot/original-hg/dovecot-1.2: src/plugins/fts-squat/squat-trie.c annotate

annotate src/plugins/fts-squat/squat-trie.c @ 7220:21d1d0ced13e HEAD

Expunge fixes

author	Timo Sirainen <tss@iki.fi>
date	Sat, 09 Feb 2008 15:11:24 +0200
parents	6c48b72c5096
children	e042b6385c7b

rev	line source
7086 7ed926ed7aa4 Updated copyright notices to include year 2008. Timo Sirainen <tss@iki.fi> parents: 7043 diff changeset	1 /* Copyright (c) 2007-2008 Dovecot authors, see the included COPYING file */
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	2
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	3 #include "lib.h"
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	4 #include "array.h"
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	5 #include "str.h"
6944 66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	6 #include "read-full.h"
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	7 #include "istream.h"
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	8 #include "ostream.h"
6911 a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	9 #include "unichar.h"
6944 66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	10 #include "file-cache.h"
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	11 #include "seq-range-array.h"
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	12 #include "squat-uidlist.h"
5068 65a4e1e61555 Some bugfixes. Moved header struct to a separate file. Timo Sirainen <tss@iki.fi> parents: 5010 diff changeset	13 #include "squat-trie-private.h"
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	14
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	15 #include <stdio.h>
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	16 #include <stdlib.h>
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	17 #include <unistd.h>
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	18 #include <sys/mman.h>
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	19
6905 8c779f3e16be Fixes to FTS handling. Timo Sirainen <tss@iki.fi> parents: 6902 diff changeset	20 #define DEFAULT_NORMALIZE_MAP_CHARS \
8c779f3e16be Fixes to FTS handling. Timo Sirainen <tss@iki.fi> parents: 6902 diff changeset	21 "EOTIRSACDNLMVUGPHBFWYXKJQZ0123456789@.-+#$%_&"
8c779f3e16be Fixes to FTS handling. Timo Sirainen <tss@iki.fi> parents: 6902 diff changeset	22 #define DEFAULT_PARTIAL_LEN 4
8c779f3e16be Fixes to FTS handling. Timo Sirainen <tss@iki.fi> parents: 6902 diff changeset	23 #define DEFAULT_FULL_LEN 4
8c779f3e16be Fixes to FTS handling. Timo Sirainen <tss@iki.fi> parents: 6902 diff changeset	24
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	25 #define MAX_FAST_LEVEL 3
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	26 #define SEQUENTIAL_COUNT 46
4878 88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	27
6944 66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	28 #define TRIE_BYTES_LEFT(n) \
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	29 ((n) * SQUAT_PACK_MAX_SIZE)
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	30 #define TRIE_READAHEAD_SIZE \
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	31 I_MAX(4096, 1 + 256 + TRIE_BYTES_LEFT(256))
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	32
4878 88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	33 struct squat_trie_build_context {
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	34 struct squat_trie *trie;
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	35 struct ostream *output;
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	36 struct squat_uidlist_build_context *uidlist_build_ctx;
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	37
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	38 struct file_lock *file_lock;
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	39
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	40 uint32_t first_uid;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	41 unsigned int compress_nodes:1;
4878 88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	42 };
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	43
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	44 struct squat_trie_iterate_node {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	45 struct squat_node *node;
7210 f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	46 ARRAY_TYPE(seq_range) shifts;
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	47 unsigned int idx;
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	48 };
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	49
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	50 struct squat_trie_iterate_context {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	51 struct squat_trie *trie;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	52 struct squat_trie_iterate_node cur;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	53 ARRAY_DEFINE(parents, struct squat_trie_iterate_node);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	54 bool failed;
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	55 };
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	56
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	57 static int squat_trie_map(struct squat_trie *trie, bool building);
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	58
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	59 void squat_trie_delete(struct squat_trie *trie)
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	60 {
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	61 if (unlink(trie->path) < 0 && errno != ENOENT)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	62 i_error("unlink(%s) failed: %m", trie->path);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	63 squat_uidlist_delete(trie->uidlist);
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	64 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	65
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	66 static void squat_trie_set_corrupted(struct squat_trie *trie)
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	67 {
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	68 trie->corrupted = TRUE;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	69 i_error("Corrupted file %s", trie->path);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	70 squat_trie_delete(trie);
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	71 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	72
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	73 static void squat_trie_normalize_map_build(struct squat_trie *trie)
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	74 {
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	75 static unsigned char valid_chars[] =
6905 8c779f3e16be Fixes to FTS handling. Timo Sirainen <tss@iki.fi> parents: 6902 diff changeset	76 DEFAULT_NORMALIZE_MAP_CHARS;
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	77 unsigned int i, j;
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	78
6905 8c779f3e16be Fixes to FTS handling. Timo Sirainen <tss@iki.fi> parents: 6902 diff changeset	79 memset(trie->default_normalize_map, 0,
8c779f3e16be Fixes to FTS handling. Timo Sirainen <tss@iki.fi> parents: 6902 diff changeset	80 sizeof(trie->default_normalize_map));
4901 5e604d6f391e Only ASCII texts were actually indexed. Now UTF-8 input is properly Timo Sirainen <tss@iki.fi> parents: 4898 diff changeset	81
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	82 #if 1
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	83 for (i = 0, j = 1; i < sizeof(valid_chars)-1; i++) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	84 unsigned char chr = valid_chars[i];
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	85
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	86 if (chr >= 'A' && chr <= 'Z')
6905 8c779f3e16be Fixes to FTS handling. Timo Sirainen <tss@iki.fi> parents: 6902 diff changeset	87 trie->default_normalize_map[chr-'A'+'a'] = j;
8c779f3e16be Fixes to FTS handling. Timo Sirainen <tss@iki.fi> parents: 6902 diff changeset	88 trie->default_normalize_map[chr] = j++;
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	89 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	90 i_assert(j <= SEQUENTIAL_COUNT);
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	91
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	92 for (i = 128; i < 256; i++)
6905 8c779f3e16be Fixes to FTS handling. Timo Sirainen <tss@iki.fi> parents: 6902 diff changeset	93 trie->default_normalize_map[i] = j++;
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	94 #else
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	95 for (i = 0; i < sizeof(valid_chars)-1; i++) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	96 unsigned char chr = valid_chars[i];
4890 cb3b4153136c 64bit and big endian fixes Timo Sirainen <tss@iki.fi> parents: 4887 diff changeset	97
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	98 if (chr >= 'A' && chr <= 'Z')
6905 8c779f3e16be Fixes to FTS handling. Timo Sirainen <tss@iki.fi> parents: 6902 diff changeset	99 trie->default_normalize_map[chr-'A'+'a'] = chr;
8c779f3e16be Fixes to FTS handling. Timo Sirainen <tss@iki.fi> parents: 6902 diff changeset	100 trie->default_normalize_map[chr] = chr;
4890 cb3b4153136c 64bit and big endian fixes Timo Sirainen <tss@iki.fi> parents: 4887 diff changeset	101 }
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	102 for (i = 128; i < 256; i++)
6905 8c779f3e16be Fixes to FTS handling. Timo Sirainen <tss@iki.fi> parents: 6902 diff changeset	103 trie->default_normalize_map[i] = i_toupper(i);
4890 cb3b4153136c 64bit and big endian fixes Timo Sirainen <tss@iki.fi> parents: 4887 diff changeset	104 #endif
cb3b4153136c 64bit and big endian fixes Timo Sirainen <tss@iki.fi> parents: 4887 diff changeset	105 }
cb3b4153136c 64bit and big endian fixes Timo Sirainen <tss@iki.fi> parents: 4887 diff changeset	106
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	107 static void node_free(struct squat_trie trie, struct squat_node node)
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	108 {
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	109 struct squat_node *children;
4890 cb3b4153136c 64bit and big endian fixes Timo Sirainen <tss@iki.fi> parents: 4887 diff changeset	110 unsigned int i;
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	111
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	112 if (NODE_IS_DYNAMIC_LEAF(node))
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	113 i_free(node->children.leaf_string);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	114 else if (!node->children_not_mapped && node->child_count > 0) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	115 children = NODE_CHILDREN_NODES(node);
4883 d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	116
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	117 trie->node_alloc_size -=
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	118 NODE_CHILDREN_ALLOC_SIZE(node->child_count);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	119 for (i = 0; i < node->child_count; i++)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	120 node_free(trie, &children[i]);
4883 d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	121
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	122 i_free(node->children.data);
4898 07a038b57946 Optimization. Timo Sirainen <tss@iki.fi> parents: 4897 diff changeset	123 }
07a038b57946 Optimization. Timo Sirainen <tss@iki.fi> parents: 4897 diff changeset	124 }
07a038b57946 Optimization. Timo Sirainen <tss@iki.fi> parents: 4897 diff changeset	125
4878 88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	126 struct squat_trie *
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	127 squat_trie_init(const char *path, uint32_t uidvalidity,
4879 b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	128 enum file_lock_method lock_method, bool mmap_disable)
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	129 {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	130 struct squat_trie *trie;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	131
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	132 trie = i_new(struct squat_trie, 1);
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	133 trie->path = i_strdup(path);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	134 trie->uidlist = squat_uidlist_init(trie);
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	135 trie->fd = -1;
4878 88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	136 trie->lock_method = lock_method;
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	137 trie->uidvalidity = uidvalidity;
6944 66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	138 trie->mmap_disable = mmap_disable;
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	139 squat_trie_normalize_map_build(trie);
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	140 return trie;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	141 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	142
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	143 static void squat_trie_close(struct squat_trie *trie)
4878 88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	144 {
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	145 trie->corrupted = FALSE;
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	146 node_free(trie, &trie->root);
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	147 memset(&trie->root, 0, sizeof(trie->root));
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	148 memset(&trie->hdr, 0, sizeof(trie->hdr));
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	149
6944 66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	150 trie->data = NULL;
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	151 trie->data_size = 0;
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	152
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	153 if (trie->file_cache != NULL)
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	154 file_cache_free(&trie->file_cache);
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	155 if (trie->mmap_size != 0) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	156 if (munmap(trie->mmap_base, trie->mmap_size) < 0)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	157 i_error("munmap(%s) failed: %m", trie->path);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	158 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	159 if (trie->fd != -1) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	160 if (close(trie->fd) < 0)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	161 i_error("close(%s) failed: %m", trie->path);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	162 trie->fd = -1;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	163 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	164 trie->locked_file_size = 0;
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	165 }
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	166
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	167 void squat_trie_deinit(struct squat_trie **_trie)
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	168 {
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	169 struct squat_trie trie = _trie;
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	170
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	171 *_trie = NULL;
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	172 squat_trie_close(trie);
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	173 squat_uidlist_deinit(trie->uidlist);
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	174 i_free(trie->path);
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	175 i_free(trie);
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	176 }
4883 d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	177
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	178 static void squat_trie_header_init(struct squat_trie *trie)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	179 {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	180 memset(&trie->hdr, 0, sizeof(trie->hdr));
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	181 trie->hdr.version = SQUAT_TRIE_VERSION;
7159 2de2058a5cdc Don't use ioloop_time. Fixes squat-test which doesn't create ioloop. Timo Sirainen <tss@iki.fi> parents: 7158 diff changeset	182 trie->hdr.indexid = time(NULL);
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	183 trie->hdr.uidvalidity = trie->uidvalidity;
6905 8c779f3e16be Fixes to FTS handling. Timo Sirainen <tss@iki.fi> parents: 6902 diff changeset	184 trie->hdr.partial_len = DEFAULT_PARTIAL_LEN;
8c779f3e16be Fixes to FTS handling. Timo Sirainen <tss@iki.fi> parents: 6902 diff changeset	185 trie->hdr.full_len = DEFAULT_FULL_LEN;
8c779f3e16be Fixes to FTS handling. Timo Sirainen <tss@iki.fi> parents: 6902 diff changeset	186
8c779f3e16be Fixes to FTS handling. Timo Sirainen <tss@iki.fi> parents: 6902 diff changeset	187 i_assert(sizeof(trie->hdr.normalize_map) ==
8c779f3e16be Fixes to FTS handling. Timo Sirainen <tss@iki.fi> parents: 6902 diff changeset	188 sizeof(trie->default_normalize_map));
8c779f3e16be Fixes to FTS handling. Timo Sirainen <tss@iki.fi> parents: 6902 diff changeset	189 memcpy(trie->hdr.normalize_map, trie->default_normalize_map,
8c779f3e16be Fixes to FTS handling. Timo Sirainen <tss@iki.fi> parents: 6902 diff changeset	190 sizeof(trie->hdr.normalize_map));
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	191 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	192
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	193 static int squat_trie_open_fd(struct squat_trie *trie)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	194 {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	195 trie->fd = open(trie->path, O_RDWR);
4883 d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	196 if (trie->fd == -1) {
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	197 if (errno == ENOENT) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	198 squat_trie_header_init(trie);
4883 d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	199 return 0;
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	200 }
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	201 i_error("open(%s) failed: %m", trie->path);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	202 return -1;
4883 d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	203 }
7218 43d2f70fb279 Squat fixes Timo Sirainen <tss@iki.fi> parents: 7210 diff changeset	204 if (trie->file_cache != NULL)
43d2f70fb279 Squat fixes Timo Sirainen <tss@iki.fi> parents: 7210 diff changeset	205 file_cache_set_fd(trie->file_cache, trie->fd);
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	206 return 0;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	207 }
4883 d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	208
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	209 static int squat_trie_open(struct squat_trie *trie)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	210 {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	211 squat_trie_close(trie);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	212
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	213 if (squat_trie_open_fd(trie) < 0)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	214 return -1;
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	215 return squat_trie_map(trie, FALSE);
4878 88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	216 }
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	217
4879 b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	218 static int squat_trie_is_file_stale(struct squat_trie *trie)
b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	219 {
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	220 struct stat st, st2;
4879 b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	221
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	222 if (stat(trie->path, &st) < 0) {
4883 d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	223 if (errno == ENOENT)
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	224 return 1;
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	225
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	226 i_error("stat(%s) failed: %m", trie->path);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	227 return -1;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	228 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	229 if (fstat(trie->fd, &st2) < 0) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	230 i_error("fstat(%s) failed: %m", trie->path);
4879 b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	231 return -1;
b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	232 }
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	233 trie->locked_file_size = st2.st_size;
4879 b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	234
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	235 return st.st_ino == st2.st_ino &&
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	236 CMP_DEV_T(st.st_dev, st2.st_dev) ? 0 : 1;
4879 b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	237 }
b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	238
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	239 void squat_trie_refresh(struct squat_trie *trie)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	240 {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	241 if (squat_trie_is_file_stale(trie) > 0)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	242 (void)squat_trie_open(trie);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	243 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	244
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	245 static int squat_trie_lock(struct squat_trie *trie, int lock_type,
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	246 struct file_lock **file_lock_r)
4886 ae2114f11a0e Memory leak fixes. Also when building a large mailbox flush once in a while Timo Sirainen <tss@iki.fi> parents: 4883 diff changeset	247 {
ae2114f11a0e Memory leak fixes. Also when building a large mailbox flush once in a while Timo Sirainen <tss@iki.fi> parents: 4883 diff changeset	248 int ret;
ae2114f11a0e Memory leak fixes. Also when building a large mailbox flush once in a while Timo Sirainen <tss@iki.fi> parents: 4883 diff changeset	249
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	250 while (trie->fd != -1) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	251 ret = file_wait_lock(trie->fd, trie->path, lock_type,
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	252 trie->lock_method, SQUAT_TRIE_LOCK_TIMEOUT,
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	253 file_lock_r);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	254 if (ret == 0) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	255 i_error("file_wait_lock(%s) failed: %m", trie->path);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	256 return 0;
4879 b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	257 }
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	258 if (ret < 0)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	259 return -1;
4879 b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	260
b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	261 /* if the trie has been compressed, we need to reopen the
b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	262 file and try to lock again */
b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	263 ret = squat_trie_is_file_stale(trie);
b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	264 if (ret == 0)
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	265 return 1;
4879 b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	266
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	267 file_unlock(file_lock_r);
4879 b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	268 if (ret < 0)
b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	269 return -1;
b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	270
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	271 squat_trie_close(trie);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	272 if (squat_trie_open_fd(trie) < 0)
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	273 return -1;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	274 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	275 return 0;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	276 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	277
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	278 static void
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	279 node_make_squential(struct squat_trie trie, struct squat_node node, int level)
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	280 {
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	281 const unsigned int alloc_size =
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	282 NODE_CHILDREN_ALLOC_SIZE(SEQUENTIAL_COUNT);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	283 struct squat_node *children;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	284 unsigned char *chars;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	285 unsigned int i;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	286
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	287 i_assert(node->child_count == 0);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	288
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	289 trie->node_alloc_size += alloc_size;
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	290
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	291 node->want_sequential = FALSE;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	292 node->have_sequential = TRUE;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	293
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	294 node->child_count = SEQUENTIAL_COUNT;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	295 node->children.data = i_malloc(alloc_size);
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	296
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	297 chars = NODE_CHILDREN_CHARS(node);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	298 for (i = 0; i < SEQUENTIAL_COUNT; i++)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	299 chars[i] = i;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	300
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	301 if (level < MAX_FAST_LEVEL) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	302 children = NODE_CHILDREN_NODES(node);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	303 for (i = 0; i < SEQUENTIAL_COUNT; i++)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	304 children[i].want_sequential = TRUE;
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	305 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	306 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	307
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	308 static unsigned int
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	309 node_add_child(struct squat_trie trie, struct squat_node node,
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	310 unsigned char chr, int level)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	311 {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	312 unsigned int old_child_count = node->child_count;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	313 struct squat_node children, old_children;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	314 unsigned char *chars;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	315 size_t old_size, new_size;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	316
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	317 i_assert(node->leaf_string_length == 0);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	318
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	319 if (node->want_sequential) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	320 node_make_squential(trie, node, level);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	321
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	322 if (chr < SEQUENTIAL_COUNT)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	323 return chr;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	324 old_child_count = SEQUENTIAL_COUNT;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	325 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	326
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	327 node->child_count++;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	328 new_size = NODE_CHILDREN_ALLOC_SIZE(node->child_count);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	329
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	330 if (old_child_count == 0) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	331 /* first child */
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	332 node->children.data = i_malloc(new_size);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	333 trie->node_alloc_size += new_size;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	334 children = NODE_CHILDREN_NODES(node);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	335 } else {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	336 old_size = NODE_CHILDREN_ALLOC_SIZE(old_child_count);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	337 if (old_size != new_size) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	338 trie->node_alloc_size += new_size - old_size;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	339 node->children.data = i_realloc(node->children.data,
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	340 old_size, new_size);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	341 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	342
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	343 children = NODE_CHILDREN_NODES(node);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	344 old_children = (void *)(NODE_CHILDREN_CHARS(node) +
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	345 MEM_ALIGN(old_child_count));
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	346 if (children != old_children) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	347 memmove(children, old_children,
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	348 old_child_count * sizeof(struct squat_node));
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	349 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	350 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	351
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	352 chars = NODE_CHILDREN_CHARS(node);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	353 chars[node->child_count - 1] = chr;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	354 return node->child_count - 1;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	355 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	356
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	357 static int
6944 66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	358 trie_file_cache_read(struct squat_trie *trie, size_t offset, size_t size)
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	359 {
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	360 if (trie->file_cache == NULL)
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	361 return 0;
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	362
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	363 if (file_cache_read(trie->file_cache, offset, size) < 0) {
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	364 i_error("read(%s) failed: %m", trie->path);
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	365 return -1;
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	366 }
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	367 trie->data = file_cache_get_map(trie->file_cache, &trie->data_size);
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	368 return 0;
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	369 }
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	370
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	371 static int
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	372 node_read_children(struct squat_trie trie, struct squat_node node, int level)
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	373 {
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	374 const uint8_t data, end;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	375 const unsigned char *child_chars;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	376 struct squat_node child, children = NULL;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	377 uoff_t node_offset;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	378 unsigned int i, child_idx, child_count;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	379 uoff_t base_offset;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	380 uint32_t num;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	381
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	382 i_assert(node->children_not_mapped);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	383 i_assert(!node->have_sequential);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	384 i_assert(trie->unmapped_child_count > 0);
6944 66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	385 i_assert(trie->data_size <= trie->locked_file_size);
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	386
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	387 trie->unmapped_child_count--;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	388 node_offset = node->children.offset;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	389 node->children_not_mapped = FALSE;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	390 node->children.data = NULL;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	391
6944 66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	392 if (trie_file_cache_read(trie, node_offset, TRIE_READAHEAD_SIZE) < 0)
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	393 return -1;
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	394 if (unlikely(node_offset >= trie->data_size)) {
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	395 squat_trie_set_corrupted(trie);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	396 return -1;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	397 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	398
6944 66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	399 data = CONST_PTR_OFFSET(trie->data, node_offset);
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	400 end = CONST_PTR_OFFSET(trie->data, trie->data_size);
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	401 child_count = *data++;
6944 66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	402 if (unlikely(node_offset + child_count >= trie->data_size)) {
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	403 squat_trie_set_corrupted(trie);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	404 return -1;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	405 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	406
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	407 if (child_count == 0)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	408 return 0;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	409
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	410 child_chars = data;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	411 data += child_count;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	412
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	413 /* get child offsets */
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	414 base_offset = node_offset;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	415 for (i = 0; i < child_count; i++) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	416 /* we always start with !have_sequential, so at i=0 this
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	417 check always goes to add the first child */
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	418 if (node->have_sequential && child_chars[i] < SEQUENTIAL_COUNT)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	419 child_idx = child_chars[i];
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	420 else {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	421 child_idx = node_add_child(trie, node, child_chars[i],
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	422 level);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	423 children = NODE_CHILDREN_NODES(node);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	424 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	425 child = &children[child_idx];
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	426
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	427 /* 1) child offset */
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	428 num = squat_unpack_num(&data, end);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	429 if (num == 0) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	430 /* no children */
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	431 } else {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	432 if ((num & 1) != 0) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	433 base_offset += num >> 1;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	434 } else {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	435 base_offset -= num >> 1;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	436 }
6944 66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	437 if (base_offset >= trie->locked_file_size) {
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	438 squat_trie_set_corrupted(trie);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	439 return -1;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	440 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	441 trie->unmapped_child_count++;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	442 child->children_not_mapped = TRUE;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	443 child->children.offset = base_offset;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	444 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	445
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	446 /* 2) uidlist */
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	447 child->uid_list_idx = squat_unpack_num(&data, end);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	448 if (child->uid_list_idx == 0) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	449 /* we don't write nodes with empty uidlists */
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	450 squat_trie_set_corrupted(trie);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	451 return -1;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	452 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	453 if (!UIDLIST_IS_SINGLETON(child->uid_list_idx)) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	454 /* 3) next uid */
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	455 child->next_uid = squat_unpack_num(&data, end) + 1;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	456 } else {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	457 uint32_t idx = child->uid_list_idx;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	458
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	459 child->next_uid = 1 +
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	460 squat_uidlist_singleton_last_uid(idx);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	461 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	462
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	463 /* 4) unused uids + leaf string flag */
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	464 num = squat_unpack_num(&data, end);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	465 child->unused_uids = num >> 1;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	466 if ((num & 1) != 0) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	467 /* leaf string */
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	468 unsigned int len;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	469 unsigned char *dest;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	470
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	471 /* 5) leaf string length */
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	472 len = child->leaf_string_length =
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	473 squat_unpack_num(&data, end) + 1;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	474 if (!NODE_IS_DYNAMIC_LEAF(child))
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	475 dest = child->children.static_leaf_string;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	476 else {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	477 dest = child->children.leaf_string =
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	478 i_malloc(len);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	479 }
6944 66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	480
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	481 if (trie->file_cache != NULL) {
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	482 /* the string may be long -
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	483 recalculate the end pos */
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	484 size_t offset, size;
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	485
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	486 offset = (const char *)data -
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	487 (const char *)trie->data;
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	488 size = len + TRIE_BYTES_LEFT(child_count - i);
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	489
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	490 if (trie_file_cache_read(trie, offset,
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	491 size) < 0)
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	492 return -1;
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	493 data = CONST_PTR_OFFSET(trie->data, offset);
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	494 end = CONST_PTR_OFFSET(trie->data,
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	495 trie->data_size);
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	496 }
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	497
7043 7708a8c166d6 Compiler warning fixes on 32bit systems. Timo Sirainen <tss@iki.fi> parents: 6945 diff changeset	498 if ((size_t)(end - data) < len) {
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	499 squat_trie_set_corrupted(trie);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	500 return -1;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	501 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	502 memcpy(dest, data, len);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	503 data += len;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	504 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	505 }
6944 66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	506 if (unlikely(data == end)) {
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	507 /* we should never get this far */
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	508 squat_trie_set_corrupted(trie);
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	509 return -1;
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	510 }
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	511 return 0;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	512 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	513
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	514 static void
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	515 node_write_children(struct squat_trie_build_context *ctx,
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	516 struct squat_node node, const uoff_t node_offsets)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	517 {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	518 struct squat_node *children;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	519 const unsigned char *chars;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	520 uint8_t child_count, buf[SQUAT_PACK_MAX_SIZE * 5], *bufp;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	521 uoff_t base_offset;
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	522 unsigned int i;
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	523
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	524 chars = NODE_CHILDREN_CHARS(node);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	525 children = NODE_CHILDREN_NODES(node);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	526
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	527 base_offset = ctx->output->offset;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	528 child_count = node->child_count;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	529 o_stream_send(ctx->output, &child_count, 1);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	530 o_stream_send(ctx->output, chars, child_count);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	531
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	532 for (i = 0; i < child_count; i++) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	533 bufp = buf;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	534 /* 1) child offset */
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	535 if (node_offsets[i] == 0)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	536 *bufp++ = 0;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	537 else if (node_offsets[i] >= base_offset) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	538 squat_pack_num(&bufp,
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	539 ((node_offsets[i] - base_offset) << 1) \| 1);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	540 base_offset = node_offsets[i];
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	541 } else {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	542 squat_pack_num(&bufp,
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	543 (base_offset - node_offsets[i]) << 1);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	544 base_offset = node_offsets[i];
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	545 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	546
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	547 /* 2) uidlist */
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	548 squat_pack_num(&bufp, children[i].uid_list_idx);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	549 if (!UIDLIST_IS_SINGLETON(children[i].uid_list_idx)) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	550 /* 3) next uid */
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	551 squat_pack_num(&bufp, children[i].next_uid - 1);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	552 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	553
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	554 if (children[i].leaf_string_length == 0) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	555 /* 4a) unused uids */
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	556 squat_pack_num(&bufp, children[i].unused_uids << 1);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	557 o_stream_send(ctx->output, buf, bufp - buf);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	558 } else {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	559 i_assert(node_offsets[i] == 0);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	560 /* 4b) unused uids + flag */
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	561 squat_pack_num(&bufp, (children[i].unused_uids << 1) \| 1);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	562 /* 5) leaf string length */
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	563 squat_pack_num(&bufp, children[i].leaf_string_length - 1);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	564 o_stream_send(ctx->output, buf, bufp - buf);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	565 o_stream_send(ctx->output,
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	566 NODE_LEAF_STRING(&children[i]),
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	567 children[i].leaf_string_length);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	568 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	569 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	570 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	571
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	572 static inline void
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	573 node_add_uid(struct squat_trie_build_context *ctx, uint32_t uid,
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	574 struct squat_node *node)
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	575 {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	576 if (uid < node->next_uid) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	577 /* duplicate */
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	578 return;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	579 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	580 node->unused_uids += uid - node->next_uid;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	581 node->next_uid = uid + 1;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	582
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	583 node->uid_list_idx =
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	584 squat_uidlist_build_add_uid(ctx->uidlist_build_ctx,
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	585 node->uid_list_idx, uid);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	586 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	587
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	588 static void
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	589 node_split_string(struct squat_trie_build_context ctx, struct squat_node node)
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	590 {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	591 struct squat_node *child;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	592 unsigned char *str;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	593 unsigned int uid, idx, str_len = node->leaf_string_length;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	594
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	595 i_assert(str_len > 0);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	596
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	597 /* make a copy of the leaf string and convert to normal node by
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	598 removing it. */
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	599 str = t_malloc(str_len);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	600 if (!NODE_IS_DYNAMIC_LEAF(node))
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	601 memcpy(str, node->children.static_leaf_string, str_len);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	602 else {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	603 memcpy(str, node->children.leaf_string, str_len);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	604 i_free(node->children.leaf_string);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	605 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	606 node->leaf_string_length = 0;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	607
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	608 /* create a new child node for the rest of the string */
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	609 idx = node_add_child(ctx->trie, node, str[0], MAX_FAST_LEVEL);
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	610 child = NODE_CHILDREN_NODES(node) + idx;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	611
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	612 /* update uidlist to contain all of parent's UIDs */
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	613 child->next_uid = node->next_uid - node->unused_uids;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	614 for (uid = 0; uid < child->next_uid; uid++) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	615 child->uid_list_idx =
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	616 squat_uidlist_build_add_uid(ctx->uidlist_build_ctx,
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	617 child->uid_list_idx, uid);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	618 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	619
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	620 i_assert(!child->have_sequential && child->children.data == NULL);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	621 if (str_len > 1) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	622 /* make the child a leaf string */
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	623 str_len--;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	624 child->leaf_string_length = str_len;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	625 if (!NODE_IS_DYNAMIC_LEAF(child)) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	626 memcpy(child->children.static_leaf_string,
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	627 str + 1, str_len);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	628 } else {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	629 child->children.leaf_string = i_malloc(str_len);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	630 memcpy(child->children.leaf_string, str + 1, str_len);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	631 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	632 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	633 }
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	634
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	635 static bool
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	636 node_leaf_string_add_or_split(struct squat_trie_build_context *ctx,
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	637 struct squat_node *node,
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	638 const unsigned char *data, unsigned int data_len)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	639 {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	640 const unsigned char *str = NODE_LEAF_STRING(node);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	641 const unsigned int str_len = node->leaf_string_length;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	642 unsigned int i;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	643
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	644 if (data_len != str_len) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	645 /* different lengths, can't match */
6940 414c9d631a81 Replaced t_push/t_pop calls with T_FRAME() macros. Timo Sirainen <tss@iki.fi>* parents: 6931 diff changeset	646 T_FRAME(
414c9d631a81 Replaced t_push/t_pop calls with T_FRAME() macros. Timo Sirainen <tss@iki.fi>* parents: 6931 diff changeset	647 node_split_string(ctx, node);
414c9d631a81 Replaced t_push/t_pop calls with T_FRAME() macros. Timo Sirainen <tss@iki.fi>* parents: 6931 diff changeset	648 );
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	649 return FALSE;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	650 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	651
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	652 for (i = 0; i < data_len; i++) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	653 if (data[i] != str[i]) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	654 /* non-match */
6940 414c9d631a81 Replaced t_push/t_pop calls with T_FRAME() macros. Timo Sirainen <tss@iki.fi>* parents: 6931 diff changeset	655 T_FRAME(
414c9d631a81 Replaced t_push/t_pop calls with T_FRAME() macros. Timo Sirainen <tss@iki.fi>* parents: 6931 diff changeset	656 node_split_string(ctx, node);
414c9d631a81 Replaced t_push/t_pop calls with T_FRAME() macros. Timo Sirainen <tss@iki.fi>* parents: 6931 diff changeset	657 );
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	658 return FALSE;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	659 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	660 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	661 return TRUE;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	662 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	663
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	664 static int squat_build_add(struct squat_trie_build_context *ctx, uint32_t uid,
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	665 const unsigned char *data, unsigned int size)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	666 {
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	667 struct squat_trie *trie = ctx->trie;
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	668 struct squat_node *node = &trie->root;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	669 const unsigned char *end = data + size;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	670 unsigned char *chars;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	671 unsigned int idx;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	672 int level = 0;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	673
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	674 for (;;) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	675 if (node->children_not_mapped) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	676 if (unlikely(node_read_children(trie, node, level) < 0))
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	677 return -1;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	678 }
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	679
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	680 if (node->leaf_string_length != 0) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	681 /* the whole string must match or we need to split
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	682 the node */
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	683 if (node_leaf_string_add_or_split(ctx, node, data,
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	684 end - data)) {
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	685 node_add_uid(ctx, uid, node);
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	686 return 0;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	687 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	688 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	689
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	690 node_add_uid(ctx, uid, node);
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	691
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	692 if (unlikely(uid < node->unused_uids)) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	693 squat_trie_set_corrupted(trie);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	694 return -1;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	695 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	696 /* child node's UIDs are relative to ours. so for example if
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	697 we're adding UID 4 and this node now has [2,4] UIDs,
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	698 unused_uids=3 and so the child node will be adding
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	699 UID 4-3 = 1. */
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	700 uid -= node->unused_uids;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	701
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	702 if (data == end)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	703 return 0;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	704 level++;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	705
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	706 if (node->have_sequential) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	707 if (*data < SEQUENTIAL_COUNT) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	708 idx = *data;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	709 goto found;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	710 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	711 idx = SEQUENTIAL_COUNT;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	712 } else {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	713 idx = 0;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	714 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	715 chars = NODE_CHILDREN_CHARS(node);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	716 for (; idx < node->child_count; idx++) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	717 if (chars[idx] == *data)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	718 goto found;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	719 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	720 break;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	721 found:
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	722 data++;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	723 node = NODE_CHILDREN_NODES(node) + idx;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	724 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	725
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	726 /* create new children */
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	727 i_assert(node->leaf_string_length == 0);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	728
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	729 for (;;) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	730 idx = node_add_child(trie, node, *data,
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	731 size - (end - data) + 1);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	732 node = NODE_CHILDREN_NODES(node) + idx;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	733
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	734 node_add_uid(ctx, uid, node);
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	735 uid = 0;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	736
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	737 if (++data == end)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	738 break;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	739
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	740 if (!node->have_sequential) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	741 /* convert the node into a leaf string */
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	742 unsigned int len = end - data;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	743
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	744 i_assert(node->children.data == NULL);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	745 node->leaf_string_length = len;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	746 if (!NODE_IS_DYNAMIC_LEAF(node)) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	747 memcpy(node->children.static_leaf_string,
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	748 data, len);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	749 } else {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	750 node->children.leaf_string = i_malloc(len);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	751 memcpy(node->children.leaf_string, data, len);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	752 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	753 break;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	754 }
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	755 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	756 return 0;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	757 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	758
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	759 static int
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	760 squat_build_word_bytes(struct squat_trie_build_context *ctx, uint32_t uid,
6911 a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	761 const unsigned char *data, unsigned int size)
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	762 {
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	763 struct squat_trie *trie = ctx->trie;
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	764 unsigned int i;
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	765
6911 a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	766 if (trie->hdr.full_len <= trie->hdr.partial_len)
a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	767 i = 0;
a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	768 else {
a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	769 /* the first word is longer than others */
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	770 if (squat_build_add(ctx, uid, data,
6911 a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	771 I_MIN(size, trie->hdr.full_len)) < 0)
a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	772 return -1;
a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	773 i = 1;
a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	774 }
a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	775
a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	776 for (; i < size; i++) {
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	777 if (squat_build_add(ctx, uid, data + i,
6911 a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	778 I_MIN(trie->hdr.partial_len, size-i)) < 0)
4883 d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	779 return -1;
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	780 }
6911 a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	781 return 0;
a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	782 }
a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	783
a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	784 static int
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	785 squat_build_word(struct squat_trie_build_context *ctx, uint32_t uid,
6911 a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	786 const unsigned char data, const uint8_t char_lengths,
a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	787 unsigned int size)
a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	788 {
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	789 struct squat_trie *trie = ctx->trie;
6911 a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	790 unsigned int i, j, bytelen;
a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	791
a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	792 if (char_lengths == NULL) {
a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	793 /* optimization path: all characters are bytes */
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	794 return squat_build_word_bytes(ctx, uid, data, size);
6911 a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	795 }
a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	796
a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	797 if (trie->hdr.full_len <= trie->hdr.partial_len)
a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	798 i = 0;
a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	799 else {
a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	800 /* the first word is longer than others */
a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	801 bytelen = 0;
a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	802 for (j = 0; j < trie->hdr.full_len && bytelen < size; j++)
a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	803 bytelen += char_lengths[bytelen];
a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	804 i_assert(bytelen <= size);
a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	805
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	806 if (squat_build_add(ctx, uid, data, bytelen) < 0)
6911 a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	807 return -1;
a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	808 i = char_lengths[0];
a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	809 }
a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	810
a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	811 for (; i < size; i += char_lengths[i]) {
a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	812 bytelen = 0;
a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	813 for (j = 0; j < trie->hdr.partial_len && i+bytelen < size; j++)
a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	814 bytelen += char_lengths[i + bytelen];
a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	815 i_assert(i + bytelen <= size);
a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	816
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	817 if (squat_build_add(ctx, uid, data + i, bytelen) < 0)
6911 a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	818 return -1;
a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	819 }
a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	820 return 0;
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	821 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	822
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	823 static unsigned char *
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	824 squat_data_normalize(struct squat_trie trie, const unsigned char data,
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	825 unsigned int size)
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	826 {
7187 d9b87e3ce6c8 Treat replacement characters as non-indexed chars. Timo Sirainen <tss@iki.fi> parents: 7161 diff changeset	827 static const unsigned char replacement_utf8[] = { 0xef, 0xbf, 0xbd };
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	828 unsigned char *dest;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	829 unsigned int i;
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	830
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	831 dest = t_malloc(size);
7187 d9b87e3ce6c8 Treat replacement characters as non-indexed chars. Timo Sirainen <tss@iki.fi> parents: 7161 diff changeset	832 for (i = 0; i < size; i++) {
d9b87e3ce6c8 Treat replacement characters as non-indexed chars. Timo Sirainen <tss@iki.fi> parents: 7161 diff changeset	833 if (data[i] == replacement_utf8[0] && i + 2 < size &&
d9b87e3ce6c8 Treat replacement characters as non-indexed chars. Timo Sirainen <tss@iki.fi> parents: 7161 diff changeset	834 data[i+1] == replacement_utf8[1] &&
d9b87e3ce6c8 Treat replacement characters as non-indexed chars. Timo Sirainen <tss@iki.fi> parents: 7161 diff changeset	835 data[i+2] == replacement_utf8[2]) {
d9b87e3ce6c8 Treat replacement characters as non-indexed chars. Timo Sirainen <tss@iki.fi> parents: 7161 diff changeset	836 /* Don't index replacement character */
d9b87e3ce6c8 Treat replacement characters as non-indexed chars. Timo Sirainen <tss@iki.fi> parents: 7161 diff changeset	837 dest[i++] = 0;
d9b87e3ce6c8 Treat replacement characters as non-indexed chars. Timo Sirainen <tss@iki.fi> parents: 7161 diff changeset	838 dest[i++] = 0;
d9b87e3ce6c8 Treat replacement characters as non-indexed chars. Timo Sirainen <tss@iki.fi> parents: 7161 diff changeset	839 dest[i] = 0;
d9b87e3ce6c8 Treat replacement characters as non-indexed chars. Timo Sirainen <tss@iki.fi> parents: 7161 diff changeset	840 } else {
d9b87e3ce6c8 Treat replacement characters as non-indexed chars. Timo Sirainen <tss@iki.fi> parents: 7161 diff changeset	841 dest[i] = trie->hdr.normalize_map[data[i]];
d9b87e3ce6c8 Treat replacement characters as non-indexed chars. Timo Sirainen <tss@iki.fi> parents: 7161 diff changeset	842 }
d9b87e3ce6c8 Treat replacement characters as non-indexed chars. Timo Sirainen <tss@iki.fi> parents: 7161 diff changeset	843 }
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	844 return dest;
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	845 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	846
6940 414c9d631a81 Replaced t_push/t_pop calls with T_FRAME() macros. Timo Sirainen <tss@iki.fi>* parents: 6931 diff changeset	847 static int
414c9d631a81 Replaced t_push/t_pop calls with T_FRAME() macros. Timo Sirainen <tss@iki.fi>* parents: 6931 diff changeset	848 squat_trie_build_more_real(struct squat_trie_build_context *ctx,
414c9d631a81 Replaced t_push/t_pop calls with T_FRAME() macros. Timo Sirainen <tss@iki.fi>* parents: 6931 diff changeset	849 uint32_t uid, enum squat_index_type type,
414c9d631a81 Replaced t_push/t_pop calls with T_FRAME() macros. Timo Sirainen <tss@iki.fi>* parents: 6931 diff changeset	850 const unsigned char *input, unsigned int size)
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	851 {
4878 88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	852 struct squat_trie *trie = ctx->trie;
6911 a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	853 const unsigned char *data;
a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	854 uint8_t *char_lengths;
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	855 unsigned int i, start = 0;
6911 a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	856 bool multibyte_chars = FALSE;
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	857 int ret = 0;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	858
6914 9c3f0e180751 Index message's all headers (including MIME headers) after the body. Timo Sirainen <tss@iki.fi> parents: 6912 diff changeset	859 uid = uid * 2 + (type == SQUAT_INDEX_TYPE_HEADER ? 1 : 0);
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	860
6911 a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	861 char_lengths = t_malloc(size);
a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	862 data = squat_data_normalize(trie, input, size);
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	863 for (i = 0; i < size; i++) {
6911 a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	864 char_lengths[i] = uni_utf8_char_bytes(input[i]);
a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	865 if (char_lengths[i] != 1)
a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	866 multibyte_chars = TRUE;
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	867 if (data[i] != '\0')
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	868 continue;
4886 ae2114f11a0e Memory leak fixes. Also when building a large mailbox flush once in a while Timo Sirainen <tss@iki.fi> parents: 4883 diff changeset	869
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	870 while (start < i && data[start] == '\0')
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	871 start++;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	872 if (i != start) {
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	873 if (squat_build_word(ctx, uid, data + start,
6911 a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	874 !multibyte_chars ? NULL :
a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	875 char_lengths + start,
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	876 i - start) < 0) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	877 ret = -1;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	878 start = i;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	879 break;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	880 }
4883 d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	881 }
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	882 start = i + 1;
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	883 }
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	884 while (start < i && data[start] == '\0')
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	885 start++;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	886 if (i != start) {
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	887 if (squat_build_word(ctx, uid, data + start,
6911 a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	888 !multibyte_chars ? NULL :
a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	889 char_lengths + start, i - start) < 0)
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	890 ret = -1;
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	891 }
6940 414c9d631a81 Replaced t_push/t_pop calls with T_FRAME() macros. Timo Sirainen <tss@iki.fi>* parents: 6931 diff changeset	892 return ret;
414c9d631a81 Replaced t_push/t_pop calls with T_FRAME() macros. Timo Sirainen <tss@iki.fi>* parents: 6931 diff changeset	893 }
414c9d631a81 Replaced t_push/t_pop calls with T_FRAME() macros. Timo Sirainen <tss@iki.fi>* parents: 6931 diff changeset	894
414c9d631a81 Replaced t_push/t_pop calls with T_FRAME() macros. Timo Sirainen <tss@iki.fi>* parents: 6931 diff changeset	895 int squat_trie_build_more(struct squat_trie_build_context *ctx,
414c9d631a81 Replaced t_push/t_pop calls with T_FRAME() macros. Timo Sirainen <tss@iki.fi>* parents: 6931 diff changeset	896 uint32_t uid, enum squat_index_type type,
414c9d631a81 Replaced t_push/t_pop calls with T_FRAME() macros. Timo Sirainen <tss@iki.fi>* parents: 6931 diff changeset	897 const unsigned char *input, unsigned int size)
414c9d631a81 Replaced t_push/t_pop calls with T_FRAME() macros. Timo Sirainen <tss@iki.fi>* parents: 6931 diff changeset	898 {
414c9d631a81 Replaced t_push/t_pop calls with T_FRAME() macros. Timo Sirainen <tss@iki.fi>* parents: 6931 diff changeset	899 int ret;
414c9d631a81 Replaced t_push/t_pop calls with T_FRAME() macros. Timo Sirainen <tss@iki.fi>* parents: 6931 diff changeset	900
414c9d631a81 Replaced t_push/t_pop calls with T_FRAME() macros. Timo Sirainen <tss@iki.fi>* parents: 6931 diff changeset	901 T_FRAME(
414c9d631a81 Replaced t_push/t_pop calls with T_FRAME() macros. Timo Sirainen <tss@iki.fi>* parents: 6931 diff changeset	902 ret = squat_trie_build_more_real(ctx, uid, type, input, size);
414c9d631a81 Replaced t_push/t_pop calls with T_FRAME() macros. Timo Sirainen <tss@iki.fi>* parents: 6931 diff changeset	903 );
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	904 return ret;
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	905 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	906
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	907 static void node_drop_unused_children(struct squat_node *node)
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	908 {
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	909 unsigned char *chars;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	910 struct squat_node children_src, children_dest;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	911 unsigned int i, j, orig_child_count = node->child_count;
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	912
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	913 chars = NODE_CHILDREN_CHARS(node);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	914 children_src = NODE_CHILDREN_NODES(node);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	915
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	916 /* move chars */
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	917 for (i = j = 0; i < orig_child_count; i++) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	918 if (children_src[i].next_uid != 0)
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	919 chars[j++] = chars[i];
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	920 }
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	921 node->child_count = j;
4890 cb3b4153136c 64bit and big endian fixes Timo Sirainen <tss@iki.fi> parents: 4887 diff changeset	922
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	923 /* move children. note that children_dest may point to different
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	924 location than children_src, although they both point to the
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	925 same node. */
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	926 children_dest = NODE_CHILDREN_NODES(node);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	927 for (i = j = 0; i < orig_child_count; i++) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	928 if (children_src[i].next_uid != 0)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	929 children_dest[j++] = children_src[i];
4890 cb3b4153136c 64bit and big endian fixes Timo Sirainen <tss@iki.fi> parents: 4887 diff changeset	930 }
cb3b4153136c 64bit and big endian fixes Timo Sirainen <tss@iki.fi> parents: 4887 diff changeset	931 }
cb3b4153136c 64bit and big endian fixes Timo Sirainen <tss@iki.fi> parents: 4887 diff changeset	932
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	933 static int
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	934 squat_write_node(struct squat_trie_build_context ctx, struct squat_node node,
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	935 uoff_t *node_offset_r, int level)
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	936 {
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	937 struct squat_trie *trie = ctx->trie;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	938 struct squat_node *children;
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	939 unsigned int i;
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	940 uoff_t *node_offsets;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	941 uint8_t child_count;
6940 414c9d631a81 Replaced t_push/t_pop calls with T_FRAME() macros. Timo Sirainen <tss@iki.fi>* parents: 6931 diff changeset	942 int ret;
4901 5e604d6f391e Only ASCII texts were actually indexed. Now UTF-8 input is properly Timo Sirainen <tss@iki.fi> parents: 4898 diff changeset	943
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	944 i_assert(node->next_uid != 0);
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	945
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	946 if (node->children_not_mapped && ctx->compress_nodes) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	947 if (node_read_children(trie, node, MAX_FAST_LEVEL) < 0)
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	948 return -1;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	949 }
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	950
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	951 node->have_sequential = FALSE;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	952 node_drop_unused_children(node);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	953
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	954 child_count = node->child_count;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	955 if (child_count == 0) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	956 i_assert(!node->children_not_mapped \|\|
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	957 node->leaf_string_length == 0);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	958 *node_offset_r = !node->children_not_mapped ? 0 :
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	959 node->children.offset;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	960 return 0;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	961 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	962 i_assert(!node->children_not_mapped);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	963
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	964 trie->hdr.node_count++;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	965
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	966 children = NODE_CHILDREN_NODES(node);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	967 node_offsets = t_new(uoff_t, child_count);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	968 for (i = 0; i < child_count; i++) {
6940 414c9d631a81 Replaced t_push/t_pop calls with T_FRAME() macros. Timo Sirainen <tss@iki.fi>* parents: 6931 diff changeset	969 T_FRAME(
414c9d631a81 Replaced t_push/t_pop calls with T_FRAME() macros. Timo Sirainen <tss@iki.fi>* parents: 6931 diff changeset	970 ret = squat_write_node(ctx, &children[i],
414c9d631a81 Replaced t_push/t_pop calls with T_FRAME() macros. Timo Sirainen <tss@iki.fi>* parents: 6931 diff changeset	971 &node_offsets[i], level + 1);
414c9d631a81 Replaced t_push/t_pop calls with T_FRAME() macros. Timo Sirainen <tss@iki.fi>* parents: 6931 diff changeset	972 );
414c9d631a81 Replaced t_push/t_pop calls with T_FRAME() macros. Timo Sirainen <tss@iki.fi>* parents: 6931 diff changeset	973 if (ret < 0)
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	974 return -1;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	975 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	976
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	977 *node_offset_r = ctx->output->offset;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	978 node_write_children(ctx, node, node_offsets);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	979 return 0;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	980 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	981
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	982 static int squat_write_nodes(struct squat_trie_build_context *ctx)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	983 {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	984 struct squat_trie *trie = ctx->trie;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	985 uoff_t node_offset;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	986 int ret;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	987
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	988 if (ctx->trie->root.next_uid == 0)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	989 return 0;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	990
6940 414c9d631a81 Replaced t_push/t_pop calls with T_FRAME() macros. Timo Sirainen <tss@iki.fi>* parents: 6931 diff changeset	991 T_FRAME(
414c9d631a81 Replaced t_push/t_pop calls with T_FRAME() macros. Timo Sirainen <tss@iki.fi>* parents: 6931 diff changeset	992 ret = squat_write_node(ctx, &ctx->trie->root, &node_offset, 0);
414c9d631a81 Replaced t_push/t_pop calls with T_FRAME() macros. Timo Sirainen <tss@iki.fi>* parents: 6931 diff changeset	993 );
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	994 if (ret < 0)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	995 return -1;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	996
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	997 trie->hdr.root_offset = node_offset;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	998 trie->hdr.root_unused_uids = trie->root.unused_uids;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	999 trie->hdr.root_next_uid = trie->root.next_uid;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1000 trie->hdr.root_uidlist_idx = trie->root.uid_list_idx;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1001 return 0;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1002 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1003
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1004 static struct squat_trie_iterate_context *
7197 bb9304dee6d4 uidlist renumbering: changed iteration code to iterate children first. This Timo Sirainen <tss@iki.fi> parents: 7187 diff changeset	1005 squat_trie_iterate_init(struct squat_trie *trie)
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1006 {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1007 struct squat_trie_iterate_context *ctx;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1008
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1009 ctx = i_new(struct squat_trie_iterate_context, 1);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1010 ctx->trie = trie;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1011 ctx->cur.node = &trie->root;
6905 8c779f3e16be Fixes to FTS handling. Timo Sirainen <tss@iki.fi> parents: 6902 diff changeset	1012 i_array_init(&ctx->parents, trie->hdr.partial_len*2);
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1013 return ctx;
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1014 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1015
4878 88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1016 static int
7197 bb9304dee6d4 uidlist renumbering: changed iteration code to iterate children first. This Timo Sirainen <tss@iki.fi> parents: 7187 diff changeset	1017 squat_trie_iterate_deinit(struct squat_trie_iterate_context *ctx)
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1018 {
7210 f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1019 struct squat_trie_iterate_node *nodes;
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1020 unsigned int i, count;
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1021 int ret = ctx->failed ? -1 : 0;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1022
7210 f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1023 if (array_is_created(&ctx->cur.shifts)) {
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1024 nodes = array_get_modifiable(&ctx->parents, &count);
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1025 for (i = 0; i < count; i++)
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1026 array_free(&nodes[i].shifts);
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1027 array_free(&ctx->cur.shifts);
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1028 }
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1029 array_free(&ctx->parents);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1030 i_free(ctx);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1031 return ret;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1032 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1033
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1034 static struct squat_node *
7197 bb9304dee6d4 uidlist renumbering: changed iteration code to iterate children first. This Timo Sirainen <tss@iki.fi> parents: 7187 diff changeset	1035 squat_trie_iterate_first(struct squat_trie_iterate_context *ctx)
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1036 {
7197 bb9304dee6d4 uidlist renumbering: changed iteration code to iterate children first. This Timo Sirainen <tss@iki.fi> parents: 7187 diff changeset	1037 if (ctx->cur.node->children_not_mapped) {
bb9304dee6d4 uidlist renumbering: changed iteration code to iterate children first. This Timo Sirainen <tss@iki.fi> parents: 7187 diff changeset	1038 if (node_read_children(ctx->trie, ctx->cur.node, 1) < 0) {
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1039 ctx->failed = TRUE;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1040 return NULL;
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1041 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1042 }
7197 bb9304dee6d4 uidlist renumbering: changed iteration code to iterate children first. This Timo Sirainen <tss@iki.fi> parents: 7187 diff changeset	1043 return ctx->cur.node;
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1044 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1045
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1046 static struct squat_node *
7210 f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1047 squat_trie_iterate_next(struct squat_trie_iterate_context *ctx,
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1048 ARRAY_TYPE(seq_range) *shifts_r)
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1049 {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1050 struct squat_trie_iterate_node *iter_nodes;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1051 struct squat_node *children;
7210 f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1052 unsigned int count, shift_count = 0;
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1053
7210 f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1054 while (ctx->cur.idx == ctx->cur.node->child_count \|\|
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1055 ctx->cur.node->uid_list_idx == 0)
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1056 {
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1057 iter_nodes = array_get_modifiable(&ctx->parents, &count);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1058 if (count == 0)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1059 return NULL;
7210 f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1060
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1061 if (array_is_created(&ctx->cur.shifts))
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1062 array_free(&ctx->cur.shifts);
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1063 ctx->cur = iter_nodes[count-1];
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1064 array_delete(&ctx->parents, count-1, 1);
7197 bb9304dee6d4 uidlist renumbering: changed iteration code to iterate children first. This Timo Sirainen <tss@iki.fi> parents: 7187 diff changeset	1065 }
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1066
7210 f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1067 *shifts_r = ctx->cur.shifts;
7220 21d1d0ced13e Expunge fixes Timo Sirainen <tss@iki.fi> parents: 7219 diff changeset	1068 if (array_is_created(&ctx->cur.shifts))
7210 f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1069 shift_count = array_count(&ctx->cur.shifts);
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1070
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1071 children = NODE_CHILDREN_NODES(ctx->cur.node);
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1072 while (children[ctx->cur.idx++].uid_list_idx == 0) {
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1073 if (ctx->cur.idx == ctx->cur.node->child_count) {
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1074 /* no more non-empty children in this node */
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1075 return squat_trie_iterate_next(ctx, shifts_r);
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1076 }
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1077 }
7197 bb9304dee6d4 uidlist renumbering: changed iteration code to iterate children first. This Timo Sirainen <tss@iki.fi> parents: 7187 diff changeset	1078 array_append(&ctx->parents, &ctx->cur, 1);
bb9304dee6d4 uidlist renumbering: changed iteration code to iterate children first. This Timo Sirainen <tss@iki.fi> parents: 7187 diff changeset	1079 ctx->cur.node = &children[ctx->cur.idx-1];
bb9304dee6d4 uidlist renumbering: changed iteration code to iterate children first. This Timo Sirainen <tss@iki.fi> parents: 7187 diff changeset	1080 ctx->cur.idx = 0;
7210 f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1081 if (shift_count != 0)
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1082 i_array_init(&ctx->cur.shifts, shift_count);
7220 21d1d0ced13e Expunge fixes Timo Sirainen <tss@iki.fi> parents: 7219 diff changeset	1083 else
21d1d0ced13e Expunge fixes Timo Sirainen <tss@iki.fi> parents: 7219 diff changeset	1084 memset(&ctx->cur.shifts, 0, sizeof(ctx->cur.shifts));
7219 6c48b72c5096 Squat fixes Timo Sirainen <tss@iki.fi> parents: 7218 diff changeset	1085 return squat_trie_iterate_first(ctx);
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1086 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1087
7210 f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1088 static void
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1089 squat_uidlist_update_expunged_uids(const ARRAY_TYPE(seq_range) *shifts_arr,
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1090 ARRAY_TYPE(seq_range) *child_shifts,
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1091 ARRAY_TYPE(seq_range) *uids_arr,
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1092 bool do_shifts)
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1093 {
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1094 const struct seq_range *shifts;
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1095 struct seq_range *uids, shift;
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1096 unsigned int i, uid_idx, uid_count, shift_count;
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1097 uint32_t child_shift_seq1, child_shift_count, seq_high;
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1098 unsigned int shift_sum = 0, child_sum = 0;
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1099
7220 21d1d0ced13e Expunge fixes Timo Sirainen <tss@iki.fi> parents: 7219 diff changeset	1100 if (!array_is_created(shifts_arr))
21d1d0ced13e Expunge fixes Timo Sirainen <tss@iki.fi> parents: 7219 diff changeset	1101 return;
21d1d0ced13e Expunge fixes Timo Sirainen <tss@iki.fi> parents: 7219 diff changeset	1102
7210 f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1103 uids = array_get_modifiable(uids_arr, &uid_count);
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1104 shifts = array_get(shifts_arr, &shift_count);
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1105 for (i = 0, uid_idx = 0, seq_high = 0;; ) {
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1106 /* skip UID ranges until we skip/overlap shifts */
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1107 while (uid_idx < uid_count &&
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1108 (i == shift_count \|\|
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1109 I_MAX(shifts[i].seq1, seq_high) > uids[uid_idx].seq2))
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1110 {
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1111 i_assert(uids[uid_idx].seq1 >= shift_sum);
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1112 uids[uid_idx].seq1 -= shift_sum;
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1113 uids[uid_idx].seq2 -= shift_sum;
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1114 child_sum += uids[uid_idx].seq2 -
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1115 uids[uid_idx].seq1 + 1;
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1116
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1117 if (uid_idx > 0 &&
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1118 uids[uid_idx-1].seq2 == uids[uid_idx].seq1 - 1) {
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1119 /* we can merge this and the previous range */
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1120 uids[uid_idx-1].seq2 = uids[uid_idx].seq2;
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1121 array_delete(uids_arr, uid_idx, 1);
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1122 uids = array_get_modifiable(uids_arr,
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1123 &uid_count);
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1124 } else {
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1125 uid_idx++;
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1126 }
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1127 }
7220 21d1d0ced13e Expunge fixes Timo Sirainen <tss@iki.fi> parents: 7219 diff changeset	1128 if (uid_idx == uid_count)
7210 f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1129 break;
7220 21d1d0ced13e Expunge fixes Timo Sirainen <tss@iki.fi> parents: 7219 diff changeset	1130
7210 f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1131 shift.seq1 = I_MAX(shifts[i].seq1, seq_high);
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1132 shift.seq2 = shifts[i].seq2;
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1133 if (shift.seq2 < uids[uid_idx].seq1) {
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1134 /* shift is entirely before UID range */
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1135 shift_sum += shift.seq2 - shift.seq1 + 1;
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1136 i++;
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1137 } else {
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1138 /* handle shifts before UID range */
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1139 if (shift.seq1 < uids[uid_idx].seq1) {
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1140 shift_sum += uids[uid_idx].seq1 - shift.seq1;
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1141 shift.seq1 = uids[uid_idx].seq1;
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1142 }
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1143 /* update child shifts */
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1144 child_shift_seq1 = child_sum +
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1145 shift.seq1 - uids[uid_idx].seq1;
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1146 child_shift_count =
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1147 I_MIN(shift.seq2, uids[uid_idx].seq2) -
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1148 shift.seq1 + 1;
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1149 seq_range_array_add_range(child_shifts,
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1150 child_shift_seq1,
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1151 child_shift_seq1 +
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1152 child_shift_count - 1);
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1153 child_sum += child_shift_count;
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1154
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1155 /* if the shifts continue after the UID range,
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1156 treat it in the next loop iteration */
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1157 if (shift.seq2 <= uids[uid_idx].seq2)
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1158 i++;
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1159 else
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1160 seq_high = uids[uid_idx].seq2 + 1;
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1161
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1162 /* update UIDs - no matter where within the UID range
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1163 the shifts happened, the result is the same:
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1164 shift number of UIDs are removed, and the rest
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1165 are decreased by shift_sum.
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1166
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1167 123 uids child_shifts
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1168 a) s -> 12 1
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1169 b) s -> 12 2
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1170 c) s -> 12 3
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1171 */
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1172 if (uids[uid_idx].seq1 +
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1173 child_shift_count > uids[uid_idx].seq2) {
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1174 /* removed completely */
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1175 array_delete(uids_arr, uid_idx, 1);
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1176 uids = array_get_modifiable(uids_arr,
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1177 &uid_count);
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1178 } else {
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1179 /* the next loop iteration fixes the UIDs */
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1180 uids[uid_idx].seq1 += child_shift_count;
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1181 }
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1182 shift_sum += child_shift_count;
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1183 }
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1184 if (!do_shifts) {
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1185 /* root node - UIDs are only removed, not shifted */
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1186 shift_sum = 0;
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1187 }
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1188 }
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1189 }
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1190
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1191 static int
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1192 squat_trie_expunge_uidlists(struct squat_trie_build_context *ctx,
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1193 struct squat_uidlist_rebuild_context *rebuild_ctx,
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1194 struct squat_trie_iterate_context *iter,
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1195 const ARRAY_TYPE(seq_range) *expunged_uids)
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1196 {
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1197 struct squat_node *node;
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1198 ARRAY_TYPE(seq_range) uid_range, shifts;
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1199 bool shift = FALSE;
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1200 int ret = 0;
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1201
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1202 node = squat_trie_iterate_first(iter);
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1203 if (node->uid_list_idx == 0)
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1204 return 0;
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1205
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1206 i_array_init(&uid_range, 1024);
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1207 i_array_init(&shifts, array_count(expunged_uids));
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1208 array_append_array(&shifts, expunged_uids);
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1209
7220 21d1d0ced13e Expunge fixes Timo Sirainen <tss@iki.fi> parents: 7219 diff changeset	1210 do {
7210 f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1211 i_assert(node->uid_list_idx != 0);
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1212 array_clear(&uid_range);
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1213 if (squat_uidlist_get_seqrange(ctx->trie->uidlist,
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1214 node->uid_list_idx,
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1215 &uid_range) < 0) {
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1216 ret = -1;
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1217 break;
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1218 }
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1219 squat_uidlist_update_expunged_uids(&shifts, &iter->cur.shifts,
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1220 &uid_range, shift);
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1221 node->uid_list_idx =
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1222 squat_uidlist_rebuild_nextu(rebuild_ctx, &uid_range);
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1223 if (node->uid_list_idx == 0) {
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1224 node->child_count = 0;
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1225 node->next_uid = 0;
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1226 }
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1227 node = squat_trie_iterate_next(iter, &shifts);
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1228 shift = TRUE;
7220 21d1d0ced13e Expunge fixes Timo Sirainen <tss@iki.fi> parents: 7219 diff changeset	1229 } while (node != NULL);
7210 f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1230 array_free(&uid_range);
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1231 return ret;
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1232 }
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1233
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1234 static int
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1235 squat_trie_renumber_uidlists2(struct squat_trie_build_context *ctx,
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1236 struct squat_uidlist_rebuild_context *rebuild_ctx,
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1237 struct squat_trie_iterate_context *iter)
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1238 {
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1239 struct squat_node *node;
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1240 ARRAY_TYPE(seq_range) shifts;
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1241 ARRAY_TYPE(uint32_t) uids;
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1242 int ret = 0;
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1243
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1244 node = squat_trie_iterate_first(iter);
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1245 if (node->uid_list_idx == 0)
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1246 return 0;
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1247
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1248 i_array_init(&uids, 1024);
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1249 while (node != NULL) {
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1250 i_assert(node->uid_list_idx != 0);
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1251 if (!UIDLIST_IS_SINGLETON(node->uid_list_idx)) {
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1252 /* rebuild the uidlist */
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1253 array_clear(&uids);
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1254 if (squat_uidlist_get(ctx->trie->uidlist,
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1255 node->uid_list_idx, &uids) < 0) {
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1256 ret = -1;
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1257 break;
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1258 }
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1259 node->uid_list_idx =
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1260 squat_uidlist_rebuild_next(rebuild_ctx, &uids);
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1261 }
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1262 node = squat_trie_iterate_next(iter, &shifts);
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1263 }
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1264 array_free(&uids);
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1265 return ret;
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1266 }
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1267
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	1268 static int
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	1269 squat_trie_renumber_uidlists(struct squat_trie_build_context *ctx,
7210 f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1270 const ARRAY_TYPE(seq_range) *expunged_uids,
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	1271 bool compress)
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1272 {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1273 struct squat_trie_iterate_context *iter;
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	1274 struct squat_uidlist_rebuild_context *rebuild_ctx;
7159 2de2058a5cdc Don't use ioloop_time. Fixes squat-test which doesn't create ioloop. Timo Sirainen <tss@iki.fi> parents: 7158 diff changeset	1275 time_t now;
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1276 int ret = 0;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1277
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	1278 if ((ret = squat_uidlist_rebuild_init(ctx->uidlist_build_ctx,
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	1279 compress, &rebuild_ctx)) <= 0)
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1280 return ret;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1281
7159 2de2058a5cdc Don't use ioloop_time. Fixes squat-test which doesn't create ioloop. Timo Sirainen <tss@iki.fi> parents: 7158 diff changeset	1282 now = time(NULL);
7043 7708a8c166d6 Compiler warning fixes on 32bit systems. Timo Sirainen <tss@iki.fi> parents: 6945 diff changeset	1283 ctx->trie->hdr.indexid =
7159 2de2058a5cdc Don't use ioloop_time. Fixes squat-test which doesn't create ioloop. Timo Sirainen <tss@iki.fi> parents: 7158 diff changeset	1284 I_MAX((unsigned int)now, ctx->trie->hdr.indexid + 1);
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	1285
7197 bb9304dee6d4 uidlist renumbering: changed iteration code to iterate children first. This Timo Sirainen <tss@iki.fi> parents: 7187 diff changeset	1286 iter = squat_trie_iterate_init(ctx->trie);
7210 f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1287 if (expunged_uids != NULL) {
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1288 ret = squat_trie_expunge_uidlists(ctx, rebuild_ctx, iter,
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1289 expunged_uids);
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1290 } else {
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1291 ret = squat_trie_renumber_uidlists2(ctx, rebuild_ctx, iter);
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1292 }
7197 bb9304dee6d4 uidlist renumbering: changed iteration code to iterate children first. This Timo Sirainen <tss@iki.fi> parents: 7187 diff changeset	1293 squat_trie_iterate_deinit(iter);
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1294
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	1295 /* lock the trie before we rename uidlist */
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	1296 if (squat_trie_lock(ctx->trie, F_WRLCK, &ctx->file_lock) <= 0)
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	1297 ret = -1;
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	1298 return squat_uidlist_rebuild_finish(rebuild_ctx, ret < 0);
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1299 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1300
6905 8c779f3e16be Fixes to FTS handling. Timo Sirainen <tss@iki.fi> parents: 6902 diff changeset	1301 static bool squat_trie_check_header(struct squat_trie *trie)
8c779f3e16be Fixes to FTS handling. Timo Sirainen <tss@iki.fi> parents: 6902 diff changeset	1302 {
8c779f3e16be Fixes to FTS handling. Timo Sirainen <tss@iki.fi> parents: 6902 diff changeset	1303 if (trie->hdr.version != SQUAT_TRIE_VERSION \|\|
8c779f3e16be Fixes to FTS handling. Timo Sirainen <tss@iki.fi> parents: 6902 diff changeset	1304 trie->hdr.uidvalidity != trie->uidvalidity)
8c779f3e16be Fixes to FTS handling. Timo Sirainen <tss@iki.fi> parents: 6902 diff changeset	1305 return FALSE;
8c779f3e16be Fixes to FTS handling. Timo Sirainen <tss@iki.fi> parents: 6902 diff changeset	1306
8c779f3e16be Fixes to FTS handling. Timo Sirainen <tss@iki.fi> parents: 6902 diff changeset	1307 if (trie->hdr.partial_len > trie->hdr.full_len) {
8c779f3e16be Fixes to FTS handling. Timo Sirainen <tss@iki.fi> parents: 6902 diff changeset	1308 i_error("Corrupted %s: partial len > full len", trie->path);
8c779f3e16be Fixes to FTS handling. Timo Sirainen <tss@iki.fi> parents: 6902 diff changeset	1309 return FALSE;
8c779f3e16be Fixes to FTS handling. Timo Sirainen <tss@iki.fi> parents: 6902 diff changeset	1310 }
8c779f3e16be Fixes to FTS handling. Timo Sirainen <tss@iki.fi> parents: 6902 diff changeset	1311 if (trie->hdr.full_len == 0) {
8c779f3e16be Fixes to FTS handling. Timo Sirainen <tss@iki.fi> parents: 6902 diff changeset	1312 i_error("Corrupted %s: full len=0", trie->path);
8c779f3e16be Fixes to FTS handling. Timo Sirainen <tss@iki.fi> parents: 6902 diff changeset	1313 return FALSE;
8c779f3e16be Fixes to FTS handling. Timo Sirainen <tss@iki.fi> parents: 6902 diff changeset	1314 }
8c779f3e16be Fixes to FTS handling. Timo Sirainen <tss@iki.fi> parents: 6902 diff changeset	1315 return TRUE;
8c779f3e16be Fixes to FTS handling. Timo Sirainen <tss@iki.fi> parents: 6902 diff changeset	1316 }
8c779f3e16be Fixes to FTS handling. Timo Sirainen <tss@iki.fi> parents: 6902 diff changeset	1317
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1318 static int squat_trie_map_header(struct squat_trie *trie)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1319 {
6944 66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	1320 int ret;
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	1321
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1322 if (trie->locked_file_size == 0) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1323 /* newly created file */
7158 f41bd3065ec8 Init header for newly created files. Timo Sirainen <tss@iki.fi> parents: 7086 diff changeset	1324 squat_trie_header_init(trie);
6944 66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	1325 return 1;
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1326 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1327 i_assert(trie->fd != -1);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1328
6944 66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	1329 if (trie->mmap_disable) {
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	1330 ret = pread_full(trie->fd, &trie->hdr, sizeof(trie->hdr), 0);
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	1331 if (ret <= 0) {
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	1332 if (ret < 0) {
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	1333 i_error("pread(%s) failed: %m", trie->path);
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	1334 return -1;
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	1335 }
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	1336 i_error("Corrupted %s: File too small", trie->path);
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	1337 return 0;
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	1338 }
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	1339 trie->data = NULL;
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	1340 trie->data_size = 0;
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	1341 } else {
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	1342 if (trie->locked_file_size < sizeof(trie->hdr)) {
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	1343 i_error("Corrupted %s: File too small", trie->path);
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	1344 return 0;
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	1345 }
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	1346 if (trie->mmap_size != 0) {
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	1347 if (munmap(trie->mmap_base, trie->mmap_size) < 0)
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	1348 i_error("munmap(%s) failed: %m", trie->path);
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	1349 }
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	1350
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	1351 trie->mmap_size = trie->locked_file_size;
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	1352 trie->mmap_base = mmap(NULL, trie->mmap_size,
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	1353 PROT_READ \| PROT_WRITE,
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	1354 MAP_SHARED, trie->fd, 0);
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	1355 if (trie->mmap_base == MAP_FAILED) {
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	1356 trie->data = trie->mmap_base = NULL;
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	1357 trie->data_size = trie->mmap_size = 0;
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	1358 i_error("mmap(%s) failed: %m", trie->path);
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	1359 return -1;
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	1360 }
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	1361 memcpy(&trie->hdr, trie->mmap_base, sizeof(trie->hdr));
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	1362 trie->data = trie->mmap_base;
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	1363 trie->data_size = trie->mmap_size;
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1364 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1365
6944 66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	1366 return squat_trie_check_header(trie) ? 1 : 0;
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1367 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1368
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	1369 static int squat_trie_map(struct squat_trie *trie, bool building)
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1370 {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1371 struct file_lock *file_lock = NULL;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1372 bool changed;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1373 int ret;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1374
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1375 if (trie->fd != -1) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1376 if (squat_trie_lock(trie, F_RDLCK, &file_lock) <= 0)
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1377 return -1;
6944 66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	1378 if (trie->mmap_disable && trie->file_cache == NULL)
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	1379 trie->file_cache = file_cache_new(trie->fd);
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1380 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1381
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1382 ret = squat_trie_map_header(trie);
6944 66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	1383 if (ret == 0) {
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	1384 file_lock_free(&file_lock);
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	1385 squat_trie_delete(trie);
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	1386 squat_trie_close(trie);
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	1387 squat_trie_header_init(trie);
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	1388 }
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1389 changed = trie->root.children.offset != trie->hdr.root_offset;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1390
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1391 if (changed \|\| trie->hdr.root_offset == 0) {
7218 43d2f70fb279 Squat fixes Timo Sirainen <tss@iki.fi> parents: 7210 diff changeset	1392 node_free(trie, &trie->root);
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1393 memset(&trie->root, 0, sizeof(trie->root));
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1394 trie->root.want_sequential = TRUE;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1395 trie->root.unused_uids = trie->hdr.root_unused_uids;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1396 trie->root.next_uid = trie->hdr.root_next_uid;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1397 trie->root.uid_list_idx = trie->hdr.root_uidlist_idx;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1398 trie->root.children.offset = trie->hdr.root_offset;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1399
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1400 if (trie->hdr.root_offset == 0) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1401 trie->unmapped_child_count = 0;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1402 trie->root.children_not_mapped = FALSE;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1403 } else {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1404 trie->unmapped_child_count = 1;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1405 trie->root.children_not_mapped = TRUE;
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1406 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1407 }
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1408
6944 66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	1409 if (ret >= 0 && !building) {
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	1410 /* do this while we're still locked */
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	1411 ret = squat_uidlist_refresh(trie->uidlist);
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	1412 }
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	1413
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1414 if (file_lock != NULL)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1415 file_unlock(&file_lock);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1416 if (ret < 0)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1417 return -1;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1418
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1419 return trie->hdr.root_offset == 0 \|\| !changed ? 0 :
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1420 node_read_children(trie, &trie->root, 1);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1421 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1422
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1423 int squat_trie_build_init(struct squat_trie trie, uint32_t last_uid_r,
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1424 struct squat_trie_build_context **ctx_r)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1425 {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1426 struct squat_trie_build_context *ctx;
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	1427 struct squat_uidlist_build_context *uidlist_build_ctx;
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1428
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1429 if (trie->fd == -1) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1430 trie->fd = open(trie->path, O_RDWR \| O_CREAT, 0600);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1431 if (trie->fd == -1) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1432 i_error("creat(%s) failed: %m", trie->path);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1433 return -1;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1434 }
7218 43d2f70fb279 Squat fixes Timo Sirainen <tss@iki.fi> parents: 7210 diff changeset	1435 if (trie->file_cache != NULL)
43d2f70fb279 Squat fixes Timo Sirainen <tss@iki.fi> parents: 7210 diff changeset	1436 file_cache_set_fd(trie->file_cache, trie->fd);
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1437 }
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1438
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1439 /* uidlist locks building */
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	1440 if (squat_uidlist_build_init(trie->uidlist, &uidlist_build_ctx) < 0)
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1441 return -1;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1442
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	1443 if (squat_trie_map(trie, TRUE) < 0) {
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	1444 squat_uidlist_build_deinit(&uidlist_build_ctx);
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1445 return -1;
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	1446 }
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1447
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1448 ctx = i_new(struct squat_trie_build_context, 1);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1449 ctx->trie = trie;
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	1450 ctx->uidlist_build_ctx = uidlist_build_ctx;
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1451 ctx->first_uid = trie->root.next_uid;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1452
6905 8c779f3e16be Fixes to FTS handling. Timo Sirainen <tss@iki.fi> parents: 6902 diff changeset	1453 *last_uid_r = I_MAX((trie->root.next_uid+1)/2, 1) - 1;
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1454 *ctx_r = ctx;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1455 return 0;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1456 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1457
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	1458 static int squat_trie_write_lock(struct squat_trie_build_context *ctx)
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	1459 {
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	1460 if (ctx->file_lock != NULL)
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	1461 return 0;
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	1462
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	1463 if (squat_trie_lock(ctx->trie, F_WRLCK, &ctx->file_lock) <= 0)
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	1464 return -1;
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	1465 return 0;
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	1466 }
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	1467
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1468 static int squat_trie_write(struct squat_trie_build_context *ctx)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1469 {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1470 struct squat_trie *trie = ctx->trie;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1471 struct ostream *output;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1472 const char *path;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1473 int fd = -1, ret = 0;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1474
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1475 if ((trie->hdr.used_file_size > sizeof(trie->hdr) &&
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1476 trie->unmapped_child_count < trie->hdr.node_count/4) \|\| 1) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1477 /* we might as well recreate the file */
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1478 ctx->compress_nodes = TRUE;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1479
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1480 path = t_strconcat(trie->path, ".tmp", NULL);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1481 fd = open(path, O_RDWR \| O_CREAT, 0600);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1482 if (fd == -1) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1483 i_error("creat(%s) failed: %m", path);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1484 return -1;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1485 }
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	1486 ret = file_wait_lock(fd, path, F_WRLCK, trie->lock_method,
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	1487 SQUAT_TRIE_LOCK_TIMEOUT, &ctx->file_lock);
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	1488 if (ret <= 0) {
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	1489 if (ret == 0)
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	1490 i_error("file_wait_lock(%s) failed: %m", path);
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	1491 (void)close(fd);
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	1492 return -1;
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	1493 }
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	1494
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1495 output = o_stream_create_fd(fd, 0, FALSE);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1496 o_stream_cork(output);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1497 o_stream_send(output, &trie->hdr, sizeof(trie->hdr));
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1498 } else {
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	1499 /* we need to lock only while header is being written */
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1500 path = trie->path;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1501 ctx->compress_nodes =
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1502 trie->hdr.used_file_size == sizeof(trie->hdr);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1503 output = o_stream_create_fd(trie->fd, 0, FALSE);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1504 o_stream_cork(output);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1505
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	1506 if (trie->hdr.used_file_size != 0)
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	1507 o_stream_seek(output, trie->hdr.used_file_size);
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	1508 else {
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	1509 if (squat_trie_write_lock(ctx) < 0) {
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	1510 o_stream_unref(&output);
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	1511 return -1;
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	1512 }
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1513 o_stream_send(output, &trie->hdr, sizeof(trie->hdr));
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	1514 }
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1515 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1516
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1517 ctx->output = output;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1518 ret = squat_write_nodes(ctx);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1519 ctx->output = NULL;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1520
6944 66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	1521 /* write 1 byte guard at the end of file, so that we can verify broken
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	1522 squat_unpack_num() input by checking if data==end */
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	1523 o_stream_send(output, "", 1);
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	1524
6905 8c779f3e16be Fixes to FTS handling. Timo Sirainen <tss@iki.fi> parents: 6902 diff changeset	1525 if (trie->corrupted)
8c779f3e16be Fixes to FTS handling. Timo Sirainen <tss@iki.fi> parents: 6902 diff changeset	1526 ret = -1;
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	1527 if (ret == 0)
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	1528 ret = squat_trie_write_lock(ctx);
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1529 if (ret == 0) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1530 trie->hdr.used_file_size = output->offset;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1531 o_stream_seek(output, 0);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1532 o_stream_send(output, &trie->hdr, sizeof(trie->hdr));
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1533 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1534 if (output->last_failed_errno != 0) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1535 errno = output->last_failed_errno;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1536 i_error("write() to %s failed: %m", path);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1537 ret = -1;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1538 }
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	1539 o_stream_destroy(&output);
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	1540
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	1541 if (fd == -1) {
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	1542 /* appended to the existing file */
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	1543 return ret;
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	1544 }
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1545
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	1546 /* recreating the trie file */
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	1547 if (ret < 0) {
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	1548 if (close(fd) < 0)
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	1549 i_error("close(%s) failed: %m", path);
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	1550 fd = -1;
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	1551 } else if (rename(path, trie->path) < 0) {
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	1552 i_error("rename(%s, %s) failed: %m", path, trie->path);
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	1553 ret = -1;
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	1554 }
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	1555
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	1556 if (ret < 0) {
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	1557 if (unlink(path) < 0 && errno != ENOENT)
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	1558 i_error("unlink(%s) failed: %m", path);
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	1559 } else {
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	1560 if (trie->fd != -1) {
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1561 if (close(trie->fd) < 0)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1562 i_error("close(%s) failed: %m", trie->path);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1563 }
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	1564 trie->fd = fd;
7218 43d2f70fb279 Squat fixes Timo Sirainen <tss@iki.fi> parents: 7210 diff changeset	1565 if (trie->file_cache != NULL)
43d2f70fb279 Squat fixes Timo Sirainen <tss@iki.fi> parents: 7210 diff changeset	1566 file_cache_set_fd(trie->file_cache, trie->fd);
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1567 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1568 return ret;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1569 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1570
7210 f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1571 int squat_trie_build_deinit(struct squat_trie_build_context **_ctx,
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1572 const ARRAY_TYPE(seq_range) *expunged_uids)
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1573 {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1574 struct squat_trie_build_context ctx = _ctx;
7218 43d2f70fb279 Squat fixes Timo Sirainen <tss@iki.fi> parents: 7210 diff changeset	1575 bool compress, unlock = TRUE;
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1576 int ret;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1577
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1578 *_ctx = NULL;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1579
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1580 compress = (ctx->trie->root.next_uid - ctx->first_uid) > 10;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1581
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	1582 /* keep trie locked while header is being written and when files are
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	1583 being renamed, so that while trie is read locked, uidlist can't
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	1584 change under. */
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	1585 squat_uidlist_build_flush(ctx->uidlist_build_ctx);
7210 f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7197 diff changeset	1586 ret = squat_trie_renumber_uidlists(ctx, expunged_uids, compress);
7218 43d2f70fb279 Squat fixes Timo Sirainen <tss@iki.fi> parents: 7210 diff changeset	1587 if (ret == 0) {
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1588 ret = squat_trie_write(ctx);
7218 43d2f70fb279 Squat fixes Timo Sirainen <tss@iki.fi> parents: 7210 diff changeset	1589 if (ret < 0)
43d2f70fb279 Squat fixes Timo Sirainen <tss@iki.fi> parents: 7210 diff changeset	1590 unlock = FALSE;
43d2f70fb279 Squat fixes Timo Sirainen <tss@iki.fi> parents: 7210 diff changeset	1591 }
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1592
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	1593 if (ret == 0)
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	1594 ret = squat_uidlist_build_finish(ctx->uidlist_build_ctx);
7218 43d2f70fb279 Squat fixes Timo Sirainen <tss@iki.fi> parents: 7210 diff changeset	1595 if (ctx->file_lock != NULL) {
43d2f70fb279 Squat fixes Timo Sirainen <tss@iki.fi> parents: 7210 diff changeset	1596 if (unlock)
43d2f70fb279 Squat fixes Timo Sirainen <tss@iki.fi> parents: 7210 diff changeset	1597 file_unlock(&ctx->file_lock);
43d2f70fb279 Squat fixes Timo Sirainen <tss@iki.fi> parents: 7210 diff changeset	1598 else
43d2f70fb279 Squat fixes Timo Sirainen <tss@iki.fi> parents: 7210 diff changeset	1599 file_lock_free(&ctx->file_lock);
43d2f70fb279 Squat fixes Timo Sirainen <tss@iki.fi> parents: 7210 diff changeset	1600 }
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	1601 squat_uidlist_build_deinit(&ctx->uidlist_build_ctx);
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6914 diff changeset	1602
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1603 i_free(ctx);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1604 return ret;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1605 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1606
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1607 int squat_trie_get_last_uid(struct squat_trie trie, uint32_t last_uid_r)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1608 {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1609 if (trie->fd == -1) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1610 if (squat_trie_open(trie) < 0)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1611 return -1;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1612 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1613
6905 8c779f3e16be Fixes to FTS handling. Timo Sirainen <tss@iki.fi> parents: 6902 diff changeset	1614 *last_uid_r = I_MAX((trie->root.next_uid+1)/2, 1) - 1;
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1615 return 0;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1616 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1617
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1618 static int
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1619 squat_trie_lookup_data(struct squat_trie trie, const unsigned char data,
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1620 unsigned int size, ARRAY_TYPE(seq_range) *uids)
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1621 {
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1622 struct squat_node *node = &trie->root;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1623 unsigned char *chars;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1624 unsigned int idx;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1625 int level = 0;
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1626
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1627 array_clear(uids);
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1628
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1629 for (;;) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1630 if (node->children_not_mapped) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1631 if (node_read_children(trie, node, level) < 0)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1632 return -1;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1633 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1634 if (node->leaf_string_length != 0) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1635 unsigned int str_len = node->leaf_string_length;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1636 const unsigned char *str;
5414 3ec03a4c0f26 fixes Timo Sirainen <tss@iki.fi> parents: 5336 diff changeset	1637
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1638 if (str_len > sizeof(node->children.static_leaf_string))
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1639 str = node->children.leaf_string;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1640 else
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1641 str = node->children.static_leaf_string;
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1642
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1643 if (size > str_len \|\| memcmp(data, str, size) != 0)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1644 return 0;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1645
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1646 /* match */
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1647 break;
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1648 }
4898 07a038b57946 Optimization. Timo Sirainen <tss@iki.fi> parents: 4897 diff changeset	1649
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1650 if (size == 0)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1651 break;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1652 level++;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1653
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1654 if (node->have_sequential) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1655 if (*data < SEQUENTIAL_COUNT) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1656 idx = *data;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1657 goto found;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1658 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1659 idx = SEQUENTIAL_COUNT;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1660 } else {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1661 idx = 0;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1662 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1663 chars = NODE_CHILDREN_CHARS(node);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1664 for (; idx < node->child_count; idx++) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1665 if (chars[idx] == *data)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1666 goto found;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1667 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1668 return 0;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1669 found:
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1670 /* follow to children */
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1671 if (level == 1) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1672 /* root level, add all UIDs */
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1673 if (squat_uidlist_get_seqrange(trie->uidlist,
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1674 node->uid_list_idx,
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1675 uids) < 0)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1676 return -1;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1677 } else {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1678 if (squat_uidlist_filter(trie->uidlist,
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1679 node->uid_list_idx, uids) < 0)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1680 return -1;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1681 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1682 data++;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1683 size--;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1684 node = NODE_CHILDREN_NODES(node) + idx;
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1685 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1686
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1687 if (squat_uidlist_filter(trie->uidlist, node->uid_list_idx, uids) < 0)
4878 88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1688 return -1;
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1689 return 1;
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1690 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1691
4878 88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1692 static void
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1693 squat_trie_filter_type(enum squat_index_type type,
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1694 const ARRAY_TYPE(seq_range) *src,
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1695 ARRAY_TYPE(seq_range) *dest)
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1696 {
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1697 const struct seq_range *src_range;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1698 struct seq_range new_range;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1699 unsigned int i, count, mask;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1700 uint32_t next_seq, uid;
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1701
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1702 array_clear(dest);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1703 src_range = array_get(src, &count);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1704 if (count == 0)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1705 return;
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1706
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1707 if ((type & SQUAT_INDEX_TYPE_HEADER) != 0 &&
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1708 (type & SQUAT_INDEX_TYPE_BODY) != 0) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1709 /* everything is fine, just fix the UIDs */
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1710 new_range.seq1 = src_range[0].seq1 / 2;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1711 new_range.seq2 = src_range[0].seq2 / 2;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1712 for (i = 1; i < count; i++) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1713 next_seq = src_range[i].seq1 / 2;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1714 if (next_seq == new_range.seq2 + 1) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1715 /* we can continue the previous range */
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1716 } else {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1717 array_append(dest, &new_range, 1);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1718 new_range.seq1 = src_range[i].seq1 / 2;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1719 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1720 new_range.seq2 = src_range[i].seq2 / 2;
4878 88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1721 }
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1722 array_append(dest, &new_range, 1);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1723 return;
4886 ae2114f11a0e Memory leak fixes. Also when building a large mailbox flush once in a while Timo Sirainen <tss@iki.fi> parents: 4883 diff changeset	1724 }
ae2114f11a0e Memory leak fixes. Also when building a large mailbox flush once in a while Timo Sirainen <tss@iki.fi> parents: 4883 diff changeset	1725
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1726 /* we'll have to drop either header or body UIDs */
6914 9c3f0e180751 Index message's all headers (including MIME headers) after the body. Timo Sirainen <tss@iki.fi> parents: 6912 diff changeset	1727 mask = (type & SQUAT_INDEX_TYPE_HEADER) != 0 ? 1 : 0;
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1728 for (i = 0; i < count; i++) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1729 for (uid = src_range[i].seq1; uid <= src_range[i].seq2; uid++) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1730 if ((uid & 1) == mask)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1731 seq_range_array_add(dest, 0, uid/2);
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1732 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1733 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1734 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1735
6902 cff5428c3c4d Fixed handling non-indexed characters in search key. Timo Sirainen <tss@iki.fi> parents: 6898 diff changeset	1736 struct squat_trie_lookup_context {
cff5428c3c4d Fixed handling non-indexed characters in search key. Timo Sirainen <tss@iki.fi> parents: 6898 diff changeset	1737 struct squat_trie *trie;
cff5428c3c4d Fixed handling non-indexed characters in search key. Timo Sirainen <tss@iki.fi> parents: 6898 diff changeset	1738 enum squat_index_type type;
cff5428c3c4d Fixed handling non-indexed characters in search key. Timo Sirainen <tss@iki.fi> parents: 6898 diff changeset	1739
cff5428c3c4d Fixed handling non-indexed characters in search key. Timo Sirainen <tss@iki.fi> parents: 6898 diff changeset	1740 ARRAY_TYPE(seq_range) definite_uids, maybe_uids;
cff5428c3c4d Fixed handling non-indexed characters in search key. Timo Sirainen <tss@iki.fi> parents: 6898 diff changeset	1741 ARRAY_TYPE(seq_range) tmp_uids, tmp_uids2;
cff5428c3c4d Fixed handling non-indexed characters in search key. Timo Sirainen <tss@iki.fi> parents: 6898 diff changeset	1742 bool first;
cff5428c3c4d Fixed handling non-indexed characters in search key. Timo Sirainen <tss@iki.fi> parents: 6898 diff changeset	1743 };
cff5428c3c4d Fixed handling non-indexed characters in search key. Timo Sirainen <tss@iki.fi> parents: 6898 diff changeset	1744
cff5428c3c4d Fixed handling non-indexed characters in search key. Timo Sirainen <tss@iki.fi> parents: 6898 diff changeset	1745 static int
cff5428c3c4d Fixed handling non-indexed characters in search key. Timo Sirainen <tss@iki.fi> parents: 6898 diff changeset	1746 squat_trie_lookup_partial(struct squat_trie_lookup_context *ctx,
6911 a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	1747 const unsigned char data, uint8_t char_lengths,
a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	1748 unsigned int size)
6902 cff5428c3c4d Fixed handling non-indexed characters in search key. Timo Sirainen <tss@iki.fi> parents: 6898 diff changeset	1749 {
6911 a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	1750 const unsigned int partial_len = ctx->trie->hdr.partial_len;
a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	1751 unsigned int char_idx, max_chars, i, j, bytelen;
6902 cff5428c3c4d Fixed handling non-indexed characters in search key. Timo Sirainen <tss@iki.fi> parents: 6898 diff changeset	1752 int ret;
cff5428c3c4d Fixed handling non-indexed characters in search key. Timo Sirainen <tss@iki.fi> parents: 6898 diff changeset	1753
6911 a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	1754 max_chars = uni_utf8_strlen_n(data, size);
a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	1755 if (max_chars > ctx->trie->hdr.partial_len)
a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	1756 max_chars = partial_len;
6902 cff5428c3c4d Fixed handling non-indexed characters in search key. Timo Sirainen <tss@iki.fi> parents: 6898 diff changeset	1757
6911 a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	1758 for (i = 0, char_idx = 0; char_idx < max_chars; char_idx++) {
a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	1759 bytelen = 0;
a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	1760 for (j = 0; j < partial_len && i+bytelen < size; j++)
a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	1761 bytelen += char_lengths[i + bytelen];
a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	1762
a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	1763 ret = squat_trie_lookup_data(ctx->trie, data + i, bytelen,
6902 cff5428c3c4d Fixed handling non-indexed characters in search key. Timo Sirainen <tss@iki.fi> parents: 6898 diff changeset	1764 &ctx->tmp_uids);
cff5428c3c4d Fixed handling non-indexed characters in search key. Timo Sirainen <tss@iki.fi> parents: 6898 diff changeset	1765 if (ret <= 0) {
cff5428c3c4d Fixed handling non-indexed characters in search key. Timo Sirainen <tss@iki.fi> parents: 6898 diff changeset	1766 array_clear(ctx->maybe_uids);
cff5428c3c4d Fixed handling non-indexed characters in search key. Timo Sirainen <tss@iki.fi> parents: 6898 diff changeset	1767 return ret;
cff5428c3c4d Fixed handling non-indexed characters in search key. Timo Sirainen <tss@iki.fi> parents: 6898 diff changeset	1768 }
cff5428c3c4d Fixed handling non-indexed characters in search key. Timo Sirainen <tss@iki.fi> parents: 6898 diff changeset	1769
cff5428c3c4d Fixed handling non-indexed characters in search key. Timo Sirainen <tss@iki.fi> parents: 6898 diff changeset	1770 if (ctx->first) {
cff5428c3c4d Fixed handling non-indexed characters in search key. Timo Sirainen <tss@iki.fi> parents: 6898 diff changeset	1771 squat_trie_filter_type(ctx->type, &ctx->tmp_uids,
cff5428c3c4d Fixed handling non-indexed characters in search key. Timo Sirainen <tss@iki.fi> parents: 6898 diff changeset	1772 ctx->maybe_uids);
cff5428c3c4d Fixed handling non-indexed characters in search key. Timo Sirainen <tss@iki.fi> parents: 6898 diff changeset	1773 ctx->first = FALSE;
cff5428c3c4d Fixed handling non-indexed characters in search key. Timo Sirainen <tss@iki.fi> parents: 6898 diff changeset	1774 } else {
cff5428c3c4d Fixed handling non-indexed characters in search key. Timo Sirainen <tss@iki.fi> parents: 6898 diff changeset	1775 squat_trie_filter_type(ctx->type, &ctx->tmp_uids,
cff5428c3c4d Fixed handling non-indexed characters in search key. Timo Sirainen <tss@iki.fi> parents: 6898 diff changeset	1776 &ctx->tmp_uids2);
cff5428c3c4d Fixed handling non-indexed characters in search key. Timo Sirainen <tss@iki.fi> parents: 6898 diff changeset	1777 seq_range_array_remove_invert_range(ctx->maybe_uids,
cff5428c3c4d Fixed handling non-indexed characters in search key. Timo Sirainen <tss@iki.fi> parents: 6898 diff changeset	1778 &ctx->tmp_uids2);
cff5428c3c4d Fixed handling non-indexed characters in search key. Timo Sirainen <tss@iki.fi> parents: 6898 diff changeset	1779 }
6911 a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	1780
a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	1781 i += char_lengths[i];
a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	1782 }
6902 cff5428c3c4d Fixed handling non-indexed characters in search key. Timo Sirainen <tss@iki.fi> parents: 6898 diff changeset	1783 return 1;
cff5428c3c4d Fixed handling non-indexed characters in search key. Timo Sirainen <tss@iki.fi> parents: 6898 diff changeset	1784 }
cff5428c3c4d Fixed handling non-indexed characters in search key. Timo Sirainen <tss@iki.fi> parents: 6898 diff changeset	1785
6905 8c779f3e16be Fixes to FTS handling. Timo Sirainen <tss@iki.fi> parents: 6902 diff changeset	1786 static void squat_trie_add_unknown(struct squat_trie *trie,
8c779f3e16be Fixes to FTS handling. Timo Sirainen <tss@iki.fi> parents: 6902 diff changeset	1787 ARRAY_TYPE(seq_range) *maybe_uids)
8c779f3e16be Fixes to FTS handling. Timo Sirainen <tss@iki.fi> parents: 6902 diff changeset	1788 {
8c779f3e16be Fixes to FTS handling. Timo Sirainen <tss@iki.fi> parents: 6902 diff changeset	1789 struct seq_range *range, new_range;
8c779f3e16be Fixes to FTS handling. Timo Sirainen <tss@iki.fi> parents: 6902 diff changeset	1790 unsigned int count;
8c779f3e16be Fixes to FTS handling. Timo Sirainen <tss@iki.fi> parents: 6902 diff changeset	1791 uint32_t last_uid;
8c779f3e16be Fixes to FTS handling. Timo Sirainen <tss@iki.fi> parents: 6902 diff changeset	1792
8c779f3e16be Fixes to FTS handling. Timo Sirainen <tss@iki.fi> parents: 6902 diff changeset	1793 last_uid = I_MAX((trie->root.next_uid+1)/2, 1) - 1;
8c779f3e16be Fixes to FTS handling. Timo Sirainen <tss@iki.fi> parents: 6902 diff changeset	1794
8c779f3e16be Fixes to FTS handling. Timo Sirainen <tss@iki.fi> parents: 6902 diff changeset	1795 range = array_get_modifiable(maybe_uids, &count);
8c779f3e16be Fixes to FTS handling. Timo Sirainen <tss@iki.fi> parents: 6902 diff changeset	1796 if (count > 0 && range[count-1].seq2 == last_uid) {
8c779f3e16be Fixes to FTS handling. Timo Sirainen <tss@iki.fi> parents: 6902 diff changeset	1797 /* increase the range */
8c779f3e16be Fixes to FTS handling. Timo Sirainen <tss@iki.fi> parents: 6902 diff changeset	1798 range[count-1].seq2 = (uint32_t)-1;
8c779f3e16be Fixes to FTS handling. Timo Sirainen <tss@iki.fi> parents: 6902 diff changeset	1799 } else {
8c779f3e16be Fixes to FTS handling. Timo Sirainen <tss@iki.fi> parents: 6902 diff changeset	1800 new_range.seq1 = last_uid + 1;
8c779f3e16be Fixes to FTS handling. Timo Sirainen <tss@iki.fi> parents: 6902 diff changeset	1801 new_range.seq2 = (uint32_t)-1;
8c779f3e16be Fixes to FTS handling. Timo Sirainen <tss@iki.fi> parents: 6902 diff changeset	1802 array_append(maybe_uids, &new_range, 1);
8c779f3e16be Fixes to FTS handling. Timo Sirainen <tss@iki.fi> parents: 6902 diff changeset	1803 }
8c779f3e16be Fixes to FTS handling. Timo Sirainen <tss@iki.fi> parents: 6902 diff changeset	1804 }
8c779f3e16be Fixes to FTS handling. Timo Sirainen <tss@iki.fi> parents: 6902 diff changeset	1805
6940 414c9d631a81 Replaced t_push/t_pop calls with T_FRAME() macros. Timo Sirainen <tss@iki.fi>* parents: 6931 diff changeset	1806 static int
414c9d631a81 Replaced t_push/t_pop calls with T_FRAME() macros. Timo Sirainen <tss@iki.fi>* parents: 6931 diff changeset	1807 squat_trie_lookup_real(struct squat_trie trie, const char str,
414c9d631a81 Replaced t_push/t_pop calls with T_FRAME() macros. Timo Sirainen <tss@iki.fi>* parents: 6931 diff changeset	1808 enum squat_index_type type,
414c9d631a81 Replaced t_push/t_pop calls with T_FRAME() macros. Timo Sirainen <tss@iki.fi>* parents: 6931 diff changeset	1809 ARRAY_TYPE(seq_range) *definite_uids,
414c9d631a81 Replaced t_push/t_pop calls with T_FRAME() macros. Timo Sirainen <tss@iki.fi>* parents: 6931 diff changeset	1810 ARRAY_TYPE(seq_range) *maybe_uids)
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1811 {
6902 cff5428c3c4d Fixed handling non-indexed characters in search key. Timo Sirainen <tss@iki.fi> parents: 6898 diff changeset	1812 struct squat_trie_lookup_context ctx;
cff5428c3c4d Fixed handling non-indexed characters in search key. Timo Sirainen <tss@iki.fi> parents: 6898 diff changeset	1813 unsigned char *data;
6911 a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	1814 uint8_t *char_lengths;
a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	1815 unsigned int i, start, bytes, str_bytelen, str_charlen;
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1816 int ret = 0;
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1817
7161 b2333f0f23af squat_trie_lookup(): Make sure we clear result arrays first. Timo Sirainen <tss@iki.fi> parents: 7159 diff changeset	1818 array_clear(definite_uids);
b2333f0f23af squat_trie_lookup(): Make sure we clear result arrays first. Timo Sirainen <tss@iki.fi> parents: 7159 diff changeset	1819 array_clear(maybe_uids);
b2333f0f23af squat_trie_lookup(): Make sure we clear result arrays first. Timo Sirainen <tss@iki.fi> parents: 7159 diff changeset	1820
6902 cff5428c3c4d Fixed handling non-indexed characters in search key. Timo Sirainen <tss@iki.fi> parents: 6898 diff changeset	1821 memset(&ctx, 0, sizeof(ctx));
cff5428c3c4d Fixed handling non-indexed characters in search key. Timo Sirainen <tss@iki.fi> parents: 6898 diff changeset	1822 ctx.trie = trie;
cff5428c3c4d Fixed handling non-indexed characters in search key. Timo Sirainen <tss@iki.fi> parents: 6898 diff changeset	1823 ctx.type = type;
cff5428c3c4d Fixed handling non-indexed characters in search key. Timo Sirainen <tss@iki.fi> parents: 6898 diff changeset	1824 ctx.definite_uids = definite_uids;
cff5428c3c4d Fixed handling non-indexed characters in search key. Timo Sirainen <tss@iki.fi> parents: 6898 diff changeset	1825 ctx.maybe_uids = maybe_uids;
6945 5915aea5f070 UID ranges can be large, don't allocate them from data stack. Timo Sirainen <tss@iki.fi> parents: 6944 diff changeset	1826 i_array_init(&ctx.tmp_uids, 128);
5915aea5f070 UID ranges can be large, don't allocate them from data stack. Timo Sirainen <tss@iki.fi> parents: 6944 diff changeset	1827 i_array_init(&ctx.tmp_uids2, 128);
6902 cff5428c3c4d Fixed handling non-indexed characters in search key. Timo Sirainen <tss@iki.fi> parents: 6898 diff changeset	1828 ctx.first = TRUE;
cff5428c3c4d Fixed handling non-indexed characters in search key. Timo Sirainen <tss@iki.fi> parents: 6898 diff changeset	1829
6911 a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	1830 str_bytelen = strlen(str);
a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	1831 char_lengths = t_malloc0(str_bytelen);
6912 b326327b1f10 Fix for searching <4 char keys. Timo Sirainen <tss@iki.fi> parents: 6911 diff changeset	1832 for (i = 0, str_charlen = 0; i < str_bytelen; str_charlen++) {
6911 a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	1833 bytes = uni_utf8_char_bytes(str[i]);
a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	1834 char_lengths[i] = bytes;
a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	1835 i += bytes;
a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	1836 }
a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	1837 data = squat_data_normalize(trie, (const unsigned char *)str,
a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	1838 str_bytelen);
6902 cff5428c3c4d Fixed handling non-indexed characters in search key. Timo Sirainen <tss@iki.fi> parents: 6898 diff changeset	1839
6911 a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	1840 for (i = start = 0; i < str_bytelen && ret >= 0; i += char_lengths[i]) {
6902 cff5428c3c4d Fixed handling non-indexed characters in search key. Timo Sirainen <tss@iki.fi> parents: 6898 diff changeset	1841 if (data[i] != '\0')
cff5428c3c4d Fixed handling non-indexed characters in search key. Timo Sirainen <tss@iki.fi> parents: 6898 diff changeset	1842 continue;
cff5428c3c4d Fixed handling non-indexed characters in search key. Timo Sirainen <tss@iki.fi> parents: 6898 diff changeset	1843
cff5428c3c4d Fixed handling non-indexed characters in search key. Timo Sirainen <tss@iki.fi> parents: 6898 diff changeset	1844 /* string has nonindexed characters.
cff5428c3c4d Fixed handling non-indexed characters in search key. Timo Sirainen <tss@iki.fi> parents: 6898 diff changeset	1845 search it in parts. */
cff5428c3c4d Fixed handling non-indexed characters in search key. Timo Sirainen <tss@iki.fi> parents: 6898 diff changeset	1846 if (i != start) {
cff5428c3c4d Fixed handling non-indexed characters in search key. Timo Sirainen <tss@iki.fi> parents: 6898 diff changeset	1847 ret = squat_trie_lookup_partial(&ctx, data + start,
6911 a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	1848 char_lengths,
6902 cff5428c3c4d Fixed handling non-indexed characters in search key. Timo Sirainen <tss@iki.fi> parents: 6898 diff changeset	1849 i - start);
cff5428c3c4d Fixed handling non-indexed characters in search key. Timo Sirainen <tss@iki.fi> parents: 6898 diff changeset	1850 }
6911 a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	1851 start = i + char_lengths[i];
6902 cff5428c3c4d Fixed handling non-indexed characters in search key. Timo Sirainen <tss@iki.fi> parents: 6898 diff changeset	1852 }
cff5428c3c4d Fixed handling non-indexed characters in search key. Timo Sirainen <tss@iki.fi> parents: 6898 diff changeset	1853
cff5428c3c4d Fixed handling non-indexed characters in search key. Timo Sirainen <tss@iki.fi> parents: 6898 diff changeset	1854 if (start != 0) {
cff5428c3c4d Fixed handling non-indexed characters in search key. Timo Sirainen <tss@iki.fi> parents: 6898 diff changeset	1855 /* string has nonindexed characters. finish the search. */
cff5428c3c4d Fixed handling non-indexed characters in search key. Timo Sirainen <tss@iki.fi> parents: 6898 diff changeset	1856 array_clear(definite_uids);
cff5428c3c4d Fixed handling non-indexed characters in search key. Timo Sirainen <tss@iki.fi> parents: 6898 diff changeset	1857 if (i != start && ret >= 0) {
cff5428c3c4d Fixed handling non-indexed characters in search key. Timo Sirainen <tss@iki.fi> parents: 6898 diff changeset	1858 ret = squat_trie_lookup_partial(&ctx, data + start,
6911 a215deb3de8d Fixed non-ASCII searches. Timo Sirainen <tss@iki.fi> parents: 6905 diff changeset	1859 char_lengths,
6902 cff5428c3c4d Fixed handling non-indexed characters in search key. Timo Sirainen <tss@iki.fi> parents: 6898 diff changeset	1860 i - start);
cff5428c3c4d Fixed handling non-indexed characters in search key. Timo Sirainen <tss@iki.fi> parents: 6898 diff changeset	1861 }
6945 5915aea5f070 UID ranges can be large, don't allocate them from data stack. Timo Sirainen <tss@iki.fi> parents: 6944 diff changeset	1862 } else {
5915aea5f070 UID ranges can be large, don't allocate them from data stack. Timo Sirainen <tss@iki.fi> parents: 6944 diff changeset	1863 if (str_charlen <= trie->hdr.partial_len \|\|
5915aea5f070 UID ranges can be large, don't allocate them from data stack. Timo Sirainen <tss@iki.fi> parents: 6944 diff changeset	1864 trie->hdr.full_len > trie->hdr.partial_len) {
5915aea5f070 UID ranges can be large, don't allocate them from data stack. Timo Sirainen <tss@iki.fi> parents: 6944 diff changeset	1865 ret = squat_trie_lookup_data(trie, data, str_bytelen,
5915aea5f070 UID ranges can be large, don't allocate them from data stack. Timo Sirainen <tss@iki.fi> parents: 6944 diff changeset	1866 &ctx.tmp_uids);
5915aea5f070 UID ranges can be large, don't allocate them from data stack. Timo Sirainen <tss@iki.fi> parents: 6944 diff changeset	1867 if (ret > 0) {
5915aea5f070 UID ranges can be large, don't allocate them from data stack. Timo Sirainen <tss@iki.fi> parents: 6944 diff changeset	1868 squat_trie_filter_type(type, &ctx.tmp_uids,
5915aea5f070 UID ranges can be large, don't allocate them from data stack. Timo Sirainen <tss@iki.fi> parents: 6944 diff changeset	1869 definite_uids);
5915aea5f070 UID ranges can be large, don't allocate them from data stack. Timo Sirainen <tss@iki.fi> parents: 6944 diff changeset	1870 }
5915aea5f070 UID ranges can be large, don't allocate them from data stack. Timo Sirainen <tss@iki.fi> parents: 6944 diff changeset	1871 } else {
5915aea5f070 UID ranges can be large, don't allocate them from data stack. Timo Sirainen <tss@iki.fi> parents: 6944 diff changeset	1872 array_clear(definite_uids);
6902 cff5428c3c4d Fixed handling non-indexed characters in search key. Timo Sirainen <tss@iki.fi> parents: 6898 diff changeset	1873 }
4878 88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1874
6945 5915aea5f070 UID ranges can be large, don't allocate them from data stack. Timo Sirainen <tss@iki.fi> parents: 6944 diff changeset	1875 if (str_charlen <= trie->hdr.partial_len \|\|
5915aea5f070 UID ranges can be large, don't allocate them from data stack. Timo Sirainen <tss@iki.fi> parents: 6944 diff changeset	1876 trie->hdr.partial_len == 0) {
5915aea5f070 UID ranges can be large, don't allocate them from data stack. Timo Sirainen <tss@iki.fi> parents: 6944 diff changeset	1877 /* we have the result */
5915aea5f070 UID ranges can be large, don't allocate them from data stack. Timo Sirainen <tss@iki.fi> parents: 6944 diff changeset	1878 array_clear(maybe_uids);
5915aea5f070 UID ranges can be large, don't allocate them from data stack. Timo Sirainen <tss@iki.fi> parents: 6944 diff changeset	1879 } else {
5915aea5f070 UID ranges can be large, don't allocate them from data stack. Timo Sirainen <tss@iki.fi> parents: 6944 diff changeset	1880 ret = squat_trie_lookup_partial(&ctx, data + start,
5915aea5f070 UID ranges can be large, don't allocate them from data stack. Timo Sirainen <tss@iki.fi> parents: 6944 diff changeset	1881 char_lengths,
5915aea5f070 UID ranges can be large, don't allocate them from data stack. Timo Sirainen <tss@iki.fi> parents: 6944 diff changeset	1882 i - start);
5915aea5f070 UID ranges can be large, don't allocate them from data stack. Timo Sirainen <tss@iki.fi> parents: 6944 diff changeset	1883 }
4878 88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1884 }
6905 8c779f3e16be Fixes to FTS handling. Timo Sirainen <tss@iki.fi> parents: 6902 diff changeset	1885 squat_trie_add_unknown(trie, maybe_uids);
6945 5915aea5f070 UID ranges can be large, don't allocate them from data stack. Timo Sirainen <tss@iki.fi> parents: 6944 diff changeset	1886 array_free(&ctx.tmp_uids);
5915aea5f070 UID ranges can be large, don't allocate them from data stack. Timo Sirainen <tss@iki.fi> parents: 6944 diff changeset	1887 array_free(&ctx.tmp_uids2);
6905 8c779f3e16be Fixes to FTS handling. Timo Sirainen <tss@iki.fi> parents: 6902 diff changeset	1888 return ret < 0 ? -1 : 0;
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1889 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1890
6940 414c9d631a81 Replaced t_push/t_pop calls with T_FRAME() macros. Timo Sirainen <tss@iki.fi>* parents: 6931 diff changeset	1891 int squat_trie_lookup(struct squat_trie trie, const char str,
414c9d631a81 Replaced t_push/t_pop calls with T_FRAME() macros. Timo Sirainen <tss@iki.fi>* parents: 6931 diff changeset	1892 enum squat_index_type type,
414c9d631a81 Replaced t_push/t_pop calls with T_FRAME() macros. Timo Sirainen <tss@iki.fi>* parents: 6931 diff changeset	1893 ARRAY_TYPE(seq_range) *definite_uids,
414c9d631a81 Replaced t_push/t_pop calls with T_FRAME() macros. Timo Sirainen <tss@iki.fi>* parents: 6931 diff changeset	1894 ARRAY_TYPE(seq_range) *maybe_uids)
414c9d631a81 Replaced t_push/t_pop calls with T_FRAME() macros. Timo Sirainen <tss@iki.fi>* parents: 6931 diff changeset	1895 {
414c9d631a81 Replaced t_push/t_pop calls with T_FRAME() macros. Timo Sirainen <tss@iki.fi>* parents: 6931 diff changeset	1896 int ret;
414c9d631a81 Replaced t_push/t_pop calls with T_FRAME() macros. Timo Sirainen <tss@iki.fi>* parents: 6931 diff changeset	1897
414c9d631a81 Replaced t_push/t_pop calls with T_FRAME() macros. Timo Sirainen <tss@iki.fi>* parents: 6931 diff changeset	1898 T_FRAME(
414c9d631a81 Replaced t_push/t_pop calls with T_FRAME() macros. Timo Sirainen <tss@iki.fi>* parents: 6931 diff changeset	1899 ret = squat_trie_lookup_real(trie, str, type,
414c9d631a81 Replaced t_push/t_pop calls with T_FRAME() macros. Timo Sirainen <tss@iki.fi>* parents: 6931 diff changeset	1900 definite_uids, maybe_uids);
414c9d631a81 Replaced t_push/t_pop calls with T_FRAME() macros. Timo Sirainen <tss@iki.fi>* parents: 6931 diff changeset	1901 );
414c9d631a81 Replaced t_push/t_pop calls with T_FRAME() macros. Timo Sirainen <tss@iki.fi>* parents: 6931 diff changeset	1902 return ret;
414c9d631a81 Replaced t_push/t_pop calls with T_FRAME() macros. Timo Sirainen <tss@iki.fi>* parents: 6931 diff changeset	1903 }
414c9d631a81 Replaced t_push/t_pop calls with T_FRAME() macros. Timo Sirainen <tss@iki.fi>* parents: 6931 diff changeset	1904
6418 46d9ee79f292 Removed _ prefix from all public APIs. Timo Sirainen <tss@iki.fi> parents: 6414 diff changeset	1905 struct squat_uidlist squat_trie_get_uidlist(struct squat_trie trie)
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1906 {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1907 return trie->uidlist;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1908 }
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1909
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1910 size_t squat_trie_mem_used(struct squat_trie trie, unsigned int count_r)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1911 {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1912 *count_r = trie->hdr.node_count;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1913 return trie->node_alloc_size;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1914 }

Mercurial > dovecot > original-hg > dovecot-1.2

annotate src/plugins/fts-squat/squat-trie.c @ 7220:21d1d0ced13e HEAD