dovecot/original-hg/dovecot-1.2: src/plugins/fts-squat/squat-trie.c annotate

annotate src/plugins/fts-squat/squat-trie.c @ 4886:ae2114f11a0e HEAD

Memory leak fixes. Also when building a large mailbox flush once in a while to free memory.

author	Timo Sirainen <tss@iki.fi>
date	Sun, 10 Dec 2006 01:01:13 +0200
parents	d8adbe93c969
children	463a4ebba685

rev	line source
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1 /* Copyright (C) 2006 Timo Sirainen */
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	2
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	3 #include "lib.h"
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	4 #include "array.h"
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	5 #include "bsearch-insert-pos.h"
4883 d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	6 #include "file-cache.h"
4878 88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	7 #include "file-lock.h"
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	8 #include "istream.h"
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	9 #include "ostream.h"
4883 d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	10 #include "read-full.h"
4879 b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	11 #include "write-full.h"
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	12 #include "mmap-util.h"
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	13 #include "squat-uidlist.h"
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	14 #include "squat-trie.h"
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	15
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	16 #include <stdio.h>
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	17 #include <stdlib.h>
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	18 #include <unistd.h>
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	19 #include <fcntl.h>
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	20 #include <ctype.h>
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	21
4878 88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	22 #define TRIE_COMPRESS_PERCENTAGE 30
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	23 #define TRIE_COMPRESS_MIN_SIZE (1024*50)
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	24
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	25 #define SQUAT_TRIE_VERSION 1
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	26 #define SQUAT_TRIE_LOCK_TIMEOUT 60
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	27
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	28 /* for non-x86 use memcpy() when accessing unaligned int* addresses */
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	29 #if defined(__i386__) \|\| defined(__x86_64__)
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	30 # define ALLOW_UNALIGNED_ACCESS
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	31 #endif
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	32
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	33 #define BLOCK_SIZE 4
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	34
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	35 #define ALIGN(size) \
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	36 (((size) + sizeof(void )-1) & ~((unsigned int) sizeof(void )-1))
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	37
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	38 struct squat_trie {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	39 char *filepath;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	40 int fd;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	41 dev_t dev;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	42 ino_t ino;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	43
4878 88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	44 enum file_lock_method lock_method;
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	45 struct file_lock *file_lock;
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	46 int lock_count;
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	47 int lock_type; /* F_RDLCK / F_WRLCK */
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	48
4883 d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	49 struct file_cache *file_cache;
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	50 uint32_t file_cache_modify_counter;
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	51
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	52 void mmap_base; / NULL with mmap_disable=yes */
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	53 const uint8_t *const_mmap_base;
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	54 size_t mmap_size;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	55
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	56 const struct squat_trie_header *hdr;
4879 b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	57 uint32_t uidvalidity;
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	58
4879 b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	59 char *uidlist_filepath;
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	60 struct squat_uidlist *uidlist;
4886 ae2114f11a0e Memory leak fixes. Also when building a large mailbox flush once in a while Timo Sirainen <tss@iki.fi> parents: 4883 diff changeset	61
ae2114f11a0e Memory leak fixes. Also when building a large mailbox flush once in a while Timo Sirainen <tss@iki.fi> parents: 4883 diff changeset	62 pool_t node_pool;
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	63 struct trie_node *root;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	64 buffer_t *buf;
4878 88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	65
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	66 unsigned int corrupted:1;
4883 d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	67 unsigned int mmap_disable:1;
4878 88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	68 };
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	69
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	70 struct squat_trie_build_context {
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	71 struct squat_trie *trie;
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	72
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	73 struct ostream *output;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	74
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	75 uint32_t prev_uid;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	76 unsigned int prev_added_size;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	77 uint16_t prev_added[BLOCK_SIZE-1];
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	78
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	79 unsigned int node_count;
4878 88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	80 unsigned int deleted_space;
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	81
4883 d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	82 unsigned int modified:1;
4878 88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	83 unsigned int failed:1;
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	84 unsigned int locked:1;
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	85 };
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	86
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	87 struct squat_trie_compress_context {
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	88 struct squat_trie *trie;
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	89
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	90 const char *tmp_path;
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	91 struct ostream *output;
4886 ae2114f11a0e Memory leak fixes. Also when building a large mailbox flush once in a while Timo Sirainen <tss@iki.fi> parents: 4883 diff changeset	92 int fd;
4878 88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	93
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	94 struct squat_uidlist_compress_ctx *uidlist_ctx;
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	95
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	96 unsigned int node_count;
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	97 };
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	98
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	99 struct squat_trie_header {
4878 88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	100 uint8_t version;
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	101 uint8_t unused[3];
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	102
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	103 uint32_t uidvalidity;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	104 uint32_t used_file_size;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	105 uint32_t deleted_space;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	106 uint32_t node_count;
4883 d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	107 uint32_t modify_counter;
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	108
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	109 uint32_t root_offset;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	110 };
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	111
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	112 /*
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	113 packed_node {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	114 packed ((8bit_chars_count << 1) \| have_16bit_chars);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	115 uint8_t 8bit_chars[8bit_chars_count];
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	116 uint32_t idx[8bit_chars_count];
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	117 if (have_16bit_chars) {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	118 packed 16bit_chars_count;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	119 uint16_t 16bit_chars[16bit_chars_count];
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	120 uint32_t idx[16bit_chars_count];
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	121 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	122 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	123 */
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	124
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	125 struct trie_node {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	126 /* new characters have been added to this node */
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	127 uint8_t resized:1;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	128 /* idx pointers have been updated */
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	129 uint8_t modified:1;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	130 uint8_t chars_8bit_count;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	131 uint16_t chars_16bit_count;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	132
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	133 uint32_t file_offset;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	134 uint32_t orig_size;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	135
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	136 /* the node pointers are valid as long as their lowest bit is 0,
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	137 otherwise they're offsets to the trie file (>> 1).
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	138
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	139 in leaf nodes the children pointers are uint32_t uid_list_idx[]; */
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	140 /* uint8_t 8bit_chars[chars_8bit_count]; */
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	141 /* struct trie_node children[chars_8bit_count]; /
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	142 /* uint16_t 16bit_chars[chars_16bit_count]; */
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	143 /* struct trie_node children[chars_16bit_count]; /
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	144 };
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	145 #define NODE_CHARS8(node) \
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	146 (uint8_t *)(node + 1)
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	147 #define NODE_CHILDREN8(node) \
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	148 (struct trie_node **) \
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	149 ((char *)((node) + 1) + \
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	150 ALIGN(sizeof(uint8_t) * ((node)->chars_8bit_count)))
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	151 #define NODE_CHARS16(node) \
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	152 (uint16_t )((char )NODE_CHILDREN8(node) + \
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	153 sizeof(struct trie_node ) ((node)->chars_8bit_count))
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	154 #define NODE_CHILDREN16(node) \
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	155 (struct trie_node **) \
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	156 ((char *)NODE_CHARS16(node) + \
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	157 ALIGN(sizeof(uint16_t) * ((node)->chars_16bit_count)))
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	158
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	159 static int
4878 88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	160 squat_trie_compress_node(struct squat_trie_compress_context *ctx,
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	161 struct trie_node *node, unsigned int level);
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	162 static int trie_write_node(struct squat_trie_build_context *ctx,
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	163 unsigned int level, struct trie_node *node);
4886 ae2114f11a0e Memory leak fixes. Also when building a large mailbox flush once in a while Timo Sirainen <tss@iki.fi> parents: 4883 diff changeset	164 static int
ae2114f11a0e Memory leak fixes. Also when building a large mailbox flush once in a while Timo Sirainen <tss@iki.fi> parents: 4883 diff changeset	165 squat_trie_build_flush(struct squat_trie_build_context *ctx, bool finish);
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	166
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	167 static int chr_8bit_cmp(const void _key, const void _chr)
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	168 {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	169 const uint8_t key = _key, chr = _chr;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	170
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	171 return key - chr;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	172 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	173
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	174 static int chr_16bit_cmp(const void _key, const void _chr)
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	175 {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	176 const uint16_t key = _key, chr = _chr;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	177
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	178 return key - chr;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	179 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	180
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	181 void _squat_trie_pack_num(buffer_t *buffer, uint32_t num)
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	182 {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	183 uint8_t c;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	184
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	185 /* number continues as long as the highest bit is set */
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	186 while (num >= 0x80) {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	187 c = (num & 0x7f) \| 0x80;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	188 num >>= 7;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	189
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	190 buffer_append(buffer, &c, 1);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	191 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	192
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	193 c = num;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	194 buffer_append(buffer, &c, 1);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	195 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	196
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	197 uint32_t _squat_trie_unpack_num(const uint8_t *p, const uint8_t end)
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	198 {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	199 const uint8_t c = p;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	200 uint32_t value = 0;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	201 unsigned int bits = 0;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	202
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	203 while (c != end && *c >= 0x80) {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	204 value \|= (*c & 0x7f) << bits;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	205 bits += 7;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	206 c++;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	207 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	208
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	209 if (c == end) {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	210 /* last number shouldn't end with high bit */
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	211 return 0;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	212 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	213 if (bits > 32-7) {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	214 /* we have only 32bit numbers */
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	215 return 0;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	216 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	217
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	218 value \|= (*c & 0x7f) << bits;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	219 *p = c + 1;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	220 return value;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	221 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	222
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	223 static const void data_normalize(const void data, size_t size, buffer_t *dest)
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	224 {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	225 const uint8_t *src = data;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	226 size_t i;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	227
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	228 buffer_set_used_size(dest, 0);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	229 for (i = 0; i < size; i++) {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	230 uint16_t chr;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	231
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	232 if (src[i] <= 32)
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	233 chr = 0;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	234 else if (src[i] > 'z')
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	235 chr = src[i] - 32 - 26;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	236 else
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	237 chr = i_toupper(src[i]) - 32;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	238 buffer_append(dest, &chr, sizeof(chr));
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	239 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	240 return dest->data;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	241 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	242
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	243 static void
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	244 squat_trie_set_syscall_error(struct squat_trie trie, const char function)
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	245 {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	246 i_error("%s failed with index search file %s: %m",
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	247 function, trie->filepath);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	248 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	249
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	250 void squat_trie_set_corrupted(struct squat_trie trie, const char reason)
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	251 {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	252 i_error("Corrupted index search file %s: %s", trie->filepath, reason);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	253
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	254 (void)unlink(trie->filepath);
4879 b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	255 (void)unlink(trie->uidlist_filepath);
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	256 trie->corrupted = TRUE;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	257 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	258
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	259 static void
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	260 trie_map_node_save_children(unsigned int level, const uint32_t *src_idx,
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	261 unsigned int count, struct trie_node **children)
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	262 {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	263 unsigned int i, file_bit;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	264
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	265 file_bit = level == BLOCK_SIZE ? 0 : 1;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	266
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	267 #ifndef ALLOW_UNALIGNED_ACCESS
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	268 if ((POINTER_CAST_TO(src_idx, size_t) & (sizeof(uint32_t)-1)) == 0) {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	269 #endif
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	270 for (i = 0; i < count; i++) {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	271 children[i] = src_idx[i] == 0 ? NULL :
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	272 POINTER_CAST(src_idx[i] \| file_bit);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	273 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	274 #ifndef ALLOW_UNALIGNED_ACCESS
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	275 } else {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	276 /* unaligned access */
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	277 uint32_t idx;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	278
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	279 for (i = 0; i < count; i++) {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	280 memcpy(&idx, &src_idx[i], sizeof(idx));
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	281 children[i] = idx == 0 ? NULL :
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	282 POINTER_CAST(idx \| file_bit);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	283 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	284 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	285 #endif
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	286 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	287
4883 d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	288 static int trie_map_area(struct squat_trie *trie, uoff_t offset, size_t len)
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	289 {
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	290 ssize_t ret;
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	291
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	292 if (trie->file_cache == NULL)
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	293 return 0;
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	294
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	295 ret = file_cache_read(trie->file_cache, offset, len);
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	296 if (ret < 0) {
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	297 squat_trie_set_syscall_error(trie, "file_cache_read()");
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	298 return -1;
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	299 }
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	300 trie->const_mmap_base =
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	301 file_cache_get_map(trie->file_cache, &trie->mmap_size);
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	302 trie->hdr = (const void *)trie->const_mmap_base;
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	303 return 0;
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	304 }
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	305
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	306 static int
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	307 trie_map_node(struct squat_trie *trie, uint32_t offset, unsigned int level,
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	308 struct trie_node **node_r)
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	309 {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	310 struct trie_node *node;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	311 const uint8_t p, end, chars8_src, chars16_src;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	312 uint32_t num, chars8_count, chars16_count;
4883 d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	313 unsigned int chars8_offset, chars8_size, chars8_memsize;
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	314 unsigned int chars16_offset, chars16_size, chars16_memsize;
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	315
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	316 i_assert(trie->fd != -1);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	317
4883 d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	318 if (trie_map_area(trie, offset, 2+256) < 0)
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	319 return -1;
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	320
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	321 if (offset >= trie->mmap_size) {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	322 squat_trie_set_corrupted(trie, "trie offset too large");
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	323 return -1;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	324 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	325
4883 d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	326 p = trie->const_mmap_base + offset;
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	327 end = trie->const_mmap_base + trie->mmap_size;
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	328
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	329 /* get 8bit char count and check that it's valid */
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	330 num = _squat_trie_unpack_num(&p, end);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	331 chars8_count = num >> 1;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	332
4883 d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	333 chars8_offset = p - trie->const_mmap_base;
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	334 chars8_size = chars8_count * (sizeof(uint8_t) + sizeof(uint32_t));
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	335
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	336 if (chars8_count > 256 \|\|
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	337 chars8_offset + chars8_size > trie->mmap_size) {
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	338 squat_trie_set_corrupted(trie, "trie offset broken");
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	339 return -1;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	340 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	341
4883 d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	342 chars8_memsize = ALIGN(chars8_count * sizeof(uint8_t)) +
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	343 chars8_count * sizeof(struct trie_node *);
4883 d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	344
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	345 if (trie_map_area(trie, chars8_offset, chars8_size + 8) < 0)
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	346 return -1;
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	347
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	348 if ((num & 1) == 0) {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	349 /* no 16bit chars */
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	350 chars16_count = 0;
4883 d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	351 chars16_memsize = 0;
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	352 chars16_offset = 0;
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	353 } else {
4883 d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	354 /* get the 16bit char count */
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	355 p = trie->const_mmap_base + chars8_offset + chars8_size;
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	356 end = trie->const_mmap_base + trie->mmap_size;
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	357
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	358 chars16_count = _squat_trie_unpack_num(&p, end);
4883 d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	359 if (chars16_count > 65536) {
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	360 squat_trie_set_corrupted(trie, "trie offset broken");
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	361 return -1;
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	362 }
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	363 chars16_offset = p - trie->const_mmap_base;
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	364
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	365 /* map the required area size and make sure it exists */
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	366 chars16_size = chars16_count *
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	367 (sizeof(uint16_t) + sizeof(uint32_t));
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	368 if (trie_map_area(trie, chars16_offset, chars16_size) < 0)
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	369 return -1;
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	370
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	371 if (chars16_offset + chars16_size > trie->mmap_size) {
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	372 squat_trie_set_corrupted(trie, "trie offset broken");
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	373 return -1;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	374 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	375
4883 d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	376 chars16_memsize = ALIGN(chars16_count * sizeof(uint16_t)) +
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	377 chars16_count * sizeof(struct trie_node *);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	378 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	379
4886 ae2114f11a0e Memory leak fixes. Also when building a large mailbox flush once in a while Timo Sirainen <tss@iki.fi> parents: 4883 diff changeset	380 node = p_malloc(trie->node_pool,
ae2114f11a0e Memory leak fixes. Also when building a large mailbox flush once in a while Timo Sirainen <tss@iki.fi> parents: 4883 diff changeset	381 sizeof(*node) + chars8_memsize + chars16_memsize);
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	382 node->chars_8bit_count = chars8_count;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	383 node->chars_16bit_count = chars16_count;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	384 node->file_offset = offset;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	385
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	386 {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	387 uint8_t *chars8 = NODE_CHARS8(node);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	388 uint16_t *chars16 = NODE_CHARS16(node);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	389 struct trie_node **children8 = NODE_CHILDREN8(node);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	390 struct trie_node **children16 = NODE_CHILDREN16(node);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	391 const uint32_t *src_idx;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	392 const void *end_offset;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	393
4883 d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	394 chars8_src = trie->const_mmap_base + chars8_offset;
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	395 chars16_src = trie->const_mmap_base + chars16_offset;
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	396
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	397 memcpy(chars8, chars8_src, sizeof(uint8_t) * chars8_count);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	398 memcpy(chars16, chars16_src, sizeof(uint16_t) * chars16_count);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	399
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	400 src_idx = CONST_PTR_OFFSET(chars8_src, chars8_count);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	401 trie_map_node_save_children(level, src_idx, chars8_count,
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	402 children8);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	403 if (chars16_count == 0)
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	404 end_offset = &src_idx[chars8_count];
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	405 else {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	406 src_idx = CONST_PTR_OFFSET(chars16_src,
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	407 chars16_count *
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	408 sizeof(uint16_t));
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	409 trie_map_node_save_children(level, src_idx,
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	410 chars16_count, children16);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	411 end_offset = &src_idx[chars16_count];
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	412 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	413
4883 d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	414 node->orig_size = ((const uint8_t *)end_offset -
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	415 trie->const_mmap_base) - offset;
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	416 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	417
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	418 *node_r = node;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	419 return 0;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	420 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	421
4879 b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	422 static void squat_trie_unmap(struct squat_trie *trie)
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	423 {
4883 d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	424 if (trie->file_cache != NULL)
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	425 file_cache_invalidate(trie->file_cache, 0, (uoff_t)-1);
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	426
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	427 if (trie->mmap_base != NULL) {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	428 if (munmap(trie->mmap_base, trie->mmap_size) < 0)
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	429 squat_trie_set_syscall_error(trie, "munmap()");
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	430 trie->mmap_base = NULL;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	431 }
4879 b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	432
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	433 trie->mmap_size = 0;
4879 b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	434 trie->hdr = NULL;
4883 d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	435 trie->const_mmap_base = NULL;
4886 ae2114f11a0e Memory leak fixes. Also when building a large mailbox flush once in a while Timo Sirainen <tss@iki.fi> parents: 4883 diff changeset	436
ae2114f11a0e Memory leak fixes. Also when building a large mailbox flush once in a while Timo Sirainen <tss@iki.fi> parents: 4883 diff changeset	437 p_clear(trie->node_pool);
ae2114f11a0e Memory leak fixes. Also when building a large mailbox flush once in a while Timo Sirainen <tss@iki.fi> parents: 4883 diff changeset	438 trie->root = NULL;
4879 b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	439 }
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	440
4879 b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	441 static void trie_file_close(struct squat_trie *trie)
b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	442 {
4883 d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	443 if (trie->file_cache != NULL)
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	444 file_cache_free(&trie->file_cache);
4879 b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	445 if (trie->file_lock != NULL)
b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	446 file_lock_free(&trie->file_lock);
b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	447
b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	448 squat_trie_unmap(trie);
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	449 if (trie->fd != -1) {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	450 if (close(trie->fd) < 0)
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	451 squat_trie_set_syscall_error(trie, "close()");
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	452 trie->fd = -1;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	453 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	454
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	455 trie->hdr = NULL;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	456 trie->corrupted = FALSE;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	457 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	458
4883 d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	459 static int
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	460 trie_map_check_header(struct squat_trie *trie,
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	461 const struct squat_trie_header *hdr, uoff_t file_size)
4878 88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	462 {
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	463 if (hdr->version != SQUAT_TRIE_VERSION)
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	464 return -1;
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	465
4883 d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	466 if (hdr->used_file_size > file_size) {
4878 88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	467 squat_trie_set_corrupted(trie, "used_file_size too large");
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	468 return -1;
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	469 }
4879 b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	470 if (hdr->root_offset != 0 &&
4883 d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	471 (hdr->root_offset > file_size \|\|
4879 b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	472 hdr->root_offset < sizeof(*hdr))) {
4878 88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	473 squat_trie_set_corrupted(trie, "invalid root_offset");
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	474 return -1;
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	475 }
4879 b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	476 if (hdr->uidvalidity != trie->uidvalidity) {
b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	477 squat_trie_set_corrupted(trie, "uidvalidity changed");
b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	478 return -1;
b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	479 }
4878 88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	480
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	481 return 0;
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	482 }
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	483
4883 d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	484 static int squat_trie_file_was_modified(struct squat_trie *trie)
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	485 {
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	486 struct squat_trie_header hdr;
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	487 int ret;
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	488
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	489 ret = pread_full(trie->fd, &hdr.modify_counter,
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	490 sizeof(hdr.modify_counter),
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	491 offsetof(struct squat_trie_header, modify_counter));
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	492 if (ret < 0) {
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	493 squat_trie_set_syscall_error(trie, "pread_full()");
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	494 return -1;
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	495 }
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	496 if (ret == 0) {
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	497 /* broken file, treat as modified */
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	498 return 1;
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	499 }
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	500 return hdr.modify_counter == trie->file_cache_modify_counter ? 0 : 1;
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	501 }
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	502
4879 b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	503 static int squat_trie_map(struct squat_trie *trie)
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	504 {
4883 d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	505 const struct squat_trie_header *hdr;
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	506 struct stat st;
4883 d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	507 ssize_t ret;
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	508
4883 d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	509 if (trie->hdr != NULL) {
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	510 if (!trie->mmap_disable) {
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	511 if (trie->hdr->used_file_size <= trie->mmap_size) {
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	512 /* everything is already mapped */
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	513 return 1;
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	514 }
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	515 } else {
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	516 ret = squat_trie_file_was_modified(trie);
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	517 if (ret <= 0)
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	518 return ret < 0 ? -1 : 1;
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	519 }
4879 b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	520 }
4878 88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	521
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	522 if (fstat(trie->fd, &st) < 0) {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	523 squat_trie_set_syscall_error(trie, "fstat()");
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	524 return -1;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	525 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	526 trie->dev = st.st_dev;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	527 trie->ino = st.st_ino;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	528
4879 b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	529 squat_trie_unmap(trie);
b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	530
4883 d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	531 if (!trie->mmap_disable) {
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	532 trie->mmap_size = st.st_size;
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	533 trie->mmap_base = mmap(NULL, trie->mmap_size,
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	534 PROT_READ \| PROT_WRITE,
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	535 MAP_SHARED, trie->fd, 0);
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	536 if (trie->mmap_base == MAP_FAILED) {
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	537 trie->mmap_size = 0;
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	538 trie->mmap_base = NULL;
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	539 squat_trie_set_syscall_error(trie, "mmap()");
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	540 return -1;
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	541 }
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	542 trie->const_mmap_base = trie->mmap_base;
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	543 } else {
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	544 ret = file_cache_read(trie->file_cache, 0, sizeof(*trie->hdr));
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	545 if (ret < 0) {
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	546 squat_trie_set_syscall_error(trie, "file_cache_read()");
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	547 return -1;
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	548 }
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	549 if ((size_t)ret < sizeof(*trie->hdr)) {
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	550 squat_trie_set_corrupted(trie, "file too small");
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	551 return -1;
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	552 }
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	553 trie->const_mmap_base =
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	554 file_cache_get_map(trie->file_cache, &trie->mmap_size);
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	555 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	556
4883 d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	557 hdr = (const void *)trie->const_mmap_base;
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	558 if (trie_map_check_header(trie, hdr, st.st_size) < 0)
4879 b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	559 return -1;
4883 d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	560 trie->hdr = hdr;
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	561 trie->file_cache_modify_counter = trie->hdr->modify_counter;
4878 88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	562
4883 d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	563 if (trie_map_node(trie, trie->hdr->root_offset, 1, &trie->root) < 0)
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	564 return 0;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	565 return 1;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	566 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	567
4886 ae2114f11a0e Memory leak fixes. Also when building a large mailbox flush once in a while Timo Sirainen <tss@iki.fi> parents: 4883 diff changeset	568 static void trie_file_open_fd(struct squat_trie *trie, int fd)
ae2114f11a0e Memory leak fixes. Also when building a large mailbox flush once in a while Timo Sirainen <tss@iki.fi> parents: 4883 diff changeset	569 {
ae2114f11a0e Memory leak fixes. Also when building a large mailbox flush once in a while Timo Sirainen <tss@iki.fi> parents: 4883 diff changeset	570 struct stat st;
ae2114f11a0e Memory leak fixes. Also when building a large mailbox flush once in a while Timo Sirainen <tss@iki.fi> parents: 4883 diff changeset	571
ae2114f11a0e Memory leak fixes. Also when building a large mailbox flush once in a while Timo Sirainen <tss@iki.fi> parents: 4883 diff changeset	572 if (fstat(fd, &st) < 0) {
ae2114f11a0e Memory leak fixes. Also when building a large mailbox flush once in a while Timo Sirainen <tss@iki.fi> parents: 4883 diff changeset	573 /* don't bother adding complexity by trying to handle this
ae2114f11a0e Memory leak fixes. Also when building a large mailbox flush once in a while Timo Sirainen <tss@iki.fi> parents: 4883 diff changeset	574 error here. we'll break later anyway in easier error
ae2114f11a0e Memory leak fixes. Also when building a large mailbox flush once in a while Timo Sirainen <tss@iki.fi> parents: 4883 diff changeset	575 handling paths. */
ae2114f11a0e Memory leak fixes. Also when building a large mailbox flush once in a while Timo Sirainen <tss@iki.fi> parents: 4883 diff changeset	576 squat_trie_set_syscall_error(trie, "fstat()");
ae2114f11a0e Memory leak fixes. Also when building a large mailbox flush once in a while Timo Sirainen <tss@iki.fi> parents: 4883 diff changeset	577 trie->ino = 0;
ae2114f11a0e Memory leak fixes. Also when building a large mailbox flush once in a while Timo Sirainen <tss@iki.fi> parents: 4883 diff changeset	578 } else {
ae2114f11a0e Memory leak fixes. Also when building a large mailbox flush once in a while Timo Sirainen <tss@iki.fi> parents: 4883 diff changeset	579 trie->dev = st.st_dev;
ae2114f11a0e Memory leak fixes. Also when building a large mailbox flush once in a while Timo Sirainen <tss@iki.fi> parents: 4883 diff changeset	580 trie->ino = st.st_ino;
ae2114f11a0e Memory leak fixes. Also when building a large mailbox flush once in a while Timo Sirainen <tss@iki.fi> parents: 4883 diff changeset	581 }
ae2114f11a0e Memory leak fixes. Also when building a large mailbox flush once in a while Timo Sirainen <tss@iki.fi> parents: 4883 diff changeset	582 trie->fd = fd;
ae2114f11a0e Memory leak fixes. Also when building a large mailbox flush once in a while Timo Sirainen <tss@iki.fi> parents: 4883 diff changeset	583
ae2114f11a0e Memory leak fixes. Also when building a large mailbox flush once in a while Timo Sirainen <tss@iki.fi> parents: 4883 diff changeset	584 if (trie->mmap_disable)
ae2114f11a0e Memory leak fixes. Also when building a large mailbox flush once in a while Timo Sirainen <tss@iki.fi> parents: 4883 diff changeset	585 trie->file_cache = file_cache_new(trie->fd);
ae2114f11a0e Memory leak fixes. Also when building a large mailbox flush once in a while Timo Sirainen <tss@iki.fi> parents: 4883 diff changeset	586 }
ae2114f11a0e Memory leak fixes. Also when building a large mailbox flush once in a while Timo Sirainen <tss@iki.fi> parents: 4883 diff changeset	587
4883 d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	588 static int trie_file_open(struct squat_trie *trie, bool create)
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	589 {
4883 d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	590 int fd;
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	591
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	592 i_assert(trie->fd == -1);
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	593
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	594 fd = open(trie->filepath, O_RDWR \| (create ? O_CREAT : 0), 0660);
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	595 if (fd == -1) {
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	596 if (errno == ENOENT)
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	597 return 0;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	598
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	599 squat_trie_set_syscall_error(trie, "open()");
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	600 return -1;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	601 }
4886 ae2114f11a0e Memory leak fixes. Also when building a large mailbox flush once in a while Timo Sirainen <tss@iki.fi> parents: 4883 diff changeset	602 trie_file_open_fd(trie, fd);
4883 d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	603 return 1;
4879 b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	604 }
b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	605
b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	606 static int trie_file_create_finish(struct squat_trie *trie)
b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	607 {
b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	608 struct squat_trie_header hdr;
b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	609 struct stat st;
b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	610
b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	611 if (fstat(trie->fd, &st) < 0) {
b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	612 squat_trie_set_syscall_error(trie, "fstat()");
b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	613 return -1;
b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	614 }
b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	615
b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	616 if (st.st_size <= sizeof(hdr)) {
b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	617 memset(&hdr, 0, sizeof(hdr));
b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	618 hdr.version = SQUAT_TRIE_VERSION;
b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	619 hdr.uidvalidity = trie->uidvalidity;
b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	620 hdr.used_file_size = sizeof(hdr);
b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	621
b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	622 if (pwrite_full(trie->fd, &hdr, sizeof(hdr), 0) < 0) {
b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	623 squat_trie_set_syscall_error(trie, "pwrite_full()");
b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	624 return -1;
b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	625 }
b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	626 }
b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	627
b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	628 return 0;
b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	629 }
b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	630
4878 88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	631 struct squat_trie *
4879 b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	632 squat_trie_open(const char *path, uint32_t uidvalidity,
b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	633 enum file_lock_method lock_method, bool mmap_disable)
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	634 {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	635 struct squat_trie *trie;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	636
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	637 trie = i_new(struct squat_trie, 1);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	638 trie->fd = -1;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	639 trie->filepath = i_strdup(path);
4879 b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	640 trie->uidvalidity = uidvalidity;
4878 88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	641 trie->lock_method = lock_method;
4879 b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	642 trie->mmap_disable = mmap_disable;
4878 88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	643 trie->buf = buffer_create_dynamic(default_pool, 1024);
4886 ae2114f11a0e Memory leak fixes. Also when building a large mailbox flush once in a while Timo Sirainen <tss@iki.fi> parents: 4883 diff changeset	644 trie->node_pool = pool_alloconly_create("trie node pool", 1024*64);
4878 88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	645
4879 b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	646 trie->uidlist_filepath = i_strconcat(path, ".uids", NULL);
b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	647 trie->uidlist =
4883 d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	648 squat_uidlist_init(trie, trie->uidlist_filepath,
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	649 uidvalidity, mmap_disable);
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	650 return trie;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	651 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	652
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	653 void squat_trie_close(struct squat_trie *trie)
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	654 {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	655 buffer_free(trie->buf);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	656 squat_uidlist_deinit(trie->uidlist);
4886 ae2114f11a0e Memory leak fixes. Also when building a large mailbox flush once in a while Timo Sirainen <tss@iki.fi> parents: 4883 diff changeset	657 pool_unref(trie->node_pool);
4879 b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	658 i_free(trie->uidlist_filepath);
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	659 i_free(trie->filepath);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	660 i_free(trie);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	661 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	662
4878 88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	663 int squat_trie_get_last_uid(struct squat_trie trie, uint32_t uid_r)
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	664 {
4883 d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	665 int ret;
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	666
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	667 if (trie->fd == -1) {
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	668 if ((ret = trie_file_open(trie, FALSE)) < 0)
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	669 return ret;
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	670 if (ret == 0) {
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	671 *uid_r = 0;
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	672 return 0;
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	673 }
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	674 }
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	675
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	676 if (squat_trie_lock(trie, F_RDLCK) <= 0)
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	677 return -1;
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	678
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	679 ret = squat_uidlist_get_last_uid(trie->uidlist, uid_r);
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	680 squat_trie_unlock(trie);
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	681 return ret;
4878 88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	682 }
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	683
4879 b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	684 static int squat_trie_is_file_stale(struct squat_trie *trie)
b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	685 {
b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	686 struct stat st;
b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	687
b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	688 if (stat(trie->filepath, &st) < 0) {
4883 d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	689 if (errno == ENOENT)
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	690 return 1;
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	691
4879 b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	692 squat_trie_set_syscall_error(trie, "stat()");
b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	693 return -1;
b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	694 }
b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	695
b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	696 return st.st_ino == trie->ino &&
b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	697 CMP_DEV_T(st.st_dev, trie->dev) ? 0 : 1;
b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	698 }
b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	699
4886 ae2114f11a0e Memory leak fixes. Also when building a large mailbox flush once in a while Timo Sirainen <tss@iki.fi> parents: 4883 diff changeset	700 static int
ae2114f11a0e Memory leak fixes. Also when building a large mailbox flush once in a while Timo Sirainen <tss@iki.fi> parents: 4883 diff changeset	701 squat_trie_file_lock(struct squat_trie trie, int fd, const char path,
ae2114f11a0e Memory leak fixes. Also when building a large mailbox flush once in a while Timo Sirainen <tss@iki.fi> parents: 4883 diff changeset	702 int lock_type, struct file_lock **lock_r)
ae2114f11a0e Memory leak fixes. Also when building a large mailbox flush once in a while Timo Sirainen <tss@iki.fi> parents: 4883 diff changeset	703 {
ae2114f11a0e Memory leak fixes. Also when building a large mailbox flush once in a while Timo Sirainen <tss@iki.fi> parents: 4883 diff changeset	704 int ret;
ae2114f11a0e Memory leak fixes. Also when building a large mailbox flush once in a while Timo Sirainen <tss@iki.fi> parents: 4883 diff changeset	705
ae2114f11a0e Memory leak fixes. Also when building a large mailbox flush once in a while Timo Sirainen <tss@iki.fi> parents: 4883 diff changeset	706 ret = file_wait_lock(fd, path, lock_type, trie->lock_method,
ae2114f11a0e Memory leak fixes. Also when building a large mailbox flush once in a while Timo Sirainen <tss@iki.fi> parents: 4883 diff changeset	707 SQUAT_TRIE_LOCK_TIMEOUT, lock_r);
ae2114f11a0e Memory leak fixes. Also when building a large mailbox flush once in a while Timo Sirainen <tss@iki.fi> parents: 4883 diff changeset	708 if (ret == 0)
ae2114f11a0e Memory leak fixes. Also when building a large mailbox flush once in a while Timo Sirainen <tss@iki.fi> parents: 4883 diff changeset	709 squat_trie_set_syscall_error(trie, "file_wait_lock()");
ae2114f11a0e Memory leak fixes. Also when building a large mailbox flush once in a while Timo Sirainen <tss@iki.fi> parents: 4883 diff changeset	710 return ret;
ae2114f11a0e Memory leak fixes. Also when building a large mailbox flush once in a while Timo Sirainen <tss@iki.fi> parents: 4883 diff changeset	711 }
ae2114f11a0e Memory leak fixes. Also when building a large mailbox flush once in a while Timo Sirainen <tss@iki.fi> parents: 4883 diff changeset	712
4878 88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	713 int squat_trie_lock(struct squat_trie *trie, int lock_type)
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	714 {
4879 b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	715 bool created = FALSE;
4878 88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	716 int ret;
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	717
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	718 i_assert(lock_type == F_RDLCK \|\| lock_type == F_WRLCK);
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	719
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	720 if (trie->lock_count > 0) {
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	721 /* read lock -> write lock would deadlock */
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	722 i_assert(trie->lock_type == lock_type \|\| lock_type == F_RDLCK);
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	723
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	724 trie->lock_count++;
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	725 return 1;
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	726 }
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	727
4879 b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	728 if (trie->fd == -1 \|\| trie->corrupted) {
b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	729 trie_file_close(trie);
b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	730 if (lock_type == F_WRLCK) {
4883 d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	731 if ((ret = trie_file_open(trie, FALSE)) < 0)
4879 b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	732 return -1;
b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	733 if (ret == 0) {
4883 d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	734 if (trie_file_open(trie, TRUE) < 0)
4879 b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	735 return -1;
b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	736 created = TRUE;
b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	737 }
b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	738 } else {
4883 d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	739 if (trie_file_open(trie, FALSE) <= 0)
4879 b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	740 return -1;
b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	741 }
b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	742 }
b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	743
b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	744 for (;;) {
b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	745 i_assert(trie->file_lock == NULL);
4886 ae2114f11a0e Memory leak fixes. Also when building a large mailbox flush once in a while Timo Sirainen <tss@iki.fi> parents: 4883 diff changeset	746 ret = squat_trie_file_lock(trie, trie->fd, trie->filepath,
ae2114f11a0e Memory leak fixes. Also when building a large mailbox flush once in a while Timo Sirainen <tss@iki.fi> parents: 4883 diff changeset	747 lock_type, &trie->file_lock);
ae2114f11a0e Memory leak fixes. Also when building a large mailbox flush once in a while Timo Sirainen <tss@iki.fi> parents: 4883 diff changeset	748 if (ret <= 0)
4879 b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	749 return ret;
b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	750
b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	751 /* if the trie has been compressed, we need to reopen the
b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	752 file and try to lock again */
b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	753 ret = squat_trie_is_file_stale(trie);
b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	754 if (ret == 0)
b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	755 break;
b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	756
b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	757 file_unlock(&trie->file_lock);
b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	758 if (ret < 0)
b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	759 return -1;
b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	760
4883 d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	761 trie_file_close(trie);
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	762 if (trie_file_open(trie, FALSE) <= 0)
4879 b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	763 return -1;
b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	764 }
b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	765
b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	766 if (created) {
b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	767 /* we possibly created this file. now that we've locked the
b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	768 file, we can safely check if someone else already wrote the
b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	769 header or if we should do it now */
b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	770 if (trie_file_create_finish(trie) < 0) {
b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	771 file_unlock(&trie->file_lock);
b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	772 return -1;
b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	773 }
b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	774 }
b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	775
b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	776 if (squat_trie_map(trie) <= 0) {
b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	777 file_unlock(&trie->file_lock);
b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	778 return -1;
b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	779 }
4883 d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	780 if (squat_uidlist_refresh(trie->uidlist) < 0) {
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	781 file_unlock(&trie->file_lock);
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	782 return -1;
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	783 }
4878 88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	784
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	785 trie->lock_count++;
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	786 trie->lock_type = lock_type;
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	787 return 1;
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	788 }
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	789
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	790 void squat_trie_unlock(struct squat_trie *trie)
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	791 {
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	792 i_assert(trie->lock_count > 0);
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	793
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	794 if (--trie->lock_count > 0)
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	795 return;
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	796
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	797 file_unlock(&trie->file_lock);
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	798 }
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	799
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	800 static struct trie_node *
4886 ae2114f11a0e Memory leak fixes. Also when building a large mailbox flush once in a while Timo Sirainen <tss@iki.fi> parents: 4883 diff changeset	801 node_alloc(struct squat_trie *trie, uint16_t chr, unsigned int level)
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	802 {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	803 struct trie_node *node;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	804 unsigned int idx_size, idx_offset = sizeof(*node);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	805
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	806 idx_size = level < BLOCK_SIZE ?
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	807 sizeof(struct trie_node *) : sizeof(uint32_t);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	808
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	809 if (chr < 256) {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	810 uint8_t *chrp;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	811
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	812 idx_offset += ALIGN(sizeof(*chrp));
4886 ae2114f11a0e Memory leak fixes. Also when building a large mailbox flush once in a while Timo Sirainen <tss@iki.fi> parents: 4883 diff changeset	813 node = p_malloc(trie->node_pool, idx_offset + idx_size);
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	814 node->chars_8bit_count = 1;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	815
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	816 chrp = PTR_OFFSET(node, sizeof(*node));
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	817 *chrp = chr;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	818 } else {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	819 uint16_t *chrp;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	820
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	821 idx_offset += ALIGN(sizeof(*chrp));
4886 ae2114f11a0e Memory leak fixes. Also when building a large mailbox flush once in a while Timo Sirainen <tss@iki.fi> parents: 4883 diff changeset	822 node = p_malloc(trie->node_pool, idx_offset + idx_size);
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	823 node->chars_16bit_count = 1;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	824
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	825 chrp = PTR_OFFSET(node, sizeof(*node));
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	826 *chrp = chr;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	827 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	828
4883 d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	829 node->modified = TRUE;
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	830 node->resized = TRUE;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	831 return node;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	832 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	833
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	834 static struct trie_node *
4886 ae2114f11a0e Memory leak fixes. Also when building a large mailbox flush once in a while Timo Sirainen <tss@iki.fi> parents: 4883 diff changeset	835 node_realloc(struct squat_trie trie, struct trie_node node,
ae2114f11a0e Memory leak fixes. Also when building a large mailbox flush once in a while Timo Sirainen <tss@iki.fi> parents: 4883 diff changeset	836 uint32_t char_idx, uint16_t chr, unsigned int level)
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	837 {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	838 struct trie_node *new_node;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	839 unsigned int old_size_8bit, old_size_16bit, old_idx_offset;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	840 unsigned int idx_size, old_size, new_size, new_idx_offset;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	841 unsigned int hole1_pos, hole2_pos, skip;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	842
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	843 idx_size = level < BLOCK_SIZE ?
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	844 sizeof(struct trie_node *) : sizeof(uint32_t);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	845
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	846 old_size_8bit = ALIGN(node->chars_8bit_count) +
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	847 node->chars_8bit_count * idx_size;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	848 old_size_16bit = ALIGN(sizeof(uint16_t) * node->chars_16bit_count) +
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	849 node->chars_16bit_count * idx_size;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	850 old_size = sizeof(*node) + old_size_8bit + old_size_16bit;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	851
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	852 if (chr < 256) {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	853 new_idx_offset = sizeof(*node) +
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	854 ALIGN(node->chars_8bit_count + sizeof(uint8_t));
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	855 new_size = new_idx_offset + old_size_16bit +
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	856 (node->chars_8bit_count + 1) * idx_size;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	857 } else {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	858 new_idx_offset = sizeof(*node) + old_size_8bit +
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	859 ALIGN((node->chars_16bit_count + 1) * sizeof(uint16_t));
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	860 new_size = new_idx_offset +
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	861 (node->chars_16bit_count + 1) * idx_size;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	862 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	863
4886 ae2114f11a0e Memory leak fixes. Also when building a large mailbox flush once in a while Timo Sirainen <tss@iki.fi> parents: 4883 diff changeset	864 new_node = p_malloc(trie->node_pool, new_size);
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	865 if (chr < 256) {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	866 hole1_pos = sizeof(*node) + char_idx;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	867 old_idx_offset = sizeof(*node) + ALIGN(node->chars_8bit_count);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	868 } else {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	869 hole1_pos = sizeof(*node) + old_size_8bit +
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	870 char_idx * sizeof(uint16_t);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	871 old_idx_offset = sizeof(*node) + old_size_8bit +
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	872 ALIGN(node->chars_16bit_count * sizeof(uint16_t));
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	873 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	874 hole2_pos = old_idx_offset + idx_size * char_idx;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	875
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	876 memcpy(new_node, node, hole1_pos);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	877 if (chr < 256) {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	878 uint8_t *chrp = PTR_OFFSET(new_node, hole1_pos);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	879 *chrp = chr;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	880 new_node->chars_8bit_count++;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	881
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	882 memcpy(PTR_OFFSET(new_node, hole1_pos + sizeof(uint8_t)),
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	883 PTR_OFFSET(node, hole1_pos), old_idx_offset - hole1_pos);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	884 } else {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	885 uint16_t *chrp = PTR_OFFSET(new_node, hole1_pos);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	886 *chrp = chr;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	887 new_node->chars_16bit_count++;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	888
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	889 memcpy(PTR_OFFSET(new_node, hole1_pos + sizeof(uint16_t)),
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	890 PTR_OFFSET(node, hole1_pos), old_idx_offset - hole1_pos);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	891 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	892
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	893 memcpy(PTR_OFFSET(new_node, new_idx_offset),
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	894 PTR_OFFSET(node, old_idx_offset),
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	895 hole2_pos - old_idx_offset);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	896
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	897 skip = new_idx_offset - old_idx_offset;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	898 memset(PTR_OFFSET(new_node, hole2_pos + skip), 0, idx_size);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	899 skip += sizeof(uint32_t);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	900 memcpy(PTR_OFFSET(new_node, hole2_pos + skip),
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	901 PTR_OFFSET(node, hole2_pos),
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	902 old_size - hole2_pos);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	903
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	904 new_node->resized = TRUE;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	905 return new_node;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	906 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	907
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	908 static int
4878 88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	909 trie_insert_node(struct squat_trie_build_context *ctx,
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	910 struct trie_node **parent,
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	911 const uint16_t *data, uint32_t uid, unsigned int level)
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	912 {
4878 88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	913 struct squat_trie *trie = ctx->trie;
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	914 struct trie_node node = parent;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	915 uint32_t char_idx, idx_base_offset;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	916 bool modified = FALSE;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	917 int ret;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	918
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	919 if (*data < 256) {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	920 unsigned int count;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	921
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	922 if (node == NULL) {
4878 88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	923 ctx->node_count++;
4886 ae2114f11a0e Memory leak fixes. Also when building a large mailbox flush once in a while Timo Sirainen <tss@iki.fi> parents: 4883 diff changeset	924 node = parent = node_alloc(trie, data, level);
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	925 char_idx = 0;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	926 count = 1;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	927 modified = TRUE;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	928 } else {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	929 uint8_t chars = PTR_OFFSET(node, sizeof(node));
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	930 uint8_t *pos;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	931
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	932 count = node->chars_8bit_count;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	933 pos = bsearch_insert_pos(data, chars, count,
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	934 sizeof(chars[0]),
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	935 chr_8bit_cmp);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	936 char_idx = pos - chars;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	937 if (char_idx == count \|\| pos != data) {
4886 ae2114f11a0e Memory leak fixes. Also when building a large mailbox flush once in a while Timo Sirainen <tss@iki.fi> parents: 4883 diff changeset	938 node = node_realloc(trie, node, char_idx,
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	939 *data, level);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	940 *parent = node;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	941 modified = TRUE;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	942 count++;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	943 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	944 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	945 idx_base_offset = sizeof(*node) + ALIGN(count);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	946 } else {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	947 unsigned int offset = sizeof(*node);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	948 unsigned int count;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	949
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	950 if (node == NULL) {
4878 88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	951 ctx->node_count++;
4886 ae2114f11a0e Memory leak fixes. Also when building a large mailbox flush once in a while Timo Sirainen <tss@iki.fi> parents: 4883 diff changeset	952 node = parent = node_alloc(trie, data, level);
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	953 char_idx = 0;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	954 count = 1;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	955 modified = TRUE;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	956 } else {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	957 unsigned int idx_size;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	958 uint16_t chars, pos;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	959
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	960 idx_size = level < BLOCK_SIZE ?
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	961 sizeof(struct trie_node *) : sizeof(uint32_t);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	962 offset += ALIGN(node->chars_8bit_count) +
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	963 idx_size * node->chars_8bit_count;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	964 chars = PTR_OFFSET(node, offset);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	965
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	966 count = node->chars_16bit_count;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	967 pos = bsearch_insert_pos(data, chars, count,
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	968 sizeof(chars[0]),
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	969 chr_16bit_cmp);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	970 char_idx = pos - chars;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	971 if (char_idx == count \|\| pos != data) {
4886 ae2114f11a0e Memory leak fixes. Also when building a large mailbox flush once in a while Timo Sirainen <tss@iki.fi> parents: 4883 diff changeset	972 node = node_realloc(trie, node, char_idx,
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	973 *data, level);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	974 *parent = node;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	975 modified = TRUE;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	976 count++;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	977 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	978 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	979
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	980 idx_base_offset = offset + ALIGN(sizeof(uint16_t) * count);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	981 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	982
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	983 if (level < BLOCK_SIZE) {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	984 struct trie_node **children = PTR_OFFSET(node, idx_base_offset);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	985 size_t child_idx = POINTER_CAST_TO(children[char_idx], size_t);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	986
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	987 if ((child_idx & 1) != 0) {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	988 if (trie_map_node(trie, child_idx & ~1, level + 1,
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	989 &children[char_idx]) < 0)
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	990 return -1;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	991 }
4878 88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	992 ret = trie_insert_node(ctx, &children[char_idx],
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	993 data + 1, uid, level + 1);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	994 if (ret < 0)
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	995 return -1;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	996 if (ret > 0)
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	997 node->modified = TRUE;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	998 } else {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	999 uint32_t *uid_lists = PTR_OFFSET(node, idx_base_offset);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1000 if (squat_uidlist_add(trie->uidlist, &uid_lists[char_idx],
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1001 uid) < 0)
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1002 return -1;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1003
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1004 node->modified = TRUE;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1005 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1006 return modified ? 1 : 0;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1007 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1008
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1009 static uint32_t
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1010 trie_lookup_node(struct squat_trie trie, struct trie_node node,
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1011 const uint16_t *data, unsigned int level)
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1012 {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1013 uint32_t char_idx, idx_base_offset;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1014
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1015 if (*data < 256) {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1016 const uint8_t chars, pos;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1017 unsigned int count;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1018
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1019 if (node == NULL)
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1020 return 0;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1021
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1022 chars = CONST_PTR_OFFSET(node, sizeof(*node));
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1023 count = node->chars_8bit_count;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1024 pos = bsearch(data, chars, count, sizeof(chars[0]),
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1025 chr_8bit_cmp);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1026 if (pos == NULL \|\| pos != data)
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1027 return 0;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1028
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1029 char_idx = pos - chars;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1030 idx_base_offset = sizeof(*node) + ALIGN(count);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1031 } else {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1032 const uint16_t chars, pos;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1033 unsigned int count, idx_size, offset;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1034
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1035 if (node == NULL)
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1036 return 0;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1037
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1038 idx_size = level < BLOCK_SIZE ?
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1039 sizeof(struct trie_node *) : sizeof(uint32_t);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1040 offset = sizeof(*node) + ALIGN(node->chars_8bit_count) +
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1041 idx_size * node->chars_8bit_count;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1042 chars = PTR_OFFSET(node, offset);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1043
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1044 count = node->chars_16bit_count;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1045 pos = bsearch(data, chars, count, sizeof(chars[0]),
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1046 chr_16bit_cmp);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1047 if (pos == NULL \|\| pos != data)
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1048 return 0;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1049
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1050 char_idx = pos - chars;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1051 idx_base_offset = offset + ALIGN(sizeof(uint16_t) * count);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1052 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1053
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1054 if (level < BLOCK_SIZE) {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1055 struct trie_node **children = PTR_OFFSET(node, idx_base_offset);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1056 size_t child_idx = POINTER_CAST_TO(children[char_idx], size_t);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1057
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1058 if ((child_idx & 1) != 0) {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1059 /* not mapped to memory yet. do it. */
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1060 if (trie_map_node(trie, child_idx & ~1, level + 1,
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1061 &children[char_idx]) < 0)
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1062 return -1;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1063 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1064
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1065 return trie_lookup_node(trie, children[char_idx],
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1066 data + 1, level + 1);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1067 } else {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1068 const uint32_t *uid_lists =
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1069 CONST_PTR_OFFSET(node, idx_base_offset);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1070
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1071 return uid_lists[char_idx];
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1072 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1073 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1074
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1075 static bool block_want_add(const uint16_t *data)
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1076 {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1077 unsigned int i;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1078
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1079 /* skip all blocks that contain spaces or control characters.
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1080 no-one searches them anyway */
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1081 for (i = 0; i < BLOCK_SIZE; i++) {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1082 if (data[i] == 0)
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1083 return FALSE;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1084 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1085 return TRUE;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1086 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1087
4878 88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1088 struct squat_trie_build_context *
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1089 squat_trie_build_init(struct squat_trie trie, uint32_t last_uid_r)
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1090 {
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1091 struct squat_trie_build_context *ctx;
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1092
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1093 ctx = i_new(struct squat_trie_build_context, 1);
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1094 ctx->trie = trie;
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1095
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1096 if (squat_trie_lock(trie, F_WRLCK) <= 0)
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1097 ctx->failed = TRUE;
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1098 else {
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1099 ctx->locked = TRUE;
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1100 ctx->node_count = trie->hdr->node_count;
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1101
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1102 if (squat_uidlist_get_last_uid(trie->uidlist, last_uid_r) < 0)
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1103 ctx->failed = TRUE;
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1104 }
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1105
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1106 if (ctx->failed)
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1107 *last_uid_r = 0;
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1108 return ctx;
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1109 }
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1110
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1111 int squat_trie_build_deinit(struct squat_trie_build_context *ctx)
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1112 {
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1113 int ret = ctx->failed ? -1 : 0;
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1114
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1115 if (ret == 0)
4886 ae2114f11a0e Memory leak fixes. Also when building a large mailbox flush once in a while Timo Sirainen <tss@iki.fi> parents: 4883 diff changeset	1116 ret = squat_trie_build_flush(ctx, TRUE);
4878 88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1117
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1118 if (ctx->locked)
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1119 squat_trie_unlock(ctx->trie);
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1120
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1121 i_free(ctx);
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1122 return ret;
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1123 }
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1124
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1125 int squat_trie_build_more(struct squat_trie_build_context *ctx, uint32_t uid,
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1126 const void *data, size_t size)
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1127 {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1128 const uint16_t *str;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1129 uint16_t buf[(BLOCK_SIZE-1)*2];
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1130 unsigned int i, tmp_size;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1131
4878 88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1132 if (ctx->failed)
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1133 return -1;
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1134
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1135 t_push();
4878 88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1136 str = data_normalize(data, size, ctx->trie->buf);
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1137
4878 88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1138 if (uid == ctx->prev_uid) {
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1139 /* @UNSAFE: continue from last block */
4878 88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1140 memcpy(buf, ctx->prev_added,
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1141 sizeof(buf[0]) * ctx->prev_added_size);
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1142 tmp_size = I_MIN(size, BLOCK_SIZE-1);
4878 88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1143 memcpy(buf + ctx->prev_added_size, str,
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1144 sizeof(buf[0]) * tmp_size);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1145
4878 88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1146 tmp_size += ctx->prev_added_size;
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1147 for (i = 0; i + BLOCK_SIZE <= tmp_size; i++) {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1148 if (block_want_add(buf+i)) {
4878 88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1149 if (trie_insert_node(ctx,
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1150 &ctx->trie->root,
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1151 buf + i, uid, 1) < 0) {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1152 t_pop();
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1153 return -1;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1154 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1155 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1156 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1157
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1158 if (size < BLOCK_SIZE) {
4878 88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1159 ctx->prev_added_size = I_MIN(tmp_size, BLOCK_SIZE-1);
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1160 memcpy(ctx->prev_added, buf + i,
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1161 sizeof(buf[0]) * ctx->prev_added_size);
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1162 t_pop();
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1163 return 0;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1164 }
4886 ae2114f11a0e Memory leak fixes. Also when building a large mailbox flush once in a while Timo Sirainen <tss@iki.fi> parents: 4883 diff changeset	1165 } else if (squat_uidlist_want_flush(ctx->trie->uidlist)) {
ae2114f11a0e Memory leak fixes. Also when building a large mailbox flush once in a while Timo Sirainen <tss@iki.fi> parents: 4883 diff changeset	1166 if (squat_trie_build_flush(ctx, FALSE) < 0) {
ae2114f11a0e Memory leak fixes. Also when building a large mailbox flush once in a while Timo Sirainen <tss@iki.fi> parents: 4883 diff changeset	1167 ctx->failed = TRUE;
ae2114f11a0e Memory leak fixes. Also when building a large mailbox flush once in a while Timo Sirainen <tss@iki.fi> parents: 4883 diff changeset	1168 t_pop();
ae2114f11a0e Memory leak fixes. Also when building a large mailbox flush once in a while Timo Sirainen <tss@iki.fi> parents: 4883 diff changeset	1169 return -1;
ae2114f11a0e Memory leak fixes. Also when building a large mailbox flush once in a while Timo Sirainen <tss@iki.fi> parents: 4883 diff changeset	1170 }
ae2114f11a0e Memory leak fixes. Also when building a large mailbox flush once in a while Timo Sirainen <tss@iki.fi> parents: 4883 diff changeset	1171 str = data_normalize(data, size, ctx->trie->buf);
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1172 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1173
4886 ae2114f11a0e Memory leak fixes. Also when building a large mailbox flush once in a while Timo Sirainen <tss@iki.fi> parents: 4883 diff changeset	1174 ctx->prev_uid = uid;
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1175 for (i = 0; i + BLOCK_SIZE <= size; i++) {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1176 if (block_want_add(str+i)) {
4878 88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1177 if (trie_insert_node(ctx, &ctx->trie->root,
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1178 str + i, uid, 1) < 0) {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1179 t_pop();
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1180 return -1;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1181 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1182 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1183 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1184
4878 88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1185 ctx->prev_added_size = I_MIN(size, BLOCK_SIZE-1);
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1186 memcpy(ctx->prev_added, str + i,
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1187 sizeof(ctx->prev_added[0]) * ctx->prev_added_size);
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1188 t_pop();
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1189 return 0;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1190 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1191
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1192 static void node_pack_children(buffer_t buf, struct trie_node *children,
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1193 unsigned int count)
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1194 {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1195 unsigned int i;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1196 size_t child_idx;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1197 uint32_t idx;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1198
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1199 for (i = 0; i < count; i++) {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1200 child_idx = POINTER_CAST_TO(children[i], size_t);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1201 if ((child_idx & 1) != 0)
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1202 idx = child_idx & ~1;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1203 else
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1204 idx = children[i]->file_offset;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1205 buffer_append(buf, &idx, sizeof(idx));
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1206 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1207 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1208
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1209 static void node_pack(buffer_t buf, struct trie_node node)
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1210 {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1211 uint8_t *chars8 = NODE_CHARS8(node);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1212 uint16_t *chars16 = NODE_CHARS16(node);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1213 struct trie_node **children8 = NODE_CHILDREN8(node);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1214 struct trie_node **children16 = NODE_CHILDREN16(node);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1215
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1216 buffer_set_used_size(buf, 0);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1217 _squat_trie_pack_num(buf, (node->chars_8bit_count << 1) \|
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1218 (node->chars_16bit_count > 0 ? 1 : 0));
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1219 buffer_append(buf, chars8, node->chars_8bit_count);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1220 node_pack_children(buf, children8, node->chars_8bit_count);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1221
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1222 if (node->chars_16bit_count > 0) {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1223 _squat_trie_pack_num(buf, node->chars_16bit_count);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1224 buffer_append(buf, chars16,
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1225 sizeof(chars16) node->chars_16bit_count);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1226 node_pack_children(buf, children16, node->chars_16bit_count);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1227 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1228 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1229
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1230 static int node_leaf_finish(struct squat_trie trie, struct trie_node node)
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1231 {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1232 uint32_t idx8 = (uint32_t )NODE_CHILDREN8(node);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1233 uint32_t idx16 = (uint32_t )NODE_CHILDREN16(node);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1234 unsigned int i;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1235
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1236 for (i = 0; i < node->chars_8bit_count; i++) {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1237 if (squat_uidlist_finish_list(trie->uidlist, &idx8[i]) < 0)
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1238 return -1;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1239 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1240 for (i = 0; i < node->chars_16bit_count; i++) {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1241 if (squat_uidlist_finish_list(trie->uidlist, &idx16[i]) < 0)
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1242 return -1;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1243 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1244 return 0;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1245 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1246
4878 88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1247 static void node_pack_leaf(buffer_t buf, struct trie_node node)
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1248 {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1249 uint8_t *chars8 = NODE_CHARS8(node);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1250 uint16_t *chars16 = NODE_CHARS16(node);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1251 uint32_t idx8 = (uint32_t )NODE_CHILDREN8(node);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1252 uint32_t idx16 = (uint32_t )NODE_CHILDREN16(node);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1253
4878 88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1254 buffer_set_used_size(buf, 0);
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1255 _squat_trie_pack_num(buf, (node->chars_8bit_count << 1) \|
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1256 (node->chars_16bit_count > 0 ? 1 : 0));
4878 88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1257 buffer_append(buf, chars8, node->chars_8bit_count);
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1258 buffer_append(buf, idx8, sizeof(idx8) node->chars_8bit_count);
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1259
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1260 if (node->chars_16bit_count > 0) {
4878 88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1261 _squat_trie_pack_num(buf, node->chars_16bit_count);
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1262 buffer_append(buf, chars16,
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1263 sizeof(chars16) node->chars_16bit_count);
4878 88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1264 buffer_append(buf, idx16,
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1265 sizeof(idx16) node->chars_16bit_count);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1266 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1267 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1268
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1269 static int
4878 88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1270 trie_write_node_children(struct squat_trie_build_context *ctx,
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1271 unsigned int level, struct trie_node **children,
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1272 unsigned int count)
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1273 {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1274 unsigned int i;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1275 size_t child_idx;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1276
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1277 for (i = 0; i < count; i++) {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1278 child_idx = POINTER_CAST_TO(children[i], size_t);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1279 if ((child_idx & 1) == 0) {
4878 88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1280 if (trie_write_node(ctx, level, children[i]) < 0)
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1281 return -1;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1282 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1283 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1284 return 0;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1285 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1286
4878 88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1287 static int trie_write_node(struct squat_trie_build_context *ctx,
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1288 unsigned int level, struct trie_node *node)
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1289 {
4878 88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1290 struct squat_trie *trie = ctx->trie;
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1291 uoff_t offset;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1292
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1293 if (level < BLOCK_SIZE) {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1294 struct trie_node **children8 = NODE_CHILDREN8(node);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1295 struct trie_node **children16 = NODE_CHILDREN16(node);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1296
4883 d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	1297 if (trie_write_node_children(ctx, level + 1,
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	1298 children8,
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	1299 node->chars_8bit_count) < 0)
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	1300 return -1;
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	1301 if (trie_write_node_children(ctx, level + 1,
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	1302 children16,
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	1303 node->chars_16bit_count) < 0)
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	1304 return -1;
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	1305 }
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	1306
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	1307 if (!node->modified)
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	1308 return 0;
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	1309
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	1310 if (level < BLOCK_SIZE)
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1311 node_pack(trie->buf, node);
4883 d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	1312 else {
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1313 if (node_leaf_finish(trie, node) < 0)
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1314 return -1;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1315
4878 88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1316 node_pack_leaf(trie->buf, node);
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1317 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1318
4878 88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1319 offset = ctx->output->offset;
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1320 if ((offset & 1) != 0) {
4878 88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1321 o_stream_send(ctx->output, "", 1);
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1322 offset++;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1323 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1324
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1325 if (node->resized && node->orig_size != trie->buf->used) {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1326 /* append to end of file. the parent node is written later. */
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1327 node->file_offset = offset;
4878 88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1328 o_stream_send(ctx->output, trie->buf->data, trie->buf->used);
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1329
4878 88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1330 ctx->deleted_space += node->orig_size;
4883 d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	1331 } else {
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1332 /* overwrite node's contents */
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1333 i_assert(node->file_offset != 0);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1334 i_assert(trie->buf->used <= node->orig_size);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1335
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1336 /* FIXME: write only the indexes if !node->resized */
4878 88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1337 o_stream_seek(ctx->output, node->file_offset);
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1338 o_stream_send(ctx->output, trie->buf->data, trie->buf->used);
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1339 o_stream_seek(ctx->output, offset);
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1340
4878 88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1341 ctx->deleted_space += trie->buf->used - node->orig_size;
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1342 }
4883 d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	1343
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	1344 ctx->modified = TRUE;
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1345 return 0;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1346 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1347
4878 88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1348 static int
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1349 trie_nodes_write(struct squat_trie_build_context ctx, uint32_t uidvalidity_r)
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1350 {
4878 88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1351 struct squat_trie *trie = ctx->trie;
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1352 struct squat_trie_header hdr;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1353
4879 b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	1354 hdr = *trie->hdr;
b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	1355 if (lseek(trie->fd, hdr.used_file_size, SEEK_SET) < 0) {
b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	1356 squat_trie_set_syscall_error(trie, "lseek()");
b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	1357 return -1;
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1358 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1359
4878 88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1360 ctx->output = o_stream_create_file(trie->fd, default_pool, 0, FALSE);
4883 d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	1361 if (hdr.used_file_size == 0) {
4878 88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1362 o_stream_send(ctx->output, &hdr, sizeof(hdr));
4883 d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	1363 ctx->modified = TRUE;
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	1364 }
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1365
4878 88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1366 ctx->deleted_space = 0;
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1367 if (trie_write_node(ctx, 1, trie->root) < 0)
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1368 return -1;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1369
4883 d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	1370 if (ctx->modified) {
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	1371 /* update the header */
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	1372 hdr.root_offset = trie->root->file_offset;
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	1373 hdr.used_file_size = ctx->output->offset;
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	1374 hdr.deleted_space += ctx->deleted_space;
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	1375 hdr.node_count = ctx->node_count;
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	1376 hdr.modify_counter++;
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	1377 o_stream_seek(ctx->output, 0);
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	1378 o_stream_send(ctx->output, &hdr, sizeof(hdr));
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	1379 }
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1380
4878 88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1381 o_stream_destroy(&ctx->output);
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1382 *uidvalidity_r = hdr.uidvalidity;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1383 return 0;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1384 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1385
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1386 static bool squat_trie_need_compress(struct squat_trie *trie,
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1387 unsigned int current_message_count)
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1388 {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1389 uint32_t max_del_space;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1390
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1391 if (trie->hdr->used_file_size >= TRIE_COMPRESS_MIN_SIZE) {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1392 /* see if we've reached the max. deleted space in file */
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1393 max_del_space = trie->hdr->used_file_size / 100 *
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1394 TRIE_COMPRESS_PERCENTAGE;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1395 if (trie->hdr->deleted_space > max_del_space)
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1396 return TRUE;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1397 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1398
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1399 return squat_uidlist_need_compress(trie->uidlist,
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1400 current_message_count);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1401 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1402
4886 ae2114f11a0e Memory leak fixes. Also when building a large mailbox flush once in a while Timo Sirainen <tss@iki.fi> parents: 4883 diff changeset	1403 static int
ae2114f11a0e Memory leak fixes. Also when building a large mailbox flush once in a while Timo Sirainen <tss@iki.fi> parents: 4883 diff changeset	1404 squat_trie_build_flush(struct squat_trie_build_context *ctx, bool finish)
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1405 {
4878 88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1406 struct squat_trie *trie = ctx->trie;
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1407 uint32_t uidvalidity;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1408
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1409 if (trie->root == NULL) {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1410 /* nothing changed */
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1411 return 0;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1412 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1413
4886 ae2114f11a0e Memory leak fixes. Also when building a large mailbox flush once in a while Timo Sirainen <tss@iki.fi> parents: 4883 diff changeset	1414 if (trie->corrupted)
ae2114f11a0e Memory leak fixes. Also when building a large mailbox flush once in a while Timo Sirainen <tss@iki.fi> parents: 4883 diff changeset	1415 return -1;
ae2114f11a0e Memory leak fixes. Also when building a large mailbox flush once in a while Timo Sirainen <tss@iki.fi> parents: 4883 diff changeset	1416
4878 88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1417 if (trie_nodes_write(ctx, &uidvalidity) < 0)
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1418 return -1;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1419 if (squat_uidlist_flush(trie->uidlist, uidvalidity) < 0)
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1420 return -1;
4879 b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	1421
b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	1422 squat_trie_unmap(trie);
b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	1423 if (squat_trie_map(trie) <= 0)
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1424 return -1;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1425
4886 ae2114f11a0e Memory leak fixes. Also when building a large mailbox flush once in a while Timo Sirainen <tss@iki.fi> parents: 4883 diff changeset	1426 /if (squat_trie_need_compress(trie, (unsigned int)-1))/ {
ae2114f11a0e Memory leak fixes. Also when building a large mailbox flush once in a while Timo Sirainen <tss@iki.fi> parents: 4883 diff changeset	1427 if (ctx->locked && finish) {
4883 d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	1428 squat_trie_unlock(ctx->trie);
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	1429 ctx->locked = FALSE;
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	1430 }
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	1431
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1432 if (squat_trie_compress(trie, NULL) < 0)
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1433 return -1;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1434 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1435 return 0;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1436 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1437
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1438 static void squat_trie_compress_chars8(struct trie_node *node)
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1439 {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1440 uint8_t *chars = NODE_CHARS8(node);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1441 struct trie_node **child_src = NODE_CHILDREN8(node);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1442 struct trie_node **child_dest;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1443 unsigned int i, j, old_count;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1444
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1445 old_count = node->chars_8bit_count;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1446 for (i = j = 0; i < old_count; i++) {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1447 if (child_src[i] != NULL)
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1448 chars[j++] = chars[i];
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1449 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1450
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1451 node->chars_8bit_count = j;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1452 child_dest = NODE_CHILDREN8(node);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1453
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1454 for (i = j = 0; i < old_count; i++) {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1455 if (child_src[i] != NULL)
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1456 child_dest[j++] = child_src[i];
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1457 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1458 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1459
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1460 static void squat_trie_compress_chars16(struct trie_node *node)
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1461 {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1462 uint16_t *chars = NODE_CHARS16(node);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1463 struct trie_node **child_src = NODE_CHILDREN16(node);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1464 struct trie_node **child_dest;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1465 unsigned int i, j, old_count;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1466
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1467 old_count = node->chars_16bit_count;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1468 for (i = j = 0; i < old_count; i++) {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1469 if (child_src[i] != NULL)
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1470 chars[j++] = chars[i];
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1471 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1472
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1473 node->chars_16bit_count = j;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1474 child_dest = NODE_CHILDREN16(node);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1475
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1476 for (i = j = 0; i < old_count; i++) {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1477 if (child_src[i] != NULL)
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1478 child_dest[j++] = child_src[i];
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1479 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1480 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1481
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1482 static int
4878 88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1483 squat_trie_compress_children(struct squat_trie_compress_context *ctx,
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1484 struct trie_node **children, unsigned int count,
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1485 unsigned int level)
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1486 {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1487 struct trie_node *child_node;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1488 size_t child_idx;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1489 unsigned int i;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1490 int ret = 0;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1491 bool need_char_compress = FALSE;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1492
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1493 for (i = 0; i < count; i++) {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1494 child_idx = POINTER_CAST_TO(children[i], size_t);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1495 i_assert((child_idx & 1) != 0);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1496 child_idx &= ~1;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1497
4878 88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1498 if (trie_map_node(ctx->trie, child_idx, level, &child_node) < 0)
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1499 return -1;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1500
4878 88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1501 ret = squat_trie_compress_node(ctx, child_node, level);
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1502 if (child_node->file_offset != 0)
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1503 children[i] = POINTER_CAST(child_node->file_offset \| 1);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1504 else {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1505 children[i] = NULL;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1506 need_char_compress = TRUE;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1507 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1508
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1509 if (ret < 0)
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1510 return -1;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1511 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1512 return need_char_compress ? 0 : 1;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1513 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1514
4878 88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1515 static int
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1516 squat_trie_compress_leaf_uidlist(struct squat_trie_compress_context *ctx,
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1517 struct trie_node *node)
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1518 {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1519 uint32_t idx8 = (uint32_t )NODE_CHILDREN8(node);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1520 uint32_t idx16 = (uint32_t )NODE_CHILDREN16(node);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1521 unsigned int i;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1522 int ret;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1523 bool compress_chars = FALSE;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1524
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1525 for (i = 0; i < node->chars_8bit_count; i++) {
4878 88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1526 ret = squat_uidlist_compress_next(ctx->uidlist_ctx, &idx8[i]);
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1527 if (ret < 0)
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1528 return -1;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1529 if (ret == 0) {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1530 idx8[i] = 0;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1531 compress_chars = TRUE;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1532 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1533 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1534 if (compress_chars) {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1535 squat_trie_compress_chars8(node);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1536 compress_chars = FALSE;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1537 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1538 for (i = 0; i < node->chars_16bit_count; i++) {
4878 88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1539 ret = squat_uidlist_compress_next(ctx->uidlist_ctx, &idx16[i]);
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1540 if (ret < 0)
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1541 return -1;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1542 if (ret == 0) {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1543 idx16[i] = 0;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1544 compress_chars = TRUE;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1545 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1546 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1547 if (compress_chars) {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1548 squat_trie_compress_chars16(node);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1549 node->chars_16bit_count = i;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1550 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1551 return 0;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1552 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1553
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1554 static int
4878 88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1555 squat_trie_compress_node(struct squat_trie_compress_context *ctx,
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1556 struct trie_node *node, unsigned int level)
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1557 {
4878 88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1558 struct squat_trie *trie = ctx->trie;
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1559 int ret;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1560
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1561 if (level == BLOCK_SIZE) {
4878 88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1562 if (squat_trie_compress_leaf_uidlist(ctx, node))
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1563 return -1;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1564
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1565 if (node->chars_8bit_count == 0 &&
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1566 node->chars_16bit_count == 0) {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1567 /* everything expunged */
4878 88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1568 ctx->node_count--;
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1569 node->file_offset = 0;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1570 return 0;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1571 }
4878 88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1572 node_pack_leaf(trie->buf, node);
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1573 } else {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1574 struct trie_node **children8 = NODE_CHILDREN8(node);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1575 struct trie_node **children16 = NODE_CHILDREN16(node);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1576
4878 88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1577 if ((ret = squat_trie_compress_children(ctx, children8,
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1578 node->chars_8bit_count,
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1579 level + 1)) < 0)
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1580 return -1;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1581 if (ret == 0)
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1582 squat_trie_compress_chars8(node);
4878 88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1583 if ((ret = squat_trie_compress_children(ctx, children16,
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1584 node->chars_16bit_count,
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1585 level + 1)) < 0)
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1586 return -1;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1587 if (ret == 0)
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1588 squat_trie_compress_chars16(node);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1589
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1590 if (node->chars_8bit_count == 0 &&
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1591 node->chars_16bit_count == 0) {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1592 /* everything expunged */
4878 88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1593 ctx->node_count--;
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1594 node->file_offset = 0;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1595 return 0;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1596 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1597 node_pack(trie->buf, node);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1598 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1599
4878 88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1600 if ((ctx->output->offset & 1) != 0)
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1601 o_stream_send(ctx->output, "", 1);
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1602 node->file_offset = ctx->output->offset;
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1603
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1604 o_stream_send(ctx->output, trie->buf->data, trie->buf->used);
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1605 return 0;
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1606 }
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1607
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1608 static int squat_trie_compress_init(struct squat_trie_compress_context *ctx,
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1609 struct squat_trie *trie)
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1610 {
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1611 struct squat_trie_header hdr;
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1612
4883 d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	1613 memset(ctx, 0, sizeof(*ctx));
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	1614
4878 88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1615 ctx->tmp_path = t_strconcat(trie->filepath, ".tmp", NULL);
4886 ae2114f11a0e Memory leak fixes. Also when building a large mailbox flush once in a while Timo Sirainen <tss@iki.fi> parents: 4883 diff changeset	1616 ctx->fd = open(ctx->tmp_path, O_RDWR \| O_CREAT \| O_TRUNC, 0600);
ae2114f11a0e Memory leak fixes. Also when building a large mailbox flush once in a while Timo Sirainen <tss@iki.fi> parents: 4883 diff changeset	1617 if (ctx->fd == -1) {
4878 88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1618 i_error("open(%s, O_CREAT) failed: %m", ctx->tmp_path);
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1619 return -1;
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1620 }
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1621
4878 88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1622 ctx->trie = trie;
4886 ae2114f11a0e Memory leak fixes. Also when building a large mailbox flush once in a while Timo Sirainen <tss@iki.fi> parents: 4883 diff changeset	1623 ctx->output = o_stream_create_file(ctx->fd, default_pool, 0, FALSE);
4878 88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1624 ctx->node_count = trie->hdr->node_count;
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1625
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1626 /* write a dummy header first */
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1627 memset(&hdr, 0, sizeof(hdr));
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1628 o_stream_send(ctx->output, &hdr, sizeof(hdr));
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1629 return 0;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1630 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1631
4878 88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1632 static void
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1633 squat_trie_compress_write_header(struct squat_trie_compress_context *ctx,
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1634 struct trie_node *root_node)
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1635 {
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1636 struct squat_trie_header hdr;
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1637
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1638 memset(&hdr, 0, sizeof(hdr));
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1639 hdr.version = SQUAT_TRIE_VERSION;
4879 b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	1640 hdr.uidvalidity = ctx->trie->uidvalidity;
4878 88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1641 hdr.root_offset = root_node->file_offset;
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1642 hdr.used_file_size = ctx->output->offset;
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1643 hdr.node_count = ctx->node_count;
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1644
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1645 o_stream_seek(ctx->output, 0);
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1646 o_stream_send(ctx->output, &hdr, sizeof(hdr));
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1647 }
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1648
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1649 int squat_trie_compress(struct squat_trie *trie,
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1650 const ARRAY_TYPE(seq_range) *existing_uids)
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1651 {
4878 88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1652 struct squat_trie_compress_context ctx;
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1653 struct trie_node *node;
4886 ae2114f11a0e Memory leak fixes. Also when building a large mailbox flush once in a while Timo Sirainen <tss@iki.fi> parents: 4883 diff changeset	1654 struct file_lock *file_lock = NULL;
ae2114f11a0e Memory leak fixes. Also when building a large mailbox flush once in a while Timo Sirainen <tss@iki.fi> parents: 4883 diff changeset	1655 unsigned int orig_lock_count;
4878 88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1656 int ret;
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1657
4886 ae2114f11a0e Memory leak fixes. Also when building a large mailbox flush once in a while Timo Sirainen <tss@iki.fi> parents: 4883 diff changeset	1658 orig_lock_count = trie->lock_count;
4878 88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1659 if (squat_trie_lock(trie, F_WRLCK) <= 0)
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1660 return -1;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1661
4878 88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1662 if (squat_trie_compress_init(&ctx, trie) < 0) {
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1663 squat_trie_unlock(trie);
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1664 return -1;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1665 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1666
4878 88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1667 ret = trie_map_node(trie, trie->hdr->root_offset, 1, &node);
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1668 if (ret == 0) {
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1669 /* do the compression */
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1670 ctx.uidlist_ctx = squat_uidlist_compress_begin(trie->uidlist,
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1671 existing_uids);
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1672 if ((ret = squat_trie_compress_node(&ctx, node, 1)) < 0)
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1673 squat_uidlist_compress_rollback(&ctx.uidlist_ctx);
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1674 else {
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1675 ret = squat_uidlist_compress_commit(&ctx.uidlist_ctx);
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1676
4878 88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1677 squat_trie_compress_write_header(&ctx, node);
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1678 }
4886 ae2114f11a0e Memory leak fixes. Also when building a large mailbox flush once in a while Timo Sirainen <tss@iki.fi> parents: 4883 diff changeset	1679 }
ae2114f11a0e Memory leak fixes. Also when building a large mailbox flush once in a while Timo Sirainen <tss@iki.fi> parents: 4883 diff changeset	1680
ae2114f11a0e Memory leak fixes. Also when building a large mailbox flush once in a while Timo Sirainen <tss@iki.fi> parents: 4883 diff changeset	1681 if (ret == 0 && orig_lock_count > 0) {
ae2114f11a0e Memory leak fixes. Also when building a large mailbox flush once in a while Timo Sirainen <tss@iki.fi> parents: 4883 diff changeset	1682 /* lock the file before renaming so we can keep it locked. */
ae2114f11a0e Memory leak fixes. Also when building a large mailbox flush once in a while Timo Sirainen <tss@iki.fi> parents: 4883 diff changeset	1683 if (squat_trie_file_lock(trie, ctx.fd, ctx.tmp_path, F_WRLCK,
ae2114f11a0e Memory leak fixes. Also when building a large mailbox flush once in a while Timo Sirainen <tss@iki.fi> parents: 4883 diff changeset	1684 &file_lock) <= 0)
ae2114f11a0e Memory leak fixes. Also when building a large mailbox flush once in a while Timo Sirainen <tss@iki.fi> parents: 4883 diff changeset	1685 ret = -1;
4878 88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1686 }
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1687
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1688 if (ret == 0) {
4878 88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1689 if (rename(ctx.tmp_path, trie->filepath) < 0) {
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1690 i_error("rename(%s, %s) failed: %m",
4878 88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1691 ctx.tmp_path, trie->filepath);
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1692 ret = -1;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1693 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1694 }
4886 ae2114f11a0e Memory leak fixes. Also when building a large mailbox flush once in a while Timo Sirainen <tss@iki.fi> parents: 4883 diff changeset	1695
4878 88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1696 o_stream_destroy(&ctx.output);
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1697 squat_trie_unlock(trie);
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1698
4886 ae2114f11a0e Memory leak fixes. Also when building a large mailbox flush once in a while Timo Sirainen <tss@iki.fi> parents: 4883 diff changeset	1699 if (ret < 0) {
ae2114f11a0e Memory leak fixes. Also when building a large mailbox flush once in a while Timo Sirainen <tss@iki.fi> parents: 4883 diff changeset	1700 if (file_lock != NULL)
ae2114f11a0e Memory leak fixes. Also when building a large mailbox flush once in a while Timo Sirainen <tss@iki.fi> parents: 4883 diff changeset	1701 file_lock_free(&file_lock);
ae2114f11a0e Memory leak fixes. Also when building a large mailbox flush once in a while Timo Sirainen <tss@iki.fi> parents: 4883 diff changeset	1702 (void)close(ctx.fd);
4878 88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1703 (void)unlink(ctx.tmp_path);
4886 ae2114f11a0e Memory leak fixes. Also when building a large mailbox flush once in a while Timo Sirainen <tss@iki.fi> parents: 4883 diff changeset	1704 } else {
4883 d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	1705 trie_file_close(trie);
4886 ae2114f11a0e Memory leak fixes. Also when building a large mailbox flush once in a while Timo Sirainen <tss@iki.fi> parents: 4883 diff changeset	1706 trie_file_open_fd(trie, ctx.fd);
ae2114f11a0e Memory leak fixes. Also when building a large mailbox flush once in a while Timo Sirainen <tss@iki.fi> parents: 4883 diff changeset	1707
ae2114f11a0e Memory leak fixes. Also when building a large mailbox flush once in a while Timo Sirainen <tss@iki.fi> parents: 4883 diff changeset	1708 trie->file_lock = file_lock;
ae2114f11a0e Memory leak fixes. Also when building a large mailbox flush once in a while Timo Sirainen <tss@iki.fi> parents: 4883 diff changeset	1709 if (squat_trie_map(trie) <= 0)
4879 b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	1710 return -1;
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1711 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1712 return ret;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1713 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1714
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1715 int squat_trie_mark_having_expunges(struct squat_trie *trie,
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1716 const ARRAY_TYPE(seq_range) *existing_uids,
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1717 unsigned int current_message_count)
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1718 {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1719 bool compress;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1720 int ret;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1721
4883 d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	1722 if ((ret = squat_trie_lock(trie, F_RDLCK)) <= 0)
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	1723 return ret;
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1724 compress = squat_trie_need_compress(trie, current_message_count);
4883 d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	1725 squat_trie_unlock(trie);
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	1726
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1727 ret = squat_uidlist_mark_having_expunges(trie->uidlist, compress);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1728
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1729 if (compress)
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1730 ret = squat_trie_compress(trie, existing_uids);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1731 return ret;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1732 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1733
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1734 size_t squat_trie_mem_used(struct squat_trie trie, unsigned int count_r)
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1735 {
4879 b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	1736 *count_r = trie->hdr == NULL ? 0 : trie->hdr->node_count;
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1737
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1738 return trie->mmap_size;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1739 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1740
4878 88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1741 static int squat_trie_lookup_init(struct squat_trie trie, const char str,
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1742 const uint16_t *data_r, unsigned int len_r)
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1743 {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1744 const uint16_t *data;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1745 unsigned int len = strlen(str);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1746
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1747 if (len < BLOCK_SIZE)
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1748 return -1;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1749
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1750 data = data_normalize(str, len, trie->buf);
4878 88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1751
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1752 /* skip the blocks that can't exist */
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1753 while (!block_want_add(data + len - BLOCK_SIZE)) {
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1754 if (--len < BLOCK_SIZE)
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1755 return -1;
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1756 }
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1757
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1758 if (squat_trie_lock(trie, F_RDLCK) <= 0)
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1759 return -1;
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1760
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1761 *data_r = data;
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1762 *len_r = len;
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1763 return 0;
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1764 }
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1765
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1766 static int
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1767 squat_trie_lookup_locked(struct squat_trie trie, ARRAY_TYPE(seq_range) result,
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1768 const uint16_t *data, unsigned int len)
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1769 {
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1770 uint32_t list;
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1771
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1772 list = trie_lookup_node(trie, trie->root, data + len - BLOCK_SIZE, 1);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1773 if (list == 0)
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1774 return 0;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1775
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1776 if (squat_uidlist_get(trie->uidlist, list, result) < 0) {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1777 squat_trie_set_corrupted(trie, "uidlist offset broken");
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1778 return -1;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1779 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1780 while (len > BLOCK_SIZE) {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1781 len--;
4878 88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1782
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1783 if (!block_want_add(data + len - BLOCK_SIZE))
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1784 continue;
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1785
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1786 list = trie_lookup_node(trie, trie->root,
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1787 data + len - BLOCK_SIZE, 1);
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1788 if (list == 0) {
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1789 array_clear(result);
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1790 return 0;
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1791 }
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1792 if (squat_uidlist_filter(trie->uidlist, list, result) < 0) {
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1793 squat_trie_set_corrupted(trie, "uidlist offset broken");
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1794 return -1;
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1795 }
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1796 }
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1797 return array_count(result) > 0 ? 1 : 0;
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1798 }
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1799
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1800 int squat_trie_lookup(struct squat_trie trie, ARRAY_TYPE(seq_range) result,
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1801 const char *str)
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1802 {
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1803 const uint16_t *data;
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1804 unsigned int len;
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1805 int ret;
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1806
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1807 if (squat_trie_lookup_init(trie, str, &data, &len) < 0)
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1808 return -1;
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1809
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1810 ret = squat_trie_lookup_locked(trie, result, data, len);
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1811 squat_trie_unlock(trie);
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1812 return ret;
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1813 }
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1814
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1815 static int
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1816 squat_trie_filter_locked(struct squat_trie trie, ARRAY_TYPE(seq_range) result,
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1817 const uint16_t *data, unsigned int len)
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1818 {
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1819 uint32_t list;
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1820
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1821 for (; len >= BLOCK_SIZE; len--) {
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1822 if (!block_want_add(data + len - BLOCK_SIZE))
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1823 continue;
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1824
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1825 list = trie_lookup_node(trie, trie->root,
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1826 data + len - BLOCK_SIZE, 1);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1827 if (list == 0) {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1828 array_clear(result);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1829 return 0;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1830 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1831 if (squat_uidlist_filter(trie->uidlist, list, result) < 0) {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1832 squat_trie_set_corrupted(trie, "uidlist offset broken");
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1833 return -1;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1834 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1835 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1836 return array_count(result) > 0 ? 1 : 0;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1837 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1838
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1839 int squat_trie_filter(struct squat_trie trie, ARRAY_TYPE(seq_range) result,
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1840 const char *str)
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1841 {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1842 const uint16_t *data;
4878 88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1843 unsigned int len;
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1844 int ret;
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1845
4878 88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1846 if (squat_trie_lookup_init(trie, str, &data, &len) < 0)
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1847 return -1;
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1848 ret = squat_trie_filter_locked(trie, result, data, len);
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1849 squat_trie_unlock(trie);
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	1850 return ret;
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1851 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1852
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1853 struct squat_uidlist _squat_trie_get_uidlist(struct squat_trie trie)
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1854 {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1855 return trie->uidlist;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1856 }

Mercurial > dovecot > original-hg > dovecot-1.2

annotate src/plugins/fts-squat/squat-trie.c @ 4886:ae2114f11a0e HEAD