dovecot/original-hg/dovecot-1.2: src/plugins/fts-squat/squat-uidlist.c annotate

annotate src/plugins/fts-squat/squat-uidlist.c @ 4883:d8adbe93c969 HEAD

Added support for mmap_disable=yes and several other fixes.

author	Timo Sirainen <tss@iki.fi>
date	Sat, 09 Dec 2006 23:08:56 +0200
parents	b0ada6e57b07
children	ae2114f11a0e

rev	line source
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1 /* Copyright (C) 2006 Timo Sirainen */
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	2
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	3 #include "lib.h"
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	4 #include "array.h"
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	5 #include "ostream.h"
4883 d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	6 #include "file-cache.h"
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	7 #include "mmap-util.h"
4883 d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	8 #include "read-full.h"
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	9 #include "write-full.h"
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	10 #include "squat-trie.h"
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	11 #include "squat-uidlist.h"
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	12
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	13 #include <stdio.h>
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	14 #include <unistd.h>
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	15 #include <fcntl.h>
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	16 #include <sys/stat.h>
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	17
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	18 #define UIDLIST_COMPRESS_PERCENTAGE 30
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	19 #define UIDLIST_UID_COMPRESS_PERCENTAGE 20
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	20 #define UIDLIST_COMPRESS_MIN_SIZE (1024*8)
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	21
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	22 #define UID_NODE_PREV_FLAG_OLD 0x00000001
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	23 #define UID_LIST_IDX_FLAG_SINGLE 0x80000000
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	24
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	25 struct squat_uidlist_header {
4879 b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	26 uint32_t uidvalidity;
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	27 uint32_t used_file_size;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	28 uint32_t deleted_space;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	29
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	30 uint32_t uid_max;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	31 uint32_t uid_count;
4878 88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	32 uint8_t uids_expunged; /* updated without locking */
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	33 uint8_t unused[3];
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	34 uint32_t node_count;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	35 };
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	36
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	37 struct uid_node {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	38 struct uid_node *prev;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	39 uint32_t uid;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	40 };
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	41
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	42 struct squat_uidlist_get_context {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	43 struct squat_uidlist *uidlist;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	44
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	45 ARRAY_TYPE(seq_range) *result;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	46
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	47 uint32_t filter_pos;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	48 };
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	49
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	50 struct squat_uidlist {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	51 struct squat_trie *trie;
4879 b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	52 uint32_t uidvalidity;
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	53
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	54 char *filepath;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	55 int fd;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	56 struct ostream *output;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	57
4883 d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	58 dev_t dev;
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	59 ino_t ino;
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	60
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	61 void *mmap_base;
4883 d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	62 const uint8_t *const_mmap_base;
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	63 size_t mmap_size;
4883 d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	64
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	65 struct file_cache *file_cache;
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	66 struct squat_uidlist_header hdr;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	67
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	68 ARRAY_DEFINE(lists, struct uid_node);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	69 uint32_t first_new_list_idx;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	70
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	71 pool_t node_pool;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	72 buffer_t tmp_buf, list_buf;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	73
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	74 unsigned int check_expunges:1;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	75 unsigned int write_failed:1;
4883 d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	76 unsigned int mmap_disable:1;
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	77 };
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	78
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	79 struct squat_uidlist_compress_ctx {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	80 struct squat_uidlist *uidlist;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	81 const ARRAY_TYPE(seq_range) *existing_uids;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	82
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	83 struct ostream *output;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	84 char *tmp_path;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	85
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	86 pool_t node_pool;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	87 struct uid_node *last_node;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	88 ARRAY_TYPE(seq_range) seen_uids;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	89
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	90 struct squat_uidlist_header hdr;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	91
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	92 unsigned int seen_expunged:1;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	93 unsigned int failed:1;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	94 };
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	95
4883 d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	96 static void squat_uidlist_close(struct squat_uidlist *uidlist);
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	97
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	98 static void
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	99 squat_uidlist_set_syscall_error(struct squat_uidlist *uidlist,
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	100 const char *function)
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	101 {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	102 i_error("%s failed with index search uidlist file %s: %m",
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	103 function, uidlist->filepath);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	104 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	105
4879 b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	106 static int squat_uidlist_check_header(struct squat_uidlist *uidlist,
4883 d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	107 const struct squat_uidlist_header *hdr,
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	108 uoff_t file_size)
4879 b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	109 {
b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	110 if (hdr->uidvalidity != uidlist->uidvalidity) {
b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	111 squat_trie_set_corrupted(uidlist->trie,
b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	112 "uidlist: uidvalidity changed");
b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	113 return -1;
b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	114 }
4883 d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	115 if (hdr->used_file_size > file_size) {
4879 b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	116 squat_trie_set_corrupted(uidlist->trie,
b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	117 "uidlist: used_file_size too large");
b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	118 return -1;
b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	119 }
b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	120
b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	121 return 0;
b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	122 }
b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	123
4883 d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	124 static int squat_uidlist_read_header(struct squat_uidlist *uidlist)
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	125 {
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	126 int ret;
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	127
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	128 ret = pread_full(uidlist->fd, &uidlist->hdr, sizeof(uidlist->hdr), 0);
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	129 if (ret < 0)
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	130 squat_uidlist_set_syscall_error(uidlist, "pread_full()");
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	131 return ret;
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	132 }
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	133
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	134 static int squat_uidlist_map(struct squat_uidlist *uidlist)
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	135 {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	136 struct stat st;
4883 d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	137 int ret;
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	138
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	139 if (!uidlist->mmap_disable) {
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	140 const struct squat_uidlist_header *hdr = uidlist->mmap_base;
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	141
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	142 if (hdr != NULL && hdr->used_file_size <= uidlist->mmap_size) {
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	143 /* everything is already mapped */
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	144 uidlist->hdr = *hdr;
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	145 return 1;
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	146 }
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	147 } else {
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	148 if ((ret = squat_uidlist_read_header(uidlist)) < 0)
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	149 return -1;
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	150 if (ret == 0)
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	151 return 0;
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	152 }
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	153
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	154 if (fstat(uidlist->fd, &st) < 0) {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	155 squat_uidlist_set_syscall_error(uidlist, "fstat()");
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	156 return -1;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	157 }
4883 d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	158 uidlist->dev = st.st_dev;
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	159 uidlist->ino = st.st_ino;
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	160
4883 d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	161 if (st.st_size <= sizeof(uidlist->hdr))
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	162 return 0;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	163
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	164 if (uidlist->mmap_base != NULL) {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	165 if (munmap(uidlist->mmap_base, uidlist->mmap_size) < 0)
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	166 squat_uidlist_set_syscall_error(uidlist, "munmap()");
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	167 }
4883 d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	168 uidlist->const_mmap_base = NULL;
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	169
4883 d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	170 if (!uidlist->mmap_disable) {
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	171 uidlist->mmap_size = st.st_size;
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	172 uidlist->mmap_base =
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	173 mmap(NULL, uidlist->mmap_size, PROT_READ \| PROT_WRITE,
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	174 MAP_SHARED, uidlist->fd, 0);
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	175 if (uidlist->mmap_base == MAP_FAILED) {
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	176 uidlist->mmap_size = 0;
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	177 uidlist->mmap_base = NULL;
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	178 squat_uidlist_set_syscall_error(uidlist, "mmap()");
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	179 return -1;
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	180 }
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	181 uidlist->const_mmap_base = uidlist->mmap_base;
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	182 memcpy(&uidlist->hdr, uidlist->mmap_base, sizeof(uidlist->hdr));
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	183 } else {
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	184 /* the header is always read separately. everything between it
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	185 and the used_file_size doesn't change */
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	186 file_cache_invalidate(uidlist->file_cache,
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	187 uidlist->hdr.used_file_size, (uoff_t)-1);
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	188 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	189
4883 d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	190 if (squat_uidlist_check_header(uidlist, &uidlist->hdr, st.st_size) < 0)
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	191 return 0;
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	192
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	193 if (uidlist->hdr.uids_expunged)
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	194 uidlist->check_expunges = TRUE;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	195
4883 d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	196 uidlist->first_new_list_idx = uidlist->hdr.used_file_size;
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	197 return 1;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	198 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	199
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	200 static int squat_uidlist_open(struct squat_uidlist *uidlist)
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	201 {
4883 d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	202 int ret;
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	203
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	204 i_assert(uidlist->fd == -1);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	205
4883 d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	206 uidlist->fd = open(uidlist->filepath, O_RDWR, 0600);
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	207 if (uidlist->fd == -1) {
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	208 if (errno == ENOENT)
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	209 return 0;
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	210
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	211 squat_uidlist_set_syscall_error(uidlist, "open()");
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	212 return -1;
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	213 }
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	214
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	215 if (uidlist->mmap_disable)
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	216 uidlist->file_cache = file_cache_new(uidlist->fd);
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	217
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	218 if ((ret = squat_uidlist_map(uidlist)) == 0) {
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	219 /* broken */
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	220 if (unlink(uidlist->filepath) < 0)
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	221 squat_uidlist_set_syscall_error(uidlist, "unlink()");
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	222 squat_uidlist_close(uidlist);
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	223 }
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	224 return ret;
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	225 }
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	226
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	227 static int squat_uidlist_create(struct squat_uidlist *uidlist)
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	228 {
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	229 i_assert(uidlist->fd == -1);
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	230
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	231 /* we should get here only if normal file opening failed */
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	232 uidlist->fd = open(uidlist->filepath, O_RDWR \| O_CREAT \| O_TRUNC, 0600);
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	233 if (uidlist->fd == -1) {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	234 squat_uidlist_set_syscall_error(uidlist, "open()");
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	235 return -1;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	236 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	237
4883 d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	238 if (uidlist->mmap_disable)
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	239 uidlist->file_cache = file_cache_new(uidlist->fd);
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	240 return 0;
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	241 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	242
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	243 static void squat_uidlist_close(struct squat_uidlist *uidlist)
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	244 {
4883 d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	245 if (uidlist->file_cache != NULL)
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	246 file_cache_free(&uidlist->file_cache);
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	247 if (uidlist->mmap_base != NULL) {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	248 if (munmap(uidlist->mmap_base, uidlist->mmap_size) < 0)
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	249 squat_uidlist_set_syscall_error(uidlist, "munmap()");
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	250 uidlist->mmap_base = NULL;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	251 }
4883 d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	252 uidlist->const_mmap_base = NULL;
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	253 uidlist->mmap_size = 0;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	254
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	255 if (uidlist->fd != -1) {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	256 if (close(uidlist->fd) < 0)
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	257 squat_uidlist_set_syscall_error(uidlist, "close()");
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	258 uidlist->fd = -1;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	259 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	260 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	261
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	262 struct squat_uidlist *
4879 b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	263 squat_uidlist_init(struct squat_trie trie, const char path,
4883 d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	264 uint32_t uidvalidity, bool mmap_disable)
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	265 {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	266 struct squat_uidlist *uidlist;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	267
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	268 uidlist = i_new(struct squat_uidlist, 1);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	269 uidlist->trie = trie;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	270 uidlist->filepath = i_strdup(path);
4879 b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	271 uidlist->uidvalidity = uidvalidity;
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	272 uidlist->fd = -1;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	273 uidlist->first_new_list_idx = 1;
4883 d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	274 uidlist->mmap_disable = mmap_disable;
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	275 i_array_init(&uidlist->lists, 65536);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	276 uidlist->node_pool =
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	277 pool_alloconly_create("squat uidlist node pool", 65536);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	278 uidlist->tmp_buf = buffer_create_dynamic(default_pool, 16);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	279 uidlist->list_buf = buffer_create_dynamic(default_pool, 256);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	280 return uidlist;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	281 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	282
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	283 void squat_uidlist_deinit(struct squat_uidlist *uidlist)
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	284 {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	285 squat_uidlist_close(uidlist);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	286
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	287 pool_unref(uidlist->node_pool);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	288 array_free(&uidlist->lists);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	289 buffer_free(uidlist->tmp_buf);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	290 buffer_free(uidlist->list_buf);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	291 i_free(uidlist);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	292 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	293
4883 d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	294 int squat_uidlist_refresh(struct squat_uidlist *uidlist)
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	295 {
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	296 struct stat st;
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	297 int ret;
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	298
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	299 if (uidlist->fd != -1) {
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	300 if (stat(uidlist->filepath, &st) < 0) {
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	301 if (errno == ENOENT)
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	302 return 0;
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	303
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	304 squat_uidlist_set_syscall_error(uidlist, "stat()");
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	305 return -1;
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	306 }
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	307 if (st.st_ino == uidlist->ino &&
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	308 CMP_DEV_T(st.st_dev, uidlist->dev)) {
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	309 /* no need to reopen, just remap */
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	310 if ((ret = squat_uidlist_map(uidlist)) != 0)
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	311 return ret < 0 ? -1 : 0;
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	312 /* broken file */
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	313 }
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	314 squat_uidlist_close(uidlist);
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	315 }
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	316
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	317 if (squat_uidlist_open(uidlist) < 0)
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	318 return -1;
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	319 return 0;
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	320 }
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	321
4878 88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	322 int squat_uidlist_get_last_uid(struct squat_uidlist uidlist, uint32_t uid_r)
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	323 {
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	324 *uid_r = uidlist->hdr.uid_max;
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	325 return 0;
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	326 }
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	327
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	328 int squat_uidlist_add(struct squat_uidlist uidlist, uint32_t _uid_list_idx,
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	329 uint32_t uid)
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	330 {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	331 uint32_t uid_list_idx = *_uid_list_idx;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	332 struct uid_node node, old_node;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	333
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	334 i_assert(uid >= uidlist->hdr.uid_max);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	335
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	336 if (uid_list_idx == 0) {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	337 *_uid_list_idx = uid \| UID_LIST_IDX_FLAG_SINGLE;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	338 return 0;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	339 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	340
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	341 if (uid > uidlist->hdr.uid_max) {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	342 uidlist->hdr.uid_max = uid;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	343 uidlist->hdr.uid_count++;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	344 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	345
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	346 if (uid_list_idx < uidlist->first_new_list_idx) {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	347 /* continue an existing list in the uidlist file */
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	348 old_node = POINTER_CAST((uid_list_idx << 1) \|
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	349 UID_NODE_PREV_FLAG_OLD);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	350 uid_list_idx = uidlist->first_new_list_idx +
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	351 array_count(&uidlist->lists);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	352 node = array_append_space(&uidlist->lists);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	353
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	354 uidlist->hdr.node_count++;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	355 } else if ((uid_list_idx & UID_LIST_IDX_FLAG_SINGLE) != 0) {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	356 uint32_t old_uid = uid_list_idx & ~UID_LIST_IDX_FLAG_SINGLE;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	357
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	358 if (uid == old_uid) {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	359 /* trying to add the same uid again */
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	360 return 0;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	361 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	362
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	363 /* convert single UID to a list */
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	364 old_node = p_new(uidlist->node_pool, struct uid_node, 1);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	365 old_node->uid = old_uid;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	366
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	367 uid_list_idx = uidlist->first_new_list_idx +
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	368 array_count(&uidlist->lists);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	369 node = array_append_space(&uidlist->lists);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	370
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	371 uidlist->hdr.node_count++;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	372 } else {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	373 /* update an in-memory list */
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	374 uint32_t arr_idx = uid_list_idx - uidlist->first_new_list_idx;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	375 if (arr_idx >= array_count(&uidlist->lists)) {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	376 /* broken */
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	377 squat_trie_set_corrupted(uidlist->trie,
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	378 "corrupted uidlist index (adding)");
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	379 return -1;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	380 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	381
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	382 node = array_idx_modifiable(&uidlist->lists, arr_idx);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	383 if (node->uid == uid) {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	384 /* trying to add the same uid again */
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	385 return 0;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	386 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	387
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	388 old_node = p_new(uidlist->node_pool, struct uid_node, 1);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	389 old_node = node;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	390 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	391
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	392 node->prev = old_node;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	393 node->uid = uid;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	394 *_uid_list_idx = uid_list_idx;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	395 return 0;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	396 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	397
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	398 static int
4883 d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	399 squat_uidlist_map_area(struct squat_uidlist *uidlist,
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	400 size_t offset, size_t size)
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	401 {
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	402 ssize_t ret;
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	403
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	404 if (uidlist->file_cache == NULL)
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	405 return 0;
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	406
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	407 ret = file_cache_read(uidlist->file_cache, offset, size);
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	408 if (ret < 0) {
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	409 squat_uidlist_set_syscall_error(uidlist, "file_cache_read()");
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	410 return -1;
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	411 }
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	412 uidlist->const_mmap_base =
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	413 file_cache_get_map(uidlist->file_cache, &uidlist->mmap_size);
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	414 return 0;
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	415 }
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	416
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	417 static int
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	418 squat_uidlist_map_list(struct squat_uidlist *uidlist, size_t offset,
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	419 const uint8_t *data_r, uint32_t size_r)
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	420 {
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	421 const uint8_t data, end;
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	422 size_t data_offset;
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	423 uint32_t size;
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	424
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	425 if (squat_uidlist_map_area(uidlist, offset, 128) < 0)
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	426 return -1;
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	427 if (offset >= uidlist->mmap_size)
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	428 return -1;
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	429
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	430 data = uidlist->const_mmap_base + offset;
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	431 end = uidlist->const_mmap_base + uidlist->mmap_size;
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	432
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	433 size = _squat_trie_unpack_num(&data, end);
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	434 data_offset = data - uidlist->const_mmap_base;
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	435
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	436 if (squat_uidlist_map_area(uidlist, data_offset, size) < 0)
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	437 return -1;
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	438 if (data_offset + size > uidlist->mmap_size)
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	439 return -1;
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	440
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	441 *data_r = uidlist->const_mmap_base + data_offset;
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	442 *size_r = size;
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	443 return 0;
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	444 }
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	445
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	446 static int
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	447 squat_uidlist_copy_existing(struct squat_uidlist *uidlist, size_t offset,
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	448 uint32_t prev_uid_r, uint32_t written_uid_r)
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	449 {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	450 const uint8_t data, data_start, end, p = NULL;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	451 uint32_t size, num, prev_uid, next_uid;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	452
4883 d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	453 if (squat_uidlist_map_list(uidlist, offset, &data, &size) < 0)
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	454 return -1;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	455
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	456 data_start = data;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	457 end = data + size;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	458
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	459 prev_uid = next_uid = _squat_trie_unpack_num(&data, end);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	460 p = data;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	461 while (data != end) {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	462 num = _squat_trie_unpack_num(&data, end);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	463 next_uid = prev_uid + (num >> 1) + 1;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	464
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	465 if ((num & 1) != 0) {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	466 /* prev_uid..next_uid */
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	467 if (data == end) {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	468 /* try to increase this range */
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	469 break;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	470 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	471
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	472 /* beginning a new uid/range */
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	473 num = _squat_trie_unpack_num(&data, end);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	474 next_uid += num + 1;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	475
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	476 prev_uid = next_uid;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	477 p = data;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	478 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	479
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	480 prev_uid = next_uid;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	481 p = data;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	482 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	483
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	484 *written_uid_r = prev_uid;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	485 *prev_uid_r = next_uid;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	486
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	487 uidlist->hdr.deleted_space +=
4883 d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	488 (end - (const uint8_t *)uidlist->const_mmap_base) - offset;
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	489
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	490 buffer_append(uidlist->list_buf, data_start, p - data_start);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	491 return 0;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	492 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	493
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	494 static int
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	495 squat_uidlist_write_range(struct squat_uidlist *uidlist,
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	496 const struct uid_node *node,
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	497 uint32_t prev_uid_r, uint32_t written_uid_r,
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	498 int level)
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	499 {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	500 buffer_t *buffer = uidlist->list_buf;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	501 uint32_t written_uid, prev_uid;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	502 uint32_t prev_idx = POINTER_CAST_TO(node->prev, uint32_t);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	503
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	504 *prev_uid_r = node->uid;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	505
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	506 if (node->prev == NULL) {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	507 /* first UID */
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	508 _squat_trie_pack_num(buffer, node->uid);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	509 } else {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	510 if ((prev_idx & UID_NODE_PREV_FLAG_OLD) != 0) {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	511 prev_idx >>= 1;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	512 if (squat_uidlist_copy_existing(uidlist, prev_idx,
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	513 &prev_uid,
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	514 &written_uid) < 0 \|\|
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	515 prev_uid >= node->uid) {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	516 squat_trie_set_corrupted(uidlist->trie,
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	517 "corrupted continued uidlist index");
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	518 return -1;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	519 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	520 } else {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	521 if (squat_uidlist_write_range(uidlist, node->prev,
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	522 &prev_uid, &written_uid,
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	523 level+1) < 0)
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	524 return -1;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	525 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	526
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	527 /* prev_uid contains the previous node's UID.
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	528 written_uid contains the last written UID. */
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	529 if (prev_uid + 1 == node->uid) {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	530 if (level != 0) {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	531 /* this node continue the range */
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	532 *written_uid_r = written_uid;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	533 return 0;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	534 } else {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	535 /* finishing range */
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	536 _squat_trie_pack_num(buffer, 1 \|
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	537 ((node->uid - written_uid - 1) << 1));
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	538 return 0;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	539 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	540 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	541 i_assert(prev_uid < node->uid);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	542 if (written_uid != prev_uid) {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	543 i_assert(written_uid < prev_uid);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	544
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	545 /* range ends at prev_uid */
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	546 _squat_trie_pack_num(buffer, 1 \|
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	547 ((prev_uid - written_uid - 1) << 1));
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	548 /* next uid/range */
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	549 _squat_trie_pack_num(buffer, node->uid - prev_uid - 1);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	550 } else {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	551 /* no range */
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	552 _squat_trie_pack_num(buffer,
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	553 ((node->uid - prev_uid - 1) << 1));
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	554 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	555 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	556
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	557 *written_uid_r = node->uid;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	558 return 0;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	559 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	560
4883 d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	561 static int squat_uidlist_write_init(struct squat_uidlist *uidlist)
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	562 {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	563 i_assert(uidlist->output == NULL);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	564
4883 d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	565 if (uidlist->fd == -1) {
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	566 if (squat_uidlist_create(uidlist) < 0)
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	567 return -1;
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	568 }
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	569
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	570 uidlist->output = o_stream_create_file(uidlist->fd, default_pool,
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	571 0, FALSE);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	572 if (uidlist->hdr.used_file_size < sizeof(uidlist->hdr)) {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	573 /* creating a new file, write a dummy header */
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	574 o_stream_seek(uidlist->output, 0);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	575 o_stream_send(uidlist->output, &uidlist->hdr,
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	576 sizeof(uidlist->hdr));
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	577 } else {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	578 o_stream_seek(uidlist->output,
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	579 uidlist->hdr.used_file_size);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	580 }
4883 d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	581 return 0;
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	582 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	583
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	584 static int squat_uidlist_write_listbuf(struct squat_uidlist *uidlist,
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	585 struct ostream *output)
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	586 {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	587 /* write size + buffer */
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	588 buffer_set_used_size(uidlist->tmp_buf, 0);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	589 _squat_trie_pack_num(uidlist->tmp_buf, uidlist->list_buf->used);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	590
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	591 if (o_stream_send(output, uidlist->tmp_buf->data,
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	592 uidlist->tmp_buf->used) < 0 \|\|
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	593 o_stream_send(output, uidlist->list_buf->data,
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	594 uidlist->list_buf->used) < 0) {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	595 return -1;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	596 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	597 return 0;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	598 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	599
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	600 int squat_uidlist_finish_list(struct squat_uidlist *uidlist,
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	601 uint32_t *_uid_list_idx)
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	602 {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	603 uint32_t uid_list_idx = *_uid_list_idx;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	604 const struct uid_node *node;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	605 uint32_t prev_uid, written_uid;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	606
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	607 if ((uid_list_idx & UID_LIST_IDX_FLAG_SINGLE) != 0) {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	608 /* this is a single UID "list" */
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	609 return 0;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	610 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	611 if (uid_list_idx < uidlist->first_new_list_idx) {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	612 /* the list hasn't changed */
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	613 return 0;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	614 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	615
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	616 uid_list_idx -= uidlist->first_new_list_idx;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	617 if (uid_list_idx >= array_count(&uidlist->lists)) {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	618 /* broken */
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	619 squat_trie_set_corrupted(uidlist->trie,
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	620 "corrupted uidlist index (finishing)");
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	621 return -1;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	622 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	623
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	624 /* write the uidlist into a buffer */
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	625 node = array_idx(&uidlist->lists, uid_list_idx);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	626 buffer_set_used_size(uidlist->list_buf, 0);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	627 if (squat_uidlist_write_range(uidlist, node,
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	628 &prev_uid, &written_uid, 0) < 0) {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	629 uidlist->write_failed = TRUE;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	630 return -1;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	631 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	632
4883 d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	633 if (uidlist->output == NULL) {
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	634 if (squat_uidlist_write_init(uidlist) < 0) {
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	635 uidlist->write_failed = TRUE;
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	636 return -1;
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	637 }
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	638 }
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	639
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	640 /* new uidlist index is the offset in uidlist file */
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	641 *_uid_list_idx = uidlist->output->offset;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	642
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	643 if (squat_uidlist_write_listbuf(uidlist, uidlist->output) < 0)
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	644 uidlist->write_failed = TRUE;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	645 return 0;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	646 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	647
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	648 static void squat_uidlist_write_header(struct squat_uidlist *uidlist)
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	649 {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	650 uidlist->hdr.used_file_size = uidlist->output->offset;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	651
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	652 o_stream_seek(uidlist->output, 0);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	653 o_stream_send(uidlist->output, &uidlist->hdr, sizeof(uidlist->hdr));
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	654 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	655
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	656 int squat_uidlist_flush(struct squat_uidlist *uidlist, uint32_t uid_validity)
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	657 {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	658 int ret = uidlist->write_failed ? -1 : 0;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	659
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	660 if (uidlist->output != NULL) {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	661 if (ret == 0) {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	662 uidlist->hdr.uidvalidity = uid_validity;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	663 squat_uidlist_write_header(uidlist);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	664 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	665 o_stream_destroy(&uidlist->output);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	666 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	667
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	668 array_clear(&uidlist->lists);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	669 p_clear(uidlist->node_pool);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	670
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	671 uidlist->write_failed = FALSE;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	672 return ret;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	673 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	674
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	675 bool squat_uidlist_need_compress(struct squat_uidlist *uidlist,
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	676 unsigned int current_message_count)
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	677 {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	678 uint32_t max_del_space, max_uid_del_count;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	679
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	680 if (uidlist->hdr.used_file_size >= UIDLIST_COMPRESS_MIN_SIZE) {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	681 /* see if we've reached the max. deleted space in file */
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	682 max_del_space = uidlist->hdr.used_file_size / 100 *
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	683 UIDLIST_COMPRESS_PERCENTAGE;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	684 if (uidlist->hdr.deleted_space > max_del_space)
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	685 return TRUE;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	686 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	687 if (uidlist->hdr.uid_count > current_message_count) {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	688 if (current_message_count == 0)
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	689 return TRUE;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	690
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	691 max_uid_del_count = uidlist->hdr.uid_count *
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	692 UIDLIST_UID_COMPRESS_PERCENTAGE / 100;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	693 if ((uidlist->hdr.uid_count - current_message_count) >
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	694 max_uid_del_count)
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	695 return TRUE;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	696 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	697 return FALSE;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	698 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	699
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	700 int squat_uidlist_mark_having_expunges(struct squat_uidlist *uidlist,
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	701 bool update_disk)
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	702 {
4878 88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	703 uint8_t flag = 1;
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	704 size_t offset;
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	705
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	706 uidlist->check_expunges = TRUE;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	707
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	708 if (update_disk) {
4878 88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	709 /* NOTE: we're writing this flag without locking */
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	710 offset = offsetof(struct squat_uidlist_header, uids_expunged);
88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	711 if (pwrite_full(uidlist->fd, &flag, sizeof(flag), offset) < 0) {
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	712 squat_uidlist_set_syscall_error(uidlist,
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	713 "pwrite_full()");
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	714 return -1;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	715 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	716 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	717 return 0;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	718 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	719
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	720 struct squat_uidlist_compress_ctx *
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	721 squat_uidlist_compress_begin(struct squat_uidlist *uidlist,
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	722 const ARRAY_TYPE(seq_range) *existing_uids)
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	723 {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	724 struct squat_uidlist_compress_ctx *ctx;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	725 int fd;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	726
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	727 ctx = i_new(struct squat_uidlist_compress_ctx, 1);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	728 ctx->uidlist = uidlist;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	729 ctx->tmp_path = i_strconcat(uidlist->filepath, ".tmp", NULL);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	730
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	731 if (existing_uids != NULL) {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	732 ctx->node_pool = pool_alloconly_create("compress node pool",
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	733 1024);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	734 ctx->existing_uids = existing_uids;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	735 i_array_init(&ctx->seen_uids,
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	736 I_MIN(128, array_count(existing_uids)));
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	737 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	738
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	739 fd = open(ctx->tmp_path, O_RDWR \| O_CREAT \| O_TRUNC, 0600);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	740 if (fd == -1) {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	741 ctx->failed = TRUE;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	742 i_error("open(%s) failed: %m", ctx->tmp_path);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	743 } else {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	744 ctx->output = o_stream_create_file(fd, default_pool, 0, TRUE);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	745 o_stream_send(ctx->output, &ctx->hdr, sizeof(ctx->hdr));
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	746 }
4883 d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	747
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	748 if (squat_uidlist_refresh(uidlist) < 0)
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	749 ctx->failed = TRUE;
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	750 return ctx;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	751 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	752
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	753 static bool
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	754 squat_uidlist_is_expunged(struct squat_uidlist_compress_ctx *ctx, uint32_t uid)
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	755 {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	756 if (ctx->existing_uids == NULL)
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	757 return FALSE;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	758
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	759 return !seq_range_exists(ctx->existing_uids, uid);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	760 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	761
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	762 static void
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	763 squat_uidlist_compress_add_uid(struct squat_uidlist_compress_ctx *ctx,
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	764 uint32_t uid)
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	765 {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	766 struct uid_node *node;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	767
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	768 if (squat_uidlist_is_expunged(ctx, uid)) {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	769 ctx->seen_expunged = TRUE;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	770 return;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	771 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	772
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	773 if (!seq_range_exists(&ctx->seen_uids, uid)) {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	774 if (uid > ctx->hdr.uid_max)
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	775 ctx->hdr.uid_max = uid;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	776 ctx->hdr.uid_count++;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	777 seq_range_array_add(&ctx->seen_uids, 0, uid);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	778 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	779
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	780 node = p_new(ctx->node_pool, struct uid_node, 1);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	781 node->prev = ctx->last_node;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	782 node->uid = uid;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	783
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	784 ctx->last_node = node;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	785 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	786
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	787 static int
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	788 squat_uidlist_remove_expunged(struct squat_uidlist_compress_ctx *ctx,
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	789 const uint8_t *data, size_t size,
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	790 bool *all_expunged_r)
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	791 {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	792 const uint8_t *end;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	793 uint32_t num, prev_uid, next_uid, written_uid;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	794
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	795 end = data + size;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	796
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	797 p_clear(ctx->node_pool);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	798 ctx->seen_expunged = FALSE;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	799 ctx->last_node = NULL;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	800
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	801 prev_uid = _squat_trie_unpack_num(&data, end);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	802 squat_uidlist_compress_add_uid(ctx, prev_uid);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	803
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	804 while (data != end) {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	805 num = _squat_trie_unpack_num(&data, end);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	806 next_uid = prev_uid + (num >> 1) + 1;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	807 if ((num & 1) != 0) {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	808 for (prev_uid++; prev_uid <= next_uid; prev_uid++)
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	809 squat_uidlist_compress_add_uid(ctx, prev_uid);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	810
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	811 if (data == end)
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	812 break;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	813 num = _squat_trie_unpack_num(&data, end);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	814 next_uid += num + 1;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	815 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	816 squat_uidlist_compress_add_uid(ctx, next_uid);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	817 prev_uid = next_uid;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	818 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	819
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	820 if (!ctx->seen_expunged) {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	821 /* no changes */
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	822 return 0;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	823 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	824 if (ctx->last_node == NULL) {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	825 /* everything expunged */
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	826 *all_expunged_r = TRUE;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	827 return 1;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	828 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	829
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	830 /* recreate the list and write it */
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	831 buffer_set_used_size(ctx->uidlist->list_buf, 0);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	832 if (squat_uidlist_write_range(ctx->uidlist, ctx->last_node,
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	833 &prev_uid, &written_uid, 0) < 0)
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	834 return -1;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	835 if (squat_uidlist_write_listbuf(ctx->uidlist, ctx->output) < 0)
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	836 return -1;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	837 *all_expunged_r = FALSE;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	838 return 1;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	839 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	840
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	841 int squat_uidlist_compress_next(struct squat_uidlist_compress_ctx *ctx,
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	842 uint32_t *uid_list_idx)
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	843 {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	844 struct squat_uidlist *uidlist = ctx->uidlist;
4883 d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	845 const uint8_t data, data_start;
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	846 uint32_t size, old_offset;
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	847 int ret;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	848
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	849 if ((*uid_list_idx & UID_LIST_IDX_FLAG_SINGLE) != 0) {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	850 uint32_t uid = *uid_list_idx & ~UID_LIST_IDX_FLAG_SINGLE;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	851
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	852 if (ctx->uidlist->check_expunges) {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	853 if (squat_uidlist_is_expunged(ctx, uid))
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	854 return 0;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	855 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	856 return 1;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	857 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	858
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	859 if (ctx->output == NULL)
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	860 return -1;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	861
4883 d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	862 if (squat_uidlist_map_list(uidlist, *uid_list_idx, &data, &size) < 0) {
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	863 squat_trie_set_corrupted(uidlist->trie,
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	864 "corrupted uidlist index (compressing)");
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	865 return -1;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	866 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	867
4883 d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	868 old_offset = *uid_list_idx;
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	869 *uid_list_idx = ctx->output->offset;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	870
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	871 if (!ctx->uidlist->check_expunges)
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	872 ret = 0;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	873 else {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	874 bool all_expunged;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	875
4883 d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	876 ret = squat_uidlist_remove_expunged(ctx, data, size,
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	877 &all_expunged);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	878 if (ret < 0) {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	879 ctx->failed = TRUE;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	880 return -1;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	881 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	882 if (ret > 0 && all_expunged)
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	883 return 0;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	884 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	885
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	886 if (ret == 0) {
4883 d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	887 data_start = data = uidlist->const_mmap_base + old_offset;
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	888 (void)_squat_trie_unpack_num(&data, NULL);
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	889
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	890 if (o_stream_send(ctx->output, data_start,
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	891 data - data_start + size) < 0) {
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	892 ctx->failed = TRUE;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	893 return -1;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	894 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	895 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	896
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	897 ctx->hdr.node_count++;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	898 return 1;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	899 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	900
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	901 void squat_uidlist_compress_rollback(struct squat_uidlist_compress_ctx **_ctx)
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	902 {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	903 struct squat_uidlist_compress_ctx ctx = _ctx;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	904
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	905 *_ctx = NULL;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	906
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	907 if (ctx->node_pool != NULL)
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	908 pool_unref(ctx->node_pool);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	909 if (array_is_created(&ctx->seen_uids))
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	910 array_free(&ctx->seen_uids);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	911 if (ctx->output != NULL) {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	912 if (ctx->failed)
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	913 (void)unlink(ctx->tmp_path);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	914 o_stream_destroy(&ctx->output);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	915 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	916 i_free(ctx->tmp_path);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	917 i_free(ctx);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	918 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	919
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	920 int squat_uidlist_compress_commit(struct squat_uidlist_compress_ctx **_ctx)
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	921 {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	922 struct squat_uidlist_compress_ctx ctx = _ctx;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	923 int ret = 0;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	924
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	925 if (ctx->failed) {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	926 squat_uidlist_compress_rollback(_ctx);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	927 return -1;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	928 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	929
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	930 /* write the header */
4879 b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	931 ctx->hdr.uidvalidity = ctx->uidlist->uidvalidity;
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	932 ctx->hdr.used_file_size = ctx->output->offset;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	933
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	934 if (ctx->existing_uids == NULL) {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	935 ctx->hdr.uid_max = ctx->uidlist->hdr.uid_max;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	936 ctx->hdr.uid_count = ctx->uidlist->hdr.uid_count;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	937 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	938
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	939 o_stream_seek(ctx->output, 0);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	940 if (o_stream_send(ctx->output, &ctx->hdr, sizeof(ctx->hdr)) < 0)
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	941 ret = -1;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	942
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	943 if (ret == 0) {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	944 if (rename(ctx->tmp_path, ctx->uidlist->filepath) < 0) {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	945 i_error("rename(%s, %s) failed: %m",
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	946 ctx->tmp_path, ctx->uidlist->filepath);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	947 ret = -1;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	948 } else {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	949 /* reopen */
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	950 ctx->uidlist->check_expunges = FALSE;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	951 squat_uidlist_close(ctx->uidlist);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	952 (void)squat_uidlist_open(ctx->uidlist);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	953 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	954 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	955
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	956 if (ret < 0)
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	957 ctx->failed = TRUE;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	958
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	959 squat_uidlist_compress_rollback(_ctx);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	960 return ret;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	961 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	962
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	963 static void
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	964 squat_uidlist_get_add_uid(struct squat_uidlist_get_context *ctx, uint32_t uid)
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	965 {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	966 if (ctx->filter_pos == 0) {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	967 seq_range_array_add(ctx->result, 0, uid);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	968 return;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	969 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	970
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	971 for (; ctx->filter_pos < uid; ctx->filter_pos++)
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	972 seq_range_array_remove(ctx->result, ctx->filter_pos);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	973 ctx->filter_pos++;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	974 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	975
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	976 static int
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	977 squat_uidlist_get_range_list(struct squat_uidlist_get_context *ctx,
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	978 size_t offset)
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	979 {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	980 const uint8_t data, end;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	981 uint32_t size, num, prev_uid, next_uid;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	982
4883 d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	983 if (squat_uidlist_map_list(ctx->uidlist, offset, &data, &size) < 0)
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	984 return -1;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	985 end = data + size;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	986
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	987 prev_uid = _squat_trie_unpack_num(&data, end);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	988 squat_uidlist_get_add_uid(ctx, prev_uid);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	989
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	990 while (data != end) {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	991 num = _squat_trie_unpack_num(&data, end);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	992 next_uid = prev_uid + (num >> 1) + 1;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	993 if ((num & 1) != 0) {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	994 for (prev_uid++; prev_uid <= next_uid; prev_uid++)
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	995 squat_uidlist_get_add_uid(ctx, prev_uid);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	996
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	997 if (data == end)
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	998 break;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	999 num = _squat_trie_unpack_num(&data, end);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1000 next_uid += num + 1;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1001 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1002 squat_uidlist_get_add_uid(ctx, next_uid);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1003 prev_uid = next_uid;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1004 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1005 return 0;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1006 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1007
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1008 static int
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1009 squat_uidlist_get_ctx(struct squat_uidlist_get_context *ctx,
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1010 uint32_t uid_list_idx)
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1011 {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1012 if ((uid_list_idx & UID_LIST_IDX_FLAG_SINGLE) != 0) {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1013 uint32_t uid = uid_list_idx & ~UID_LIST_IDX_FLAG_SINGLE;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1014 squat_uidlist_get_add_uid(ctx, uid);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1015 return 0;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1016 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1017
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1018 return squat_uidlist_get_range_list(ctx, uid_list_idx);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1019 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1020
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1021 int squat_uidlist_get(struct squat_uidlist *uidlist, uint32_t uid_list_idx,
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1022 ARRAY_TYPE(seq_range) *result)
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1023 {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1024 struct squat_uidlist_get_context ctx;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1025
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1026 memset(&ctx, 0, sizeof(ctx));
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1027 ctx.uidlist = uidlist;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1028 ctx.result = result;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1029
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1030 return squat_uidlist_get_ctx(&ctx, uid_list_idx);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1031 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1032
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1033 int squat_uidlist_filter(struct squat_uidlist *uidlist, uint32_t uid_list_idx,
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1034 ARRAY_TYPE(seq_range) *result)
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1035 {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1036 struct squat_uidlist_get_context ctx;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1037 const struct seq_range *range;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1038 unsigned int count;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1039
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1040 memset(&ctx, 0, sizeof(ctx));
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1041 ctx.uidlist = uidlist;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1042 ctx.result = result;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1043 ctx.filter_pos = 1;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1044
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1045 return squat_uidlist_get_ctx(&ctx, uid_list_idx);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1046
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1047 range = array_get(ctx.result, &count);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1048 if (count > 0) {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1049 for (; ctx.filter_pos <= range[count-1].seq2; ctx.filter_pos++)
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1050 seq_range_array_remove(result, ctx.filter_pos);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1051 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1052 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1053
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1054 size_t squat_uidlist_mem_used(struct squat_uidlist *uidlist,
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1055 unsigned int *count_r)
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1056 {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1057 *count_r = uidlist->hdr.node_count;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1058
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1059 return uidlist->hdr.used_file_size;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1060 }

Mercurial > dovecot > original-hg > dovecot-1.2

annotate src/plugins/fts-squat/squat-uidlist.c @ 4883:d8adbe93c969 HEAD