dovecot/original-hg/dovecot-1.2: src/plugins/fts-squat/squat-uidlist.c annotate

annotate src/plugins/fts-squat/squat-uidlist.c @ 7218:43d2f70fb279 HEAD

Squat fixes

author	Timo Sirainen <tss@iki.fi>
date	Sat, 09 Feb 2008 13:53:54 +0200
parents	f5f77a3ae203
children	21d1d0ced13e

rev	line source
7086 7ed926ed7aa4 Updated copyright notices to include year 2008. Timo Sirainen <tss@iki.fi> parents: 6945 diff changeset	1 /* Copyright (c) 2007-2008 Dovecot authors, see the included COPYING file */
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	2
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	3 #include "lib.h"
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	4 #include "array.h"
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	5 #include "bsearch-insert-pos.h"
6944 66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	6 #include "file-cache.h"
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	7 #include "file-lock.h"
6944 66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	8 #include "read-full.h"
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	9 #include "write-full.h"
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	10 #include "ostream.h"
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	11 #include "squat-trie-private.h"
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	12 #include "squat-uidlist.h"
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	13
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	14 #include <stdio.h>
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	15 #include <sys/stat.h>
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	16 #include <sys/mman.h>
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	17
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	18 #define UIDLIST_LIST_SIZE 31
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	19 #define UIDLIST_BLOCK_LIST_COUNT 100
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	20 #define UID_LIST_MASK_RANGE 0x80000000
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	21
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	22 /* set = points to uidlist index number, unset = points to uidlist offset */
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	23 #define UID_LIST_POINTER_MASK_LIST_IDX 0x80000000
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	24
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	25 #define UIDLIST_PACKED_FLAG_BITMASK 1
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	26 #define UIDLIST_PACKED_FLAG_BEGINS_WITH_POINTER 2
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	27
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	28 struct uidlist_list {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	29 uint32_t uid_count:31;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	30 uint32_t uid_begins_with_pointer:1;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	31 uint32_t uid_list[UIDLIST_LIST_SIZE];
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	32 };
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	33
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	34 struct squat_uidlist {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	35 struct squat_trie *trie;
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	36
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	37 char *path;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	38 int fd;
6944 66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	39 struct file_cache *file_cache;
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	40
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	41 struct file_lock *file_lock;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	42 uoff_t locked_file_size;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	43
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	44 void *mmap_base;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	45 size_t mmap_size;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	46 struct squat_uidlist_file_header hdr;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	47
6944 66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	48 const void *data;
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	49 size_t data_size;
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	50
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	51 unsigned int cur_block_count;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	52 const uint32_t *cur_block_offsets;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	53 const uint32_t *cur_block_end_indexes;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	54
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	55 size_t max_size;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	56 unsigned int corrupted:1;
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	57 unsigned int building:1;
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	58 };
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	59
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	60 struct squat_uidlist_build_context {
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	61 struct squat_uidlist *uidlist;
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	62 struct ostream *output;
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	63
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	64 ARRAY_TYPE(uint32_t) block_offsets;
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	65 ARRAY_TYPE(uint32_t) block_end_indexes;
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	66
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	67 ARRAY_DEFINE(lists, struct uidlist_list);
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	68 uint32_t list_start_idx;
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	69
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	70 struct squat_uidlist_file_header build_hdr;
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	71 unsigned int need_reopen:1;
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	72 };
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	73
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	74 struct squat_uidlist_rebuild_context {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	75 struct squat_uidlist *uidlist;
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	76 struct squat_uidlist_build_context *build_ctx;
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	77
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	78 int fd;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	79 struct ostream *output;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	80
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	81 ARRAY_TYPE(uint32_t) new_block_offsets, new_block_end_indexes;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	82 uoff_t cur_block_start_offset;
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	83
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	84 uint32_t list_sizes[UIDLIST_BLOCK_LIST_COUNT];
7210 f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7086 diff changeset	85 uint32_t next_uid_list_idx;
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	86 unsigned int list_idx;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	87 unsigned int new_count;
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	88 };
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	89
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	90 static void squat_uidlist_close(struct squat_uidlist *uidlist);
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	91
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	92 void squat_uidlist_delete(struct squat_uidlist *uidlist)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	93 {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	94 if (unlink(uidlist->path) < 0 && errno != ENOENT)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	95 i_error("unlink(%s) failed: %m", uidlist->path);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	96 }
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	97
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	98 static void squat_uidlist_set_corrupted(struct squat_uidlist *uidlist,
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	99 const char *reason)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	100 {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	101 if (uidlist->corrupted)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	102 return;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	103 uidlist->corrupted = TRUE;
4883 d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	104
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	105 i_error("Corrupted squat uidlist file %s: %s", uidlist->path, reason);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	106 squat_trie_delete(uidlist->trie);
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	107 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	108
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	109 static int
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	110 uidlist_write_array(struct ostream output, const uint32_t uid_list,
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	111 unsigned int uid_count, uint32_t packed_flags,
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	112 uint32_t offset, bool write_size, uint32_t *size_r)
4879 b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	113 {
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	114 uint8_t uidbuf, bufp, sizebuf[SQUAT_PACK_MAX_SIZE], *sizebufp;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	115 uint8_t listbuf[SQUAT_PACK_MAX_SIZE], *listbufp = listbuf;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	116 uint32_t uid, uid2, prev, base_uid, size_value;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	117 unsigned int i, bitmask_len, uid_list_len;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	118 unsigned int idx, max_idx, mask;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	119 bool datastack;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	120 int num;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	121
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	122 if ((packed_flags & UIDLIST_PACKED_FLAG_BEGINS_WITH_POINTER) != 0)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	123 squat_pack_num(&listbufp, offset);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	124
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	125 /* @UNSAFE */
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	126 base_uid = uid_list[0] & ~UID_LIST_MASK_RANGE;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	127 datastack = uid_count < 1024*8/SQUAT_PACK_MAX_SIZE;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	128 if (datastack)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	129 uidbuf = t_malloc(SQUAT_PACK_MAX_SIZE * uid_count);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	130 else
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	131 uidbuf = i_malloc(SQUAT_PACK_MAX_SIZE * uid_count);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	132 bufp = uidbuf;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	133 squat_pack_num(&bufp, base_uid);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	134
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	135 bitmask_len = (uid_list[uid_count-1] - base_uid + 7) / 8 +
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	136 (bufp - uidbuf);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	137 if (bitmask_len < uid_count) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	138 bitmask_build:
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	139 i_assert(bitmask_len < SQUAT_PACK_MAX_SIZE*uid_count);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	140
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	141 memset(bufp, 0, bitmask_len - (bufp - uidbuf));
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	142 if ((uid_list[0] & UID_LIST_MASK_RANGE) == 0) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	143 i = 1;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	144 uid = i == uid_count ? 0 : uid_list[i];
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	145 } else {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	146 i = 0;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	147 uid = uid_list[0] + 1;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	148 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	149 base_uid++;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	150
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	151 for (; i < uid_count; i++) {
7218 43d2f70fb279 Squat fixes Timo Sirainen <tss@iki.fi> parents: 7210 diff changeset	152 i_assert((uid & ~UID_LIST_MASK_RANGE) >= base_uid);
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	153 if ((uid & UID_LIST_MASK_RANGE) == 0) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	154 uid -= base_uid;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	155 uid2 = uid;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	156 } else {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	157 uid &= ~UID_LIST_MASK_RANGE;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	158 uid -= base_uid;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	159 uid2 = uid_list[i+1] - base_uid;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	160 i++;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	161 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	162
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	163 if (uid2 - uid < 3*8) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	164 for (; uid <= uid2; uid++)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	165 bufp[uid / 8] \|= 1 << (uid % 8);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	166 } else {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	167 /* first byte */
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	168 idx = uid / 8;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	169 num = uid % 8;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	170 if (num != 0) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	171 uid += 8 - num;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	172 for (mask = 0; num < 8; num++)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	173 mask \|= 1 << num;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	174 bufp[idx++] \|= mask;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	175 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	176
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	177 /* middle bytes */
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	178 num = uid2 % 8;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	179 max_idx = idx + (uid2 - num - uid)/8;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	180 for (; idx < max_idx; idx++, uid += 8)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	181 bufp[idx] = 0xff;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	182
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	183 /* last byte */
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	184 for (mask = 0; num >= 0; num--)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	185 mask \|= 1 << num;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	186 bufp[idx] \|= mask;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	187 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	188 uid = i+1 == uid_count ? 0 : uid_list[i+1];
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	189 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	190 uid_list_len = bitmask_len;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	191 packed_flags \|= UIDLIST_PACKED_FLAG_BITMASK;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	192 } else {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	193 bufp = uidbuf;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	194 prev = 0;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	195 for (i = 0; i < uid_count; i++) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	196 uid = uid_list[i];
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	197 if (unlikely((uid & ~UID_LIST_MASK_RANGE) < prev))
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	198 return -1;
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	199 if ((uid & UID_LIST_MASK_RANGE) == 0) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	200 squat_pack_num(&bufp, (uid - prev) << 1);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	201 prev = uid + 1;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	202 } else {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	203 uid &= ~UID_LIST_MASK_RANGE;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	204 squat_pack_num(&bufp, 1 \| (uid - prev) << 1);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	205 squat_pack_num(&bufp, uid_list[i+1] - uid - 1);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	206 prev = uid_list[i+1] + 1;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	207 i++;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	208 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	209 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	210 uid_list_len = bufp - uidbuf;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	211 if (uid_list_len > bitmask_len) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	212 bufp = uidbuf;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	213 squat_pack_num(&bufp, base_uid);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	214 goto bitmask_build;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	215 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	216 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	217
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	218 size_value = ((uid_list_len +
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	219 (listbufp - listbuf)) << 2) \| packed_flags;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	220 if (write_size) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	221 sizebufp = sizebuf;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	222 squat_pack_num(&sizebufp, size_value);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	223 o_stream_send(output, sizebuf, sizebufp - sizebuf);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	224 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	225 o_stream_send(output, listbuf, listbufp - listbuf);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	226 o_stream_send(output, uidbuf, uid_list_len);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	227 if (!datastack)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	228 i_free(uidbuf);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	229
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	230 *size_r = size_value;
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	231 return 0;
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	232 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	233
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	234 static int
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	235 uidlist_write(struct ostream output, const struct uidlist_list list,
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	236 bool write_size, uint32_t *size_r)
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	237 {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	238 const uint32_t *uid_list = list->uid_list;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	239 uint8_t buf[SQUAT_PACK_MAX_SIZE], *bufp;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	240 uint32_t uid_count = list->uid_count;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	241 uint32_t packed_flags = 0;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	242 uint32_t offset = 0;
6940 414c9d631a81 Replaced t_push/t_pop calls with T_FRAME() macros. Timo Sirainen <tss@iki.fi>* parents: 6931 diff changeset	243 int ret;
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	244
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	245 if (list->uid_begins_with_pointer) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	246 /* continued UID list */
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	247 packed_flags \|= UIDLIST_PACKED_FLAG_BEGINS_WITH_POINTER;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	248 if ((uid_list[0] & UID_LIST_POINTER_MASK_LIST_IDX) != 0) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	249 offset = ((uid_list[0] & ~UID_LIST_POINTER_MASK_LIST_IDX) << 1) \| 1;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	250 if (list->uid_count == 1) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	251 bufp = buf;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	252 squat_pack_num(&bufp, offset);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	253 o_stream_send(output, buf, bufp - buf);
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	254 *size_r = (bufp - buf) << 2 \| packed_flags;
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	255 return 0;
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	256 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	257 } else {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	258 i_assert(list->uid_count > 1);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	259 i_assert(output->offset > uid_list[0]);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	260 offset = (output->offset - uid_list[0]) << 1;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	261 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	262 uid_list++;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	263 uid_count--;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	264 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	265
6940 414c9d631a81 Replaced t_push/t_pop calls with T_FRAME() macros. Timo Sirainen <tss@iki.fi>* parents: 6931 diff changeset	266 T_FRAME(
414c9d631a81 Replaced t_push/t_pop calls with T_FRAME() macros. Timo Sirainen <tss@iki.fi>* parents: 6931 diff changeset	267 ret = uidlist_write_array(output, uid_list, uid_count,
414c9d631a81 Replaced t_push/t_pop calls with T_FRAME() macros. Timo Sirainen <tss@iki.fi>* parents: 6931 diff changeset	268 packed_flags, offset,
414c9d631a81 Replaced t_push/t_pop calls with T_FRAME() macros. Timo Sirainen <tss@iki.fi>* parents: 6931 diff changeset	269 write_size, size_r);
414c9d631a81 Replaced t_push/t_pop calls with T_FRAME() macros. Timo Sirainen <tss@iki.fi>* parents: 6931 diff changeset	270 );
414c9d631a81 Replaced t_push/t_pop calls with T_FRAME() macros. Timo Sirainen <tss@iki.fi>* parents: 6931 diff changeset	271 return ret;
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	272 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	273
6944 66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	274 static void squat_uidlist_map_blocks_set_pointers(struct squat_uidlist *uidlist)
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	275 {
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	276 const void *base;
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	277 size_t end_index_size, end_size;
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	278
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	279 base = CONST_PTR_OFFSET(uidlist->data, uidlist->hdr.block_list_offset +
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	280 sizeof(uint32_t));
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	281
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	282 end_index_size = uidlist->cur_block_count * sizeof(uint32_t);
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	283 end_size = end_index_size + uidlist->cur_block_count * sizeof(uint32_t);
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	284 if (end_size <= uidlist->data_size) {
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	285 uidlist->cur_block_end_indexes = base;
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	286 uidlist->cur_block_offsets =
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	287 CONST_PTR_OFFSET(base, end_index_size);
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	288 } else {
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	289 uidlist->cur_block_end_indexes = NULL;
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	290 uidlist->cur_block_offsets = NULL;
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	291 }
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	292 }
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	293
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	294 static int uidlist_file_cache_read(struct squat_uidlist *uidlist,
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	295 size_t offset, size_t size)
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	296 {
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	297 if (uidlist->file_cache == NULL)
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	298 return 0;
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	299
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	300 if (file_cache_read(uidlist->file_cache, offset, size) < 0) {
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	301 i_error("read(%s) failed: %m", uidlist->path);
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	302 return -1;
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	303 }
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	304 uidlist->data = file_cache_get_map(uidlist->file_cache,
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	305 &uidlist->data_size);
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	306 squat_uidlist_map_blocks_set_pointers(uidlist);
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	307 return 0;
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	308 }
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	309
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	310 static int squat_uidlist_map_blocks(struct squat_uidlist *uidlist)
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	311 {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	312 const struct squat_uidlist_file_header *hdr = &uidlist->hdr;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	313 const void *base;
6944 66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	314 uint32_t block_count, blocks_offset, blocks_size, i, verify_count;
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	315
7218 43d2f70fb279 Squat fixes Timo Sirainen <tss@iki.fi> parents: 7210 diff changeset	316 if (hdr->block_list_offset == 0) {
43d2f70fb279 Squat fixes Timo Sirainen <tss@iki.fi> parents: 7210 diff changeset	317 /* empty file */
43d2f70fb279 Squat fixes Timo Sirainen <tss@iki.fi> parents: 7210 diff changeset	318 uidlist->cur_block_count = 0;
43d2f70fb279 Squat fixes Timo Sirainen <tss@iki.fi> parents: 7210 diff changeset	319 return 1;
43d2f70fb279 Squat fixes Timo Sirainen <tss@iki.fi> parents: 7210 diff changeset	320 }
43d2f70fb279 Squat fixes Timo Sirainen <tss@iki.fi> parents: 7210 diff changeset	321
6944 66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	322 /* get number of blocks */
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	323 if (uidlist_file_cache_read(uidlist, hdr->block_list_offset,
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	324 sizeof(block_count)) < 0)
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	325 return -1;
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	326 blocks_offset = hdr->block_list_offset + sizeof(block_count);
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	327 if (blocks_offset > uidlist->data_size) {
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	328 squat_uidlist_set_corrupted(uidlist, "block list outside file");
7218 43d2f70fb279 Squat fixes Timo Sirainen <tss@iki.fi> parents: 7210 diff changeset	329 return 0;
5414 3ec03a4c0f26 fixes Timo Sirainen <tss@iki.fi> parents: 5010 diff changeset	330 }
3ec03a4c0f26 fixes Timo Sirainen <tss@iki.fi> parents: 5010 diff changeset	331
6944 66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	332 base = CONST_PTR_OFFSET(uidlist->data, hdr->block_list_offset);
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	333 memcpy(&block_count, base, sizeof(block_count));
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	334
6944 66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	335 /* map the blocks */
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	336 blocks_size = block_count * sizeof(uint32_t)*2;
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	337 if (uidlist_file_cache_read(uidlist, blocks_offset, blocks_size) < 0)
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	338 return -1;
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	339 if (blocks_offset + blocks_size > uidlist->data_size) {
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	340 squat_uidlist_set_corrupted(uidlist, "block list outside file");
7218 43d2f70fb279 Squat fixes Timo Sirainen <tss@iki.fi> parents: 7210 diff changeset	341 return 0;
4879 b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	342 }
b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	343
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	344 uidlist->cur_block_count = block_count;
6944 66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	345 squat_uidlist_map_blocks_set_pointers(uidlist);
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	346
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	347 /* verify just a couple of the end indexes to make sure they
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	348 look correct */
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	349 verify_count = I_MIN(block_count, 8);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	350 for (i = 1; i < verify_count; i++) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	351 if (unlikely(uidlist->cur_block_end_indexes[i-1] >=
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	352 uidlist->cur_block_end_indexes[i])) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	353 squat_uidlist_set_corrupted(uidlist,
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	354 "block list corrupted");
7218 43d2f70fb279 Squat fixes Timo Sirainen <tss@iki.fi> parents: 7210 diff changeset	355 return 0;
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	356 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	357 }
7218 43d2f70fb279 Squat fixes Timo Sirainen <tss@iki.fi> parents: 7210 diff changeset	358 return 1;
4879 b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	359 }
b0ada6e57b07 Fixes Timo Sirainen <tss@iki.fi> parents: 4878 diff changeset	360
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	361 static int squat_uidlist_map_header(struct squat_uidlist *uidlist)
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	362 {
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	363 if (uidlist->hdr.indexid == 0) {
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	364 /* still being built */
7218 43d2f70fb279 Squat fixes Timo Sirainen <tss@iki.fi> parents: 7210 diff changeset	365 return 1;
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	366 }
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	367 if (uidlist->hdr.indexid != uidlist->trie->hdr.indexid) {
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	368 /* see if trie was recreated */
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	369 squat_trie_refresh(uidlist->trie);
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	370 }
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	371 if (uidlist->hdr.indexid != uidlist->trie->hdr.indexid) {
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	372 squat_uidlist_set_corrupted(uidlist, "wrong indexid");
7218 43d2f70fb279 Squat fixes Timo Sirainen <tss@iki.fi> parents: 7210 diff changeset	373 return 0;
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	374 }
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	375 if (uidlist->hdr.used_file_size < sizeof(uidlist->hdr) \|\|
6944 66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	376 (uidlist->hdr.used_file_size > uidlist->mmap_size &&
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	377 uidlist->mmap_base != NULL)) {
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	378 squat_uidlist_set_corrupted(uidlist, "broken used_file_size");
7218 43d2f70fb279 Squat fixes Timo Sirainen <tss@iki.fi> parents: 7210 diff changeset	379 return 0;
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	380 }
6944 66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	381 return squat_uidlist_map_blocks(uidlist);
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	382 }
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	383
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	384 static void squat_uidlist_unmap(struct squat_uidlist *uidlist)
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	385 {
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	386 if (uidlist->mmap_size != 0) {
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	387 if (munmap(uidlist->mmap_base, uidlist->mmap_size) < 0)
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	388 i_error("munmap(%s) failed: %m", uidlist->path);
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	389 uidlist->mmap_base = NULL;
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	390 uidlist->mmap_size = 0;
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	391 }
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	392 uidlist->cur_block_count = 0;
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	393 uidlist->cur_block_end_indexes = NULL;
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	394 uidlist->cur_block_offsets = NULL;
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	395 }
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	396
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	397 static int squat_uidlist_mmap(struct squat_uidlist *uidlist)
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	398 {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	399 struct stat st;
4883 d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	400
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	401 if (fstat(uidlist->fd, &st) < 0) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	402 i_error("fstat(%s) failed: %m", uidlist->path);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	403 return -1;
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	404 }
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	405 if (st.st_size < (off_t)sizeof(uidlist->hdr)) {
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	406 if (st.st_size == 0)
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	407 return 0;
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	408 squat_uidlist_set_corrupted(uidlist, "File too small");
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	409 return -1;
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	410 }
4883 d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	411
6944 66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	412 squat_uidlist_unmap(uidlist);
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	413 uidlist->mmap_size = st.st_size;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	414 uidlist->mmap_base = mmap(NULL, uidlist->mmap_size,
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	415 PROT_READ \| PROT_WRITE,
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	416 MAP_SHARED, uidlist->fd, 0);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	417 if (uidlist->mmap_base == MAP_FAILED) {
6944 66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	418 uidlist->data = uidlist->mmap_base = NULL;
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	419 uidlist->data_size = uidlist->mmap_size = 0;
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	420 i_error("mmap(%s) failed: %m", uidlist->path);
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	421 return -1;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	422 }
6944 66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	423 uidlist->data = uidlist->mmap_base;
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	424 uidlist->data_size = uidlist->mmap_size;
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	425 return 0;
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	426 }
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	427
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	428 static int squat_uidlist_map(struct squat_uidlist *uidlist)
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	429 {
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	430 const struct squat_uidlist_file_header *mmap_hdr = uidlist->mmap_base;
6944 66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	431 int ret;
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	432
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	433 if (mmap_hdr != NULL && !uidlist->building &&
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	434 uidlist->hdr.block_list_offset == mmap_hdr->block_list_offset) {
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	435 /* file hasn't changed */
7218 43d2f70fb279 Squat fixes Timo Sirainen <tss@iki.fi> parents: 7210 diff changeset	436 return 1;
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	437 }
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	438
6944 66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	439 if (!uidlist->trie->mmap_disable) {
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	440 if (mmap_hdr == NULL \|\| uidlist->building \|\|
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	441 uidlist->mmap_size < mmap_hdr->used_file_size) {
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	442 if (squat_uidlist_mmap(uidlist) < 0)
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	443 return -1;
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	444 }
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	445
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	446 if (!uidlist->building) {
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	447 memcpy(&uidlist->hdr, uidlist->mmap_base,
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	448 sizeof(uidlist->hdr));
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	449 }
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	450 } else if (uidlist->building) {
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	451 /* we want to update blocks mapping, but using the header
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	452 in memory */
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	453 } else {
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	454 ret = pread_full(uidlist->fd, &uidlist->hdr,
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	455 sizeof(uidlist->hdr), 0);
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	456 if (ret <= 0) {
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	457 if (ret < 0) {
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	458 i_error("pread(%s) failed: %m", uidlist->path);
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	459 return -1;
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	460 }
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	461 i_error("Corrupted %s: File too small", uidlist->path);
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	462 return 0;
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	463 }
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	464 uidlist->data = NULL;
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	465 uidlist->data_size = 0;
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	466 }
6944 66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	467 if (uidlist->file_cache == NULL && uidlist->trie->mmap_disable)
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	468 uidlist->file_cache = file_cache_new(uidlist->fd);
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	469 return squat_uidlist_map_header(uidlist);
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	470 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	471
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	472 struct squat_uidlist squat_uidlist_init(struct squat_trie trie)
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	473 {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	474 struct squat_uidlist *uidlist;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	475
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	476 uidlist = i_new(struct squat_uidlist, 1);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	477 uidlist->trie = trie;
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	478 uidlist->path = i_strconcat(trie->path, ".uids", NULL);
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	479 uidlist->fd = -1;
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	480
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	481 return uidlist;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	482 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	483
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	484 void squat_uidlist_deinit(struct squat_uidlist *uidlist)
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	485 {
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	486 squat_uidlist_close(uidlist);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	487
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	488 i_free(uidlist->path);
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	489 i_free(uidlist);
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	490 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	491
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	492 static int squat_uidlist_open(struct squat_uidlist *uidlist)
4883 d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	493 {
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	494 squat_uidlist_close(uidlist);
4878 88a91d9a867b Fixes. Should be pretty much working now. Timo Sirainen <tss@iki.fi> parents: 4855 diff changeset	495
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	496 uidlist->fd = open(uidlist->path, O_RDWR);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	497 if (uidlist->fd == -1) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	498 if (errno == ENOENT) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	499 memset(&uidlist->hdr, 0, sizeof(uidlist->hdr));
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	500 return 0;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	501 }
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	502 i_error("open(%s) failed: %m", uidlist->path);
4883 d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	503 return -1;
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	504 }
7218 43d2f70fb279 Squat fixes Timo Sirainen <tss@iki.fi> parents: 7210 diff changeset	505 return squat_uidlist_map(uidlist) <= 0 ? -1 : 0;
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	506 }
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	507
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	508 static void squat_uidlist_close(struct squat_uidlist *uidlist)
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	509 {
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	510 i_assert(!uidlist->building);
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	511
6944 66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	512 squat_uidlist_unmap(uidlist);
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	513 if (uidlist->file_cache != NULL)
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	514 file_cache_free(&uidlist->file_cache);
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	515 if (uidlist->file_lock != NULL)
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	516 file_lock_free(&uidlist->file_lock);
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	517 if (uidlist->fd != -1) {
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	518 if (close(uidlist->fd) < 0)
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	519 i_error("close(%s) failed: %m", uidlist->path);
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	520 uidlist->fd = -1;
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	521 }
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	522 uidlist->corrupted = FALSE;
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	523 }
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	524
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	525 int squat_uidlist_refresh(struct squat_uidlist *uidlist)
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	526 {
6944 66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	527 /* we assume here that trie is locked, so that we don't need to worry
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	528 about it when reading the header */
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	529 if (uidlist->fd == -1 \|\|
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	530 uidlist->hdr.indexid != uidlist->trie->hdr.indexid) {
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	531 if (squat_uidlist_open(uidlist) < 0)
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	532 return -1;
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	533 } else {
7218 43d2f70fb279 Squat fixes Timo Sirainen <tss@iki.fi> parents: 7210 diff changeset	534 if (squat_uidlist_map(uidlist) <= 0)
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	535 return -1;
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	536 }
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	537 return 0;
4883 d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	538 }
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	539
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	540 static int squat_uidlist_is_file_stale(struct squat_uidlist *uidlist)
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	541 {
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	542 struct stat st, st2;
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	543
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	544 i_assert(uidlist->fd != -1);
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	545
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	546 if (stat(uidlist->path, &st) < 0) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	547 if (errno == ENOENT)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	548 return 1;
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	549
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	550 i_error("stat(%s) failed: %m", uidlist->path);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	551 return -1;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	552 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	553 if (fstat(uidlist->fd, &st2) < 0) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	554 i_error("fstat(%s) failed: %m", uidlist->path);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	555 return -1;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	556 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	557 uidlist->locked_file_size = st2.st_size;
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	558
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	559 return st.st_ino == st2.st_ino &&
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	560 CMP_DEV_T(st.st_dev, st2.st_dev) ? 0 : 1;
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	561 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	562
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	563 static int squat_uidlist_lock(struct squat_uidlist *uidlist)
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	564 {
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	565 int ret;
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	566
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	567 for (;;) {
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	568 i_assert(uidlist->fd != -1);
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	569 i_assert(uidlist->file_lock == NULL);
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	570
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	571 ret = file_wait_lock(uidlist->fd, uidlist->path, F_WRLCK,
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	572 uidlist->trie->lock_method,
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	573 SQUAT_TRIE_LOCK_TIMEOUT,
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	574 &uidlist->file_lock);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	575 if (ret == 0) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	576 i_error("file_wait_lock(%s) failed: %m", uidlist->path);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	577 return 0;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	578 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	579 if (ret < 0)
4883 d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	580 return -1;
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	581
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	582 ret = squat_uidlist_is_file_stale(uidlist);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	583 if (ret == 0)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	584 break;
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	585
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	586 file_unlock(&uidlist->file_lock);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	587 if (ret < 0)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	588 return -1;
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	589
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	590 squat_uidlist_close(uidlist);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	591 uidlist->fd = open(uidlist->path, O_RDWR \| O_CREAT, 0600);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	592 if (uidlist->fd == -1) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	593 i_error("open(%s) failed: %m", uidlist->path);
4883 d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	594 return -1;
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	595 }
d8adbe93c969 Added support for mmap_disable=yes and several other fixes. Timo Sirainen <tss@iki.fi> parents: 4879 diff changeset	596 }
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	597 return 1;
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	598 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	599
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	600 static int squat_uidlist_open_or_create(struct squat_uidlist *uidlist)
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	601 {
7218 43d2f70fb279 Squat fixes Timo Sirainen <tss@iki.fi> parents: 7210 diff changeset	602 int ret;
43d2f70fb279 Squat fixes Timo Sirainen <tss@iki.fi> parents: 7210 diff changeset	603
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	604 if (uidlist->fd == -1) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	605 uidlist->fd = open(uidlist->path, O_RDWR \| O_CREAT, 0600);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	606 if (uidlist->fd == -1) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	607 i_error("creat(%s) failed: %m", uidlist->path);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	608 return -1;
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	609 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	610 }
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	611 if (squat_uidlist_lock(uidlist) <= 0)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	612 return -1;
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	613
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	614 if (uidlist->locked_file_size != 0) {
7218 43d2f70fb279 Squat fixes Timo Sirainen <tss@iki.fi> parents: 7210 diff changeset	615 if ((ret = squat_uidlist_map(uidlist)) < 0)
43d2f70fb279 Squat fixes Timo Sirainen <tss@iki.fi> parents: 7210 diff changeset	616 return -1;
43d2f70fb279 Squat fixes Timo Sirainen <tss@iki.fi> parents: 7210 diff changeset	617 if (ret == 0) {
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	618 /* broken file, truncate */
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	619 if (ftruncate(uidlist->fd, 0) < 0) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	620 i_error("ftruncate(%s) failed: %m",
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	621 uidlist->path);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	622 return -1;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	623 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	624 uidlist->locked_file_size = 0;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	625 }
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	626 }
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	627 if (uidlist->locked_file_size == 0) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	628 /* write using 0 until we're finished */
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	629 memset(&uidlist->hdr, 0, sizeof(uidlist->hdr));
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	630 if (write_full(uidlist->fd, &uidlist->hdr,
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	631 sizeof(uidlist->hdr)) < 0) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	632 i_error("write(%s) failed: %m", uidlist->path);
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	633 return -1;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	634 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	635 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	636 return 0;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	637 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	638
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	639 int squat_uidlist_build_init(struct squat_uidlist *uidlist,
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	640 struct squat_uidlist_build_context **ctx_r)
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	641 {
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	642 struct squat_uidlist_build_context *ctx;
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	643
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	644 i_assert(!uidlist->building);
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	645
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	646 if (squat_uidlist_open_or_create(uidlist) < 0) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	647 if (uidlist->file_lock != NULL)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	648 file_unlock(&uidlist->file_lock);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	649 return -1;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	650 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	651 if (lseek(uidlist->fd, uidlist->hdr.used_file_size, SEEK_SET) < 0) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	652 i_error("lseek(%s) failed: %m", uidlist->path);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	653 if (uidlist->file_lock != NULL)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	654 file_unlock(&uidlist->file_lock);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	655 return -1;
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	656 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	657
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	658 ctx = i_new(struct squat_uidlist_build_context, 1);
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	659 ctx->uidlist = uidlist;
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	660 ctx->output = o_stream_create_fd(uidlist->fd, 0, FALSE);
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	661 if (ctx->output->offset == 0) {
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	662 struct squat_uidlist_file_header hdr;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	663
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	664 memset(&hdr, 0, sizeof(hdr));
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	665 o_stream_send(ctx->output, &hdr, sizeof(hdr));
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	666 }
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	667 o_stream_cork(ctx->output);
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	668 i_array_init(&ctx->lists, 10240);
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	669 i_array_init(&ctx->block_offsets, 128);
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	670 i_array_init(&ctx->block_end_indexes, 128);
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	671 ctx->list_start_idx = uidlist->hdr.count;
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	672 ctx->build_hdr = uidlist->hdr;
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	673
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	674 uidlist->building = TRUE;
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	675 *ctx_r = ctx;
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	676 return 0;
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	677 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	678
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	679 static void
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	680 uidlist_write_block_list_and_header(struct squat_uidlist_build_context *ctx,
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	681 struct ostream *output,
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	682 ARRAY_TYPE(uint32_t) *block_offsets,
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	683 ARRAY_TYPE(uint32_t) *block_end_indexes,
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	684 bool write_old_blocks)
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	685 {
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	686 struct squat_uidlist *uidlist = ctx->uidlist;
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	687 unsigned int align, old_block_count, new_block_count;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	688 uint32_t block_offset_count;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	689 uoff_t block_list_offset;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	690
7218 43d2f70fb279 Squat fixes Timo Sirainen <tss@iki.fi> parents: 7210 diff changeset	691 i_assert(uidlist->trie->hdr.indexid != 0);
43d2f70fb279 Squat fixes Timo Sirainen <tss@iki.fi> parents: 7210 diff changeset	692 ctx->build_hdr.indexid = uidlist->trie->hdr.indexid;
43d2f70fb279 Squat fixes Timo Sirainen <tss@iki.fi> parents: 7210 diff changeset	693
43d2f70fb279 Squat fixes Timo Sirainen <tss@iki.fi> parents: 7210 diff changeset	694 if (array_count(block_end_indexes) == 0) {
43d2f70fb279 Squat fixes Timo Sirainen <tss@iki.fi> parents: 7210 diff changeset	695 ctx->build_hdr.used_file_size = output->offset;
43d2f70fb279 Squat fixes Timo Sirainen <tss@iki.fi> parents: 7210 diff changeset	696 uidlist->hdr = ctx->build_hdr;
43d2f70fb279 Squat fixes Timo Sirainen <tss@iki.fi> parents: 7210 diff changeset	697 return;
43d2f70fb279 Squat fixes Timo Sirainen <tss@iki.fi> parents: 7210 diff changeset	698 }
43d2f70fb279 Squat fixes Timo Sirainen <tss@iki.fi> parents: 7210 diff changeset	699
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	700 align = output->offset % sizeof(uint32_t);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	701 if (align != 0) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	702 static char null[sizeof(uint32_t)-1] = { 0, };
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	703
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	704 o_stream_send(output, null, sizeof(uint32_t) - align);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	705 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	706 block_list_offset = output->offset;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	707
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	708 new_block_count = array_count(block_offsets);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	709 old_block_count = write_old_blocks ? uidlist->cur_block_count : 0;
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	710
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	711 block_offset_count = new_block_count + old_block_count;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	712 o_stream_send(output, &block_offset_count, sizeof(block_offset_count));
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	713 /* write end indexes */
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	714 o_stream_send(output, uidlist->cur_block_end_indexes,
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	715 old_block_count * sizeof(uint32_t));
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	716 o_stream_send(output, array_idx(block_end_indexes, 0),
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	717 new_block_count * sizeof(uint32_t));
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	718 /* write offsets */
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	719 o_stream_send(output, uidlist->cur_block_offsets,
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	720 old_block_count * sizeof(uint32_t));
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	721 o_stream_send(output, array_idx(block_offsets, 0),
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	722 new_block_count * sizeof(uint32_t));
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	723 o_stream_flush(output);
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	724
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	725 /* update header - it's written later when trie is locked */
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	726 ctx->build_hdr.block_list_offset = block_list_offset;
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	727 ctx->build_hdr.used_file_size = output->offset;
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	728 uidlist->hdr = ctx->build_hdr;
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	729 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	730
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	731 void squat_uidlist_build_flush(struct squat_uidlist_build_context *ctx)
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	732 {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	733 struct uidlist_list *lists;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	734 uint8_t buf[SQUAT_PACK_MAX_SIZE], *bufp;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	735 unsigned int i, j, count, max;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	736 uint32_t block_offset, block_end_idx, start_offset;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	737 uint32_t list_sizes[UIDLIST_BLOCK_LIST_COUNT];
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	738 size_t mem_size;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	739
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	740 if (ctx->uidlist->corrupted)
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	741 return;
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	742
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	743 lists = array_get_modifiable(&ctx->lists, &count);
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	744 if (count == 0)
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	745 return;
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	746
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	747 /* write the lists and save the written sizes to uid_list[0] */
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	748 for (i = 0; i < count; i += UIDLIST_BLOCK_LIST_COUNT) {
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	749 start_offset = ctx->output->offset;
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	750 max = I_MIN(count - i, UIDLIST_BLOCK_LIST_COUNT);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	751 for (j = 0; j < max; j++) {
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	752 if (uidlist_write(ctx->output, &lists[i+j],
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	753 FALSE, &list_sizes[j]) < 0) {
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	754 squat_uidlist_set_corrupted(ctx->uidlist,
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	755 "Broken uidlists");
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	756 return;
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	757 }
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	758 }
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	759
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	760 block_offset = ctx->output->offset;
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	761 block_end_idx = ctx->list_start_idx + i + max;
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	762 array_append(&ctx->block_offsets, &block_offset, 1);
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	763 array_append(&ctx->block_end_indexes, &block_end_idx, 1);
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	764
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	765 /* write the full size of the uidlists */
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	766 bufp = buf;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	767 squat_pack_num(&bufp, block_offset - start_offset);
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	768 o_stream_send(ctx->output, buf, bufp - buf);
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	769
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	770 /* write the sizes/flags */
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	771 for (j = 0; j < max; j++) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	772 bufp = buf;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	773 squat_pack_num(&bufp, list_sizes[j]);
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	774 o_stream_send(ctx->output, buf, bufp - buf);
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	775 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	776 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	777
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	778 mem_size = ctx->lists.arr.buffer->used +
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	779 ctx->block_offsets.arr.buffer->used +
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	780 ctx->block_end_indexes.arr.buffer->used;
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	781 if (ctx->uidlist->max_size < mem_size)
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	782 ctx->uidlist->max_size = mem_size;
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	783
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	784 ctx->list_start_idx += count;
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	785 array_clear(&ctx->lists);
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	786
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	787 uidlist_write_block_list_and_header(ctx, ctx->output,
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	788 &ctx->block_offsets,
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	789 &ctx->block_end_indexes, TRUE);
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	790
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	791 (void)squat_uidlist_map(ctx->uidlist);
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	792
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	793 array_clear(&ctx->block_offsets);
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	794 array_clear(&ctx->block_end_indexes);
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	795 }
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	796
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	797 int squat_uidlist_build_finish(struct squat_uidlist_build_context *ctx)
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	798 {
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	799 if (ctx->uidlist->corrupted)
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	800 return -1;
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	801
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	802 o_stream_seek(ctx->output, 0);
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	803 o_stream_send(ctx->output, &ctx->build_hdr, sizeof(ctx->build_hdr));
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	804 o_stream_seek(ctx->output, ctx->build_hdr.used_file_size);
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	805 o_stream_flush(ctx->output);
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	806
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	807 if (ctx->output->last_failed_errno != 0) {
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	808 errno = ctx->output->last_failed_errno;
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	809 i_error("write() to %s failed: %m", ctx->uidlist->path);
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	810 return -1;
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	811 }
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	812 return 0;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	813 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	814
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	815 void squat_uidlist_build_deinit(struct squat_uidlist_build_context **_ctx)
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	816 {
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	817 struct squat_uidlist_build_context ctx = _ctx;
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	818
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	819 *_ctx = NULL;
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	820
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	821 i_assert(array_count(&ctx->lists) == 0 \|\| ctx->uidlist->corrupted);
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	822 i_assert(ctx->uidlist->building);
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	823 ctx->uidlist->building = FALSE;
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	824
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	825 file_unlock(&ctx->uidlist->file_lock);
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	826
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	827 if (ctx->need_reopen)
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	828 (void)squat_uidlist_open(ctx->uidlist);
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	829
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	830 array_free(&ctx->block_offsets);
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	831 array_free(&ctx->block_end_indexes);
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	832 array_free(&ctx->lists);
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	833 o_stream_unref(&ctx->output);
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	834 i_free(ctx);
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	835 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	836
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	837 int squat_uidlist_rebuild_init(struct squat_uidlist_build_context *build_ctx,
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	838 bool compress,
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	839 struct squat_uidlist_rebuild_context **ctx_r)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	840 {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	841 struct squat_uidlist_rebuild_context *ctx;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	842 struct squat_uidlist_file_header hdr;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	843 const char *temp_path;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	844 int fd;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	845
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	846 if (build_ctx->build_hdr.link_count == 0)
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	847 return 0;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	848
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	849 if (!compress) {
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	850 if (build_ctx->build_hdr.link_count <
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	851 build_ctx->build_hdr.count*2/3)
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	852 return 0;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	853 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	854
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	855 temp_path = t_strconcat(build_ctx->uidlist->path, ".tmp", NULL);
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	856 fd = open(temp_path, O_RDWR \| O_TRUNC \| O_CREAT, 0600);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	857 if (fd < 0) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	858 i_error("open(%s) failed: %m", temp_path);
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	859 return -1;
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	860 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	861
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	862 ctx = i_new(struct squat_uidlist_rebuild_context, 1);
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	863 ctx->uidlist = build_ctx->uidlist;
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	864 ctx->build_ctx = build_ctx;
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	865 ctx->fd = fd;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	866 ctx->output = o_stream_create_fd(ctx->fd, 0, FALSE);
7210 f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7086 diff changeset	867 ctx->next_uid_list_idx = 0x100;
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	868 o_stream_cork(ctx->output);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	869
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	870 memset(&hdr, 0, sizeof(hdr));
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	871 o_stream_send(ctx->output, &hdr, sizeof(hdr));
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	872
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	873 ctx->cur_block_start_offset = ctx->output->offset;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	874 i_array_init(&ctx->new_block_offsets,
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	875 build_ctx->build_hdr.count / UIDLIST_BLOCK_LIST_COUNT);
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	876 i_array_init(&ctx->new_block_end_indexes,
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	877 build_ctx->build_hdr.count / UIDLIST_BLOCK_LIST_COUNT);
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	878 *ctx_r = ctx;
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	879 return 1;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	880 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	881
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	882 static void
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	883 uidlist_rebuild_flush_block(struct squat_uidlist_rebuild_context *ctx)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	884 {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	885 uint8_t buf[SQUAT_PACK_MAX_SIZE], *bufp;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	886 uint32_t block_offset, block_end_idx;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	887 unsigned int i;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	888
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	889 ctx->new_count += ctx->list_idx;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	890
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	891 block_offset = ctx->output->offset;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	892 block_end_idx = ctx->new_count;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	893 array_append(&ctx->new_block_offsets, &block_offset, 1);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	894 array_append(&ctx->new_block_end_indexes, &block_end_idx, 1);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	895
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	896 /* this block's contents started from cur_block_start_offset and
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	897 ended to current offset. write the size of this area. */
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	898 bufp = buf;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	899 squat_pack_num(&bufp, block_offset - ctx->cur_block_start_offset);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	900 o_stream_send(ctx->output, buf, bufp - buf);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	901
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	902 /* write the sizes/flags */
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	903 for (i = 0; i < ctx->list_idx; i++) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	904 bufp = buf;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	905 squat_pack_num(&bufp, ctx->list_sizes[i]);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	906 o_stream_send(ctx->output, buf, bufp - buf);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	907 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	908 ctx->cur_block_start_offset = ctx->output->offset;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	909 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	910
7210 f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7086 diff changeset	911 uint32_t squat_uidlist_rebuild_next(struct squat_uidlist_rebuild_context *ctx,
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7086 diff changeset	912 const ARRAY_TYPE(uint32_t) *uids)
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	913 {
6940 414c9d631a81 Replaced t_push/t_pop calls with T_FRAME() macros. Timo Sirainen <tss@iki.fi>* parents: 6931 diff changeset	914 int ret;
414c9d631a81 Replaced t_push/t_pop calls with T_FRAME() macros. Timo Sirainen <tss@iki.fi>* parents: 6931 diff changeset	915
414c9d631a81 Replaced t_push/t_pop calls with T_FRAME() macros. Timo Sirainen <tss@iki.fi>* parents: 6931 diff changeset	916 T_FRAME(
414c9d631a81 Replaced t_push/t_pop calls with T_FRAME() macros. Timo Sirainen <tss@iki.fi>* parents: 6931 diff changeset	917 ret = uidlist_write_array(ctx->output, array_idx(uids, 0),
414c9d631a81 Replaced t_push/t_pop calls with T_FRAME() macros. Timo Sirainen <tss@iki.fi>* parents: 6931 diff changeset	918 array_count(uids), 0, 0, FALSE,
414c9d631a81 Replaced t_push/t_pop calls with T_FRAME() macros. Timo Sirainen <tss@iki.fi>* parents: 6931 diff changeset	919 &ctx->list_sizes[ctx->list_idx]);
414c9d631a81 Replaced t_push/t_pop calls with T_FRAME() macros. Timo Sirainen <tss@iki.fi>* parents: 6931 diff changeset	920 );
414c9d631a81 Replaced t_push/t_pop calls with T_FRAME() macros. Timo Sirainen <tss@iki.fi>* parents: 6931 diff changeset	921 if (ret < 0)
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	922 squat_uidlist_set_corrupted(ctx->uidlist, "Broken uidlists");
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	923
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	924 if (++ctx->list_idx == UIDLIST_BLOCK_LIST_COUNT) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	925 uidlist_rebuild_flush_block(ctx);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	926 ctx->list_idx = 0;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	927 }
7210 f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7086 diff changeset	928 return ctx->next_uid_list_idx++ << 1;
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7086 diff changeset	929 }
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7086 diff changeset	930
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7086 diff changeset	931 uint32_t squat_uidlist_rebuild_nextu(struct squat_uidlist_rebuild_context *ctx,
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7086 diff changeset	932 const ARRAY_TYPE(seq_range) *uids)
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7086 diff changeset	933 {
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7086 diff changeset	934 const struct seq_range *range;
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7086 diff changeset	935 ARRAY_TYPE(uint32_t) tmp_uids;
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7086 diff changeset	936 uint32_t seq, uid1, ret;
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7086 diff changeset	937 unsigned int i, count;
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7086 diff changeset	938
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7086 diff changeset	939 range = array_get(uids, &count);
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7086 diff changeset	940 if (count == 0)
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7086 diff changeset	941 return 0;
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7086 diff changeset	942
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7086 diff changeset	943 if (count == 1 && range[0].seq1 == range[0].seq2) {
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7086 diff changeset	944 /* single UID */
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7086 diff changeset	945 return (range[0].seq1 << 1) \| 1;
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7086 diff changeset	946 }
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7086 diff changeset	947 if (range[count-1].seq2 < 8) {
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7086 diff changeset	948 /* we can use a singleton bitmask */
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7086 diff changeset	949 ret = 0;
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7086 diff changeset	950 for (i = 0; i < count; i++) {
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7086 diff changeset	951 for (seq = range[i].seq1; seq <= range[i].seq2; seq++)
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7086 diff changeset	952 ret \|= 1 << (seq+1);
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7086 diff changeset	953 }
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7086 diff changeset	954 return ret;
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7086 diff changeset	955 }
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7086 diff changeset	956
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7086 diff changeset	957 /* convert seq range to our internal representation and use the
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7086 diff changeset	958 normal _rebuild_next() to write it */
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7086 diff changeset	959 i_array_init(&tmp_uids, 128);
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7086 diff changeset	960 for (i = 0; i < count; i++) {
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7086 diff changeset	961 if (range[i].seq1 == range[i].seq2)
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7086 diff changeset	962 array_append(&tmp_uids, &range[i].seq1, 1);
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7086 diff changeset	963 else {
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7086 diff changeset	964 uid1 = range[i].seq1 \| UID_LIST_MASK_RANGE;
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7086 diff changeset	965 array_append(&tmp_uids, &uid1, 1);
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7086 diff changeset	966 array_append(&tmp_uids, &range[i].seq2, 1);
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7086 diff changeset	967 }
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7086 diff changeset	968 }
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7086 diff changeset	969 ret = squat_uidlist_rebuild_next(ctx, &tmp_uids);
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7086 diff changeset	970 array_free(&tmp_uids);
f5f77a3ae203 Initial code to support expunging from squat indexes, probably still buggy. Timo Sirainen <tss@iki.fi> parents: 7086 diff changeset	971 return ret;
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	972 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	973
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	974 int squat_uidlist_rebuild_finish(struct squat_uidlist_rebuild_context *ctx,
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	975 bool cancel)
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	976 {
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	977 const char *temp_path;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	978 int ret = 1;
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	979
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	980 if (ctx->list_idx != 0)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	981 uidlist_rebuild_flush_block(ctx);
7218 43d2f70fb279 Squat fixes Timo Sirainen <tss@iki.fi> parents: 7210 diff changeset	982 if (cancel \|\| ctx->uidlist->corrupted)
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	983 ret = 0;
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	984
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	985 temp_path = t_strconcat(ctx->uidlist->path, ".tmp", NULL);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	986 if (ret > 0) {
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	987 ctx->build_ctx->build_hdr.indexid =
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	988 ctx->uidlist->trie->hdr.indexid;
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	989 ctx->build_ctx->build_hdr.count = ctx->new_count;
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	990 ctx->build_ctx->build_hdr.link_count = 0;
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	991 uidlist_write_block_list_and_header(ctx->build_ctx, ctx->output,
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	992 &ctx->new_block_offsets,
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	993 &ctx->new_block_end_indexes,
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	994 FALSE);
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	995 o_stream_seek(ctx->output, 0);
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	996 o_stream_send(ctx->output, &ctx->build_ctx->build_hdr,
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	997 sizeof(ctx->build_ctx->build_hdr));
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	998 o_stream_seek(ctx->output,
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	999 ctx->build_ctx->build_hdr.used_file_size);
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	1000 o_stream_flush(ctx->output);
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	1001
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	1002 if (ctx->uidlist->corrupted)
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	1003 ret = -1;
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	1004 else if (ctx->output->last_failed_errno != 0) {
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1005 errno = ctx->output->last_failed_errno;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1006 i_error("write() to %s failed: %m", temp_path);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1007 ret = -1;
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	1008 } else if (rename(temp_path, ctx->uidlist->path) < 0) {
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1009 i_error("rename(%s, %s) failed: %m",
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	1010 temp_path, ctx->uidlist->path);
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1011 ret = -1;
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1012 }
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	1013 ctx->build_ctx->need_reopen = TRUE;
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1014 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1015
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	1016 o_stream_unref(&ctx->output);
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	1017 if (close(ctx->fd) < 0)
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	1018 i_error("close(%s) failed: %m", temp_path);
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	1019
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1020 if (ret <= 0) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1021 if (unlink(temp_path) < 0)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1022 i_error("unlink(%s) failed: %m", temp_path);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1023 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1024 array_free(&ctx->new_block_offsets);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1025 array_free(&ctx->new_block_end_indexes);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1026 i_free(ctx);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1027 return ret < 0 ? -1 : 0;
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1028 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1029
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	1030 static void
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	1031 uidlist_flush(struct squat_uidlist_build_context *ctx,
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	1032 struct uidlist_list *list, uint32_t uid)
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1033 {
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	1034 uint32_t size, offset = ctx->output->offset;
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1035
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	1036 ctx->build_hdr.link_count++;
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	1037 if (uidlist_write(ctx->output, list, TRUE, &size) < 0)
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	1038 squat_uidlist_set_corrupted(ctx->uidlist, "Broken uidlists");
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1039
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1040 list->uid_count = 2;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1041 list->uid_begins_with_pointer = TRUE;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1042
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1043 list->uid_list[0] = offset;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1044 list->uid_list[1] = uid;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1045 }
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1046
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1047 static struct uidlist_list *
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	1048 uidlist_add_new(struct squat_uidlist_build_context *ctx, unsigned int count,
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1049 uint32_t *uid_list_idx_r)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1050 {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1051 struct uidlist_list *list;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1052
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	1053 i_assert(array_count(&ctx->lists) +
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	1054 ctx->list_start_idx == ctx->build_hdr.count);
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	1055 *uid_list_idx_r = (ctx->build_hdr.count + 0x100) << 1;
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	1056 list = array_append_space(&ctx->lists);
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	1057 ctx->build_hdr.count++;
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1058
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1059 list->uid_count = count;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1060 return list;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1061 }
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1062
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	1063 uint32_t squat_uidlist_build_add_uid(struct squat_uidlist_build_context *ctx,
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1064 uint32_t uid_list_idx, uint32_t uid)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1065 {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1066 struct uidlist_list *list;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1067 unsigned int idx, mask;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1068 uint32_t *p;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1069
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1070 if ((uid_list_idx & 1) != 0) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1071 /* adding second UID */
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1072 uint32_t prev_uid = uid_list_idx >> 1;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1073
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1074 i_assert(prev_uid != uid);
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	1075 list = uidlist_add_new(ctx, 2, &uid_list_idx);
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1076 list->uid_list[0] = prev_uid;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1077 if (prev_uid + 1 == uid)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1078 list->uid_list[0] \|= UID_LIST_MASK_RANGE;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1079 list->uid_list[1] = uid;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1080 return uid_list_idx;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1081 } else if (uid_list_idx < (0x100 << 1)) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1082 uint32_t old_list_idx;
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1083
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1084 if (uid < 8) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1085 /* UID lists containing only UIDs 0-7 are saved as
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1086 uidlist values 2..511. think of it as a bitmask. */
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1087 uid_list_idx \|= 1 << (uid + 1);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1088 i_assert((uid_list_idx & 1) == 0);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1089 return uid_list_idx;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1090 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1091
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1092 if (uid_list_idx == 0) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1093 /* first UID */
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1094 return (uid << 1) \| 1;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1095 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1096
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1097 /* create a new list */
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1098 old_list_idx = uid_list_idx >> 1;
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	1099 list = uidlist_add_new(ctx, 1, &uid_list_idx);
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1100 /* add the first UID ourself */
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1101 idx = 0;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1102 i_assert((old_list_idx & 0xff) != 0);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1103 for (mask = 1; mask <= 128; mask <<= 1, idx++) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1104 if ((old_list_idx & mask) != 0) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1105 list->uid_list[0] = idx;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1106 idx++; mask <<= 1;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1107 break;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1108 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1109 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1110 for (; mask <= 128; mask <<= 1, idx++) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1111 if ((old_list_idx & mask) != 0) {
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	1112 squat_uidlist_build_add_uid(ctx,
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1113 uid_list_idx, idx);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1114 }
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1115 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1116 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1117
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1118 /* add to existing list */
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1119 idx = (uid_list_idx >> 1) - 0x100;
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	1120 if (idx < ctx->list_start_idx) {
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	1121 list = uidlist_add_new(ctx, 2, &uid_list_idx);
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1122 list->uid_list[0] = UID_LIST_POINTER_MASK_LIST_IDX \| idx;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1123 list->uid_list[1] = uid;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1124 list->uid_begins_with_pointer = TRUE;
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	1125 ctx->build_hdr.link_count++;
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1126 return uid_list_idx;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1127 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1128
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	1129 idx -= ctx->list_start_idx;
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	1130 if (idx >= array_count(&ctx->lists)) {
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	1131 squat_uidlist_set_corrupted(ctx->uidlist,
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	1132 "missing/broken uidlist");
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1133 return 0;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1134 }
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	1135 list = array_idx_modifiable(&ctx->lists, idx);
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1136 i_assert(list->uid_count > 0);
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1137
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1138 p = &list->uid_list[list->uid_count-1];
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	1139 i_assert(uid != *p \|\| ctx->uidlist->corrupted \|\|
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1140 (list->uid_count == 1 && list->uid_begins_with_pointer));
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1141 if (uid == *p + 1 &&
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1142 (list->uid_count > 1 \|\| !list->uid_begins_with_pointer)) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1143 /* use a range */
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1144 if (list->uid_count > 1 && (p[-1] & UID_LIST_MASK_RANGE) != 0 &&
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1145 (list->uid_count > 2 \|\| !list->uid_begins_with_pointer)) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1146 /* increase the existing range */
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1147 *p += 1;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1148 return uid_list_idx;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1149 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1150
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1151 if (list->uid_count == UIDLIST_LIST_SIZE) {
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	1152 uidlist_flush(ctx, list, uid);
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1153 return uid_list_idx;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1154 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1155 /* create a new range */
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1156 *p \|= UID_LIST_MASK_RANGE;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1157 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1158
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1159 if (list->uid_count == UIDLIST_LIST_SIZE) {
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	1160 uidlist_flush(ctx, list, uid);
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1161 return uid_list_idx;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1162 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1163
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1164 p++;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1165 list->uid_count++;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1166
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1167 *p = uid;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1168 return uid_list_idx;
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1169 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1170
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1171 static void uidlist_array_append(ARRAY_TYPE(uint32_t) *uids, uint32_t uid)
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1172 {
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1173 uint32_t *uidlist;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1174 unsigned int count;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1175
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1176 uidlist = array_get_modifiable(uids, &count);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1177 if (count == 0) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1178 array_append(uids, &uid, 1);
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1179 return;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1180 }
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1181 if (uidlist[count-1] + 1 == uid) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1182 if (count > 1 && (uidlist[count-2] &
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1183 UID_LIST_MASK_RANGE) != 0) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1184 uidlist[count-1]++;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1185 return;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1186 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1187 uidlist[count-1] \|= UID_LIST_MASK_RANGE;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1188 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1189 array_append(uids, &uid, 1);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1190 }
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1191
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1192 static void uidlist_array_append_range(ARRAY_TYPE(uint32_t) *uids,
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1193 uint32_t uid1, uint32_t uid2)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1194 {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1195 uint32_t *uidlist;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1196 unsigned int count;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1197
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1198 i_assert(uid1 < uid2);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1199
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1200 uidlist = array_get_modifiable(uids, &count);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1201 if (count == 0) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1202 uid1 \|= UID_LIST_MASK_RANGE;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1203 array_append(uids, &uid1, 1);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1204 array_append(uids, &uid2, 1);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1205 return;
4933 44a0e7606f7a Use seq_range_array_remove_range() instead of slowly iterating ourself. Timo Sirainen <tss@iki.fi> parents: 4931 diff changeset	1206 }
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1207 if (uidlist[count-1] + 1 == uid1) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1208 if (count > 1 && (uidlist[count-2] &
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1209 UID_LIST_MASK_RANGE) != 0) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1210 uidlist[count-1] = uid2;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1211 return;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1212 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1213 uidlist[count-1] \|= UID_LIST_MASK_RANGE;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1214 } else {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1215 uid1 \|= UID_LIST_MASK_RANGE;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1216 array_append(uids, &uid1, 1);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1217 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1218 array_append(uids, &uid2, 1);
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1219 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1220
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1221 static int
6944 66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	1222 squat_uidlist_get_at_offset(struct squat_uidlist *uidlist, uoff_t offset,
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	1223 uint32_t num, ARRAY_TYPE(uint32_t) *uids)
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1224 {
7218 43d2f70fb279 Squat fixes Timo Sirainen <tss@iki.fi> parents: 7210 diff changeset	1225 const uint32_t *uid_list;
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1226 const uint8_t p, end;
7218 43d2f70fb279 Squat fixes Timo Sirainen <tss@iki.fi> parents: 7210 diff changeset	1227 uint32_t size, base_uid, next_uid, flags, prev;
6944 66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	1228 uoff_t uidlist_data_offset;
7218 43d2f70fb279 Squat fixes Timo Sirainen <tss@iki.fi> parents: 7210 diff changeset	1229 unsigned int i, j, count;
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1230
6944 66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	1231 if (num != 0)
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	1232 uidlist_data_offset = offset;
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	1233 else {
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	1234 /* not given, read it */
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	1235 if (uidlist_file_cache_read(uidlist, offset,
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	1236 SQUAT_PACK_MAX_SIZE) < 0)
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	1237 return -1;
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1238
6944 66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	1239 p = CONST_PTR_OFFSET(uidlist->data, offset);
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	1240 end = CONST_PTR_OFFSET(uidlist->data, uidlist->data_size);
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1241 num = squat_unpack_num(&p, end);
6944 66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	1242 uidlist_data_offset = p - (const uint8_t *)uidlist->data;
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1243 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1244 size = num >> 2;
6944 66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	1245
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	1246 if (uidlist_file_cache_read(uidlist, uidlist_data_offset, size) < 0)
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	1247 return -1;
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	1248 if (uidlist_data_offset + size > uidlist->data_size) {
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1249 squat_uidlist_set_corrupted(uidlist,
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1250 "size points outside file");
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1251 return -1;
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1252 }
6944 66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	1253
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	1254 p = CONST_PTR_OFFSET(uidlist->data, uidlist_data_offset);
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1255 end = p + size;
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1256
7218 43d2f70fb279 Squat fixes Timo Sirainen <tss@iki.fi> parents: 7210 diff changeset	1257 flags = num;
43d2f70fb279 Squat fixes Timo Sirainen <tss@iki.fi> parents: 7210 diff changeset	1258 if ((flags & UIDLIST_PACKED_FLAG_BEGINS_WITH_POINTER) != 0) {
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1259 /* link to the file */
7218 43d2f70fb279 Squat fixes Timo Sirainen <tss@iki.fi> parents: 7210 diff changeset	1260 prev = squat_unpack_num(&p, end);
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1261
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1262 if ((prev & 1) != 0) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1263 /* pointer to uidlist */
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1264 prev = ((prev >> 1) + 0x100) << 1;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1265 if (squat_uidlist_get(uidlist, prev, uids) < 0)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1266 return -1;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1267 } else {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1268 prev = offset - (prev >> 1);
6944 66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	1269 if (squat_uidlist_get_at_offset(uidlist, prev,
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	1270 0, uids) < 0)
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1271 return -1;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1272 }
7218 43d2f70fb279 Squat fixes Timo Sirainen <tss@iki.fi> parents: 7210 diff changeset	1273 uid_list = array_get(uids, &count);
43d2f70fb279 Squat fixes Timo Sirainen <tss@iki.fi> parents: 7210 diff changeset	1274 next_uid = count == 0 ? 0 : uid_list[count-1] + 1;
43d2f70fb279 Squat fixes Timo Sirainen <tss@iki.fi> parents: 7210 diff changeset	1275 } else {
43d2f70fb279 Squat fixes Timo Sirainen <tss@iki.fi> parents: 7210 diff changeset	1276 next_uid = 0;
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1277 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1278
7218 43d2f70fb279 Squat fixes Timo Sirainen <tss@iki.fi> parents: 7210 diff changeset	1279 num = base_uid = squat_unpack_num(&p, end);
43d2f70fb279 Squat fixes Timo Sirainen <tss@iki.fi> parents: 7210 diff changeset	1280 if ((flags & UIDLIST_PACKED_FLAG_BITMASK) == 0)
43d2f70fb279 Squat fixes Timo Sirainen <tss@iki.fi> parents: 7210 diff changeset	1281 base_uid >>= 1;
43d2f70fb279 Squat fixes Timo Sirainen <tss@iki.fi> parents: 7210 diff changeset	1282 if (base_uid < next_uid) {
43d2f70fb279 Squat fixes Timo Sirainen <tss@iki.fi> parents: 7210 diff changeset	1283 squat_uidlist_set_corrupted(uidlist,
43d2f70fb279 Squat fixes Timo Sirainen <tss@iki.fi> parents: 7210 diff changeset	1284 "broken continued uidlist");
43d2f70fb279 Squat fixes Timo Sirainen <tss@iki.fi> parents: 7210 diff changeset	1285 return -1;
43d2f70fb279 Squat fixes Timo Sirainen <tss@iki.fi> parents: 7210 diff changeset	1286 }
43d2f70fb279 Squat fixes Timo Sirainen <tss@iki.fi> parents: 7210 diff changeset	1287
43d2f70fb279 Squat fixes Timo Sirainen <tss@iki.fi> parents: 7210 diff changeset	1288 if ((flags & UIDLIST_PACKED_FLAG_BITMASK) != 0) {
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1289 /* bitmask */
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1290 size = end - p;
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1291
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1292 uidlist_array_append(uids, base_uid++);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1293 for (i = 0; i < size; i++) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1294 for (j = 0; j < 8; j++, base_uid++) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1295 if ((p[i] & (1 << j)) != 0)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1296 uidlist_array_append(uids, base_uid);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1297 }
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1298 }
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1299 } else {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1300 /* range */
7218 43d2f70fb279 Squat fixes Timo Sirainen <tss@iki.fi> parents: 7210 diff changeset	1301 for (;;) {
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1302 if ((num & 1) == 0) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1303 uidlist_array_append(uids, base_uid);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1304 } else {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1305 /* range */
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1306 uint32_t seq1 = base_uid;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1307 base_uid += squat_unpack_num(&p, end) + 1;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1308 uidlist_array_append_range(uids, seq1,
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1309 base_uid);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1310 }
7218 43d2f70fb279 Squat fixes Timo Sirainen <tss@iki.fi> parents: 7210 diff changeset	1311 if (p == end)
43d2f70fb279 Squat fixes Timo Sirainen <tss@iki.fi> parents: 7210 diff changeset	1312 break;
43d2f70fb279 Squat fixes Timo Sirainen <tss@iki.fi> parents: 7210 diff changeset	1313
43d2f70fb279 Squat fixes Timo Sirainen <tss@iki.fi> parents: 7210 diff changeset	1314 num = squat_unpack_num(&p, end);
43d2f70fb279 Squat fixes Timo Sirainen <tss@iki.fi> parents: 7210 diff changeset	1315 base_uid += (num >> 1) + 1;
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1316 }
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1317 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1318 return 0;
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1319 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1320
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1321 static int uint32_cmp(const void key, const void data)
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1322 {
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1323 const uint32_t i1 = key, i2 = data;
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1324
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1325 return (int)i1 - (int)i2;
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1326 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1327
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1328 static int
6944 66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	1329 squat_uidlist_get_offset(struct squat_uidlist *uidlist, uint32_t uid_list_idx,
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	1330 uint32_t offset_r, uint32_t num_r)
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1331 {
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1332 const uint8_t p, end;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1333 unsigned int idx;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1334 uint32_t num, skip_bytes, uidlists_offset;
6944 66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	1335 size_t max_map_size;
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1336
6931 97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	1337 if (uidlist->fd == -1) {
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	1338 squat_uidlist_set_corrupted(uidlist, "no uidlists");
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	1339 return -1;
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	1340 }
97702c9c4111 Locking and error handling fixes. Timo Sirainen <tss@iki.fi> parents: 6917 diff changeset	1341
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1342 if (bsearch_insert_pos(&uid_list_idx, uidlist->cur_block_end_indexes,
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1343 uidlist->cur_block_count,
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1344 sizeof(uint32_t), uint32_cmp, &idx))
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1345 idx++;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1346 if (unlikely(idx == uidlist->cur_block_count)) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1347 squat_uidlist_set_corrupted(uidlist, "uidlist not found");
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1348 return -1;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1349 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1350 if (unlikely(idx > 0 &&
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1351 uidlist->cur_block_end_indexes[idx-1] > uid_list_idx)) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1352 squat_uidlist_set_corrupted(uidlist, "broken block list");
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1353 return -1;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1354 }
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1355
6944 66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	1356 /* make sure everything is mapped */
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	1357 uid_list_idx -= idx == 0 ? 0 : uidlist->cur_block_end_indexes[idx-1];
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	1358 max_map_size = SQUAT_PACK_MAX_SIZE * (1+uid_list_idx);
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	1359 if (uidlist_file_cache_read(uidlist, uidlist->cur_block_offsets[idx],
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	1360 max_map_size) < 0)
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	1361 return -1;
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	1362
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1363 /* find the uidlist inside the block */
6944 66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	1364 p = CONST_PTR_OFFSET(uidlist->data, uidlist->cur_block_offsets[idx]);
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	1365 end = CONST_PTR_OFFSET(uidlist->data, uidlist->data_size);
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1366
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1367 uidlists_offset = uidlist->cur_block_offsets[idx] -
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1368 squat_unpack_num(&p, end);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1369 for (skip_bytes = 0; uid_list_idx > 0; uid_list_idx--) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1370 num = squat_unpack_num(&p, end);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1371 skip_bytes += num >> 2;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1372 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1373 *offset_r = uidlists_offset + skip_bytes;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1374 *num_r = squat_unpack_num(&p, end);
4931 59ce1d6c0b8c Actually the code wasn't unneeded, put it back and actually execute it now. Timo Sirainen <tss@iki.fi> parents: 4930 diff changeset	1375
6944 66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	1376 if (unlikely(p == end)) {
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	1377 squat_uidlist_set_corrupted(uidlist, "broken file");
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	1378 return -1;
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	1379 }
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	1380 if (unlikely(*offset_r > uidlist->mmap_size &&
66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	1381 uidlist->mmap_base != NULL)) {
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1382 squat_uidlist_set_corrupted(uidlist, "broken offset");
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1383 return -1;
4931 59ce1d6c0b8c Actually the code wasn't unneeded, put it back and actually execute it now. Timo Sirainen <tss@iki.fi> parents: 4930 diff changeset	1384 }
59ce1d6c0b8c Actually the code wasn't unneeded, put it back and actually execute it now. Timo Sirainen <tss@iki.fi> parents: 4930 diff changeset	1385 return 0;
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1386 }
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1387
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1388 int squat_uidlist_get(struct squat_uidlist *uidlist, uint32_t uid_list_idx,
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1389 ARRAY_TYPE(uint32_t) *uids)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1390 {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1391 unsigned int mask;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1392 uint32_t uid, offset, num;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1393
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1394 if ((uid_list_idx & 1) != 0) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1395 /* single UID */
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1396 uid = uid_list_idx >> 1;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1397 uidlist_array_append(uids, uid);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1398 return 0;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1399 } else if (uid_list_idx < (0x100 << 1)) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1400 /* bitmask */
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1401 for (uid = 0, mask = 2; mask <= 256; mask <<= 1, uid++) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1402 if ((uid_list_idx & mask) != 0)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1403 uidlist_array_append(uids, uid);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1404 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1405 return 0;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1406 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1407
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1408 uid_list_idx = (uid_list_idx >> 1) - 0x100;
6944 66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	1409 if (squat_uidlist_get_offset(uidlist, uid_list_idx, &offset, &num) < 0)
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1410 return -1;
6944 66b3e894041b Support mmap_disable=yes and some error handling improvements. Timo Sirainen <tss@iki.fi> parents: 6940 diff changeset	1411 return squat_uidlist_get_at_offset(uidlist, offset, num, uids);
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1412 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1413
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1414 uint32_t squat_uidlist_singleton_last_uid(uint32_t uid_list_idx)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1415 {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1416 unsigned int idx, mask;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1417
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1418 if ((uid_list_idx & 1) != 0) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1419 /* single UID */
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1420 return uid_list_idx >> 1;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1421 } else if (uid_list_idx < (0x100 << 1)) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1422 /* bitmask */
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1423 if (uid_list_idx == 2) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1424 /* just a quick optimization */
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1425 return 0;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1426 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1427 for (idx = 7, mask = 256; mask > 2; mask >>= 1, idx--) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1428 if ((uid_list_idx & mask) != 0)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1429 return idx;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1430 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1431 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1432
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1433 i_unreached();
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1434 return 0;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1435 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1436
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1437 int squat_uidlist_get_seqrange(struct squat_uidlist *uidlist,
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1438 uint32_t uid_list_idx,
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1439 ARRAY_TYPE(seq_range) *seq_range_arr)
4886 ae2114f11a0e Memory leak fixes. Also when building a large mailbox flush once in a while Timo Sirainen <tss@iki.fi> parents: 4883 diff changeset	1440 {
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1441 ARRAY_TYPE(uint32_t) tmp_uid_arr;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1442 struct seq_range range;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1443 const uint32_t *tmp_uids;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1444 unsigned int i, count;
6945 5915aea5f070 UID ranges can be large, don't allocate them from data stack. Timo Sirainen <tss@iki.fi> parents: 6944 diff changeset	1445 int ret;
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1446
6945 5915aea5f070 UID ranges can be large, don't allocate them from data stack. Timo Sirainen <tss@iki.fi> parents: 6944 diff changeset	1447 i_array_init(&tmp_uid_arr, 128);
5915aea5f070 UID ranges can be large, don't allocate them from data stack. Timo Sirainen <tss@iki.fi> parents: 6944 diff changeset	1448 ret = squat_uidlist_get(uidlist, uid_list_idx, &tmp_uid_arr);
5915aea5f070 UID ranges can be large, don't allocate them from data stack. Timo Sirainen <tss@iki.fi> parents: 6944 diff changeset	1449 if (ret == 0) {
5915aea5f070 UID ranges can be large, don't allocate them from data stack. Timo Sirainen <tss@iki.fi> parents: 6944 diff changeset	1450 tmp_uids = array_get(&tmp_uid_arr, &count);
5915aea5f070 UID ranges can be large, don't allocate them from data stack. Timo Sirainen <tss@iki.fi> parents: 6944 diff changeset	1451 for (i = 0; i < count; i++) {
5915aea5f070 UID ranges can be large, don't allocate them from data stack. Timo Sirainen <tss@iki.fi> parents: 6944 diff changeset	1452 if ((tmp_uids[i] & UID_LIST_MASK_RANGE) == 0)
5915aea5f070 UID ranges can be large, don't allocate them from data stack. Timo Sirainen <tss@iki.fi> parents: 6944 diff changeset	1453 range.seq1 = range.seq2 = tmp_uids[i];
5915aea5f070 UID ranges can be large, don't allocate them from data stack. Timo Sirainen <tss@iki.fi> parents: 6944 diff changeset	1454 else {
5915aea5f070 UID ranges can be large, don't allocate them from data stack. Timo Sirainen <tss@iki.fi> parents: 6944 diff changeset	1455 range.seq1 = tmp_uids[i] & ~UID_LIST_MASK_RANGE;
5915aea5f070 UID ranges can be large, don't allocate them from data stack. Timo Sirainen <tss@iki.fi> parents: 6944 diff changeset	1456 range.seq2 = tmp_uids[++i];
5915aea5f070 UID ranges can be large, don't allocate them from data stack. Timo Sirainen <tss@iki.fi> parents: 6944 diff changeset	1457 }
5915aea5f070 UID ranges can be large, don't allocate them from data stack. Timo Sirainen <tss@iki.fi> parents: 6944 diff changeset	1458 array_append(seq_range_arr, &range, 1);
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1459 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1460 }
6945 5915aea5f070 UID ranges can be large, don't allocate them from data stack. Timo Sirainen <tss@iki.fi> parents: 6944 diff changeset	1461 array_free(&tmp_uid_arr);
5915aea5f070 UID ranges can be large, don't allocate them from data stack. Timo Sirainen <tss@iki.fi> parents: 6944 diff changeset	1462 return ret;
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1463 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1464
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1465 int squat_uidlist_filter(struct squat_uidlist *uidlist, uint32_t uid_list_idx,
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1466 ARRAY_TYPE(seq_range) *uids)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1467 {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1468 const struct seq_range *parent_range;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1469 ARRAY_TYPE(seq_range) dest_uids;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1470 ARRAY_TYPE(uint32_t) relative_uids;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1471 const uint32_t *rel_range;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1472 unsigned int i, rel_count, parent_idx, parent_count, diff, parent_uid;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1473 uint32_t prev_seq, seq1, seq2;
7218 43d2f70fb279 Squat fixes Timo Sirainen <tss@iki.fi> parents: 7210 diff changeset	1474 int ret = 0;
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1475
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1476 parent_range = array_get(uids, &parent_count);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1477 if (parent_count == 0)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1478 return 0;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1479
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1480 i_array_init(&relative_uids, 128);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1481 i_array_init(&dest_uids, 128);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1482 squat_uidlist_get(uidlist, uid_list_idx, &relative_uids);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1483
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1484 parent_idx = 0;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1485 rel_range = array_get(&relative_uids, &rel_count);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1486 prev_seq = 0; parent_uid = parent_range[0].seq1;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1487 for (i = 0; i < rel_count; i++) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1488 if (unlikely(parent_uid == (uint32_t)-1)) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1489 i_error("broken UID ranges");
7218 43d2f70fb279 Squat fixes Timo Sirainen <tss@iki.fi> parents: 7210 diff changeset	1490 ret = -1;
43d2f70fb279 Squat fixes Timo Sirainen <tss@iki.fi> parents: 7210 diff changeset	1491 break;
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1492 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1493 if ((rel_range[i] & UID_LIST_MASK_RANGE) == 0)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1494 seq1 = seq2 = rel_range[i];
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1495 else {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1496 seq1 = (rel_range[i] & ~UID_LIST_MASK_RANGE);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1497 seq2 = rel_range[++i];
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1498 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1499 i_assert(seq1 >= prev_seq);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1500 diff = seq1 - prev_seq;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1501 while (diff > 0) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1502 if (unlikely(parent_uid == (uint32_t)-1)) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1503 i_error("broken UID ranges");
7218 43d2f70fb279 Squat fixes Timo Sirainen <tss@iki.fi> parents: 7210 diff changeset	1504 ret = -1;
43d2f70fb279 Squat fixes Timo Sirainen <tss@iki.fi> parents: 7210 diff changeset	1505 break;
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1506 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1507
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1508 for (; parent_idx < parent_count; parent_idx++) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1509 if (parent_range[parent_idx].seq2 <= parent_uid)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1510 continue;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1511 if (parent_uid < parent_range[parent_idx].seq1)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1512 parent_uid = parent_range[parent_idx].seq1;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1513 else
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1514 parent_uid++;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1515 break;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1516 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1517 diff--;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1518 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1519 diff = seq2 - seq1 + 1;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1520 while (diff > 0) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1521 if (unlikely(parent_uid == (uint32_t)-1)) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1522 i_error("broken UID ranges");
7218 43d2f70fb279 Squat fixes Timo Sirainen <tss@iki.fi> parents: 7210 diff changeset	1523 ret = -1;
43d2f70fb279 Squat fixes Timo Sirainen <tss@iki.fi> parents: 7210 diff changeset	1524 break;
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1525 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1526 seq_range_array_add(&dest_uids, 0, parent_uid);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1527 for (; parent_idx < parent_count; parent_idx++) {
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1528 if (parent_range[parent_idx].seq2 <= parent_uid)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1529 continue;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1530 if (parent_uid < parent_range[parent_idx].seq1)
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1531 parent_uid = parent_range[parent_idx].seq1;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1532 else
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1533 parent_uid++;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1534 break;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1535 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1536 diff--;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1537 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1538
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1539 prev_seq = seq2 + 1;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1540 }
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1541
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1542 buffer_set_used_size(uids->arr.buffer, 0);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1543 array_append_array(uids, &dest_uids);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1544
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1545 array_free(&relative_uids);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1546 array_free(&dest_uids);
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1547 return 0;
4886 ae2114f11a0e Memory leak fixes. Also when building a large mailbox flush once in a while Timo Sirainen <tss@iki.fi> parents: 4883 diff changeset	1548 }
ae2114f11a0e Memory leak fixes. Also when building a large mailbox flush once in a while Timo Sirainen <tss@iki.fi> parents: 4883 diff changeset	1549
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1550 size_t squat_uidlist_mem_used(struct squat_uidlist *uidlist,
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1551 unsigned int *count_r)
5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1552 {
6898 e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1553 *count_r = uidlist->hdr.count;
e739cffd05ef FTS API changes and Squat rewrite. Squat is still missing expunge handling. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	1554 return uidlist->max_size;
4855 5bc593f1a8f6 Added "squat" full text search indexer backend. Its name and basic ideas Timo Sirainen <tss@iki.fi> parents: diff changeset	1555 }

Mercurial > dovecot > original-hg > dovecot-1.2

annotate src/plugins/fts-squat/squat-uidlist.c @ 7218:43d2f70fb279 HEAD