dovecot/original-hg/dovecot-1.2: src/lib-index/mail-cache-fields.c annotate

annotate src/lib-index/mail-cache-fields.c @ 6707:4752637a6ad4 HEAD

Don't bother compressing cache file just to drop fields from header that have never been used.

author	Timo Sirainen <tss@iki.fi>
date	Tue, 06 Nov 2007 21:54:37 +0200
parents	68bb5bec043e
children	02014f5b0068

rev	line source
6429 65c69a53a7be Replaced my Copyright notices. The year range always ends with 2007 now. Timo Sirainen <tss@iki.fi> parents: 6296 diff changeset	1 /* Copyright (c) 2004-2007 Dovecot authors, see the included COPYING file */
2327 7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: diff changeset	2
7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: diff changeset	3 #include "lib.h"
7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: diff changeset	4 #include "buffer.h"
7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: diff changeset	5 #include "hash.h"
2866 bf1e718e7370 Cache file works now with mmap_disable=yes. Still needs a few optimizations. Timo Sirainen <tss@iki.fi> parents: 2708 diff changeset	6 #include "file-cache.h"
6704 b0e8403b4bb1 mmap_disable=yes: Find the latest cache field header by pread()ing the file Timo Sirainen <tss@iki.fi> parents: 6695 diff changeset	7 #include "read-full.h"
2339 406692edc12d Cache fixes. Decisions are saved again. Timo Sirainen <tss@iki.fi> parents: 2327 diff changeset	8 #include "write-full.h"
6694 c7742d109a67 mmap_disable=yes: When following cache field headers, don't invalidate the Timo Sirainen <tss@iki.fi> parents: 6443 diff changeset	9 #include "mmap-util.h"
2327 7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: diff changeset	10 #include "mail-cache-private.h"
7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: diff changeset	11
7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: diff changeset	12 #include <stddef.h>
7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: diff changeset	13
6443 53313d1e7e4f Code cleanups and some minor fixes. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	14 #define CACHE_FIELD_IS_NEWLY_WANTED(cache, field_idx) \
53313d1e7e4f Code cleanups and some minor fixes. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	15 ((cache)->field_file_map[field_idx] == (uint32_t)-1 && \
53313d1e7e4f Code cleanups and some minor fixes. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	16 (cache)->fields[field_idx].used)
53313d1e7e4f Code cleanups and some minor fixes. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	17
4654 35d9433de1ca When registering fields, if they were already exited in the cache file mark Timo Sirainen <tss@iki.fi> parents: 4345 diff changeset	18 static bool field_has_fixed_size(enum mail_cache_field_type type)
35d9433de1ca When registering fields, if they were already exited in the cache file mark Timo Sirainen <tss@iki.fi> parents: 4345 diff changeset	19 {
35d9433de1ca When registering fields, if they were already exited in the cache file mark Timo Sirainen <tss@iki.fi> parents: 4345 diff changeset	20 switch (type) {
35d9433de1ca When registering fields, if they were already exited in the cache file mark Timo Sirainen <tss@iki.fi> parents: 4345 diff changeset	21 case MAIL_CACHE_FIELD_FIXED_SIZE:
35d9433de1ca When registering fields, if they were already exited in the cache file mark Timo Sirainen <tss@iki.fi> parents: 4345 diff changeset	22 case MAIL_CACHE_FIELD_BITMASK:
35d9433de1ca When registering fields, if they were already exited in the cache file mark Timo Sirainen <tss@iki.fi> parents: 4345 diff changeset	23 return TRUE;
35d9433de1ca When registering fields, if they were already exited in the cache file mark Timo Sirainen <tss@iki.fi> parents: 4345 diff changeset	24 case MAIL_CACHE_FIELD_VARIABLE_SIZE:
35d9433de1ca When registering fields, if they were already exited in the cache file mark Timo Sirainen <tss@iki.fi> parents: 4345 diff changeset	25 case MAIL_CACHE_FIELD_STRING:
35d9433de1ca When registering fields, if they were already exited in the cache file mark Timo Sirainen <tss@iki.fi> parents: 4345 diff changeset	26 case MAIL_CACHE_FIELD_HEADER:
35d9433de1ca When registering fields, if they were already exited in the cache file mark Timo Sirainen <tss@iki.fi> parents: 4345 diff changeset	27 return FALSE;
4722 b0cf3e4d6c96 If cache file contains broken field type or decision type mark the cache Timo Sirainen <tss@iki.fi> parents: 4654 diff changeset	28
b0cf3e4d6c96 If cache file contains broken field type or decision type mark the cache Timo Sirainen <tss@iki.fi> parents: 4654 diff changeset	29 case MAIL_CACHE_FIELD_COUNT:
b0cf3e4d6c96 If cache file contains broken field type or decision type mark the cache Timo Sirainen <tss@iki.fi> parents: 4654 diff changeset	30 break;
4654 35d9433de1ca When registering fields, if they were already exited in the cache file mark Timo Sirainen <tss@iki.fi> parents: 4345 diff changeset	31 }
35d9433de1ca When registering fields, if they were already exited in the cache file mark Timo Sirainen <tss@iki.fi> parents: 4345 diff changeset	32
35d9433de1ca When registering fields, if they were already exited in the cache file mark Timo Sirainen <tss@iki.fi> parents: 4345 diff changeset	33 i_unreached();
35d9433de1ca When registering fields, if they were already exited in the cache file mark Timo Sirainen <tss@iki.fi> parents: 4345 diff changeset	34 return FALSE;
35d9433de1ca When registering fields, if they were already exited in the cache file mark Timo Sirainen <tss@iki.fi> parents: 4345 diff changeset	35 }
35d9433de1ca When registering fields, if they were already exited in the cache file mark Timo Sirainen <tss@iki.fi> parents: 4345 diff changeset	36
4722 b0cf3e4d6c96 If cache file contains broken field type or decision type mark the cache Timo Sirainen <tss@iki.fi> parents: 4654 diff changeset	37 static bool field_decision_is_valid(enum mail_cache_decision_type type)
b0cf3e4d6c96 If cache file contains broken field type or decision type mark the cache Timo Sirainen <tss@iki.fi> parents: 4654 diff changeset	38 {
b0cf3e4d6c96 If cache file contains broken field type or decision type mark the cache Timo Sirainen <tss@iki.fi> parents: 4654 diff changeset	39 switch (type & ~MAIL_CACHE_DECISION_FORCED) {
b0cf3e4d6c96 If cache file contains broken field type or decision type mark the cache Timo Sirainen <tss@iki.fi> parents: 4654 diff changeset	40 case MAIL_CACHE_DECISION_NO:
b0cf3e4d6c96 If cache file contains broken field type or decision type mark the cache Timo Sirainen <tss@iki.fi> parents: 4654 diff changeset	41 case MAIL_CACHE_DECISION_TEMP:
b0cf3e4d6c96 If cache file contains broken field type or decision type mark the cache Timo Sirainen <tss@iki.fi> parents: 4654 diff changeset	42 case MAIL_CACHE_DECISION_YES:
b0cf3e4d6c96 If cache file contains broken field type or decision type mark the cache Timo Sirainen <tss@iki.fi> parents: 4654 diff changeset	43 return TRUE;
b0cf3e4d6c96 If cache file contains broken field type or decision type mark the cache Timo Sirainen <tss@iki.fi> parents: 4654 diff changeset	44 default:
b0cf3e4d6c96 If cache file contains broken field type or decision type mark the cache Timo Sirainen <tss@iki.fi> parents: 4654 diff changeset	45 return FALSE;
b0cf3e4d6c96 If cache file contains broken field type or decision type mark the cache Timo Sirainen <tss@iki.fi> parents: 4654 diff changeset	46 }
b0cf3e4d6c96 If cache file contains broken field type or decision type mark the cache Timo Sirainen <tss@iki.fi> parents: 4654 diff changeset	47 }
b0cf3e4d6c96 If cache file contains broken field type or decision type mark the cache Timo Sirainen <tss@iki.fi> parents: 4654 diff changeset	48
4654 35d9433de1ca When registering fields, if they were already exited in the cache file mark Timo Sirainen <tss@iki.fi> parents: 4345 diff changeset	49 static int field_type_verify(struct mail_cache *cache, unsigned int idx,
35d9433de1ca When registering fields, if they were already exited in the cache file mark Timo Sirainen <tss@iki.fi> parents: 4345 diff changeset	50 enum mail_cache_field_type type, unsigned int size)
35d9433de1ca When registering fields, if they were already exited in the cache file mark Timo Sirainen <tss@iki.fi> parents: 4345 diff changeset	51 {
35d9433de1ca When registering fields, if they were already exited in the cache file mark Timo Sirainen <tss@iki.fi> parents: 4345 diff changeset	52 const struct mail_cache_field *field = &cache->fields[idx].field;
35d9433de1ca When registering fields, if they were already exited in the cache file mark Timo Sirainen <tss@iki.fi> parents: 4345 diff changeset	53
35d9433de1ca When registering fields, if they were already exited in the cache file mark Timo Sirainen <tss@iki.fi> parents: 4345 diff changeset	54 if (field->type != type) {
35d9433de1ca When registering fields, if they were already exited in the cache file mark Timo Sirainen <tss@iki.fi> parents: 4345 diff changeset	55 mail_cache_set_corrupted(cache,
35d9433de1ca When registering fields, if they were already exited in the cache file mark Timo Sirainen <tss@iki.fi> parents: 4345 diff changeset	56 "registered field %s type changed", field->name);
35d9433de1ca When registering fields, if they were already exited in the cache file mark Timo Sirainen <tss@iki.fi> parents: 4345 diff changeset	57 return -1;
35d9433de1ca When registering fields, if they were already exited in the cache file mark Timo Sirainen <tss@iki.fi> parents: 4345 diff changeset	58 }
35d9433de1ca When registering fields, if they were already exited in the cache file mark Timo Sirainen <tss@iki.fi> parents: 4345 diff changeset	59 if (field->field_size != size && field_has_fixed_size(type)) {
35d9433de1ca When registering fields, if they were already exited in the cache file mark Timo Sirainen <tss@iki.fi> parents: 4345 diff changeset	60 mail_cache_set_corrupted(cache,
35d9433de1ca When registering fields, if they were already exited in the cache file mark Timo Sirainen <tss@iki.fi> parents: 4345 diff changeset	61 "registered field %s size changed", field->name);
35d9433de1ca When registering fields, if they were already exited in the cache file mark Timo Sirainen <tss@iki.fi> parents: 4345 diff changeset	62 return -1;
35d9433de1ca When registering fields, if they were already exited in the cache file mark Timo Sirainen <tss@iki.fi> parents: 4345 diff changeset	63 }
35d9433de1ca When registering fields, if they were already exited in the cache file mark Timo Sirainen <tss@iki.fi> parents: 4345 diff changeset	64 return 0;
35d9433de1ca When registering fields, if they were already exited in the cache file mark Timo Sirainen <tss@iki.fi> parents: 4345 diff changeset	65 }
35d9433de1ca When registering fields, if they were already exited in the cache file mark Timo Sirainen <tss@iki.fi> parents: 4345 diff changeset	66
2327 7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: diff changeset	67 void mail_cache_register_fields(struct mail_cache *cache,
7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: diff changeset	68 struct mail_cache_field *fields,
3278 df372eecc5d0 Several size_t -> unsigned int changes. Try to keep "unsigned int" for Timo Sirainen <tss@iki.fi> parents: 2968 diff changeset	69 unsigned int fields_count)
2327 7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: diff changeset	70 {
7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: diff changeset	71 void orig_key, orig_value;
4020 fcfd44f56b04 While casting const pointers to something else, the const was often Timo Sirainen <tss@iki.fi> parents: 3999 diff changeset	72 char *name;
2327 7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: diff changeset	73 unsigned int new_idx;
3999 f1e29736e60d Don't register cache fields twice. Fixes "duplicate field in header" errors. Timo Sirainen <tss@iki.fi> parents: 3627 diff changeset	74 unsigned int i, j;
2327 7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: diff changeset	75
7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: diff changeset	76 new_idx = cache->fields_count;
7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: diff changeset	77 for (i = 0; i < fields_count; i++) {
7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: diff changeset	78 if (hash_lookup_full(cache->field_name_hash, fields[i].name,
7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: diff changeset	79 &orig_key, &orig_value)) {
4722 b0cf3e4d6c96 If cache file contains broken field type or decision type mark the cache Timo Sirainen <tss@iki.fi> parents: 4654 diff changeset	80 i_assert(fields[i].type < MAIL_CACHE_FIELD_COUNT);
b0cf3e4d6c96 If cache file contains broken field type or decision type mark the cache Timo Sirainen <tss@iki.fi> parents: 4654 diff changeset	81
2327 7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: diff changeset	82 fields[i].idx =
7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: diff changeset	83 POINTER_CAST_TO(orig_value, unsigned int);
4654 35d9433de1ca When registering fields, if they were already exited in the cache file mark Timo Sirainen <tss@iki.fi> parents: 4345 diff changeset	84 (void)field_type_verify(cache, fields[i].idx,
35d9433de1ca When registering fields, if they were already exited in the cache file mark Timo Sirainen <tss@iki.fi> parents: 4345 diff changeset	85 fields[i].type,
35d9433de1ca When registering fields, if they were already exited in the cache file mark Timo Sirainen <tss@iki.fi> parents: 4345 diff changeset	86 fields[i].field_size);
2327 7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: diff changeset	87 continue;
7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: diff changeset	88 }
7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: diff changeset	89
3999 f1e29736e60d Don't register cache fields twice. Fixes "duplicate field in header" errors. Timo Sirainen <tss@iki.fi> parents: 3627 diff changeset	90 /* check if the same header is being registered in the
f1e29736e60d Don't register cache fields twice. Fixes "duplicate field in header" errors. Timo Sirainen <tss@iki.fi> parents: 3627 diff changeset	91 same field array */
f1e29736e60d Don't register cache fields twice. Fixes "duplicate field in header" errors. Timo Sirainen <tss@iki.fi> parents: 3627 diff changeset	92 for (j = 0; j < i; j++) {
f1e29736e60d Don't register cache fields twice. Fixes "duplicate field in header" errors. Timo Sirainen <tss@iki.fi> parents: 3627 diff changeset	93 if (strcasecmp(fields[i].name, fields[j].name) == 0) {
f1e29736e60d Don't register cache fields twice. Fixes "duplicate field in header" errors. Timo Sirainen <tss@iki.fi> parents: 3627 diff changeset	94 fields[i].idx = fields[j].idx;
f1e29736e60d Don't register cache fields twice. Fixes "duplicate field in header" errors. Timo Sirainen <tss@iki.fi> parents: 3627 diff changeset	95 break;
f1e29736e60d Don't register cache fields twice. Fixes "duplicate field in header" errors. Timo Sirainen <tss@iki.fi> parents: 3627 diff changeset	96 }
f1e29736e60d Don't register cache fields twice. Fixes "duplicate field in header" errors. Timo Sirainen <tss@iki.fi> parents: 3627 diff changeset	97 }
f1e29736e60d Don't register cache fields twice. Fixes "duplicate field in header" errors. Timo Sirainen <tss@iki.fi> parents: 3627 diff changeset	98
f1e29736e60d Don't register cache fields twice. Fixes "duplicate field in header" errors. Timo Sirainen <tss@iki.fi> parents: 3627 diff changeset	99 if (j == i)
f1e29736e60d Don't register cache fields twice. Fixes "duplicate field in header" errors. Timo Sirainen <tss@iki.fi> parents: 3627 diff changeset	100 fields[i].idx = new_idx++;
2327 7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: diff changeset	101 }
7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: diff changeset	102
7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: diff changeset	103 if (new_idx == cache->fields_count)
7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: diff changeset	104 return;
7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: diff changeset	105
7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: diff changeset	106 /* @UNSAFE */
2362 f2acbe281ac1 Allocate cache->fields and cache->fields_file_map with malloc rather than Timo Sirainen <tss@iki.fi> parents: 2342 diff changeset	107 cache->fields = i_realloc(cache->fields,
2327 7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: diff changeset	108 cache->fields_count * sizeof(*cache->fields),
7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: diff changeset	109 new_idx * sizeof(*cache->fields));
7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: diff changeset	110 cache->field_file_map =
2362 f2acbe281ac1 Allocate cache->fields and cache->fields_file_map with malloc rather than Timo Sirainen <tss@iki.fi> parents: 2342 diff changeset	111 i_realloc(cache->field_file_map,
2327 7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: diff changeset	112 cache->fields_count * sizeof(*cache->field_file_map),
7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: diff changeset	113 new_idx * sizeof(*cache->field_file_map));
7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: diff changeset	114
7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: diff changeset	115 for (i = 0; i < fields_count; i++) {
7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: diff changeset	116 unsigned int idx = fields[i].idx;
7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: diff changeset	117
7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: diff changeset	118 if (idx < cache->fields_count)
7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: diff changeset	119 continue;
7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: diff changeset	120
7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: diff changeset	121 /* new index - save it */
4020 fcfd44f56b04 While casting const pointers to something else, the const was often Timo Sirainen <tss@iki.fi> parents: 3999 diff changeset	122 name = p_strdup(cache->field_pool, fields[i].name);
2339 406692edc12d Cache fixes. Decisions are saved again. Timo Sirainen <tss@iki.fi> parents: 2327 diff changeset	123 cache->fields[idx].field = fields[i];
4020 fcfd44f56b04 While casting const pointers to something else, the const was often Timo Sirainen <tss@iki.fi> parents: 3999 diff changeset	124 cache->fields[idx].field.name = name;
2327 7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: diff changeset	125 cache->field_file_map[idx] = (uint32_t)-1;
7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: diff changeset	126
4654 35d9433de1ca When registering fields, if they were already exited in the cache file mark Timo Sirainen <tss@iki.fi> parents: 4345 diff changeset	127 if (!field_has_fixed_size(cache->fields[idx].field.type))
2339 406692edc12d Cache fixes. Decisions are saved again. Timo Sirainen <tss@iki.fi> parents: 2327 diff changeset	128 cache->fields[idx].field.field_size = (unsigned int)-1;
2327 7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: diff changeset	129
4020 fcfd44f56b04 While casting const pointers to something else, the const was often Timo Sirainen <tss@iki.fi> parents: 3999 diff changeset	130 hash_insert(cache->field_name_hash, name, POINTER_CAST(idx));
2327 7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: diff changeset	131 }
7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: diff changeset	132 cache->fields_count = new_idx;
7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: diff changeset	133 }
7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: diff changeset	134
7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: diff changeset	135 unsigned int
7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: diff changeset	136 mail_cache_register_lookup(struct mail_cache cache, const char name)
7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: diff changeset	137 {
7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: diff changeset	138 void orig_key, orig_value;
7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: diff changeset	139
7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: diff changeset	140 if (hash_lookup_full(cache->field_name_hash, name,
7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: diff changeset	141 &orig_key, &orig_value))
7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: diff changeset	142 return POINTER_CAST_TO(orig_value, unsigned int);
7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: diff changeset	143 else
7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: diff changeset	144 return (unsigned int)-1;
7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: diff changeset	145 }
7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: diff changeset	146
3461 b7ce2532250a Added mail_cache_register_get_list() to get a list of all registered fields Timo Sirainen <tss@iki.fi> parents: 3375 diff changeset	147 const struct mail_cache_field *
b7ce2532250a Added mail_cache_register_get_list() to get a list of all registered fields Timo Sirainen <tss@iki.fi> parents: 3375 diff changeset	148 mail_cache_register_get_list(struct mail_cache *cache, pool_t pool,
b7ce2532250a Added mail_cache_register_get_list() to get a list of all registered fields Timo Sirainen <tss@iki.fi> parents: 3375 diff changeset	149 unsigned int *count_r)
b7ce2532250a Added mail_cache_register_get_list() to get a list of all registered fields Timo Sirainen <tss@iki.fi> parents: 3375 diff changeset	150 {
b7ce2532250a Added mail_cache_register_get_list() to get a list of all registered fields Timo Sirainen <tss@iki.fi> parents: 3375 diff changeset	151 struct mail_cache_field *list;
b7ce2532250a Added mail_cache_register_get_list() to get a list of all registered fields Timo Sirainen <tss@iki.fi> parents: 3375 diff changeset	152 unsigned int i;
b7ce2532250a Added mail_cache_register_get_list() to get a list of all registered fields Timo Sirainen <tss@iki.fi> parents: 3375 diff changeset	153
6003 6f66ce9491ad When getting cache fields list, make sure cache file is opened. Timo Sirainen <tss@iki.fi> parents: 5952 diff changeset	154 if (!cache->opened)
6f66ce9491ad When getting cache fields list, make sure cache file is opened. Timo Sirainen <tss@iki.fi> parents: 5952 diff changeset	155 (void)mail_cache_open_and_verify(cache);
6f66ce9491ad When getting cache fields list, make sure cache file is opened. Timo Sirainen <tss@iki.fi> parents: 5952 diff changeset	156
3461 b7ce2532250a Added mail_cache_register_get_list() to get a list of all registered fields Timo Sirainen <tss@iki.fi> parents: 3375 diff changeset	157 list = p_new(pool, struct mail_cache_field, cache->fields_count);
3494 7cab5dc6a0cd mail_cache_register_get_list(): Allocate the returned strings also from Timo Sirainen <tss@iki.fi> parents: 3461 diff changeset	158 for (i = 0; i < cache->fields_count; i++) {
3461 b7ce2532250a Added mail_cache_register_get_list() to get a list of all registered fields Timo Sirainen <tss@iki.fi> parents: 3375 diff changeset	159 list[i] = cache->fields[i].field;
3494 7cab5dc6a0cd mail_cache_register_get_list(): Allocate the returned strings also from Timo Sirainen <tss@iki.fi> parents: 3461 diff changeset	160 list[i].name = p_strdup(pool, list[i].name);
7cab5dc6a0cd mail_cache_register_get_list(): Allocate the returned strings also from Timo Sirainen <tss@iki.fi> parents: 3461 diff changeset	161 }
3461 b7ce2532250a Added mail_cache_register_get_list() to get a list of all registered fields Timo Sirainen <tss@iki.fi> parents: 3375 diff changeset	162
b7ce2532250a Added mail_cache_register_get_list() to get a list of all registered fields Timo Sirainen <tss@iki.fi> parents: 3375 diff changeset	163 *count_r = cache->fields_count;
b7ce2532250a Added mail_cache_register_get_list() to get a list of all registered fields Timo Sirainen <tss@iki.fi> parents: 3375 diff changeset	164 return list;
b7ce2532250a Added mail_cache_register_get_list() to get a list of all registered fields Timo Sirainen <tss@iki.fi> parents: 3375 diff changeset	165 }
b7ce2532250a Added mail_cache_register_get_list() to get a list of all registered fields Timo Sirainen <tss@iki.fi> parents: 3375 diff changeset	166
2327 7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: diff changeset	167 static int mail_cache_header_fields_get_offset(struct mail_cache *cache,
6704 b0e8403b4bb1 mmap_disable=yes: Find the latest cache field header by pread()ing the file Timo Sirainen <tss@iki.fi> parents: 6695 diff changeset	168 uint32_t *offset_r, bool map)
2327 7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: diff changeset	169 {
7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: diff changeset	170 const struct mail_cache_header_fields *field_hdr;
6704 b0e8403b4bb1 mmap_disable=yes: Find the latest cache field header by pread()ing the file Timo Sirainen <tss@iki.fi> parents: 6695 diff changeset	171 struct mail_cache_header_fields tmp_field_hdr;
6705 68bb5bec043e When reading the latest fields, begin finding the fields from the last known Timo Sirainen <tss@iki.fi> parents: 6704 diff changeset	172 uint32_t offset = 0, next_offset;
6695 76c0e3a4df7a If we need to do more than 4 reads to get to the cached fields header, Timo Sirainen <tss@iki.fi> parents: 6694 diff changeset	173 unsigned int next_count = 0;
6704 b0e8403b4bb1 mmap_disable=yes: Find the latest cache field header by pread()ing the file Timo Sirainen <tss@iki.fi> parents: 6695 diff changeset	174 int ret;
2327 7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: diff changeset	175
7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: diff changeset	176 if (MAIL_CACHE_IS_UNUSABLE(cache)) {
7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: diff changeset	177 *offset_r = 0;
7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: diff changeset	178 return 0;
7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: diff changeset	179 }
7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: diff changeset	180
7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: diff changeset	181 /* find the latest header */
7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: diff changeset	182 offset = 0;
6705 68bb5bec043e When reading the latest fields, begin finding the fields from the last known Timo Sirainen <tss@iki.fi> parents: 6704 diff changeset	183 next_offset = cache->last_field_header_offset != 0 ?
68bb5bec043e When reading the latest fields, begin finding the fields from the last known Timo Sirainen <tss@iki.fi> parents: 6704 diff changeset	184 cache->last_field_header_offset :
2407 fed6d07bd8ee Transaction log file is now read-lockless. Timo Sirainen <tss@iki.fi> parents: 2362 diff changeset	185 mail_index_offset_to_uint32(cache->hdr->field_header_offset);
2327 7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: diff changeset	186 while (next_offset != 0) {
2339 406692edc12d Cache fixes. Decisions are saved again. Timo Sirainen <tss@iki.fi> parents: 2327 diff changeset	187 if (next_offset == offset) {
406692edc12d Cache fixes. Decisions are saved again. Timo Sirainen <tss@iki.fi> parents: 2327 diff changeset	188 mail_cache_set_corrupted(cache,
406692edc12d Cache fixes. Decisions are saved again. Timo Sirainen <tss@iki.fi> parents: 2327 diff changeset	189 "next_offset in field header loops");
406692edc12d Cache fixes. Decisions are saved again. Timo Sirainen <tss@iki.fi> parents: 2327 diff changeset	190 return -1;
406692edc12d Cache fixes. Decisions are saved again. Timo Sirainen <tss@iki.fi> parents: 2327 diff changeset	191 }
2327 7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: diff changeset	192 offset = next_offset;
7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: diff changeset	193
6704 b0e8403b4bb1 mmap_disable=yes: Find the latest cache field header by pread()ing the file Timo Sirainen <tss@iki.fi> parents: 6695 diff changeset	194 if (cache->mmap_base != NULL) {
b0e8403b4bb1 mmap_disable=yes: Find the latest cache field header by pread()ing the file Timo Sirainen <tss@iki.fi> parents: 6695 diff changeset	195 if (mail_cache_map(cache, offset,
b0e8403b4bb1 mmap_disable=yes: Find the latest cache field header by pread()ing the file Timo Sirainen <tss@iki.fi> parents: 6695 diff changeset	196 sizeof(*field_hdr)) < 0)
b0e8403b4bb1 mmap_disable=yes: Find the latest cache field header by pread()ing the file Timo Sirainen <tss@iki.fi> parents: 6695 diff changeset	197 return -1;
6694 c7742d109a67 mmap_disable=yes: When following cache field headers, don't invalidate the Timo Sirainen <tss@iki.fi> parents: 6443 diff changeset	198
6704 b0e8403b4bb1 mmap_disable=yes: Find the latest cache field header by pread()ing the file Timo Sirainen <tss@iki.fi> parents: 6695 diff changeset	199 field_hdr = CONST_PTR_OFFSET(cache->data, offset);
b0e8403b4bb1 mmap_disable=yes: Find the latest cache field header by pread()ing the file Timo Sirainen <tss@iki.fi> parents: 6695 diff changeset	200 } else {
b0e8403b4bb1 mmap_disable=yes: Find the latest cache field header by pread()ing the file Timo Sirainen <tss@iki.fi> parents: 6695 diff changeset	201 /* if we need to follow multiple offsets to get to
b0e8403b4bb1 mmap_disable=yes: Find the latest cache field header by pread()ing the file Timo Sirainen <tss@iki.fi> parents: 6695 diff changeset	202 the last one, it's faster to just pread() the file
b0e8403b4bb1 mmap_disable=yes: Find the latest cache field header by pread()ing the file Timo Sirainen <tss@iki.fi> parents: 6695 diff changeset	203 instead of going through cache */
b0e8403b4bb1 mmap_disable=yes: Find the latest cache field header by pread()ing the file Timo Sirainen <tss@iki.fi> parents: 6695 diff changeset	204 ret = pread_full(cache->fd, &tmp_field_hdr,
b0e8403b4bb1 mmap_disable=yes: Find the latest cache field header by pread()ing the file Timo Sirainen <tss@iki.fi> parents: 6695 diff changeset	205 sizeof(tmp_field_hdr), offset);
b0e8403b4bb1 mmap_disable=yes: Find the latest cache field header by pread()ing the file Timo Sirainen <tss@iki.fi> parents: 6695 diff changeset	206 if (ret < 0) {
b0e8403b4bb1 mmap_disable=yes: Find the latest cache field header by pread()ing the file Timo Sirainen <tss@iki.fi> parents: 6695 diff changeset	207 mail_cache_set_syscall_error(cache, "pread()");
b0e8403b4bb1 mmap_disable=yes: Find the latest cache field header by pread()ing the file Timo Sirainen <tss@iki.fi> parents: 6695 diff changeset	208 return -1;
b0e8403b4bb1 mmap_disable=yes: Find the latest cache field header by pread()ing the file Timo Sirainen <tss@iki.fi> parents: 6695 diff changeset	209 }
b0e8403b4bb1 mmap_disable=yes: Find the latest cache field header by pread()ing the file Timo Sirainen <tss@iki.fi> parents: 6695 diff changeset	210 if (ret == 0) {
b0e8403b4bb1 mmap_disable=yes: Find the latest cache field header by pread()ing the file Timo Sirainen <tss@iki.fi> parents: 6695 diff changeset	211 mail_cache_set_corrupted(cache,
b0e8403b4bb1 mmap_disable=yes: Find the latest cache field header by pread()ing the file Timo Sirainen <tss@iki.fi> parents: 6695 diff changeset	212 "next_offset points outside file");
b0e8403b4bb1 mmap_disable=yes: Find the latest cache field header by pread()ing the file Timo Sirainen <tss@iki.fi> parents: 6695 diff changeset	213 return -1;
b0e8403b4bb1 mmap_disable=yes: Find the latest cache field header by pread()ing the file Timo Sirainen <tss@iki.fi> parents: 6695 diff changeset	214 }
b0e8403b4bb1 mmap_disable=yes: Find the latest cache field header by pread()ing the file Timo Sirainen <tss@iki.fi> parents: 6695 diff changeset	215 field_hdr = &tmp_field_hdr;
6694 c7742d109a67 mmap_disable=yes: When following cache field headers, don't invalidate the Timo Sirainen <tss@iki.fi> parents: 6443 diff changeset	216 }
c7742d109a67 mmap_disable=yes: When following cache field headers, don't invalidate the Timo Sirainen <tss@iki.fi> parents: 6443 diff changeset	217
2327 7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: diff changeset	218 next_offset =
2407 fed6d07bd8ee Transaction log file is now read-lockless. Timo Sirainen <tss@iki.fi> parents: 2362 diff changeset	219 mail_index_offset_to_uint32(field_hdr->next_offset);
6695 76c0e3a4df7a If we need to do more than 4 reads to get to the cached fields header, Timo Sirainen <tss@iki.fi> parents: 6694 diff changeset	220 next_count++;
2327 7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: diff changeset	221 }
7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: diff changeset	222
6705 68bb5bec043e When reading the latest fields, begin finding the fields from the last known Timo Sirainen <tss@iki.fi> parents: 6704 diff changeset	223 if (offset == 0) {
68bb5bec043e When reading the latest fields, begin finding the fields from the last known Timo Sirainen <tss@iki.fi> parents: 6704 diff changeset	224 mail_cache_set_corrupted(cache, "missing header fields");
68bb5bec043e When reading the latest fields, begin finding the fields from the last known Timo Sirainen <tss@iki.fi> parents: 6704 diff changeset	225 return -1;
68bb5bec043e When reading the latest fields, begin finding the fields from the last known Timo Sirainen <tss@iki.fi> parents: 6704 diff changeset	226 }
68bb5bec043e When reading the latest fields, begin finding the fields from the last known Timo Sirainen <tss@iki.fi> parents: 6704 diff changeset	227 cache->last_field_header_offset = offset;
68bb5bec043e When reading the latest fields, begin finding the fields from the last known Timo Sirainen <tss@iki.fi> parents: 6704 diff changeset	228
6695 76c0e3a4df7a If we need to do more than 4 reads to get to the cached fields header, Timo Sirainen <tss@iki.fi> parents: 6694 diff changeset	229 if (next_count > MAIL_CACHE_HEADER_FIELD_CONTINUE_COUNT)
76c0e3a4df7a If we need to do more than 4 reads to get to the cached fields header, Timo Sirainen <tss@iki.fi> parents: 6694 diff changeset	230 cache->need_compress_file_seq = cache->hdr->file_seq;
76c0e3a4df7a If we need to do more than 4 reads to get to the cached fields header, Timo Sirainen <tss@iki.fi> parents: 6694 diff changeset	231
6704 b0e8403b4bb1 mmap_disable=yes: Find the latest cache field header by pread()ing the file Timo Sirainen <tss@iki.fi> parents: 6695 diff changeset	232 if (map) {
b0e8403b4bb1 mmap_disable=yes: Find the latest cache field header by pread()ing the file Timo Sirainen <tss@iki.fi> parents: 6695 diff changeset	233 if (cache->file_cache != NULL) {
b0e8403b4bb1 mmap_disable=yes: Find the latest cache field header by pread()ing the file Timo Sirainen <tss@iki.fi> parents: 6695 diff changeset	234 /* we can't trust that the cached data is valid */
b0e8403b4bb1 mmap_disable=yes: Find the latest cache field header by pread()ing the file Timo Sirainen <tss@iki.fi> parents: 6695 diff changeset	235 file_cache_invalidate(cache->file_cache, offset,
b0e8403b4bb1 mmap_disable=yes: Find the latest cache field header by pread()ing the file Timo Sirainen <tss@iki.fi> parents: 6695 diff changeset	236 field_hdr->size);
b0e8403b4bb1 mmap_disable=yes: Find the latest cache field header by pread()ing the file Timo Sirainen <tss@iki.fi> parents: 6695 diff changeset	237 }
b0e8403b4bb1 mmap_disable=yes: Find the latest cache field header by pread()ing the file Timo Sirainen <tss@iki.fi> parents: 6695 diff changeset	238 if (mail_cache_map(cache, offset, field_hdr->size) < 0)
b0e8403b4bb1 mmap_disable=yes: Find the latest cache field header by pread()ing the file Timo Sirainen <tss@iki.fi> parents: 6695 diff changeset	239 return -1;
b0e8403b4bb1 mmap_disable=yes: Find the latest cache field header by pread()ing the file Timo Sirainen <tss@iki.fi> parents: 6695 diff changeset	240 }
b0e8403b4bb1 mmap_disable=yes: Find the latest cache field header by pread()ing the file Timo Sirainen <tss@iki.fi> parents: 6695 diff changeset	241
2327 7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: diff changeset	242 *offset_r = offset;
7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: diff changeset	243 return 0;
7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: diff changeset	244 }
7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: diff changeset	245
7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: diff changeset	246 int mail_cache_header_fields_read(struct mail_cache *cache)
7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: diff changeset	247 {
7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: diff changeset	248 const struct mail_cache_header_fields *field_hdr = NULL;
7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: diff changeset	249 struct mail_cache_field field;
7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: diff changeset	250 const uint32_t last_used, sizes;
7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: diff changeset	251 const uint8_t types, decisions;
7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: diff changeset	252 const char p, names, *end;
2339 406692edc12d Cache fixes. Decisions are saved again. Timo Sirainen <tss@iki.fi> parents: 2327 diff changeset	253 void orig_key, orig_value;
5952 ba118a9eeb50 Some paranoia fixes for memory allocation. Don't access a mmaped variable Timo Sirainen <tss@iki.fi> parents: 5947 diff changeset	254 unsigned int new_fields_count;
6296 205ee38f10d1 Drop fields that haven't been used for 30 days when compressing. Timo Sirainen <tss@iki.fi> parents: 6003 diff changeset	255 time_t max_drop_time;
2327 7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: diff changeset	256 uint32_t offset, i;
7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: diff changeset	257
6704 b0e8403b4bb1 mmap_disable=yes: Find the latest cache field header by pread()ing the file Timo Sirainen <tss@iki.fi> parents: 6695 diff changeset	258 if (mail_cache_header_fields_get_offset(cache, &offset, TRUE) < 0)
2327 7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: diff changeset	259 return -1;
7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: diff changeset	260
7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: diff changeset	261 if (offset == 0) {
7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: diff changeset	262 /* no fields - the file is empty */
7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: diff changeset	263 return 0;
7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: diff changeset	264 }
7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: diff changeset	265
2866 bf1e718e7370 Cache file works now with mmap_disable=yes. Still needs a few optimizations. Timo Sirainen <tss@iki.fi> parents: 2708 diff changeset	266 field_hdr = CONST_PTR_OFFSET(cache->data, offset);
2327 7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: diff changeset	267 if (offset + field_hdr->size > cache->mmap_length) {
7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: diff changeset	268 mail_cache_set_corrupted(cache,
7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: diff changeset	269 "field header points outside file");
7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: diff changeset	270 return -1;
7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: diff changeset	271 }
7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: diff changeset	272
7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: diff changeset	273 /* check the fixed size of the header. name[] has to be checked
7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: diff changeset	274 separately */
7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: diff changeset	275 if (field_hdr->size < sizeof(*field_hdr) +
7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: diff changeset	276 field_hdr->fields_count * (sizeof(uint32_t)*2 + 1 + 2)) {
7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: diff changeset	277 mail_cache_set_corrupted(cache, "invalid field header size");
7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: diff changeset	278 return -1;
7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: diff changeset	279 }
7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: diff changeset	280
2866 bf1e718e7370 Cache file works now with mmap_disable=yes. Still needs a few optimizations. Timo Sirainen <tss@iki.fi> parents: 2708 diff changeset	281 field_hdr = CONST_PTR_OFFSET(cache->data, offset);
5952 ba118a9eeb50 Some paranoia fixes for memory allocation. Don't access a mmaped variable Timo Sirainen <tss@iki.fi> parents: 5947 diff changeset	282 new_fields_count = field_hdr->fields_count;
2327 7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: diff changeset	283
5952 ba118a9eeb50 Some paranoia fixes for memory allocation. Don't access a mmaped variable Timo Sirainen <tss@iki.fi> parents: 5947 diff changeset	284 if (new_fields_count != 0) {
4110 9662de8fc649 Don't crash with "can't allocate 0 bytes of memory" if there are zero Timo Sirainen <tss@iki.fi> parents: 4020 diff changeset	285 cache->file_field_map =
9662de8fc649 Don't crash with "can't allocate 0 bytes of memory" if there are zero Timo Sirainen <tss@iki.fi> parents: 4020 diff changeset	286 i_realloc(cache->file_field_map,
9662de8fc649 Don't crash with "can't allocate 0 bytes of memory" if there are zero Timo Sirainen <tss@iki.fi> parents: 4020 diff changeset	287 cache->file_fields_count *
9662de8fc649 Don't crash with "can't allocate 0 bytes of memory" if there are zero Timo Sirainen <tss@iki.fi> parents: 4020 diff changeset	288 sizeof(unsigned int),
5952 ba118a9eeb50 Some paranoia fixes for memory allocation. Don't access a mmaped variable Timo Sirainen <tss@iki.fi> parents: 5947 diff changeset	289 new_fields_count * sizeof(unsigned int));
4110 9662de8fc649 Don't crash with "can't allocate 0 bytes of memory" if there are zero Timo Sirainen <tss@iki.fi> parents: 4020 diff changeset	290 } else {
9662de8fc649 Don't crash with "can't allocate 0 bytes of memory" if there are zero Timo Sirainen <tss@iki.fi> parents: 4020 diff changeset	291 i_free_and_null(cache->file_field_map);
9662de8fc649 Don't crash with "can't allocate 0 bytes of memory" if there are zero Timo Sirainen <tss@iki.fi> parents: 4020 diff changeset	292 }
5952 ba118a9eeb50 Some paranoia fixes for memory allocation. Don't access a mmaped variable Timo Sirainen <tss@iki.fi> parents: 5947 diff changeset	293 cache->file_fields_count = new_fields_count;
2327 7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: diff changeset	294
2339 406692edc12d Cache fixes. Decisions are saved again. Timo Sirainen <tss@iki.fi> parents: 2327 diff changeset	295 last_used = CONST_PTR_OFFSET(field_hdr, MAIL_CACHE_FIELD_LAST_USED());
406692edc12d Cache fixes. Decisions are saved again. Timo Sirainen <tss@iki.fi> parents: 2327 diff changeset	296 sizes = CONST_PTR_OFFSET(field_hdr,
406692edc12d Cache fixes. Decisions are saved again. Timo Sirainen <tss@iki.fi> parents: 2327 diff changeset	297 MAIL_CACHE_FIELD_SIZE(field_hdr->fields_count));
406692edc12d Cache fixes. Decisions are saved again. Timo Sirainen <tss@iki.fi> parents: 2327 diff changeset	298 types = CONST_PTR_OFFSET(field_hdr,
406692edc12d Cache fixes. Decisions are saved again. Timo Sirainen <tss@iki.fi> parents: 2327 diff changeset	299 MAIL_CACHE_FIELD_TYPE(field_hdr->fields_count));
406692edc12d Cache fixes. Decisions are saved again. Timo Sirainen <tss@iki.fi> parents: 2327 diff changeset	300 decisions = CONST_PTR_OFFSET(field_hdr,
406692edc12d Cache fixes. Decisions are saved again. Timo Sirainen <tss@iki.fi> parents: 2327 diff changeset	301 MAIL_CACHE_FIELD_DECISION(field_hdr->fields_count));
406692edc12d Cache fixes. Decisions are saved again. Timo Sirainen <tss@iki.fi> parents: 2327 diff changeset	302 names = CONST_PTR_OFFSET(field_hdr,
406692edc12d Cache fixes. Decisions are saved again. Timo Sirainen <tss@iki.fi> parents: 2327 diff changeset	303 MAIL_CACHE_FIELD_NAMES(field_hdr->fields_count));
2327 7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: diff changeset	304 end = CONST_PTR_OFFSET(field_hdr, field_hdr->size);
7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: diff changeset	305
7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: diff changeset	306 /* clear the old mapping */
7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: diff changeset	307 for (i = 0; i < cache->fields_count; i++)
7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: diff changeset	308 cache->field_file_map[i] = (uint32_t)-1;
7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: diff changeset	309
6296 205ee38f10d1 Drop fields that haven't been used for 30 days when compressing. Timo Sirainen <tss@iki.fi> parents: 6003 diff changeset	310 max_drop_time = cache->index->map->hdr.day_stamp -
205ee38f10d1 Drop fields that haven't been used for 30 days when compressing. Timo Sirainen <tss@iki.fi> parents: 6003 diff changeset	311 MAIL_CACHE_FIELD_DROP_SECS;
205ee38f10d1 Drop fields that haven't been used for 30 days when compressing. Timo Sirainen <tss@iki.fi> parents: 6003 diff changeset	312
2327 7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: diff changeset	313 memset(&field, 0, sizeof(field));
7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: diff changeset	314 for (i = 0; i < field_hdr->fields_count; i++) {
7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: diff changeset	315 for (p = names; p != end && *p != '\0'; p++) ;
2339 406692edc12d Cache fixes. Decisions are saved again. Timo Sirainen <tss@iki.fi> parents: 2327 diff changeset	316 if (p == end \|\| *names == '\0') {
2327 7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: diff changeset	317 mail_cache_set_corrupted(cache,
7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: diff changeset	318 "field header names corrupted");
7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: diff changeset	319 return -1;
7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: diff changeset	320 }
7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: diff changeset	321
4722 b0cf3e4d6c96 If cache file contains broken field type or decision type mark the cache Timo Sirainen <tss@iki.fi> parents: 4654 diff changeset	322 if (types[i] > MAIL_CACHE_FIELD_COUNT) {
b0cf3e4d6c96 If cache file contains broken field type or decision type mark the cache Timo Sirainen <tss@iki.fi> parents: 4654 diff changeset	323 mail_cache_set_corrupted(cache, "field type corrupted");
b0cf3e4d6c96 If cache file contains broken field type or decision type mark the cache Timo Sirainen <tss@iki.fi> parents: 4654 diff changeset	324 return -1;
b0cf3e4d6c96 If cache file contains broken field type or decision type mark the cache Timo Sirainen <tss@iki.fi> parents: 4654 diff changeset	325 }
b0cf3e4d6c96 If cache file contains broken field type or decision type mark the cache Timo Sirainen <tss@iki.fi> parents: 4654 diff changeset	326 if (!field_decision_is_valid(decisions[i])) {
b0cf3e4d6c96 If cache file contains broken field type or decision type mark the cache Timo Sirainen <tss@iki.fi> parents: 4654 diff changeset	327 mail_cache_set_corrupted(cache,
b0cf3e4d6c96 If cache file contains broken field type or decision type mark the cache Timo Sirainen <tss@iki.fi> parents: 4654 diff changeset	328 "field decision type corrupted");
b0cf3e4d6c96 If cache file contains broken field type or decision type mark the cache Timo Sirainen <tss@iki.fi> parents: 4654 diff changeset	329 return -1;
b0cf3e4d6c96 If cache file contains broken field type or decision type mark the cache Timo Sirainen <tss@iki.fi> parents: 4654 diff changeset	330 }
b0cf3e4d6c96 If cache file contains broken field type or decision type mark the cache Timo Sirainen <tss@iki.fi> parents: 4654 diff changeset	331
2339 406692edc12d Cache fixes. Decisions are saved again. Timo Sirainen <tss@iki.fi> parents: 2327 diff changeset	332 if (hash_lookup_full(cache->field_name_hash, names,
406692edc12d Cache fixes. Decisions are saved again. Timo Sirainen <tss@iki.fi> parents: 2327 diff changeset	333 &orig_key, &orig_value)) {
406692edc12d Cache fixes. Decisions are saved again. Timo Sirainen <tss@iki.fi> parents: 2327 diff changeset	334 /* already exists, see if decision can be updated */
406692edc12d Cache fixes. Decisions are saved again. Timo Sirainen <tss@iki.fi> parents: 2327 diff changeset	335 field.idx = POINTER_CAST_TO(orig_value, unsigned int);
406692edc12d Cache fixes. Decisions are saved again. Timo Sirainen <tss@iki.fi> parents: 2327 diff changeset	336 if (!cache->fields[field.idx].decision_dirty) {
406692edc12d Cache fixes. Decisions are saved again. Timo Sirainen <tss@iki.fi> parents: 2327 diff changeset	337 cache->fields[field.idx].field.decision =
406692edc12d Cache fixes. Decisions are saved again. Timo Sirainen <tss@iki.fi> parents: 2327 diff changeset	338 decisions[i];
406692edc12d Cache fixes. Decisions are saved again. Timo Sirainen <tss@iki.fi> parents: 2327 diff changeset	339 }
4654 35d9433de1ca When registering fields, if they were already exited in the cache file mark Timo Sirainen <tss@iki.fi> parents: 4345 diff changeset	340 if (field_type_verify(cache, field.idx,
35d9433de1ca When registering fields, if they were already exited in the cache file mark Timo Sirainen <tss@iki.fi> parents: 4345 diff changeset	341 types[i], sizes[i]) < 0)
4345 f5d9c2b4d118 If cache field's size is wrong, rebuild the cache file instead of crashing Timo Sirainen <tss@iki.fi> parents: 4110 diff changeset	342 return -1;
2339 406692edc12d Cache fixes. Decisions are saved again. Timo Sirainen <tss@iki.fi> parents: 2327 diff changeset	343 } else {
406692edc12d Cache fixes. Decisions are saved again. Timo Sirainen <tss@iki.fi> parents: 2327 diff changeset	344 field.name = names;
406692edc12d Cache fixes. Decisions are saved again. Timo Sirainen <tss@iki.fi> parents: 2327 diff changeset	345 field.type = types[i];
406692edc12d Cache fixes. Decisions are saved again. Timo Sirainen <tss@iki.fi> parents: 2327 diff changeset	346 field.field_size = sizes[i];
406692edc12d Cache fixes. Decisions are saved again. Timo Sirainen <tss@iki.fi> parents: 2327 diff changeset	347 field.decision = decisions[i];
406692edc12d Cache fixes. Decisions are saved again. Timo Sirainen <tss@iki.fi> parents: 2327 diff changeset	348 mail_cache_register_fields(cache, &field, 1);
406692edc12d Cache fixes. Decisions are saved again. Timo Sirainen <tss@iki.fi> parents: 2327 diff changeset	349 }
2968 2ab037df6cf3 Check for duplicate fields in header. Timo Sirainen <tss@iki.fi> parents: 2929 diff changeset	350 if (cache->field_file_map[field.idx] != (uint32_t)-1) {
2ab037df6cf3 Check for duplicate fields in header. Timo Sirainen <tss@iki.fi> parents: 2929 diff changeset	351 mail_cache_set_corrupted(cache,
2ab037df6cf3 Check for duplicate fields in header. Timo Sirainen <tss@iki.fi> parents: 2929 diff changeset	352 "Duplicated field in header: %s", names);
2ab037df6cf3 Check for duplicate fields in header. Timo Sirainen <tss@iki.fi> parents: 2929 diff changeset	353 return -1;
2ab037df6cf3 Check for duplicate fields in header. Timo Sirainen <tss@iki.fi> parents: 2929 diff changeset	354 }
6296 205ee38f10d1 Drop fields that haven't been used for 30 days when compressing. Timo Sirainen <tss@iki.fi> parents: 6003 diff changeset	355 cache->fields[field.idx].used = TRUE;
205ee38f10d1 Drop fields that haven't been used for 30 days when compressing. Timo Sirainen <tss@iki.fi> parents: 6003 diff changeset	356
2327 7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: diff changeset	357 cache->field_file_map[field.idx] = i;
7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: diff changeset	358 cache->file_field_map[i] = field.idx;
7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: diff changeset	359
2339 406692edc12d Cache fixes. Decisions are saved again. Timo Sirainen <tss@iki.fi> parents: 2327 diff changeset	360 /* update last_used if it's newer than ours */
2342 fa529dd77176 Fixes for big endian systems. Timo Sirainen <tss@iki.fi> parents: 2339 diff changeset	361 if (last_used[i] > cache->fields[field.idx].last_used)
fa529dd77176 Fixes for big endian systems. Timo Sirainen <tss@iki.fi> parents: 2339 diff changeset	362 cache->fields[field.idx].last_used = last_used[i];
2339 406692edc12d Cache fixes. Decisions are saved again. Timo Sirainen <tss@iki.fi> parents: 2327 diff changeset	363
6707 4752637a6ad4 Don't bother compressing cache file just to drop fields from header that Timo Sirainen <tss@iki.fi> parents: 6705 diff changeset	364 if (cache->fields[field.idx].last_used < max_drop_time &&
4752637a6ad4 Don't bother compressing cache file just to drop fields from header that Timo Sirainen <tss@iki.fi> parents: 6705 diff changeset	365 cache->fields[field.idx].last_used != 0) {
4752637a6ad4 Don't bother compressing cache file just to drop fields from header that Timo Sirainen <tss@iki.fi> parents: 6705 diff changeset	366 /* time to drop this field. don't bother dropping
4752637a6ad4 Don't bother compressing cache file just to drop fields from header that Timo Sirainen <tss@iki.fi> parents: 6705 diff changeset	367 fields that have never been used. */
6296 205ee38f10d1 Drop fields that haven't been used for 30 days when compressing. Timo Sirainen <tss@iki.fi> parents: 6003 diff changeset	368 cache->need_compress_file_seq = cache->hdr->file_seq;
205ee38f10d1 Drop fields that haven't been used for 30 days when compressing. Timo Sirainen <tss@iki.fi> parents: 6003 diff changeset	369 }
205ee38f10d1 Drop fields that haven't been used for 30 days when compressing. Timo Sirainen <tss@iki.fi> parents: 6003 diff changeset	370
2339 406692edc12d Cache fixes. Decisions are saved again. Timo Sirainen <tss@iki.fi> parents: 2327 diff changeset	371 names = p + 1;
2327 7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: diff changeset	372 }
7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: diff changeset	373 return 0;
7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: diff changeset	374 }
7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: diff changeset	375
6443 53313d1e7e4f Code cleanups and some minor fixes. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	376 static void copy_to_buf(struct mail_cache cache, buffer_t dest, bool add_new,
2339 406692edc12d Cache fixes. Decisions are saved again. Timo Sirainen <tss@iki.fi> parents: 2327 diff changeset	377 size_t offset, size_t size)
406692edc12d Cache fixes. Decisions are saved again. Timo Sirainen <tss@iki.fi> parents: 2327 diff changeset	378 {
406692edc12d Cache fixes. Decisions are saved again. Timo Sirainen <tss@iki.fi> parents: 2327 diff changeset	379 const void *data;
406692edc12d Cache fixes. Decisions are saved again. Timo Sirainen <tss@iki.fi> parents: 2327 diff changeset	380 unsigned int i, field;
406692edc12d Cache fixes. Decisions are saved again. Timo Sirainen <tss@iki.fi> parents: 2327 diff changeset	381
6443 53313d1e7e4f Code cleanups and some minor fixes. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	382 /* copy the existing fields */
2339 406692edc12d Cache fixes. Decisions are saved again. Timo Sirainen <tss@iki.fi> parents: 2327 diff changeset	383 for (i = 0; i < cache->file_fields_count; i++) {
406692edc12d Cache fixes. Decisions are saved again. Timo Sirainen <tss@iki.fi> parents: 2327 diff changeset	384 field = cache->file_field_map[i];
406692edc12d Cache fixes. Decisions are saved again. Timo Sirainen <tss@iki.fi> parents: 2327 diff changeset	385 data = CONST_PTR_OFFSET(&cache->fields[field], offset);
406692edc12d Cache fixes. Decisions are saved again. Timo Sirainen <tss@iki.fi> parents: 2327 diff changeset	386 buffer_append(dest, data, size);
406692edc12d Cache fixes. Decisions are saved again. Timo Sirainen <tss@iki.fi> parents: 2327 diff changeset	387 }
6443 53313d1e7e4f Code cleanups and some minor fixes. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	388 if (!add_new)
53313d1e7e4f Code cleanups and some minor fixes. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	389 return;
53313d1e7e4f Code cleanups and some minor fixes. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	390
53313d1e7e4f Code cleanups and some minor fixes. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	391 /* copy newly wanted fields */
2339 406692edc12d Cache fixes. Decisions are saved again. Timo Sirainen <tss@iki.fi> parents: 2327 diff changeset	392 for (i = 0; i < cache->fields_count; i++) {
6443 53313d1e7e4f Code cleanups and some minor fixes. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	393 if (CACHE_FIELD_IS_NEWLY_WANTED(cache, i)) {
53313d1e7e4f Code cleanups and some minor fixes. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	394 data = CONST_PTR_OFFSET(&cache->fields[i], offset);
53313d1e7e4f Code cleanups and some minor fixes. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	395 buffer_append(dest, data, size);
53313d1e7e4f Code cleanups and some minor fixes. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	396 }
2339 406692edc12d Cache fixes. Decisions are saved again. Timo Sirainen <tss@iki.fi> parents: 2327 diff changeset	397 }
406692edc12d Cache fixes. Decisions are saved again. Timo Sirainen <tss@iki.fi> parents: 2327 diff changeset	398 }
406692edc12d Cache fixes. Decisions are saved again. Timo Sirainen <tss@iki.fi> parents: 2327 diff changeset	399
2342 fa529dd77176 Fixes for big endian systems. Timo Sirainen <tss@iki.fi> parents: 2339 diff changeset	400 static void copy_to_buf_byte(struct mail_cache cache, buffer_t dest,
6443 53313d1e7e4f Code cleanups and some minor fixes. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	401 bool add_new, size_t offset)
2342 fa529dd77176 Fixes for big endian systems. Timo Sirainen <tss@iki.fi> parents: 2339 diff changeset	402 {
fa529dd77176 Fixes for big endian systems. Timo Sirainen <tss@iki.fi> parents: 2339 diff changeset	403 const int *data;
fa529dd77176 Fixes for big endian systems. Timo Sirainen <tss@iki.fi> parents: 2339 diff changeset	404 unsigned int i, field;
fa529dd77176 Fixes for big endian systems. Timo Sirainen <tss@iki.fi> parents: 2339 diff changeset	405 uint8_t byte;
fa529dd77176 Fixes for big endian systems. Timo Sirainen <tss@iki.fi> parents: 2339 diff changeset	406
6443 53313d1e7e4f Code cleanups and some minor fixes. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	407 /* copy the existing fields */
2342 fa529dd77176 Fixes for big endian systems. Timo Sirainen <tss@iki.fi> parents: 2339 diff changeset	408 for (i = 0; i < cache->file_fields_count; i++) {
fa529dd77176 Fixes for big endian systems. Timo Sirainen <tss@iki.fi> parents: 2339 diff changeset	409 field = cache->file_field_map[i];
fa529dd77176 Fixes for big endian systems. Timo Sirainen <tss@iki.fi> parents: 2339 diff changeset	410 data = CONST_PTR_OFFSET(&cache->fields[field], offset);
fa529dd77176 Fixes for big endian systems. Timo Sirainen <tss@iki.fi> parents: 2339 diff changeset	411 byte = (uint8_t)*data;
fa529dd77176 Fixes for big endian systems. Timo Sirainen <tss@iki.fi> parents: 2339 diff changeset	412 buffer_append(dest, &byte, 1);
fa529dd77176 Fixes for big endian systems. Timo Sirainen <tss@iki.fi> parents: 2339 diff changeset	413 }
6443 53313d1e7e4f Code cleanups and some minor fixes. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	414 if (!add_new)
53313d1e7e4f Code cleanups and some minor fixes. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	415 return;
53313d1e7e4f Code cleanups and some minor fixes. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	416
53313d1e7e4f Code cleanups and some minor fixes. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	417 /* copy newly wanted fields */
2342 fa529dd77176 Fixes for big endian systems. Timo Sirainen <tss@iki.fi> parents: 2339 diff changeset	418 for (i = 0; i < cache->fields_count; i++) {
6443 53313d1e7e4f Code cleanups and some minor fixes. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	419 if (CACHE_FIELD_IS_NEWLY_WANTED(cache, i)) {
53313d1e7e4f Code cleanups and some minor fixes. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	420 data = CONST_PTR_OFFSET(&cache->fields[i], offset);
53313d1e7e4f Code cleanups and some minor fixes. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	421 byte = (uint8_t)*data;
53313d1e7e4f Code cleanups and some minor fixes. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	422 buffer_append(dest, &byte, 1);
53313d1e7e4f Code cleanups and some minor fixes. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	423 }
2342 fa529dd77176 Fixes for big endian systems. Timo Sirainen <tss@iki.fi> parents: 2339 diff changeset	424 }
fa529dd77176 Fixes for big endian systems. Timo Sirainen <tss@iki.fi> parents: 2339 diff changeset	425 }
fa529dd77176 Fixes for big endian systems. Timo Sirainen <tss@iki.fi> parents: 2339 diff changeset	426
2929 ba9062032877 Locking fixes and cleanups Timo Sirainen <tss@iki.fi> parents: 2867 diff changeset	427 static int mail_cache_header_fields_update_locked(struct mail_cache *cache)
2327 7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: diff changeset	428 {
2339 406692edc12d Cache fixes. Decisions are saved again. Timo Sirainen <tss@iki.fi> parents: 2327 diff changeset	429 buffer_t *buffer;
6443 53313d1e7e4f Code cleanups and some minor fixes. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	430 uint32_t i, offset, dec_offset;
2339 406692edc12d Cache fixes. Decisions are saved again. Timo Sirainen <tss@iki.fi> parents: 2327 diff changeset	431 int ret = 0;
2327 7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: diff changeset	432
2339 406692edc12d Cache fixes. Decisions are saved again. Timo Sirainen <tss@iki.fi> parents: 2327 diff changeset	433 if (mail_cache_header_fields_read(cache) < 0 \|\|
6704 b0e8403b4bb1 mmap_disable=yes: Find the latest cache field header by pread()ing the file Timo Sirainen <tss@iki.fi> parents: 6695 diff changeset	434 mail_cache_header_fields_get_offset(cache, &offset, FALSE) < 0)
2339 406692edc12d Cache fixes. Decisions are saved again. Timo Sirainen <tss@iki.fi> parents: 2327 diff changeset	435 return -1;
406692edc12d Cache fixes. Decisions are saved again. Timo Sirainen <tss@iki.fi> parents: 2327 diff changeset	436
406692edc12d Cache fixes. Decisions are saved again. Timo Sirainen <tss@iki.fi> parents: 2327 diff changeset	437 t_push();
2708 f1e9f3ec8135 Buffer API change: we no longer support limited sized buffers where Timo Sirainen <tss@iki.fi> parents: 2604 diff changeset	438 buffer = buffer_create_dynamic(pool_datastack_create(), 256);
2339 406692edc12d Cache fixes. Decisions are saved again. Timo Sirainen <tss@iki.fi> parents: 2327 diff changeset	439
6443 53313d1e7e4f Code cleanups and some minor fixes. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	440 copy_to_buf(cache, buffer, FALSE,
2339 406692edc12d Cache fixes. Decisions are saved again. Timo Sirainen <tss@iki.fi> parents: 2327 diff changeset	441 offsetof(struct mail_cache_field_private, last_used),
406692edc12d Cache fixes. Decisions are saved again. Timo Sirainen <tss@iki.fi> parents: 2327 diff changeset	442 sizeof(uint32_t));
6443 53313d1e7e4f Code cleanups and some minor fixes. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	443 ret = mail_cache_write(cache, buffer->data, buffer->used,
3375 9e2abc4c341c Whenever writing to cache file, also updated file_cache. Fixes problems with Timo Sirainen <tss@iki.fi> parents: 3278 diff changeset	444 offset + MAIL_CACHE_FIELD_LAST_USED());
2339 406692edc12d Cache fixes. Decisions are saved again. Timo Sirainen <tss@iki.fi> parents: 2327 diff changeset	445 if (ret == 0) {
406692edc12d Cache fixes. Decisions are saved again. Timo Sirainen <tss@iki.fi> parents: 2327 diff changeset	446 buffer_set_used_size(buffer, 0);
6443 53313d1e7e4f Code cleanups and some minor fixes. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	447 copy_to_buf_byte(cache, buffer, FALSE,
2342 fa529dd77176 Fixes for big endian systems. Timo Sirainen <tss@iki.fi> parents: 2339 diff changeset	448 offsetof(struct mail_cache_field, decision));
2339 406692edc12d Cache fixes. Decisions are saved again. Timo Sirainen <tss@iki.fi> parents: 2327 diff changeset	449
6443 53313d1e7e4f Code cleanups and some minor fixes. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	450 dec_offset = offset +
53313d1e7e4f Code cleanups and some minor fixes. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	451 MAIL_CACHE_FIELD_DECISION(cache->file_fields_count);
53313d1e7e4f Code cleanups and some minor fixes. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	452 ret = mail_cache_write(cache, buffer->data, buffer->used,
53313d1e7e4f Code cleanups and some minor fixes. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	453 dec_offset);
2339 406692edc12d Cache fixes. Decisions are saved again. Timo Sirainen <tss@iki.fi> parents: 2327 diff changeset	454 if (ret == 0) {
2592 5edda9783458 minor fix Timo Sirainen <tss@iki.fi> parents: 2407 diff changeset	455 for (i = 0; i < cache->file_fields_count; i++)
2339 406692edc12d Cache fixes. Decisions are saved again. Timo Sirainen <tss@iki.fi> parents: 2327 diff changeset	456 cache->fields[i].decision_dirty = FALSE;
406692edc12d Cache fixes. Decisions are saved again. Timo Sirainen <tss@iki.fi> parents: 2327 diff changeset	457 }
406692edc12d Cache fixes. Decisions are saved again. Timo Sirainen <tss@iki.fi> parents: 2327 diff changeset	458 }
406692edc12d Cache fixes. Decisions are saved again. Timo Sirainen <tss@iki.fi> parents: 2327 diff changeset	459 t_pop();
406692edc12d Cache fixes. Decisions are saved again. Timo Sirainen <tss@iki.fi> parents: 2327 diff changeset	460
2867 d416b6d0a7ee More correct file cache invalidating. Timo Sirainen <tss@iki.fi> parents: 2866 diff changeset	461 if (ret == 0)
2339 406692edc12d Cache fixes. Decisions are saved again. Timo Sirainen <tss@iki.fi> parents: 2327 diff changeset	462 cache->field_header_write_pending = FALSE;
2327 7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: diff changeset	463
2929 ba9062032877 Locking fixes and cleanups Timo Sirainen <tss@iki.fi> parents: 2867 diff changeset	464 return ret;
ba9062032877 Locking fixes and cleanups Timo Sirainen <tss@iki.fi> parents: 2867 diff changeset	465 }
ba9062032877 Locking fixes and cleanups Timo Sirainen <tss@iki.fi> parents: 2867 diff changeset	466
ba9062032877 Locking fixes and cleanups Timo Sirainen <tss@iki.fi> parents: 2867 diff changeset	467 int mail_cache_header_fields_update(struct mail_cache *cache)
ba9062032877 Locking fixes and cleanups Timo Sirainen <tss@iki.fi> parents: 2867 diff changeset	468 {
ba9062032877 Locking fixes and cleanups Timo Sirainen <tss@iki.fi> parents: 2867 diff changeset	469 int ret;
ba9062032877 Locking fixes and cleanups Timo Sirainen <tss@iki.fi> parents: 2867 diff changeset	470
ba9062032877 Locking fixes and cleanups Timo Sirainen <tss@iki.fi> parents: 2867 diff changeset	471 if (cache->locked)
ba9062032877 Locking fixes and cleanups Timo Sirainen <tss@iki.fi> parents: 2867 diff changeset	472 return mail_cache_header_fields_update_locked(cache);
ba9062032877 Locking fixes and cleanups Timo Sirainen <tss@iki.fi> parents: 2867 diff changeset	473
5947 7ebe0593f488 Don't create cache file until something is actually being added to it. Timo Sirainen <tss@iki.fi> parents: 4722 diff changeset	474 if (mail_cache_lock(cache, NULL) <= 0)
2929 ba9062032877 Locking fixes and cleanups Timo Sirainen <tss@iki.fi> parents: 2867 diff changeset	475 return -1;
ba9062032877 Locking fixes and cleanups Timo Sirainen <tss@iki.fi> parents: 2867 diff changeset	476
ba9062032877 Locking fixes and cleanups Timo Sirainen <tss@iki.fi> parents: 2867 diff changeset	477 ret = mail_cache_header_fields_update_locked(cache);
3627 c0e35566dd14 Unlocking cache file can also corrupt it since it modifies it. Added a Timo Sirainen <tss@iki.fi> parents: 3617 diff changeset	478 if (mail_cache_unlock(cache) < 0)
c0e35566dd14 Unlocking cache file can also corrupt it since it modifies it. Added a Timo Sirainen <tss@iki.fi> parents: 3617 diff changeset	479 ret = -1;
2339 406692edc12d Cache fixes. Decisions are saved again. Timo Sirainen <tss@iki.fi> parents: 2327 diff changeset	480 return ret;
2327 7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: diff changeset	481 }
7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: diff changeset	482
7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: diff changeset	483 void mail_cache_header_fields_get(struct mail_cache cache, buffer_t dest)
7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: diff changeset	484 {
7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: diff changeset	485 struct mail_cache_header_fields hdr;
7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: diff changeset	486 unsigned int field;
7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: diff changeset	487 const char *name;
2339 406692edc12d Cache fixes. Decisions are saved again. Timo Sirainen <tss@iki.fi> parents: 2327 diff changeset	488 uint32_t i;
2327 7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: diff changeset	489
7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: diff changeset	490 memset(&hdr, 0, sizeof(hdr));
6296 205ee38f10d1 Drop fields that haven't been used for 30 days when compressing. Timo Sirainen <tss@iki.fi> parents: 6003 diff changeset	491 hdr.fields_count = cache->file_fields_count;
205ee38f10d1 Drop fields that haven't been used for 30 days when compressing. Timo Sirainen <tss@iki.fi> parents: 6003 diff changeset	492 for (i = 0; i < cache->fields_count; i++) {
6443 53313d1e7e4f Code cleanups and some minor fixes. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	493 if (CACHE_FIELD_IS_NEWLY_WANTED(cache, i))
6296 205ee38f10d1 Drop fields that haven't been used for 30 days when compressing. Timo Sirainen <tss@iki.fi> parents: 6003 diff changeset	494 hdr.fields_count++;
205ee38f10d1 Drop fields that haven't been used for 30 days when compressing. Timo Sirainen <tss@iki.fi> parents: 6003 diff changeset	495 }
2327 7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: diff changeset	496 buffer_append(dest, &hdr, sizeof(hdr));
7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: diff changeset	497
7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: diff changeset	498 /* we have to keep the field order for the existing fields. */
6443 53313d1e7e4f Code cleanups and some minor fixes. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	499 copy_to_buf(cache, dest, TRUE,
2339 406692edc12d Cache fixes. Decisions are saved again. Timo Sirainen <tss@iki.fi> parents: 2327 diff changeset	500 offsetof(struct mail_cache_field_private, last_used),
406692edc12d Cache fixes. Decisions are saved again. Timo Sirainen <tss@iki.fi> parents: 2327 diff changeset	501 sizeof(uint32_t));
6443 53313d1e7e4f Code cleanups and some minor fixes. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	502 copy_to_buf(cache, dest, TRUE,
53313d1e7e4f Code cleanups and some minor fixes. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	503 offsetof(struct mail_cache_field, field_size),
2339 406692edc12d Cache fixes. Decisions are saved again. Timo Sirainen <tss@iki.fi> parents: 2327 diff changeset	504 sizeof(uint32_t));
6443 53313d1e7e4f Code cleanups and some minor fixes. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	505 copy_to_buf_byte(cache, dest, TRUE,
53313d1e7e4f Code cleanups and some minor fixes. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	506 offsetof(struct mail_cache_field, type));
53313d1e7e4f Code cleanups and some minor fixes. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	507 copy_to_buf_byte(cache, dest, TRUE,
2342 fa529dd77176 Fixes for big endian systems. Timo Sirainen <tss@iki.fi> parents: 2339 diff changeset	508 offsetof(struct mail_cache_field, decision));
2339 406692edc12d Cache fixes. Decisions are saved again. Timo Sirainen <tss@iki.fi> parents: 2327 diff changeset	509
6443 53313d1e7e4f Code cleanups and some minor fixes. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	510 i_assert(dest->used == sizeof(hdr) +
2339 406692edc12d Cache fixes. Decisions are saved again. Timo Sirainen <tss@iki.fi> parents: 2327 diff changeset	511 (sizeof(uint32_t)2 + 2) hdr.fields_count);
2327 7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: diff changeset	512
6443 53313d1e7e4f Code cleanups and some minor fixes. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	513 /* add existing fields' names */
2327 7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: diff changeset	514 for (i = 0; i < cache->file_fields_count; i++) {
7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: diff changeset	515 field = cache->file_field_map[i];
2339 406692edc12d Cache fixes. Decisions are saved again. Timo Sirainen <tss@iki.fi> parents: 2327 diff changeset	516 name = cache->fields[field].field.name;
2327 7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: diff changeset	517 buffer_append(dest, name, strlen(name)+1);
7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: diff changeset	518 }
6443 53313d1e7e4f Code cleanups and some minor fixes. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	519 /* add newly wanted fields' names */
2327 7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: diff changeset	520 for (i = 0; i < cache->fields_count; i++) {
6443 53313d1e7e4f Code cleanups and some minor fixes. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	521 if (CACHE_FIELD_IS_NEWLY_WANTED(cache, i)) {
53313d1e7e4f Code cleanups and some minor fixes. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	522 name = cache->fields[i].field.name;
53313d1e7e4f Code cleanups and some minor fixes. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	523 buffer_append(dest, name, strlen(name)+1);
53313d1e7e4f Code cleanups and some minor fixes. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	524 }
2327 7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: diff changeset	525 }
7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: diff changeset	526
6443 53313d1e7e4f Code cleanups and some minor fixes. Timo Sirainen <tss@iki.fi> parents: 6429 diff changeset	527 hdr.size = dest->used;
2327 7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: diff changeset	528 buffer_write(dest, 0, &hdr, sizeof(hdr));
7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: diff changeset	529
7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: diff changeset	530 if ((hdr.size & 3) != 0)
3617 904849549eac Removed null4 and replaced the few uses of it with buffer_append_zero(). Timo Sirainen <tss@iki.fi> parents: 3494 diff changeset	531 buffer_append_zero(dest, 4 - (hdr.size & 3));
2327 7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: diff changeset	532 }
7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: diff changeset	533
7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: diff changeset	534 int mail_cache_header_fields_get_next_offset(struct mail_cache *cache,
7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: diff changeset	535 uint32_t *offset_r)
7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: diff changeset	536 {
6704 b0e8403b4bb1 mmap_disable=yes: Find the latest cache field header by pread()ing the file Timo Sirainen <tss@iki.fi> parents: 6695 diff changeset	537 if (mail_cache_header_fields_get_offset(cache, offset_r, FALSE) < 0)
2327 7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: diff changeset	538 return -1;
7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: diff changeset	539
7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: diff changeset	540 if (*offset_r == 0) {
7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: diff changeset	541 *offset_r = offsetof(struct mail_cache_header,
7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: diff changeset	542 field_header_offset);
7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: diff changeset	543 } else {
7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: diff changeset	544 *offset_r += offsetof(struct mail_cache_header_fields,
7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: diff changeset	545 next_offset);
7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: diff changeset	546 }
7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: diff changeset	547 return 0;
7d02e2a7672d Header caching redesigned. New design allows caching decisions per field, so Timo Sirainen <tss@iki.fi> parents: diff changeset	548 }

Mercurial > dovecot > original-hg > dovecot-1.2

annotate src/lib-index/mail-cache-fields.c @ 6707:4752637a6ad4 HEAD