# HG changeset patch # User Timo Sirainen # Date 1090342256 -10800 # Node ID 406692edc12d5bc8c8312eb7f08fe0f08e22321f # Parent 3347a7e133ccefa5a3da5556b8d7878cc3ce63a0 Cache fixes. Decisions are saved again. diff -r 3347a7e133cc -r 406692edc12d src/lib-index/mail-cache-compress.c --- a/src/lib-index/mail-cache-compress.c Tue Jul 20 15:45:13 2004 +0300 +++ b/src/lib-index/mail-cache-compress.c Tue Jul 20 19:50:56 2004 +0300 @@ -32,7 +32,8 @@ *field_seen = ctx->field_seen_value; field = view->cache->file_field_map[file_field]; - dec = view->cache->fields[field].decision & ~MAIL_CACHE_DECISION_FORCED; + dec = view->cache->fields[field].field.decision & + ~MAIL_CACHE_DECISION_FORCED; if (ctx->new_msg) { if (dec == MAIL_CACHE_DECISION_NO) return 1; @@ -43,7 +44,7 @@ buffer_append(ctx->buffer, &file_field, sizeof(file_field)); - if (view->cache->fields[field].field_size == (unsigned int)-1) { + if (view->cache->fields[field].field.field_size == (unsigned int)-1) { size32 = (uint32_t)data_size; buffer_append(ctx->buffer, &size32, sizeof(size32)); } @@ -65,7 +66,6 @@ struct mail_cache_record cache_rec; struct ostream *output; buffer_t *buffer; - size_t size; uint32_t message_count, seq, first_new_seq, old_offset; uoff_t offset; @@ -93,14 +93,12 @@ hdr.version = MAIL_CACHE_VERSION; hdr.indexid = idx_hdr->indexid; hdr.file_seq = idx_hdr->cache_file_seq + 1; + o_stream_send(output, &hdr, sizeof(hdr)); if (cache->fields_count != 0) { hdr.field_header_offset = - mail_cache_uint32_to_offset(sizeof(hdr)); - } - o_stream_send(output, &hdr, sizeof(hdr)); + mail_cache_uint32_to_offset(output->offset); - if (cache->fields_count != 0) { t_push(); buffer = buffer_create_dynamic(pool_datastack_create(), 256, (size_t)-1); diff -r 3347a7e133cc -r 406692edc12d src/lib-index/mail-cache-decisions.c --- a/src/lib-index/mail-cache-decisions.c Tue Jul 20 15:45:13 2004 +0300 +++ b/src/lib-index/mail-cache-decisions.c Tue Jul 20 19:50:56 2004 +0300 @@ -79,7 +79,7 @@ i_assert(field < cache->fields_count); - if (cache->fields[field].decision != MAIL_CACHE_DECISION_TEMP) { + if (cache->fields[field].field.decision != MAIL_CACHE_DECISION_TEMP) { /* a) forced decision b) not cached, mail_cache_decision_add() will handle this c) permanently cached already, okay. */ @@ -106,7 +106,7 @@ client with no local cache. if it was just a new client generating the local cache for the first time, we'll drop back to TEMP within few months. */ - cache->fields[field].decision = MAIL_CACHE_DECISION_YES; + cache->fields[field].field.decision = MAIL_CACHE_DECISION_YES; cache->field_header_write_pending = TRUE; } else { cache->fields[field].uid_highwater = uid; @@ -124,14 +124,14 @@ if (MAIL_CACHE_IS_UNUSABLE(cache)) return; - if (cache->fields[field].decision != MAIL_CACHE_DECISION_NO) { + if (cache->fields[field].field.decision != MAIL_CACHE_DECISION_NO) { /* a) forced decision b) we're already caching it, so it just wasn't in cache */ return; } /* field used the first time */ - cache->fields[field].decision = MAIL_CACHE_DECISION_TEMP; + cache->fields[field].field.decision = MAIL_CACHE_DECISION_TEMP; cache->field_header_write_pending = TRUE; if (mail_index_lookup_uid(view->view, seq, &uid) == 0) diff -r 3347a7e133cc -r 406692edc12d src/lib-index/mail-cache-fields.c --- a/src/lib-index/mail-cache-fields.c Tue Jul 20 15:45:13 2004 +0300 +++ b/src/lib-index/mail-cache-fields.c Tue Jul 20 19:50:56 2004 +0300 @@ -3,6 +3,7 @@ #include "lib.h" #include "buffer.h" #include "hash.h" +#include "write-full.h" #include "mail-cache-private.h" #include @@ -50,24 +51,24 @@ continue; /* new index - save it */ - cache->fields[idx] = fields[i]; - cache->fields[idx].name = + cache->fields[idx].field = fields[i]; + cache->fields[idx].field.name = p_strdup(cache->field_pool, fields[i].name); cache->field_file_map[idx] = (uint32_t)-1; - switch (cache->fields[idx].type) { + switch (cache->fields[idx].field.type) { case MAIL_CACHE_FIELD_FIXED_SIZE: case MAIL_CACHE_FIELD_BITMASK: break; case MAIL_CACHE_FIELD_VARIABLE_SIZE: case MAIL_CACHE_FIELD_STRING: case MAIL_CACHE_FIELD_HEADER: - cache->fields[idx].field_size = (unsigned int)-1; + cache->fields[idx].field.field_size = (unsigned int)-1; break; } hash_insert(cache->field_name_hash, - (char *)cache->fields[idx].name, + (char *)cache->fields[idx].field.name, POINTER_CAST(idx)); } cache->fields_count = new_idx; @@ -101,6 +102,11 @@ next_offset = mail_cache_offset_to_uint32(cache->hdr->field_header_offset); while (next_offset != 0) { + if (next_offset == offset) { + mail_cache_set_corrupted(cache, + "next_offset in field header loops"); + return -1; + } offset = next_offset; if (mail_cache_map(cache, offset, @@ -123,6 +129,7 @@ const uint32_t *last_used, *sizes; const uint8_t *types, *decisions; const char *p, *names, *end; + void *orig_key, *orig_value; uint32_t offset, i; if (mail_cache_header_fields_get_offset(cache, &offset) < 0) @@ -160,11 +167,15 @@ field_hdr->fields_count * sizeof(unsigned int)); cache->file_fields_count = field_hdr->fields_count; - last_used = MAIL_CACHE_FIELD_LAST_USED(field_hdr); - sizes = MAIL_CACHE_FIELD_SIZE(field_hdr); - types = MAIL_CACHE_FIELD_TYPE(field_hdr); - decisions = MAIL_CACHE_FIELD_DECISION(field_hdr); - names = MAIL_CACHE_FIELD_NAMES(field_hdr); + last_used = CONST_PTR_OFFSET(field_hdr, MAIL_CACHE_FIELD_LAST_USED()); + sizes = CONST_PTR_OFFSET(field_hdr, + MAIL_CACHE_FIELD_SIZE(field_hdr->fields_count)); + types = CONST_PTR_OFFSET(field_hdr, + MAIL_CACHE_FIELD_TYPE(field_hdr->fields_count)); + decisions = CONST_PTR_OFFSET(field_hdr, + MAIL_CACHE_FIELD_DECISION(field_hdr->fields_count)); + names = CONST_PTR_OFFSET(field_hdr, + MAIL_CACHE_FIELD_NAMES(field_hdr->fields_count)); end = CONST_PTR_OFFSET(field_hdr, field_hdr->size); /* clear the old mapping */ @@ -174,81 +185,152 @@ memset(&field, 0, sizeof(field)); for (i = 0; i < field_hdr->fields_count; i++) { for (p = names; p != end && *p != '\0'; p++) ; - if (p == end) { + if (p == end || *names == '\0') { mail_cache_set_corrupted(cache, "field header names corrupted"); return -1; } - field.name = names; - field.type = types[i]; - field.field_size = sizes[i]; - field.decision = decisions[i]; - field.last_used = (time_t)last_used[i]; - mail_cache_register_fields(cache, &field, 1); + if (hash_lookup_full(cache->field_name_hash, names, + &orig_key, &orig_value)) { + /* already exists, see if decision can be updated */ + field.idx = POINTER_CAST_TO(orig_value, unsigned int); + if (!cache->fields[field.idx].decision_dirty) { + cache->fields[field.idx].field.decision = + decisions[i]; + } + if (cache->fields[field.idx].field.type != types[i]) { + mail_cache_set_corrupted(cache, + "registered field type changed"); + return -1; + } + } else { + field.name = names; + field.type = types[i]; + field.field_size = sizes[i]; + field.decision = decisions[i]; + mail_cache_register_fields(cache, &field, 1); + } cache->field_file_map[field.idx] = i; cache->file_field_map[i] = field.idx; - names = p + 1; + /* update last_used if it's newer than ours */ + if ((time_t)last_used[i] > cache->fields[field.idx].last_used) { + cache->fields[field.idx].last_used = + (time_t)last_used[i]; + } + + names = p + 1; } return 0; } +static void copy_to_buf(struct mail_cache *cache, buffer_t *dest, + size_t offset, size_t size) +{ + const void *data; + unsigned int i, field; + + for (i = 0; i < cache->file_fields_count; i++) { + field = cache->file_field_map[i]; + data = CONST_PTR_OFFSET(&cache->fields[field], offset); + buffer_append(dest, data, size); + } + for (i = 0; i < cache->fields_count; i++) { + if (cache->field_file_map[i] != (uint32_t)-1) + continue; + data = CONST_PTR_OFFSET(&cache->fields[i], offset); + buffer_append(dest, data, size); + } +} + int mail_cache_header_fields_update(struct mail_cache *cache) { int locked = cache->locked; + buffer_t *buffer; + uint32_t i, offset; + int ret = 0; if (!locked) { if (mail_cache_lock(cache) <= 0) return -1; } - // FIXME + if (mail_cache_header_fields_read(cache) < 0 || + mail_cache_header_fields_get_offset(cache, &offset) < 0) { + mail_cache_unlock(cache); + return -1; + } + + t_push(); + buffer = buffer_create_dynamic(pool_datastack_create(), + 256, (size_t)-1); + + copy_to_buf(cache, buffer, + offsetof(struct mail_cache_field_private, last_used), + sizeof(uint32_t)); + ret = pwrite_full(cache->fd, buffer_get_data(buffer, NULL), + sizeof(uint32_t) * cache->file_fields_count, + offset + MAIL_CACHE_FIELD_LAST_USED()); + if (ret == 0) { + buffer_set_used_size(buffer, 0); + copy_to_buf(cache, buffer, + offsetof(struct mail_cache_field, decision), + sizeof(uint8_t)); + + ret = pwrite_full(cache->fd, buffer_get_data(buffer, NULL), + sizeof(uint8_t) * cache->file_fields_count, offset + + MAIL_CACHE_FIELD_DECISION(cache->file_fields_count)); + + if (ret == 0) { + for (i = 0; i < cache->fields_count; i++) + cache->fields[i].decision_dirty = FALSE; + } + } + t_pop(); + + if (ret == 0) + cache->field_header_write_pending = FALSE; if (!locked) mail_cache_unlock(cache); + return ret; } -#define UGLY_COPY_MACRO(field_name, type) \ - for (i = 0; i < cache->file_fields_count; i++) { \ - field = cache->file_field_map[i]; \ - field_name = (type)cache->fields[field].field_name; \ - buffer_append(dest, &field_name, sizeof(field_name)); \ - } \ - for (i = 0; i < cache->fields_count; i++) { \ - if (cache->field_file_map[i] != (uint32_t)-1) \ - continue; \ - field_name = (type)cache->fields[i].field_name; \ - buffer_append(dest, &field_name, sizeof(field_name)); \ - } - void mail_cache_header_fields_get(struct mail_cache *cache, buffer_t *dest) { struct mail_cache_header_fields hdr; unsigned int field; const char *name; - uint32_t i, last_used, field_size; - uint8_t type, decision; + uint32_t i; memset(&hdr, 0, sizeof(hdr)); hdr.fields_count = cache->fields_count; buffer_append(dest, &hdr, sizeof(hdr)); /* we have to keep the field order for the existing fields. */ - UGLY_COPY_MACRO(last_used, uint32_t); - UGLY_COPY_MACRO(field_size, uint32_t); - UGLY_COPY_MACRO(type, uint8_t); - UGLY_COPY_MACRO(decision, uint8_t); + copy_to_buf(cache, dest, + offsetof(struct mail_cache_field_private, last_used), + sizeof(uint32_t)); + copy_to_buf(cache, dest, offsetof(struct mail_cache_field, field_size), + sizeof(uint32_t)); + copy_to_buf(cache, dest, offsetof(struct mail_cache_field, type), + sizeof(uint8_t)); + copy_to_buf(cache, dest, offsetof(struct mail_cache_field, decision), + sizeof(uint8_t)); + + i_assert(buffer_get_used_size(dest) == sizeof(hdr) + + (sizeof(uint32_t)*2 + 2) * hdr.fields_count); for (i = 0; i < cache->file_fields_count; i++) { field = cache->file_field_map[i]; - name = cache->fields[field].name; + name = cache->fields[field].field.name; buffer_append(dest, name, strlen(name)+1); } for (i = 0; i < cache->fields_count; i++) { if (cache->field_file_map[i] != (uint32_t)-1) continue; - name = cache->fields[i].name; + name = cache->fields[i].field.name; buffer_append(dest, name, strlen(name)+1); } diff -r 3347a7e133cc -r 406692edc12d src/lib-index/mail-cache-lookup.c --- a/src/lib-index/mail-cache-lookup.c Tue Jul 20 15:45:13 2004 +0300 +++ b/src/lib-index/mail-cache-lookup.c Tue Jul 20 19:50:56 2004 +0300 @@ -103,7 +103,7 @@ } field = cache->file_field_map[file_field]; - data_size = cache->fields[field].field_size; + data_size = cache->fields[field].field.field_size; if (data_size == (unsigned int)-1) { data_size = *((const uint32_t *) CONST_PTR_OFFSET(cache_rec, pos)); @@ -225,7 +225,7 @@ { i_assert(field < cache->fields_count); - return cache->fields[field].decision; + return cache->fields[field].field.decision; } struct mail_cache_lookup_context { diff -r 3347a7e133cc -r 406692edc12d src/lib-index/mail-cache-private.h --- a/src/lib-index/mail-cache-private.h Tue Jul 20 15:45:13 2004 +0300 +++ b/src/lib-index/mail-cache-private.h Tue Jul 20 19:50:56 2004 +0300 @@ -73,20 +73,16 @@ #endif }; -#define MAIL_CACHE_FIELD_LAST_USED(field_hdr) \ - CONST_PTR_OFFSET(field_hdr, sizeof(uint32_t) * 3) -#define MAIL_CACHE_FIELD_SIZE(field_hdr) \ - CONST_PTR_OFFSET(MAIL_CACHE_FIELD_LAST_USED(field_hdr), \ - sizeof(uint32_t) * (field_hdr)->fields_count) -#define MAIL_CACHE_FIELD_TYPE(field_hdr) \ - CONST_PTR_OFFSET(MAIL_CACHE_FIELD_SIZE(field_hdr), \ - sizeof(uint32_t) * (field_hdr)->fields_count) -#define MAIL_CACHE_FIELD_DECISION(field_hdr) \ - CONST_PTR_OFFSET(MAIL_CACHE_FIELD_TYPE(field_hdr), \ - sizeof(uint8_t) * (field_hdr)->fields_count) -#define MAIL_CACHE_FIELD_NAMES(field_hdr) \ - CONST_PTR_OFFSET(MAIL_CACHE_FIELD_DECISION(field_hdr), \ - sizeof(uint8_t) * (field_hdr)->fields_count) +#define MAIL_CACHE_FIELD_LAST_USED() \ + (sizeof(uint32_t) * 3) +#define MAIL_CACHE_FIELD_SIZE(count) \ + (MAIL_CACHE_FIELD_LAST_USED() + sizeof(uint32_t) * (count)) +#define MAIL_CACHE_FIELD_TYPE(count) \ + (MAIL_CACHE_FIELD_SIZE(count) + sizeof(uint32_t) * (count)) +#define MAIL_CACHE_FIELD_DECISION(count) \ + (MAIL_CACHE_FIELD_TYPE(count) + sizeof(uint8_t) * (count)) +#define MAIL_CACHE_FIELD_NAMES(count) \ + (MAIL_CACHE_FIELD_DECISION(count) + sizeof(uint8_t) * (count)) struct mail_cache_record { uint32_t prev_offset; @@ -105,6 +101,15 @@ uint32_t magic; }; +struct mail_cache_field_private { + struct mail_cache_field field; + + uint32_t uid_highwater; + time_t last_used; + + unsigned int decision_dirty:1; +}; + struct mail_cache { struct mail_index *index; @@ -118,7 +123,7 @@ struct mail_cache_header hdr_copy; pool_t field_pool; - struct mail_cache_field *fields; + struct mail_cache_field_private *fields; uint32_t *field_file_map; unsigned int fields_count; struct hash_table *field_name_hash; /* name -> idx */ diff -r 3347a7e133cc -r 406692edc12d src/lib-index/mail-cache-transaction.c --- a/src/lib-index/mail-cache-transaction.c Tue Jul 20 15:45:13 2004 +0300 +++ b/src/lib-index/mail-cache-transaction.c Tue Jul 20 19:50:56 2004 +0300 @@ -555,7 +555,7 @@ i_assert(field < ctx->cache->fields_count); i_assert(data_size < (uint32_t)-1); - if (ctx->cache->fields[field].decision == + if (ctx->cache->fields[field].field.decision == (MAIL_CACHE_DECISION_NO | MAIL_CACHE_DECISION_FORCED)) return; @@ -571,7 +571,7 @@ mail_cache_decision_add(ctx->view, seq, field); - fixed_size = ctx->cache->fields[field].field_size; + fixed_size = ctx->cache->fields[field].field.field_size; i_assert(fixed_size == (unsigned int)-1 || fixed_size == data_size); data_size32 = (uint32_t)data_size; diff -r 3347a7e133cc -r 406692edc12d src/lib-index/mail-cache.c --- a/src/lib-index/mail-cache.c Tue Jul 20 15:45:13 2004 +0300 +++ b/src/lib-index/mail-cache.c Tue Jul 20 19:50:56 2004 +0300 @@ -327,6 +327,9 @@ { i_assert(cache->locked); + if (cache->field_header_write_pending) + (void)mail_cache_header_fields_update(cache); + cache->locked = FALSE; if (cache->hdr_modified) { @@ -358,6 +361,9 @@ void mail_cache_view_close(struct mail_cache_view *view) { + if (view->cache->field_header_write_pending) + (void)mail_cache_header_fields_update(view->cache); + buffer_free(view->cached_exists_buf); i_free(view); } diff -r 3347a7e133cc -r 406692edc12d src/lib-index/mail-cache.h --- a/src/lib-index/mail-cache.h Tue Jul 20 15:45:13 2004 +0300 +++ b/src/lib-index/mail-cache.h Tue Jul 20 19:50:56 2004 +0300 @@ -36,10 +36,6 @@ enum mail_cache_field_type type; unsigned int field_size; enum mail_cache_decision_type decision; - - /* internal: */ - uint32_t uid_highwater; - time_t last_used; }; struct mail_cache *mail_cache_open_or_create(struct mail_index *index); diff -r 3347a7e133cc -r 406692edc12d src/lib-storage/index/index-mail-headers.c --- a/src/lib-storage/index/index-mail-headers.c Tue Jul 20 15:45:13 2004 +0300 +++ b/src/lib-storage/index/index-mail-headers.c Tue Jul 20 19:50:56 2004 +0300 @@ -91,6 +91,7 @@ lines[i].end_pos - lines[i].start_pos); } + i--; } else { buffer_append(buf, header + lines[i].start_pos, lines[j-1].end_pos - lines[i].start_pos); @@ -359,7 +360,7 @@ { struct mail_cache_field header_field = { NULL, 0, MAIL_CACHE_FIELD_HEADER, 0, - MAIL_CACHE_DECISION_TEMP, 0, 0 + MAIL_CACHE_DECISION_TEMP }; const char *cache_field_name; unsigned int field_idx; @@ -491,7 +492,7 @@ struct index_mailbox *ibox = (struct index_mailbox *)box; struct mail_cache_field *fields, header_field = { NULL, 0, MAIL_CACHE_FIELD_HEADER, 0, - MAIL_CACHE_DECISION_TEMP, 0, 0 + MAIL_CACHE_DECISION_TEMP }; struct index_header_lookup_ctx *ctx; const char *const *name; diff -r 3347a7e133cc -r 406692edc12d src/lib-storage/index/index-mail.c --- a/src/lib-storage/index/index-mail.c Tue Jul 20 15:45:13 2004 +0300 +++ b/src/lib-storage/index/index-mail.c Tue Jul 20 19:50:56 2004 +0300 @@ -14,19 +14,18 @@ #include "index-mail.h" struct mail_cache_field cache_fields[MAIL_CACHE_FIELD_COUNT] = { - { "index.flags", 0, MAIL_CACHE_FIELD_BITMASK, - sizeof(uint32_t), 0, 0, 0 }, + { "index.flags", 0, MAIL_CACHE_FIELD_BITMASK, sizeof(uint32_t), 0 }, { "date.sent", 0, MAIL_CACHE_FIELD_FIXED_SIZE, - sizeof(struct mail_sent_date), 0, 0, 0 }, + sizeof(struct mail_sent_date), 0 }, { "date.received", 0, MAIL_CACHE_FIELD_FIXED_SIZE, - sizeof(time_t), 0, 0, 0 }, + sizeof(time_t), 0 }, { "size.virtual", 0, MAIL_CACHE_FIELD_FIXED_SIZE, - sizeof(uoff_t), 0, 0, 0 }, - { "imap.body", 0, MAIL_CACHE_FIELD_STRING, 0, 0, 0, 0 }, - { "imap.bodystructure", 0, MAIL_CACHE_FIELD_STRING, 0, 0, 0, 0 }, - { "imap.envelope", 0, MAIL_CACHE_FIELD_STRING, 0, 0, 0, 0 }, - { "mime.parts", 0, MAIL_CACHE_FIELD_VARIABLE_SIZE, 0, 0, 0, 0 }, - { "mail.uid", 0, MAIL_CACHE_FIELD_STRING, 0, 0, 0, 0 } + sizeof(uoff_t), 0 }, + { "imap.body", 0, MAIL_CACHE_FIELD_STRING, 0, 0 }, + { "imap.bodystructure", 0, MAIL_CACHE_FIELD_STRING, 0, 0 }, + { "imap.envelope", 0, MAIL_CACHE_FIELD_STRING, 0, 0 }, + { "mime.parts", 0, MAIL_CACHE_FIELD_VARIABLE_SIZE, 0, 0 }, + { "mail.uid", 0, MAIL_CACHE_FIELD_STRING, 0, 0 } }; static void index_mail_parse_body(struct index_mail *mail, int need_parts);