Mercurial > dovecot > core-2.2
changeset 2563:5752e5ac88f0 HEAD
Save extra record/header infos into index file permanently.
line wrap: on
line diff
--- a/src/lib-index/mail-index-lock.c Sun Sep 05 20:33:19 2004 +0300 +++ b/src/lib-index/mail-index-lock.c Sun Sep 05 20:53:45 2004 +0300 @@ -19,6 +19,7 @@ */ #include "lib.h" +#include "buffer.h" #include "mmap-util.h" #include "file-lock.h" #include "write-full.h" @@ -193,7 +194,26 @@ } } - ret = write_full(fd, index->map->hdr, sizeof(*index->map->hdr)); + if (index->map->hdr->base_header_size >= sizeof(*index->map->hdr)) { + /* header size is what's expected */ + ret = write_full(fd, index->map->hdr, + index->map->hdr->header_size); + } else { + /* write base header */ + ret = write_full(fd, index->map->hdr, + index->map->hdr->base_header_size); + if (ret == 0) { + /* write extended headers */ + const struct mail_index_header *hdr; + const void *hdr_ext; + + hdr = index->map->hdr_base; + hdr_ext = CONST_PTR_OFFSET(hdr, hdr->base_header_size); + ret = write_full(fd, hdr_ext, hdr->header_size - + hdr->base_header_size); + } + } + if (ret < 0 || write_full(fd, index->map->records, index->map->records_count * index->map->hdr->record_size) < 0) { @@ -264,7 +284,7 @@ /* if header size is smaller than what we have, we'll have to recreate the index to grow it. so don't even try regular locking. */ - if (index->map->hdr != &index->map->hdr_copy && + if (index->map->hdr != index->map->hdr_copy_buf->data && index->map->base_header_size == sizeof(*index->hdr)) { /* wait two seconds for exclusive lock */ ret = mail_index_lock(index, F_WRLCK, 2, TRUE, lock_id_r);
--- a/src/lib-index/mail-index-private.h Sun Sep 05 20:33:19 2004 +0300 +++ b/src/lib-index/mail-index-private.h Sun Sep 05 20:53:45 2004 +0300 @@ -6,11 +6,6 @@ struct mail_transaction_header; -/* Maximum number of extra record data items we allowed. Raising this limit - only means it takes a few bytes more memory, but 32 should be enough for a - long time, right? :) */ -#define MAIL_INDEX_MAX_EXTRA_RECORDS 32 - /* Index file is grown exponentially when we're adding less than this many records. */ #define MAIL_INDEX_MAX_POWER_GROW (1024*1024 / sizeof(struct mail_index_record)) @@ -28,19 +23,29 @@ struct mail_index_extra_record_info { const char *name; - uint16_t offset; - uint16_t size; + uint32_t hdr_offset; + uint32_t hdr_size; + uint32_t record_offset; + uint32_t record_size; +}; + +struct mail_index_extra_record_info_header { + uint32_t hdr_size; + uint32_t record_offset; + uint32_t record_size; }; struct mail_index_map { int refcount; const struct mail_index_header *hdr; + const void *hdr_base; void *records; /* struct mail_index_record[] */ unsigned int records_count; - struct mail_index_extra_record_info * - extra_record_map[MAIL_INDEX_MAX_EXTRA_RECORDS]; + pool_t extra_records_pool; + buffer_t *extra_infos; /* struct mail_index_extra_record_info[] */ + buffer_t *extra_infos_id_map; /* uint32_t[] (index -> file) */ void *mmap_base; size_t mmap_size, mmap_used_size; @@ -50,7 +55,7 @@ uint32_t log_file_seq; uoff_t log_file_offset; - struct mail_index_header hdr_copy; + buffer_t *hdr_copy_buf; uint32_t base_header_size; /* so we don't need lock to access it */ unsigned int write_to_disk:1; @@ -65,11 +70,8 @@ mode_t mode; gid_t gid; - pool_t extra_records_pool; - buffer_t *extra_records_buf; - const struct mail_index_extra_record_info *extra_records; - unsigned int extra_records_count; - unsigned int max_record_size; + pool_t extra_infos_pool; + buffer_t *extra_infos; /* struct mail_index_extra_record_info[] */ char *filepath; int fd; @@ -126,6 +128,14 @@ void mail_index_unmap(struct mail_index *index, struct mail_index_map *map); struct mail_index_map * mail_index_map_to_memory(struct mail_index_map *map, uint32_t new_record_size); +uint32_t mail_index_map_register_extra_info(struct mail_index *index, + struct mail_index_map *map, + const char *name, + uint32_t hdr_offset, + uint32_t hdr_size, + uint32_t record_size); +int mail_index_map_get_extra_info_idx(struct mail_index_map *map, + uint32_t data_id, uint32_t *idx_r); int mail_index_lookup_full(struct mail_index_view *view, uint32_t seq, struct mail_index_map **map_r,
--- a/src/lib-index/mail-index-sync-update.c Sun Sep 05 20:33:19 2004 +0300 +++ b/src/lib-index/mail-index-sync-update.c Sun Sep 05 20:53:45 2004 +0300 @@ -13,6 +13,8 @@ #include <time.h> +static const unsigned char *null4[] = { 0, 0, 0, 0 }; + static void mail_index_header_update_counts(struct mail_index_header *hdr, uint8_t old_flags, uint8_t new_flags) @@ -77,7 +79,7 @@ struct mail_index_sync_map_ctx *ctx = context; struct mail_index_view *view = ctx->view; struct mail_index_map *map = view->map; - struct mail_index_header *hdr = &map->hdr_copy; + struct mail_index_header *hdr; struct mail_index_record *rec; uint32_t count, seq, seq1, seq2; @@ -90,6 +92,7 @@ if (seq1 == 0) return 1; + hdr = buffer_get_modifyable_data(map->hdr_copy_buf, NULL); for (seq = seq1; seq <= seq2; seq++) { rec = MAIL_INDEX_MAP_IDX(map, seq-1); mail_index_header_update_counts(hdr, rec->flags, 0); @@ -115,20 +118,19 @@ return 1; } -static int sync_append(const struct mail_transaction_append_header *hdr, - const struct mail_index_record *rec, void *context) +static int sync_append(const struct mail_index_record *rec, void *context) { struct mail_index_sync_map_ctx *ctx = context; struct mail_index_view *view = ctx->view; struct mail_index_map *map = view->map; + struct mail_index_header *hdr; void *dest; - i_assert(hdr->record_size <= map->hdr->record_size); - - if (rec->uid < map->hdr_copy.next_uid) { + hdr = buffer_get_modifyable_data(map->hdr_copy_buf, NULL); + if (rec->uid < hdr->next_uid) { mail_transaction_log_view_set_corrupted(view->log_view, "Append with UID %u, but next_uid = %u", - rec->uid, map->hdr_copy.next_uid); + rec->uid, hdr->next_uid); return -1; } @@ -143,17 +145,17 @@ map->mmap_size); dest = MAIL_INDEX_MAP_IDX(map, map->records_count); } - memcpy(dest, rec, hdr->record_size); - memset(PTR_OFFSET(dest, hdr->record_size), 0, - map->hdr->record_size - hdr->record_size); + memcpy(dest, rec, sizeof(*rec)); + memset(PTR_OFFSET(dest, sizeof(*rec)), 0, + map->hdr->record_size - sizeof(*rec)); - map->hdr_copy.messages_count++; - map->hdr_copy.next_uid = rec->uid+1; + hdr->messages_count++; + hdr->next_uid = rec->uid+1; view->messages_count++; map->records_count++; - mail_index_header_update_counts(&map->hdr_copy, 0, rec->flags); - mail_index_header_update_lowwaters(&map->hdr_copy, rec); + mail_index_header_update_counts(hdr, 0, rec->flags); + mail_index_header_update_lowwaters(hdr, rec); return 1; } @@ -162,8 +164,8 @@ { struct mail_index_sync_map_ctx *ctx = context; struct mail_index_view *view = ctx->view; + struct mail_index_header *hdr; struct mail_index_record *rec; - struct mail_index_header *hdr; uint8_t flag_mask, old_flags; keywords_mask_t keyword_mask; uint32_t i, idx, seq1, seq2; @@ -176,7 +178,7 @@ if (seq1 == 0) return 1; - hdr = &view->map->hdr_copy; + hdr = buffer_get_modifyable_data(view->map->hdr_copy_buf, NULL); if ((u->add_flags & MAIL_INDEX_MAIL_FLAG_DIRTY) != 0) hdr->flags |= MAIL_INDEX_HDR_FLAG_HAVE_DIRTY; @@ -213,9 +215,11 @@ { struct mail_index_sync_map_ctx *ctx = context; struct mail_index_view *view = ctx->view; + struct mail_index_header *hdr; uint32_t i; - view->map->hdr_copy.cache_file_seq = u->new_file_seq; + hdr = buffer_get_modifyable_data(view->map->hdr_copy_buf, NULL); + hdr->cache_file_seq = u->new_file_seq; for (i = 0; i < view->messages_count; i++) MAIL_INDEX_MAP_IDX(view->map, i)->cache_offset = 0; @@ -259,11 +263,129 @@ static int sync_header_update(const struct mail_transaction_header_update *u, void *context) { - struct mail_index_sync_map_ctx *ctx = context; - void *data; + struct mail_index_sync_map_ctx *ctx = context; + const struct mail_index_header *hdr = ctx->view->map->hdr; + + if (u->offset >= hdr->base_header_size || + u->offset + u->size > hdr->base_header_size) { + mail_transaction_log_view_set_corrupted(ctx->view->log_view, + "Header update outside range: %u + %u > %u", + u->offset, u->size, hdr->base_header_size); + return -1; + } + + buffer_write(ctx->view->map->hdr_copy_buf, u->offset, u + 1, u->size); + ctx->view->map->hdr = ctx->view->map->hdr_copy_buf->data; + return 1; +} + +static int sync_extra_intro(const struct mail_transaction_extra_intro *u, + void *context) +{ + struct mail_index_sync_map_ctx *ctx = context; + struct mail_index_extra_record_info_header einfo_hdr; + const struct mail_index_extra_record_info *einfo; + struct mail_index_header *hdr; + const char *name; + buffer_t *hdr_buf; + uint32_t data_id; + + t_push(); + name = t_strndup(u + 1, u->name_size); + + hdr_buf = ctx->view->map->hdr_copy_buf; + data_id = mail_index_map_register_extra_info(ctx->view->index, + ctx->view->map, name, + hdr_buf->used, u->hdr_size, + u->record_size); + + einfo = ctx->view->index->extra_infos->data; + einfo += data_id; + + /* name NUL [padding] einfo_hdr [header data] */ + buffer_append(hdr_buf, name, strlen(name)+1); + if ((hdr_buf->used % 4) != 0) + buffer_append(hdr_buf, null4, 4 - (hdr_buf->used % 4)); + + memset(&einfo_hdr, 0, sizeof(einfo_hdr)); + einfo_hdr.hdr_size = einfo->hdr_size; + einfo_hdr.record_offset = einfo->record_offset; + einfo_hdr.record_size = einfo->record_size; + buffer_append(hdr_buf, &einfo_hdr, sizeof(einfo_hdr)); + buffer_set_used_size(hdr_buf, hdr_buf->used + einfo->hdr_size); + + hdr = buffer_get_modifyable_data(hdr_buf, NULL); + hdr->header_size = hdr_buf->used; + + ctx->view->map->hdr = hdr; + + t_pop(); - data = PTR_OFFSET(&ctx->view->map->hdr_copy, u->offset); - memcpy(data, u + 1, u->size); + if (data_id != u->data_id) { + mail_transaction_log_view_set_corrupted(ctx->view->log_view, + "Introduced extra with invalid data id: %u != %u", + u->data_id, data_id); + return -1; + } + return 1; +} + +static int sync_extra_reset(const struct mail_transaction_extra_rec_header *u, + void *context) +{ + struct mail_index_sync_map_ctx *ctx = context; + struct mail_index_view *view = ctx->view; + struct mail_index_map *map = view->map; + const struct mail_index_extra_record_info *einfo; + struct mail_index_record *rec; + uint32_t i; + + if (map->extra_infos == NULL || + u->data_id >= map->extra_infos->used / sizeof(*einfo)) { + mail_transaction_log_view_set_corrupted(view->log_view, + "Extra reset for unknown data id %u", + u->data_id); + return -1; + } + + einfo = map->extra_infos->data; + einfo += u->data_id; + + memset(buffer_get_space_unsafe(map->hdr_copy_buf, + einfo->hdr_offset, einfo->hdr_size), + 0, einfo->hdr_size); + map->hdr = map->hdr_copy_buf->data; + + for (i = 0; i < view->messages_count; i++) { + rec = MAIL_INDEX_MAP_IDX(view->map, i); + memset(PTR_OFFSET(rec, einfo->record_offset), 0, + einfo->record_size); + } + return 1; +} + +static int +sync_extra_hdr_update(const struct mail_transaction_extra_hdr_update *u, + void *context) +{ + struct mail_index_sync_map_ctx *ctx = context; + struct mail_index_map *map = ctx->view->map; + const struct mail_index_extra_record_info *einfo; + + if (map->extra_infos == NULL || + u->data_id >= map->extra_infos->used / sizeof(*einfo)) { + mail_transaction_log_view_set_corrupted(ctx->view->log_view, + "Extra header update for unknown data id %u", + u->data_id); + return -1; + } + + einfo = map->extra_infos->data; + einfo += u->data_id; + + buffer_write(map->hdr_copy_buf, einfo->hdr_offset + u->offset, + u + 1, u->size); + map->hdr = map->hdr_copy_buf->data; return 1; } @@ -275,20 +397,27 @@ struct mail_index_sync_map_ctx *ctx = context; struct mail_index_view *view = ctx->view; struct mail_index_record *rec; + const struct mail_index_extra_record_info *einfo; uint32_t seq; - uint16_t offset, size; - /* FIXME: do data_id mapping conversion */ + if (view->map->extra_infos == NULL || + hdr->data_id >= view->map->extra_infos->used / sizeof(*einfo)) { + mail_transaction_log_view_set_corrupted(view->log_view, + "Extra record update for unknown data id %u", + hdr->data_id); + return -1; + } if (mail_index_lookup_uid_range(view, u->uid, u->uid, &seq, &seq) < 0) return -1; if (seq != 0) { - offset = view->index->extra_records[hdr->data_id].offset; - size = view->index->extra_records[hdr->data_id].size; + einfo = view->map->extra_infos->data; + einfo += hdr->data_id; rec = MAIL_INDEX_MAP_IDX(view->map, seq-1); - memcpy(PTR_OFFSET(rec, offset), u + 1, size); + memcpy(PTR_OFFSET(rec, einfo->record_offset), + u + 1, einfo->record_size); } return 1; } @@ -296,8 +425,8 @@ static int mail_index_grow(struct mail_index *index, struct mail_index_map *map, unsigned int count) { - struct mail_index_header hdr; - size_t size; + void *hdr_copy; + size_t size, hdr_copy_size; if (MAIL_INDEX_MAP_IS_IN_MEMORY(map)) return 0; @@ -324,17 +453,24 @@ /* we only wish to grow the file, but mail_index_map() updates the headers as well and may break our modified hdr_copy. so, take a backup of it and put it back afterwards */ - hdr = map->hdr_copy; + t_push(); + hdr_copy_size = map->hdr_copy_buf->used; + hdr_copy = t_malloc(hdr_copy_size); + memcpy(hdr_copy, map->hdr_copy_buf->data, hdr_copy_size); - if (mail_index_map(index, TRUE) <= 0) + if (mail_index_map(index, TRUE) <= 0) { + t_pop(); return -1; + } map = index->map; - map->hdr_copy = hdr; - map->hdr = &map->hdr_copy; + buffer_reset(map->hdr_copy_buf); + buffer_append(map->hdr_copy_buf, hdr_copy, hdr_copy_size); + map->hdr = map->hdr_copy_buf->data; map->records_count = map->hdr->messages_count; i_assert(map->mmap_size >= size); + t_pop(); return 0; } @@ -393,7 +529,8 @@ struct mail_index_view *view = sync_ctx->view; struct mail_index_map *map; struct mail_index_sync_map_ctx sync_map_ctx; - const struct mail_transaction_header *hdr; + const struct mail_transaction_header *thdr; + struct mail_index_header *tmphdr; const void *data; unsigned int count, old_lock_id; uint32_t seq, i, first_append_uid; @@ -402,7 +539,7 @@ memset(&sync_map_ctx, 0, sizeof(sync_map_ctx)); sync_map_ctx.view = view; - sync_map_ctx.update_cache = TRUE; + sync_map_ctx.update_cache = TRUE; /* we'll have to update view->lock_id to avoid mail_index_view_lock() trying to update the file later. */ @@ -411,27 +548,32 @@ return -1; mail_index_unlock(index, old_lock_id); - /* NOTE: locking may change index->map so make sure assignment + /* NOTE: locking may change index->map so make sure the assignment is after locking */ map = index->map; if (MAIL_INDEX_MAP_IS_IN_MEMORY(map)) map->write_to_disk = TRUE; - map->hdr_copy = *map->hdr; - map->hdr = &map->hdr_copy; + if (map->hdr != map->hdr_copy_buf->data) { + buffer_reset(map->hdr_copy_buf); + buffer_append(map->hdr_copy_buf, map->hdr, + map->hdr->header_size); + map->hdr = map->hdr_copy_buf->data; + } mail_index_unmap(index, view->map); view->map = map; view->map->refcount++; - first_append_uid = 0; - had_dirty = (map->hdr_copy.flags & MAIL_INDEX_HDR_FLAG_HAVE_DIRTY) != 0; + tmphdr = buffer_get_modifyable_data(map->hdr_copy_buf, NULL); + had_dirty = (tmphdr->flags & MAIL_INDEX_HDR_FLAG_HAVE_DIRTY) != 0; if (had_dirty) - map->hdr_copy.flags &= ~MAIL_INDEX_HDR_FLAG_HAVE_DIRTY; + tmphdr->flags &= ~MAIL_INDEX_HDR_FLAG_HAVE_DIRTY; - while ((ret = mail_transaction_log_view_next(view->log_view, &hdr, + first_append_uid = 0; + while ((ret = mail_transaction_log_view_next(view->log_view, &thdr, &data, &skipped)) > 0) { - if ((hdr->type & MAIL_TRANSACTION_EXPUNGE) != 0 && + if ((thdr->type & MAIL_TRANSACTION_EXPUNGE) != 0 && !map->write_to_disk) { /* expunges have to be atomic. so we'll have to copy the mapping, do the changes there and then finally @@ -443,23 +585,13 @@ mail_index_sync_replace_map(view, map); } - if ((hdr->type & MAIL_TRANSACTION_APPEND) != 0) { - const struct mail_transaction_append_header *append_hdr; - const struct mail_index_record *rec; + if ((thdr->type & MAIL_TRANSACTION_APPEND) != 0) { + const struct mail_index_record *rec = data; - rec = CONST_PTR_OFFSET(data, sizeof(*append_hdr)); if (first_append_uid == 0) first_append_uid = rec->uid; - append_hdr = data; - if (append_hdr->record_size > map->hdr->record_size) { - /* we have to grow our record size */ - map = mail_index_map_to_memory(map, - append_hdr->record_size); - mail_index_sync_replace_map(view, map); - } - count = (hdr->size - sizeof(*append_hdr)) / - append_hdr->record_size; + count = thdr->size / sizeof(*rec); if (mail_index_grow(index, map, count) < 0) { ret = -1; break; @@ -472,12 +604,24 @@ } } - if (mail_transaction_map(index, hdr, data, + if (mail_transaction_map(view->map, thdr, data, &mail_index_map_sync_funcs, &sync_map_ctx) < 0) { ret = -1; break; } + if ((thdr->type & MAIL_TRANSACTION_EXTRA_INTRO) != 0) { + const struct mail_index_extra_record_info *einfo; + size_t size; + + einfo = buffer_get_data(map->extra_infos, &size); + einfo += (size / sizeof(*einfo)) - 1; + + map = mail_index_map_to_memory(map, + einfo->record_offset + + einfo->record_size); + mail_index_sync_replace_map(view, map); + } } if (sync_map_ctx.cache_locked) { @@ -495,13 +639,15 @@ mail_transaction_log_get_head(index->log, &seq, &offset); - map->hdr_copy.log_file_seq = seq; - map->hdr_copy.log_file_offset = offset; + /* hdr pointer may have changed, update it */ + tmphdr = buffer_get_modifyable_data(map->hdr_copy_buf, NULL); + tmphdr->log_file_seq = seq; + tmphdr->log_file_offset = offset; if (first_append_uid != 0) - mail_index_update_day_headers(&map->hdr_copy, first_append_uid); + mail_index_update_day_headers(tmphdr, first_append_uid); - if ((map->hdr_copy.flags & MAIL_INDEX_HDR_FLAG_HAVE_DIRTY) == 0 && + if ((tmphdr->flags & MAIL_INDEX_HDR_FLAG_HAVE_DIRTY) == 0 && had_dirty) { /* do we have dirty flags anymore? */ const struct mail_index_record *rec; @@ -509,8 +655,7 @@ for (i = 0; i < map->records_count; i++) { rec = MAIL_INDEX_MAP_IDX(map, i); if ((rec->flags & MAIL_INDEX_MAIL_FLAG_DIRTY) != 0) { - map->hdr_copy.flags |= - MAIL_INDEX_HDR_FLAG_HAVE_DIRTY; + tmphdr->flags |= MAIL_INDEX_HDR_FLAG_HAVE_DIRTY; break; } } @@ -520,7 +665,7 @@ map->mmap_used_size = index->hdr->header_size + map->records_count * map->hdr->record_size; - memcpy(map->mmap_base, &map->hdr_copy, sizeof(map->hdr_copy)); + memcpy(map->mmap_base, tmphdr, tmphdr->header_size); if (msync(map->mmap_base, map->mmap_used_size, MS_SYNC) < 0) { mail_index_set_syscall_error(index, "msync()"); ret = -1; @@ -535,5 +680,6 @@ struct mail_transaction_map_functions mail_index_map_sync_funcs = { sync_expunge, sync_append, sync_flag_update, sync_cache_reset, sync_cache_update, sync_header_update, - sync_extra_rec_update + sync_extra_intro, sync_extra_reset, + sync_extra_hdr_update, sync_extra_rec_update };
--- a/src/lib-index/mail-index-sync.c Sun Sep 05 20:33:19 2004 +0300 +++ b/src/lib-index/mail-index-sync.c Sun Sep 05 20:53:45 2004 +0300 @@ -125,7 +125,6 @@ ctx->hdr->size); break; case MAIL_TRANSACTION_APPEND: { - const struct mail_transaction_append_header *hdr = ctx->data; const struct mail_index_record *rec = ctx->data; if (ctx->append_uid_first == 0 || @@ -133,7 +132,7 @@ ctx->append_uid_first = rec->uid; rec = CONST_PTR_OFFSET(ctx->data, - ctx->hdr->size - hdr->record_size); + ctx->hdr->size - sizeof(*rec)); if (rec->uid > ctx->append_uid_last) ctx->append_uid_last = rec->uid;
--- a/src/lib-index/mail-index-transaction-private.h Sun Sep 05 20:33:19 2004 +0300 +++ b/src/lib-index/mail-index-transaction-private.h Sun Sep 05 20:53:45 2004 +0300 @@ -9,7 +9,6 @@ buffer_t *appends; uint32_t first_new_seq, last_new_seq; - unsigned int append_record_size; buffer_t *expunges; @@ -20,7 +19,9 @@ unsigned char hdr_change[sizeof(struct mail_index_header)]; unsigned char hdr_mask[sizeof(struct mail_index_header)]; - buffer_t *extra_rec_updates[MAIL_INDEX_MAX_EXTRA_RECORDS]; + buffer_t *extra_rec_updates; /* buffer[] */ + buffer_t *extra_intros; + uint32_t extra_intros_max_id; uint32_t new_cache_file_seq, last_cache_file_seq; buffer_t *cache_updates; @@ -36,4 +37,7 @@ void mail_index_transaction_ref(struct mail_index_transaction *t); void mail_index_transaction_unref(struct mail_index_transaction *t); +int mail_index_seq_buffer_lookup(buffer_t *buffer, uint32_t seq, + size_t record_size, size_t *pos_r); + #endif
--- a/src/lib-index/mail-index-transaction-view.c Sun Sep 05 20:33:19 2004 +0300 +++ b/src/lib-index/mail-index-transaction-view.c Sun Sep 05 20:53:45 2004 +0300 @@ -124,17 +124,48 @@ rec = buffer_get_data(tview->t->appends, NULL); seq = tview->t->first_new_seq; message_count = tview->t->last_new_seq; - for (; seq <= message_count; seq++) { + for (; seq <= message_count; seq++, rec++) { if ((rec->flags & flags_mask) == (uint8_t)flags) { *seq_r = seq; break; } - rec = CONST_PTR_OFFSET(rec, view->index->max_record_size); } return 0; } +static int _tview_lookup_extra(struct mail_index_view *view, uint32_t seq, + uint32_t data_id, const void **data_r) +{ + struct mail_index_view_transaction *tview = + (struct mail_index_view_transaction *)view; + const struct mail_index_extra_record_info *einfo; + buffer_t *const *extra_bufs; + size_t size, pos; + + if (seq < tview->t->first_new_seq) + return tview->parent->lookup_extra(view, seq, data_id, data_r); + + i_assert(seq <= tview->t->last_new_seq); + i_assert(data_id < view->index->extra_infos->used / sizeof(*einfo)); + + einfo = view->index->extra_infos->data; + einfo += data_id; + + extra_bufs = buffer_get_data(tview->t->extra_rec_updates, &size); + size /= sizeof(*extra_bufs); + + if (size <= data_id || extra_bufs[data_id] == NULL || + !mail_index_seq_buffer_lookup(extra_bufs[data_id], seq, + einfo->record_size, &pos)) { + *data_r = NULL; + return 1; + } + + *data_r = CONST_PTR_OFFSET(extra_bufs[data_id]->data, pos); + return 1; +} + static struct mail_index_view_methods view_methods = { _tview_close, _tview_get_message_count, @@ -142,7 +173,8 @@ _tview_lookup_full, _tview_lookup_uid, _tview_lookup_uid_range, - _tview_lookup_first + _tview_lookup_first, + _tview_lookup_extra }; struct mail_index_view *
--- a/src/lib-index/mail-index-transaction.c Sun Sep 05 20:33:19 2004 +0300 +++ b/src/lib-index/mail-index-transaction.c Sun Sep 05 20:53:45 2004 +0300 @@ -33,13 +33,19 @@ static void mail_index_transaction_free(struct mail_index_transaction *t) { - unsigned int i; + buffer_t **recs; + unsigned int i, size; mail_index_view_transaction_unref(t->view); - for (i = 0; i < t->view->index->extra_records_count; i++) { - if (t->extra_rec_updates[i] != NULL) - buffer_free(t->extra_rec_updates[i]); + if (t->extra_rec_updates != NULL) { + recs = buffer_get_modifyable_data(t->extra_rec_updates, &size); + size /= sizeof(*recs); + + for (i = 0; i < size; i++) { + if (recs[i] != NULL) + buffer_free(recs[i]); + } } if (t->appends != NULL) @@ -50,6 +56,8 @@ buffer_free(t->updates); if (t->cache_updates != NULL) buffer_free(t->cache_updates); + if (t->extra_intros != NULL) + buffer_free(t->extra_intros); i_free(t); } @@ -65,12 +73,15 @@ } static void -mail_index_buffer_convert_to_uids(struct mail_index_view *view, +mail_index_buffer_convert_to_uids(struct mail_index_transaction *t, buffer_t *buf, size_t record_size, int range) { + struct mail_index_view *view = t->view; + const struct mail_index_record *rec; unsigned char *data; size_t size, i; uint32_t *seq; + int j; if (buf == NULL) return; @@ -79,12 +90,16 @@ data = buffer_get_modifyable_data(buf, &size); for (i = 0; i < size; i += record_size) { seq = (uint32_t *)&data[i]; - i_assert(seq[0] <= view->map->records_count); - seq[0] = MAIL_INDEX_MAP_IDX(view->map, seq[0]-1)->uid; - if (range) { - i_assert(seq[1] <= view->map->records_count); - seq[1] = MAIL_INDEX_MAP_IDX(view->map, seq[1]-1)->uid; + for (j = 0; j <= range; j++, seq++) { + if (*seq >= t->first_new_seq) { + rec = mail_index_transaction_lookup(t, *seq); + *seq = rec->uid; + } else { + *seq = MAIL_INDEX_MAP_IDX(view->map, + *seq - 1)->uid; + } + i_assert(*seq != 0); } } } @@ -93,27 +108,34 @@ mail_index_transaction_convert_to_uids(struct mail_index_transaction *t) { struct mail_index *index = t->view->index; - unsigned int i; + const struct mail_index_extra_record_info *einfos; + buffer_t **updates; + unsigned int i, size; if (mail_index_view_lock(t->view) < 0) return -1; - for (i = 0; i < index->extra_records_count; i++) { - if (t->extra_rec_updates[i] == NULL) - continue; + if (t->extra_rec_updates != NULL) { + einfos = buffer_get_data(index->extra_infos, NULL); + updates = buffer_get_modifyable_data(t->extra_rec_updates, + &size); + size /= sizeof(*updates); - mail_index_buffer_convert_to_uids(t->view, - t->extra_rec_updates[i], - sizeof(uint32_t) + - index->extra_records[i].size, - FALSE); + for (i = 0; i < size; i++) { + if (updates[i] == NULL) + continue; + + mail_index_buffer_convert_to_uids(t, updates[i], + sizeof(uint32_t) + einfos[i].record_size, + FALSE); + } } - mail_index_buffer_convert_to_uids(t->view, t->expunges, + mail_index_buffer_convert_to_uids(t, t->expunges, sizeof(struct mail_transaction_expunge), TRUE); - mail_index_buffer_convert_to_uids(t->view, t->updates, + mail_index_buffer_convert_to_uids(t, t->updates, sizeof(struct mail_transaction_flag_update), TRUE); - mail_index_buffer_convert_to_uids(t->view, t->cache_updates, + mail_index_buffer_convert_to_uids(t, t->cache_updates, sizeof(struct mail_transaction_cache_update), FALSE); return 0; } @@ -157,42 +179,6 @@ mail_index_transaction_unref(t); } -static void -mail_index_transaction_update_append_size(struct mail_index_transaction *t) -{ - buffer_t *new_buf; - unsigned int new_record_size; - const void *src; - void *dest; - size_t i, size; - - new_record_size = t->view->index->max_record_size; - if (t->append_record_size == new_record_size) - return; - - i_assert(t->append_record_size < new_record_size); - - if (t->append_record_size != 0) { - /* resize the records in buffer */ - src = buffer_get_data(t->appends, &size); - size /= t->append_record_size; - - new_buf = buffer_create_dynamic(default_pool, - (size + 10) * new_record_size, - (size_t)-1); - for (i = 0; i < size; i++) { - dest = buffer_append_space_unsafe(new_buf, - new_record_size); - memcpy(dest, src, t->append_record_size); - src = CONST_PTR_OFFSET(src, t->append_record_size); - } - buffer_free(t->appends); - t->appends = new_buf; - } - - t->append_record_size = new_record_size; -} - struct mail_index_record * mail_index_transaction_lookup(struct mail_index_transaction *t, uint32_t seq) { @@ -200,10 +186,9 @@ i_assert(seq >= t->first_new_seq && seq <= t->last_new_seq); - mail_index_transaction_update_append_size(t); - - pos = (seq - t->first_new_seq) * t->append_record_size; - return buffer_get_space_unsafe(t->appends, pos, t->append_record_size); + pos = (seq - t->first_new_seq) * sizeof(struct mail_index_record); + return buffer_get_space_unsafe(t->appends, pos, + sizeof(struct mail_index_record)); } void mail_index_append(struct mail_index_transaction *t, uint32_t uid, @@ -214,9 +199,7 @@ if (t->appends == NULL) { t->appends = buffer_create_dynamic(default_pool, 4096, (size_t)-1); - t->append_record_size = t->view->index->max_record_size; } - mail_index_transaction_update_append_size(t); /* sequence number is visible only inside given view, so let it generate it */ @@ -225,8 +208,8 @@ else *seq_r = t->last_new_seq = t->first_new_seq; - rec = buffer_append_space_unsafe(t->appends, t->append_record_size); - memset(rec, 0, t->append_record_size); + rec = buffer_append_space_unsafe(t->appends, sizeof(*rec)); + memset(rec, 0, sizeof(*rec)); rec->uid = uid; } @@ -243,16 +226,14 @@ end = PTR_OFFSET(rec, size); /* find the first mail with uid = 0 */ - while (rec != end) { + for (; rec != end; rec++) { if (rec->uid == 0) break; - rec = PTR_OFFSET(rec, t->append_record_size); } - while (rec != end) { + for (; rec != end; rec++) { i_assert(rec->uid == 0); rec->uid = first_uid++; - rec = PTR_OFFSET(rec, t->append_record_size); } *next_uid_r = first_uid; @@ -511,9 +492,8 @@ &update, sizeof(update)); } -static int -mail_index_seq_buffer_lookup(buffer_t *buffer, uint32_t seq, - size_t record_size, size_t *pos_r) +int mail_index_seq_buffer_lookup(buffer_t *buffer, uint32_t seq, + size_t record_size, size_t *pos_r) { unsigned int idx, left_idx, right_idx; void *data; @@ -657,20 +637,22 @@ const void *data) { struct mail_index *index = t->view->index; - struct mail_index_record *rec; + buffer_t **buf; + const struct mail_index_extra_record_info *einfo; - i_assert(data_id < index->extra_records_count); + i_assert(data_id < index->extra_infos->used / sizeof(*einfo)); + + einfo = index->extra_infos->data; + einfo += data_id; - if (seq >= t->first_new_seq) { - /* just appended message, modify it directly */ - /* FIXME: do data_id mapping conversion */ - rec = mail_index_transaction_lookup(t, seq); - memcpy(PTR_OFFSET(rec, index->extra_records[data_id].offset), - data, index->extra_records[data_id].size); - } else { - mail_index_update_seq_buffer(&t->extra_rec_updates[data_id], - seq, data, index->extra_records[data_id].size, NULL); + if (t->extra_rec_updates == NULL) { + t->extra_rec_updates = + buffer_create_dynamic(default_pool, 128, (size_t)-1); } + buf = buffer_get_space_unsafe(t->extra_rec_updates, + data_id * sizeof(buffer_t *), + sizeof(buffer_t *)); + mail_index_update_seq_buffer(buf, seq, data, einfo->record_size, NULL); } void mail_index_update_header(struct mail_index_transaction *t,
--- a/src/lib-index/mail-index-view-private.h Sun Sep 05 20:33:19 2004 +0300 +++ b/src/lib-index/mail-index-view-private.h Sun Sep 05 20:53:45 2004 +0300 @@ -18,6 +18,8 @@ uint32_t *first_seq_r, uint32_t *last_seq_r); int (*lookup_first)(struct mail_index_view *view, enum mail_flags flags, uint8_t flags_mask, uint32_t *seq_r); + int (*lookup_extra)(struct mail_index_view *view, uint32_t seq, + uint32_t data_id, const void **data_r); }; struct mail_index_view {
--- a/src/lib-index/mail-index-view-sync.c Sun Sep 05 20:33:19 2004 +0300 +++ b/src/lib-index/mail-index-view-sync.c Sun Sep 05 20:53:45 2004 +0300 @@ -206,7 +206,7 @@ memset(&sync_map_ctx, 0, sizeof(sync_map_ctx)); sync_map_ctx.view = view; - if (mail_transaction_map(view->index, ctx->hdr, ctx->data, + if (mail_transaction_map(view->map, ctx->hdr, ctx->data, &mail_index_map_sync_funcs, &sync_map_ctx) < 0) return -1;
--- a/src/lib-index/mail-index-view.c Sun Sep 05 20:33:19 2004 +0300 +++ b/src/lib-index/mail-index-view.c Sun Sep 05 20:53:45 2004 +0300 @@ -407,6 +407,32 @@ return 0; } +static int _view_lookup_extra(struct mail_index_view *view, uint32_t seq, + uint32_t data_id, const void **data_r) +{ + const struct mail_index_extra_record_info *einfo; + const struct mail_index_record *rec; + struct mail_index_map *map; + uint32_t idx, offset; + int ret; + + if ((ret = mail_index_lookup_full(view, seq, &map, &rec)) < 0) + return -1; + + if (rec == NULL || + !mail_index_map_get_extra_info_idx(view->map, data_id, &idx)) { + *data_r = NULL; + return ret; + } + + einfo = view->map->extra_infos->data; + einfo += idx; + + offset = einfo->record_offset; + *data_r = offset == 0 ? NULL : CONST_PTR_OFFSET(rec, offset); + return ret; +} + void mail_index_view_close(struct mail_index_view *view) { view->methods.close(view); @@ -431,35 +457,19 @@ return mail_index_lookup_full(view, seq, &map, rec_r); } +int mail_index_lookup_full(struct mail_index_view *view, uint32_t seq, + struct mail_index_map **map_r, + const struct mail_index_record **rec_r) +{ + return view->methods.lookup_full(view, seq, map_r, rec_r); +} + int mail_index_lookup_uid(struct mail_index_view *view, uint32_t seq, uint32_t *uid_r) { return view->methods.lookup_uid(view, seq, uid_r); } -int mail_index_lookup_extra(struct mail_index_view *view, uint32_t seq, - uint32_t data_id, const void **data_r) -{ - const struct mail_index_record *rec; - struct mail_index_map *map; - uint32_t offset; - int ret; - - if ((ret = mail_index_lookup_full(view, seq, &map, &rec)) < 0) - return -1; - - if (rec == NULL) { - *data_r = NULL; - return ret; - } - - /* FIXME: do data_id mapping conversion */ - - offset = view->index->extra_records[data_id].offset; - *data_r = CONST_PTR_OFFSET(rec, offset); - return ret; -} - int mail_index_lookup_uid_range(struct mail_index_view *view, uint32_t first_uid, uint32_t last_uid, uint32_t *first_seq_r, uint32_t *last_seq_r) @@ -474,11 +484,10 @@ return view->methods.lookup_first(view, flags, flags_mask, seq_r); } -int mail_index_lookup_full(struct mail_index_view *view, uint32_t seq, - struct mail_index_map **map_r, - const struct mail_index_record **rec_r) +int mail_index_lookup_extra(struct mail_index_view *view, uint32_t seq, + uint32_t data_id, const void **data_r) { - return view->methods.lookup_full(view, seq, map_r, rec_r); + return view->methods.lookup_extra(view, seq, data_id, data_r); } static struct mail_index_view_methods view_methods = { @@ -488,7 +497,8 @@ _view_lookup_full, _view_lookup_uid, _view_lookup_uid_range, - _view_lookup_first + _view_lookup_first, + _view_lookup_extra }; struct mail_index_view *mail_index_view_open(struct mail_index *index)
--- a/src/lib-index/mail-index.c Sun Sep 05 20:33:19 2004 +0300 +++ b/src/lib-index/mail-index.c Sun Sep 05 20:53:45 2004 +0300 @@ -26,12 +26,11 @@ index->prefix = i_strdup(prefix); index->fd = -1; - index->extra_records_pool = - pool_alloconly_create("extra_record_pool", 256); - index->extra_records_buf = - buffer_create_dynamic(index->extra_records_pool, + index->extra_infos_pool = + pool_alloconly_create("extra_infos_pool", 256); + index->extra_infos = + buffer_create_dynamic(index->extra_infos_pool, 64, (size_t)-1); - index->max_record_size = sizeof(struct mail_index_record); index->mode = 0600; index->gid = (gid_t)-1; @@ -41,7 +40,7 @@ void mail_index_free(struct mail_index *index) { mail_index_close(index); - pool_unref(index->extra_records_pool); + pool_unref(index->extra_infos_pool); i_free(index->error); i_free(index->dir); @@ -50,42 +49,171 @@ } uint32_t mail_index_register_record_extra(struct mail_index *index, - const char *name, uint16_t size) + const char *name, uint32_t hdr_size, + uint16_t record_size) { + const struct mail_index_extra_record_info *einfos; struct mail_index_extra_record_info info; - size_t buf_size; + size_t extra_count; unsigned int i; + einfos = buffer_get_data(index->extra_infos, &extra_count); + extra_count /= sizeof(*einfos); + /* see if it's there already */ - for (i = 0; i < index->extra_records_count; i++) { - if (strcmp(index->extra_records[i].name, name) == 0) { - i_assert(index->extra_records[i].size == size); + for (i = 0; i < extra_count; i++) { + if (strcmp(einfos[i].name, name) == 0) { + i_assert(einfos[i].hdr_size == hdr_size); + i_assert(einfos[i].record_size == record_size); return i; } } - i_assert(size % 4 == 0); - i_assert(index->max_record_size + size <= 65535); - - if (index->extra_records_count >= MAIL_INDEX_MAX_EXTRA_RECORDS) { - i_panic("Maximum extra record count reached, " - "you'll need to recompile with larger limit. " - "MAIL_INDEX_MAX_EXTRA_RECORDS = %d", - MAIL_INDEX_MAX_EXTRA_RECORDS); - } + i_assert(hdr_size % 4 == 0); + i_assert(record_size % 4 == 0); memset(&info, 0, sizeof(info)); - info.name = p_strdup(index->extra_records_pool, name); - info.size = size; - info.offset = index->max_record_size; + info.name = p_strdup(index->extra_infos_pool, name); + info.hdr_size = hdr_size; + info.record_size = record_size; + + buffer_append(index->extra_infos, &info, sizeof(info)); + return extra_count; +} + +static void mail_index_map_create_extra_infos(struct mail_index_map *map, + unsigned int initial_count) +{ + size_t extra_infos_size, extra_infos_id_map_size, size; + + extra_infos_size = initial_count * + sizeof(struct mail_index_extra_record_info); + extra_infos_id_map_size = initial_count * sizeof(uint32_t); + if (map->extra_records_pool == NULL) { + size = extra_infos_size + extra_infos_id_map_size + + initial_count * 20; + map->extra_records_pool = + pool_alloconly_create("extra_infos", + nearest_power(size)); + } + + map->extra_infos = buffer_create_dynamic(map->extra_records_pool, + extra_infos_size, (size_t)-1); + map->extra_infos_id_map = buffer_create_dynamic(map->extra_records_pool, + extra_infos_id_map_size, + (size_t)-1); +} + +uint32_t mail_index_map_register_extra_info(struct mail_index *index, + struct mail_index_map *map, + const char *name, + uint32_t hdr_offset, + uint32_t hdr_size, + uint32_t record_size) +{ + const struct mail_index_extra_record_info *last_einfo; + struct mail_index_extra_record_info *einfo; + size_t size; + uint32_t idx, data_id; + + if (map->extra_infos == NULL) { + mail_index_map_create_extra_infos(map, 5); + last_einfo = NULL; + idx = 0; + } else { + last_einfo = buffer_get_data(map->extra_infos, &size); + idx = size / sizeof(*last_einfo); + if (idx == 0) + last_einfo = NULL; + else + last_einfo += idx - 1; + } + + einfo = buffer_append_space_unsafe(map->extra_infos, sizeof(*einfo)); + memset(einfo, 0, sizeof(*einfo)); + + einfo->name = p_strdup(map->extra_records_pool, name); + einfo->hdr_offset = hdr_offset; + einfo->hdr_size = hdr_size; + einfo->record_size = record_size; + + if (last_einfo != NULL) { + einfo->record_offset = last_einfo->record_offset + + last_einfo->record_size; + } else { + einfo->record_offset = sizeof(struct mail_index_record); + } - buffer_append(index->extra_records_buf, &info, sizeof(info)); - index->extra_records = - buffer_get_data(index->extra_records_buf, &buf_size); - index->extra_records_count = buf_size / sizeof(info); + data_id = mail_index_register_record_extra(index, name, + hdr_size, record_size); + buffer_write(map->extra_infos_id_map, data_id * sizeof(uint32_t), + &idx, sizeof(idx)); + return idx; +} + +static int mail_index_read_extra_infos(struct mail_index *index, + struct mail_index_map *map) +{ + const struct mail_index_extra_record_info_header *einfo_hdr; + unsigned int i, old_count; + const char *name; + uint32_t data_id, offset, name_offset; + + offset = map->hdr->base_header_size; + if (offset == map->hdr->header_size && + map->extra_records_pool == NULL) { + /* nothing to do, skip allocatations and all */ + return 1; + } + + old_count = index->extra_infos->used / + sizeof(struct mail_index_extra_record_info); + + if (map->extra_records_pool != NULL) + p_clear(map->extra_records_pool); + mail_index_map_create_extra_infos(map, old_count + 5); + data_id = (uint32_t)-1; + for (i = 0; i < old_count; i++) { + buffer_append(map->extra_infos_id_map, + &data_id, sizeof(data_id)); + } - index->max_record_size += size; - return index->extra_records_count-1; + name = map->hdr_base; + while (offset < map->hdr->header_size) { + name_offset = offset; + + while (offset < map->hdr->header_size && name[offset] != '\0') + offset++; + if (offset == map->hdr->header_size) { + mail_index_set_error(index, "Corrupted index file %s: " + "Header extension name doesn't end with NUL", + index->filepath); + return -1; + } + offset++; + while (offset < map->hdr->header_size && (offset % 4) != 0) + offset++; + + einfo_hdr = CONST_PTR_OFFSET(map->hdr_base, offset); + + if (offset + sizeof(*einfo_hdr) > map->hdr->header_size || + offset + sizeof(*einfo_hdr) + einfo_hdr->hdr_size > + map->hdr->header_size) { + mail_index_set_error(index, "Corrupted index file %s: " + "Header extension goes outside header", + index->filepath); + return -1; + } + + mail_index_map_register_extra_info(index, map, + name + name_offset, + offset + sizeof(*einfo_hdr), + einfo_hdr->hdr_size, + einfo_hdr->record_size); + + offset += sizeof(*einfo_hdr) + einfo_hdr->hdr_size; + } + return 1; } static int mail_index_check_header(struct mail_index *index, @@ -151,7 +279,7 @@ hdr->first_deleted_uid_lowwater > hdr->next_uid) return 0; - return 1; + return mail_index_read_extra_infos(index, map); } static void mail_index_map_clear(struct mail_index *index, @@ -184,6 +312,9 @@ i_assert(map->refcount == 0); mail_index_map_clear(index, map); + if (map->extra_records_pool != NULL) + pool_unref(map->extra_records_pool); + buffer_free(map->hdr_copy_buf); i_free(map); } @@ -197,6 +328,7 @@ static int mail_index_mmap(struct mail_index *index, struct mail_index_map *map) { const struct mail_index_header *hdr; + struct mail_index_header *mhdr; unsigned int records_count; i_assert(!map->write_to_disk); @@ -241,25 +373,35 @@ if (map->hdr->base_header_size < sizeof(*map->hdr)) { /* header smaller than ours, make a copy so our newer headers won't have garbage in them */ - memcpy(&map->hdr_copy, map->hdr, map->hdr->base_header_size); - map->hdr = &map->hdr_copy; - } + buffer_reset(map->hdr_copy_buf); + buffer_append(map->hdr_copy_buf, + map->hdr, map->hdr->base_header_size); + buffer_set_used_size(map->hdr_copy_buf, sizeof(*map->hdr)); + mhdr = buffer_get_modifyable_data(map->hdr_copy_buf, NULL); + mhdr->base_header_size = sizeof(*map->hdr); + mhdr->header_size = map->hdr_copy_buf->used; + + map->hdr = mhdr; + } + + map->hdr_base = map->mmap_base; map->records = PTR_OFFSET(map->mmap_base, map->hdr->header_size); map->records_count = map->hdr->messages_count; return 1; } static int mail_index_read_map(struct mail_index *index, - struct mail_index_map *map) + struct mail_index_map *map, int *retry_r) { - struct mail_index_header hdr; + struct mail_index_header hdr, *hdrp; void *data = NULL; ssize_t ret; size_t pos, records_size; i_assert(map->mmap_base == NULL); + *retry_r = FALSE; memset(&hdr, 0, sizeof(hdr)); ret = 1; @@ -269,7 +411,50 @@ if (ret > 0) pos += ret; } - if (ret >= 0 && pos >= MAIL_INDEX_HEADER_MIN_SIZE) { + + if (ret >= 0 && pos >= MAIL_INDEX_HEADER_MIN_SIZE && + (ret > 0 || pos >= hdr.base_header_size)) { + if (hdr.base_header_size < MAIL_INDEX_HEADER_MIN_SIZE || + hdr.header_size < hdr.base_header_size) { + mail_index_set_error(index, "Corrupted index file %s: " + "Corrupted header sizes (base %u, full %u)", + index->filepath, hdr.base_header_size, + hdr.header_size); + return 0; + } + + buffer_reset(map->hdr_copy_buf); + if (hdr.base_header_size < sizeof(hdr)) { + buffer_append(map->hdr_copy_buf, &hdr, + hdr.base_header_size); + buffer_set_used_size(map->hdr_copy_buf, sizeof(hdr) + + hdr.header_size - + hdr.base_header_size); + + /* @UNSAFE */ + ret = pread_full(index->fd, + PTR_OFFSET(map->hdr_copy_buf->data, + sizeof(hdr)), + hdr.header_size - hdr.base_header_size, + hdr.base_header_size); + + hdrp = buffer_get_modifyable_data(map->hdr_copy_buf, + NULL); + hdrp->base_header_size = sizeof(hdr); + hdrp->header_size = map->hdr_copy_buf->used; + } else { + buffer_append(map->hdr_copy_buf, &hdr, pos); + buffer_set_used_size(map->hdr_copy_buf, + hdr.header_size); + /* @UNSAFE */ + ret = pread_full(index->fd, + PTR_OFFSET(map->hdr_copy_buf->data, + pos), + hdr.header_size - pos, pos); + } + } + + if (ret > 0) { records_size = hdr.messages_count * hdr.record_size; if (map->buffer == NULL) { @@ -287,33 +472,36 @@ } if (ret < 0) { - if (errno == ESTALE) + if (errno == ESTALE) { + *retry_r = TRUE; return 0; + } mail_index_set_syscall_error(index, "pread_full()"); return -1; } if (ret == 0) { mail_index_set_error(index, - "Unexpected EOF while reading index file"); + "Corrupted index file %s: File too small", + index->filepath); return -1; } map->records = data; map->records_count = hdr.messages_count; - map->hdr_copy = hdr; - map->hdr = &map->hdr_copy; + map->hdr = map->hdr_copy_buf->data; + map->hdr_base = map->hdr_copy_buf->data; return 1; } static int mail_index_read_map_with_retry(struct mail_index *index, struct mail_index_map *map) { - int i, ret; + int i, ret, retry; for (i = 0; i < MAIL_INDEX_ESTALE_RETRY_COUNT; i++) { - ret = mail_index_read_map(index, map); - if (ret != 0) + ret = mail_index_read_map(index, map, &retry); + if (ret != 0 || !retry) return ret; /* ESTALE - reopen index file */ @@ -381,6 +569,9 @@ if (map == NULL) { map = i_new(struct mail_index_map, 1); map->refcount = 1; + map->hdr_copy_buf = + buffer_create_dynamic(default_pool, + sizeof(*map->hdr), (size_t)-1); } else if (MAIL_INDEX_MAP_IS_IN_MEMORY(map)) { if (map->write_to_disk) { /* we have modified this mapping and it's waiting to @@ -432,9 +623,11 @@ mail_index_map_to_memory(struct mail_index_map *map, uint32_t new_record_size) { struct mail_index_map *mem_map; + struct mail_index_header *hdr; + struct mail_index_extra_record_info *einfos; void *src, *dest; size_t size, copy_size; - unsigned int i; + unsigned int i, count; if (MAIL_INDEX_MAP_IS_IN_MEMORY(map)) { map->refcount++; @@ -462,12 +655,50 @@ mem_map->records = buffer_get_modifyable_data(mem_map->buffer, NULL); mem_map->records_count = map->records_count; - mem_map->hdr_copy = *map->hdr; - mem_map->hdr_copy.record_size = new_record_size; - mem_map->hdr = &mem_map->hdr_copy; + mem_map->hdr_copy_buf = buffer_create_dynamic(default_pool, + map->hdr->header_size, + (size_t)-1); + buffer_append(mem_map->hdr_copy_buf, map->hdr, map->hdr->header_size); + + hdr = buffer_get_modifyable_data(mem_map->hdr_copy_buf, NULL); + hdr->record_size = new_record_size; + mem_map->hdr = hdr; + + /* copy extra_infos */ + if (map->extra_infos_id_map != NULL) { + count = map->extra_infos_id_map->used / sizeof(uint32_t); + mail_index_map_create_extra_infos(mem_map, count); + + buffer_append_buf(mem_map->extra_infos, map->extra_infos, + 0, (size_t)-1); + buffer_append_buf(mem_map->extra_infos_id_map, + map->extra_infos_id_map, 0, (size_t)-1); + + /* fix the name pointers to use our own pool */ + einfos = buffer_get_modifyable_data(mem_map->extra_infos, NULL); + for (i = 0; i < count; i++) { + einfos[i].name = p_strdup(mem_map->extra_records_pool, + einfos[i].name); + } + } + return mem_map; } +int mail_index_map_get_extra_info_idx(struct mail_index_map *map, + uint32_t data_id, uint32_t *idx_r) +{ + const uint32_t *id_map; + + if (map->extra_infos_id_map == NULL || + map->extra_infos_id_map->used / sizeof(*id_map) <= data_id) + return 0; + + id_map = map->extra_infos_id_map->data; + *idx_r = id_map[data_id]; + return *idx_r != (uint32_t)-1; +} + static int mail_index_try_open_only(struct mail_index *index) { int i; @@ -550,8 +781,8 @@ return -1; } - index->map->hdr_copy = *hdr; - index->hdr = &index->map->hdr_copy; + buffer_write(index->map->hdr_copy_buf, 0, hdr, hdr_size); + i_assert(index->hdr == index->map->hdr_copy_buf->data); } return 0; @@ -631,8 +862,7 @@ return ret; } -static void mail_index_header_init(struct mail_index *index, - struct mail_index_header *hdr) +static void mail_index_header_init(struct mail_index_header *hdr) { time_t now = time(NULL); @@ -642,7 +872,7 @@ hdr->minor_version = MAIL_INDEX_MINOR_VERSION; hdr->base_header_size = sizeof(*hdr); hdr->header_size = sizeof(*hdr); - hdr->record_size = index->max_record_size; + hdr->record_size = sizeof(struct mail_index_record); hdr->keywords_mask_size = sizeof(keywords_mask_t); #ifndef WORDS_BIGENDIAN @@ -671,7 +901,7 @@ /* doesn't exist, or corrupted */ if ((flags & MAIL_INDEX_OPEN_FLAG_CREATE) == 0) return 0; - mail_index_header_init(index, &hdr); + mail_index_header_init(&hdr); index->hdr = &hdr; } else if (ret < 0) return -1;
--- a/src/lib-index/mail-index.h Sun Sep 05 20:33:19 2004 +0300 +++ b/src/lib-index/mail-index.h Sun Sep 05 20:53:45 2004 +0300 @@ -3,7 +3,7 @@ #include "mail-types.h" -#define MAIL_INDEX_MAJOR_VERSION 4 +#define MAIL_INDEX_MAJOR_VERSION 5 #define MAIL_INDEX_MINOR_VERSION 0 #define MAIL_INDEX_HEADER_MIN_SIZE 120 @@ -97,7 +97,6 @@ uint64_t sync_size; uint32_t cache_file_seq; - uint32_t extra_records_hdr_offset; /* daily first UIDs that have been added to index. */ uint32_t day_stamp; @@ -142,7 +141,8 @@ identifier for the data. if same name is tried to be registered multiple times, the rest are ignored. returns identifier for the name. */ uint32_t mail_index_register_record_extra(struct mail_index *index, - const char *name, uint16_t size); + const char *name, uint32_t hdr_size, + uint16_t record_size); int mail_index_open(struct mail_index *index, enum mail_index_open_flags flags); void mail_index_close(struct mail_index *index);
--- a/src/lib-index/mail-transaction-log-view.c Sun Sep 05 20:33:19 2004 +0300 +++ b/src/lib-index/mail-transaction-log-view.c Sun Sep 05 20:53:45 2004 +0300 @@ -22,6 +22,8 @@ uint32_t prev_file_seq; uoff_t prev_file_offset; + uint32_t max_extra_data_id; + unsigned int broken:1; }; @@ -286,25 +288,35 @@ ", size=%u, end=%"PRIuSIZE_T")", hdr->type & MAIL_TRANSACTION_TYPE_MASK, view->cur_offset, hdr_size, file_size); - view->cur_offset = file_size; return -1; } if (hdr_size < sizeof(*hdr)) { + type_rec = NULL; + record_size = 0; + } else { + type_rec = mail_transaction_type_lookup(hdr->type); + if (type_rec != NULL) + record_size = type_rec->record_size; + else { + mail_transaction_log_file_set_corrupted(file, + "unknown record type 0x%x", + hdr->type & MAIL_TRANSACTION_TYPE_MASK); + return -1; + } + } + + if (hdr_size < sizeof(*hdr) + record_size) { mail_transaction_log_file_set_corrupted(file, "record size too small (type=0x%x, size=%u)", hdr->type & MAIL_TRANSACTION_TYPE_MASK, hdr_size); - view->cur_offset = file_size; return -1; } - type_rec = mail_transaction_type_lookup(hdr->type); - if (type_rec != NULL) - record_size = type_rec->record_size; - else { + if ((hdr_size - sizeof(*hdr)) % record_size != 0) { mail_transaction_log_file_set_corrupted(file, - "unknown record type 0x%x", - hdr->type & MAIL_TRANSACTION_TYPE_MASK); - view->cur_offset = file->sync_offset; + "record size wrong (type 0x%x, %u %% %u != 0)", + hdr->type & MAIL_TRANSACTION_TYPE_MASK, + (hdr_size - sizeof(*hdr)), record_size); return -1; } @@ -320,25 +332,34 @@ "extra bits in header type: 0x%x", hdr->type & MAIL_TRANSACTION_TYPE_MASK); return -1; - } else if (hdr->type == MAIL_TRANSACTION_EXTRA_REC_UPDATE) { - const struct mail_transaction_extra_rec_header *ehdr = data; + } else if (hdr->type == MAIL_TRANSACTION_EXTRA_INTRO) { + const struct mail_transaction_extra_intro *intro = data; - if (ehdr->data_id >= view->log->index->extra_records_count) { + if (intro->data_id > view->max_extra_data_id) + view->max_extra_data_id = intro->data_id; + if (intro->name_size > + hdr_size - sizeof(*hdr) - sizeof(*intro)) { mail_transaction_log_file_set_corrupted(file, - "extra record update out of range (%u > %u)", - ehdr->data_id, - view->log->index->extra_records_count); + "extra intro: name_size too large"); return -1; } - } + } else if (hdr->type == MAIL_TRANSACTION_EXTRA_REC_UPDATE || + hdr->type == MAIL_TRANSACTION_EXTRA_HDR_UPDATE || + hdr->type == MAIL_TRANSACTION_EXTRA_RESET) { + const uint32_t *data_id = data; + uint32_t max_data_id; - if ((hdr_size - sizeof(*hdr)) % record_size != 0) { - mail_transaction_log_file_set_corrupted(file, - "record size wrong (type 0x%x, %u %% %u != 0)", - hdr->type & MAIL_TRANSACTION_TYPE_MASK, - (hdr_size - sizeof(*hdr)), record_size); - view->cur_offset = file->sync_offset; - return -1; + max_data_id = view->log->index->map->extra_infos->used / + sizeof(struct mail_index_extra_record_info); + if (view->max_extra_data_id > max_data_id) + max_data_id = view->max_extra_data_id; + + if (*data_id >= max_data_id) { + mail_transaction_log_file_set_corrupted(file, + "extra record update out of range (%u > %u)", + *data_id, max_data_id); + return -1; + } } *hdr_r = hdr; @@ -372,8 +393,12 @@ append isn't in mask */ } - if (ret <= 0) - return ret; + if (ret < 0) { + view->cur_offset = view->cur->sync_offset; + return -1; + } + if (ret == 0) + return 0; view->tmp_hdr = *hdr; view->tmp_hdr.size =
--- a/src/lib-index/mail-transaction-log.c Sun Sep 05 20:33:19 2004 +0300 +++ b/src/lib-index/mail-transaction-log.c Sun Sep 05 20:53:45 2004 +0300 @@ -925,10 +925,8 @@ const struct mail_transaction_header *hdr, const void *data) { - const struct mail_transaction_append_header *append_hdr = data; const struct mail_index_record *old, *old_end; struct mail_index_record *appends, *end, *rec, *dest; - uint32_t record_size = t->append_record_size; size_t size; int deleted = FALSE; @@ -944,11 +942,10 @@ /* we'll just check that none of the appends are already in transaction log. this could happen if we crashed before we had a chance to update index file */ - old = CONST_PTR_OFFSET(data, sizeof(*append_hdr)); old_end = CONST_PTR_OFFSET(data, hdr->size); - while (old != old_end) { + for (old = data; old != old_end; old++) { /* appends are sorted */ - for (rec = appends; rec != end; ) { + for (rec = appends; rec != end; rec++) { if (rec->uid >= old->uid) { if (rec->uid == old->uid) { rec->uid = 0; @@ -956,25 +953,56 @@ } break; } - rec = PTR_OFFSET(rec, record_size); } - old = CONST_PTR_OFFSET(old, append_hdr->record_size); } if (deleted) { /* compress deleted appends away */ - for (rec = dest = appends; rec != end; ) { + for (rec = dest = appends; rec != end; rec++) { if (rec->uid != 0) dest++; else if (rec != dest) *rec = *dest; - rec = PTR_OFFSET(rec, record_size); } buffer_set_used_size(t->appends, (char *)dest - (char *)appends); } } +static void +transaction_save_extra_intro(struct mail_index_transaction *t, + const struct mail_transaction_extra_intro *intro) +{ + const char *name; + void *p; + uint32_t data_id; + size_t pos; + + if (t->extra_intros == NULL) { + t->extra_intros = + buffer_create_dynamic(default_pool, 128, (size_t)-1); + } + + t_push(); + name = t_strndup((const char *)(intro+1), intro->name_size); + data_id = mail_index_register_record_extra(t->view->index, name, + intro->hdr_size, + intro->record_size); + pos = data_id * sizeof(intro->data_id); + if (pos > t->extra_intros->used) { + /* unused records are -1 */ + p = buffer_append_space_unsafe(t->extra_intros, + pos - t->extra_intros->used); + memset(p, 0xff, pos - t->extra_intros->used); + } + + buffer_write(t->extra_intros, pos, + &intro->data_id, sizeof(intro->data_id)); + if (intro->data_id > t->extra_intros_max_id) + t->extra_intros_max_id = intro->data_id; + t_pop(); +} + static int mail_transaction_log_scan_pending(struct mail_transaction_log *log, struct mail_index_transaction *t) { @@ -1001,6 +1029,12 @@ max_cache_file_seq = reset->new_file_seq; break; } + case MAIL_TRANSACTION_EXTRA_INTRO: { + const struct mail_transaction_extra_intro *intro = data; + + transaction_save_extra_intro(t, intro); + break; + } } } @@ -1114,6 +1148,49 @@ return buf; } +static int +mail_transaction_log_register_extra(struct mail_transaction_log_file *file, + struct mail_index_transaction *t, + uint32_t data_id, uint32_t *idx_r) +{ + const struct mail_index_extra_record_info *einfo; + struct mail_transaction_extra_intro *intro; + const uint32_t *id_map; + buffer_t *buf; + size_t size; + int ret; + + /* first check if it's already in nonsynced part of transaction log */ + if (t->extra_intros != NULL) { + id_map = buffer_get_data(t->extra_intros, &size); + size /= sizeof(*id_map); + + if (data_id < size && id_map[data_id] != (uint32_t)-1) { + *idx_r = id_map[data_id]; + return 0; + } + } + *idx_r = t->extra_intros_max_id++; + + einfo = t->view->index->extra_infos->data; + einfo += data_id; + + /* nope, register */ + t_push(); + buf = buffer_create_dynamic(pool_datastack_create(), 128, (size_t)-1); + intro = buffer_append_space_unsafe(buf, sizeof(*intro)); + intro->data_id = *idx_r; + intro->hdr_size = einfo->hdr_size; + intro->record_size = einfo->record_size; + intro->name_size = strlen(einfo->name); + buffer_append(buf, einfo->name, intro->name_size); + + ret = log_append_buffer(file, buf, NULL, MAIL_TRANSACTION_EXTRA_INTRO, + t->view->external); + t_pop(); + return ret; +} + int mail_transaction_log_append(struct mail_index_transaction *t, uint32_t *log_file_seq_r, uoff_t *log_file_offset_r) @@ -1125,8 +1202,10 @@ struct mail_transaction_log_file *file; struct mail_index_header idx_hdr; uoff_t append_offset; - buffer_t *hdr_buf; + buffer_t *hdr_buf, **updates; unsigned int i, lock_id; + uint32_t idx; + size_t size; int ret; index = mail_index_view_get_index(view); @@ -1189,8 +1268,13 @@ t->cache_updates = NULL; } + t->extra_intros_max_id = t->view->index->map->extra_infos == NULL ? 0 : + (t->view->index->map->extra_infos->used / + sizeof(struct mail_index_extra_record_info)); + if (t->appends != NULL || - (t->cache_updates != NULL && t->new_cache_file_seq == 0)) { + (t->cache_updates != NULL && t->new_cache_file_seq == 0) || + (t->extra_rec_updates != NULL && t->extra_rec_updates->used > 0)) { if (mail_transaction_log_scan_pending(log, t) < 0) { if (!log->index->log_locked) mail_transaction_log_file_unlock(file); @@ -1200,15 +1284,7 @@ ret = 0; if (t->appends != NULL) { - struct mail_transaction_append_header hdr; - - memset(&hdr, 0, sizeof(hdr)); - hdr.record_size = t->append_record_size; - - hdr_buf = buffer_create_data(pool_datastack_create(), - &hdr, sizeof(hdr)); - buffer_set_used_size(hdr_buf, sizeof(hdr)); - ret = log_append_buffer(file, t->appends, hdr_buf, + ret = log_append_buffer(file, t->appends, NULL, MAIL_TRANSACTION_APPEND, view->external); } @@ -1228,16 +1304,32 @@ view->external); } + if (t->extra_rec_updates == NULL) { + updates = NULL; + size = 0; + } else { + updates = buffer_get_modifyable_data(t->extra_rec_updates, + &size); + size /= sizeof(*updates); + } + hdr_buf = buffer_create_data(pool_datastack_create(), &extra_rec_hdr, sizeof(extra_rec_hdr)); buffer_set_used_size(hdr_buf, sizeof(extra_rec_hdr)); - for (i = 0; i < view->index->extra_records_count; i++) { - if (t->extra_rec_updates[i] == NULL || ret != 0) + for (i = 0; i < size && ret == 0; i++) { + if (updates[i] == NULL) continue; - /* FIXME: do data_id mapping conversion */ - extra_rec_hdr.data_id = i; - ret = log_append_buffer(file, t->extra_rec_updates[i], hdr_buf, + if (!mail_index_map_get_extra_info_idx(index->map, i, &idx)) { + /* new one */ + ret = mail_transaction_log_register_extra(file, t, i, + &idx); + if (ret < 0) + break; + } + + extra_rec_hdr.data_id = idx; + ret = log_append_buffer(file, updates[i], hdr_buf, MAIL_TRANSACTION_EXTRA_REC_UPDATE, view->external); }
--- a/src/lib-index/mail-transaction-log.h Sun Sep 05 20:33:19 2004 +0300 +++ b/src/lib-index/mail-transaction-log.h Sun Sep 05 20:53:45 2004 +0300 @@ -19,7 +19,10 @@ MAIL_TRANSACTION_CACHE_RESET = 0x00000008, MAIL_TRANSACTION_CACHE_UPDATE = 0x00000010, MAIL_TRANSACTION_HEADER_UPDATE = 0x00000020, - MAIL_TRANSACTION_EXTRA_REC_UPDATE = 0x00000040, + MAIL_TRANSACTION_EXTRA_INTRO = 0x00000040, + MAIL_TRANSACTION_EXTRA_RESET = 0x00000080, + MAIL_TRANSACTION_EXTRA_HDR_UPDATE = 0x00000100, + MAIL_TRANSACTION_EXTRA_REC_UPDATE = 0x00000200, MAIL_TRANSACTION_TYPE_MASK = 0x0000ffff, @@ -65,8 +68,23 @@ /* unsigned char data[]; */ }; +struct mail_transaction_extra_intro { + uint32_t data_id; /* must be first */ + uint32_t hdr_size; + uint16_t record_size; + uint16_t name_size; + /* unsigned char name[]; */ +}; + +struct mail_transaction_extra_hdr_update { + uint32_t data_id; /* must be first */ + uint16_t offset; + uint16_t size; + /* unsigned char data[]; */ +}; + struct mail_transaction_extra_rec_header { - uint32_t data_id; + uint32_t data_id; /* must be first */ }; struct mail_transaction_extra_rec_update { @@ -74,10 +92,6 @@ /* unsigned char data[]; */ }; -struct mail_transaction_append_header { - uint32_t record_size; -}; - struct mail_transaction_log * mail_transaction_log_open_or_create(struct mail_index *index); void mail_transaction_log_close(struct mail_transaction_log *log);
--- a/src/lib-index/mail-transaction-util.c Sun Sep 05 20:33:19 2004 +0300 +++ b/src/lib-index/mail-transaction-util.c Sun Sep 05 20:53:45 2004 +0300 @@ -25,6 +25,10 @@ { MAIL_TRANSACTION_CACHE_UPDATE, 0, sizeof(struct mail_transaction_cache_update) }, { MAIL_TRANSACTION_HEADER_UPDATE, 0, 1 }, /* variable size, use 1 */ + { MAIL_TRANSACTION_EXTRA_INTRO, 0, 1 }, + { MAIL_TRANSACTION_EXTRA_RESET, 0, + sizeof(struct mail_transaction_extra_rec_header) }, + { MAIL_TRANSACTION_EXTRA_HDR_UPDATE, 0, 1 }, { MAIL_TRANSACTION_EXTRA_REC_UPDATE, 0, 1 }, { 0, 0, 0 } }; @@ -54,29 +58,26 @@ return type; } -int mail_transaction_map(struct mail_index *index, +int mail_transaction_map(struct mail_index_map *map, const struct mail_transaction_header *hdr, const void *data, - struct mail_transaction_map_functions *map, + struct mail_transaction_map_functions *func_map, void *context) { int ret = 0; switch (hdr->type & MAIL_TRANSACTION_TYPE_MASK) { case MAIL_TRANSACTION_APPEND: { - const struct mail_transaction_append_header *append_hdr = data; const struct mail_index_record *rec, *end; - if (map->append == NULL) + if (func_map->append == NULL) break; - rec = CONST_PTR_OFFSET(data, sizeof(*append_hdr)); end = CONST_PTR_OFFSET(data, hdr->size); - while (rec != end) { - ret = map->append(append_hdr, rec, context); + for (rec = data; rec != end; rec++) { + ret = func_map->append(rec, context); if (ret <= 0) break; - rec = CONST_PTR_OFFSET(rec, append_hdr->record_size); } break; } @@ -84,12 +85,12 @@ case MAIL_TRANSACTION_EXPUNGE|MAIL_TRANSACTION_EXPUNGE_PROT: { const struct mail_transaction_expunge *rec, *end; - if (map->expunge == NULL) + if (func_map->expunge == NULL) break; end = CONST_PTR_OFFSET(data, hdr->size); for (rec = data; rec != end; rec++) { - ret = map->expunge(rec, context); + ret = func_map->expunge(rec, context); if (ret <= 0) break; } @@ -98,33 +99,33 @@ case MAIL_TRANSACTION_FLAG_UPDATE: { const struct mail_transaction_flag_update *rec, *end; - if (map->flag_update == NULL) + if (func_map->flag_update == NULL) break; end = CONST_PTR_OFFSET(data, hdr->size); for (rec = data; rec != end; rec++) { - ret = map->flag_update(rec, context); + ret = func_map->flag_update(rec, context); if (ret <= 0) break; } break; } case MAIL_TRANSACTION_CACHE_RESET: { - const struct mail_transaction_cache_reset *u = data; + const struct mail_transaction_cache_reset *rec = data; - if (map->cache_reset != NULL) - ret = map->cache_reset(u, context); + if (func_map->cache_reset != NULL) + ret = func_map->cache_reset(rec, context); break; } case MAIL_TRANSACTION_CACHE_UPDATE: { const struct mail_transaction_cache_update *rec, *end; - if (map->cache_update == NULL) + if (func_map->cache_update == NULL) break; end = CONST_PTR_OFFSET(data, hdr->size); for (rec = data; rec != end; rec++) { - ret = map->cache_update(rec, context); + ret = func_map->cache_update(rec, context); if (ret <= 0) break; } @@ -134,12 +135,61 @@ const struct mail_transaction_header_update *rec; unsigned int i; - if (map->header_update == NULL) + if (func_map->header_update == NULL) + break; + + for (i = 0; i < hdr->size; ) { + rec = CONST_PTR_OFFSET(data, i); + ret = func_map->header_update(rec, context); + if (ret <= 0) + break; + + i += sizeof(*rec) + rec->size; + } + break; + } + case MAIL_TRANSACTION_EXTRA_INTRO: { + const struct mail_transaction_extra_intro *rec = data; + unsigned int i; + + if (func_map->extra_intro == NULL) break; for (i = 0; i < hdr->size; ) { rec = CONST_PTR_OFFSET(data, i); - ret = map->header_update(rec, context); + ret = func_map->extra_intro(rec, context); + if (ret <= 0) + break; + + i += sizeof(*rec) + rec->name_size; + } + break; + } + case MAIL_TRANSACTION_EXTRA_RESET: { + const struct mail_transaction_extra_rec_header *rec = data; + unsigned int i, size; + + if (func_map->extra_reset == NULL) + break; + + size = hdr->size / sizeof(*rec); + for (i = 0; i < size; i++) { + ret = func_map->extra_reset(&rec[i], context); + if (ret <= 0) + break; + } + break; + } + case MAIL_TRANSACTION_EXTRA_HDR_UPDATE: { + const struct mail_transaction_extra_hdr_update *rec = data; + unsigned int i; + + if (func_map->extra_hdr_update == NULL) + break; + + for (i = 0; i < hdr->size; ) { + rec = CONST_PTR_OFFSET(data, i); + ret = func_map->extra_hdr_update(rec, context); if (ret <= 0) break; @@ -150,19 +200,31 @@ case MAIL_TRANSACTION_EXTRA_REC_UPDATE: { const struct mail_transaction_extra_rec_header *ehdr = data; const struct mail_transaction_extra_rec_update *rec, *end; + const struct mail_index_extra_record_info *einfo; unsigned int record_size; - if (map->extra_rec_update == NULL) + if (func_map->extra_rec_update == NULL) break; - i_assert(ehdr->data_id < index->extra_records_count); - record_size = sizeof(*ehdr) + - index->extra_records[ehdr->data_id].size; + rec = CONST_PTR_OFFSET(data, sizeof(*ehdr)); - rec = CONST_PTR_OFFSET(data, sizeof(*ehdr)); + if (map->extra_infos == NULL || + ehdr->data_id >= map->extra_infos->used / sizeof(*einfo)) { + /* broken. let the extra_rec_update handler do the + error handling. */ + ret = func_map->extra_rec_update(ehdr, rec, context); + if (ret >= 0) + i_unreached(); + break; + } + + einfo = map->extra_infos->data; + einfo += ehdr->data_id; + record_size = sizeof(*ehdr) + einfo->record_size; + end = CONST_PTR_OFFSET(data, hdr->size); while (rec != end) { - ret = map->extra_rec_update(ehdr, rec, context); + ret = func_map->extra_rec_update(ehdr, rec, context); if (ret <= 0) break;
--- a/src/lib-index/mail-transaction-util.h Sun Sep 05 20:33:19 2004 +0300 +++ b/src/lib-index/mail-transaction-util.h Sun Sep 05 20:53:45 2004 +0300 @@ -10,8 +10,7 @@ struct mail_transaction_map_functions { int (*expunge)(const struct mail_transaction_expunge *e, void *context); - int (*append)(const struct mail_transaction_append_header *hdr, - const struct mail_index_record *rec, void *context); + int (*append)(const struct mail_index_record *rec, void *context); int (*flag_update)(const struct mail_transaction_flag_update *u, void *context); int (*cache_reset)(const struct mail_transaction_cache_reset *u, @@ -20,6 +19,13 @@ void *context); int (*header_update)(const struct mail_transaction_header_update *u, void *context); + int (*extra_intro)(const struct mail_transaction_extra_intro *u, + void *context); + int (*extra_reset)(const struct mail_transaction_extra_rec_header *u, + void *context); + int (*extra_hdr_update) + (const struct mail_transaction_extra_hdr_update *u, + void *context); int (*extra_rec_update) (const struct mail_transaction_extra_rec_header *hdr, const struct mail_transaction_extra_rec_update *u, @@ -31,10 +37,10 @@ enum mail_transaction_type mail_transaction_type_mask_get(enum mail_index_sync_type sync_type); -int mail_transaction_map(struct mail_index *index, +int mail_transaction_map(struct mail_index_map *map, const struct mail_transaction_header *hdr, const void *data, - struct mail_transaction_map_functions *map, + struct mail_transaction_map_functions *func_map, void *context); void
--- a/src/lib-storage/index/mbox/mbox-storage.c Sun Sep 05 20:33:19 2004 +0300 +++ b/src/lib-storage/index/mbox/mbox-storage.c Sun Sep 05 20:53:45 2004 +0300 @@ -403,7 +403,7 @@ index = index_storage_alloc(index_dir, path, MBOX_INDEX_PREFIX); mbox_extra_idx = mail_index_register_record_extra(index, "mbox", - sizeof(uint64_t)); + 0, sizeof(uint64_t)); ibox = index_storage_mailbox_init(storage, &mbox_mailbox, index, name, flags); if (ibox == NULL)
--- a/src/lib-storage/index/mbox/mbox-sync-parse.c Sun Sep 05 20:33:19 2004 +0300 +++ b/src/lib-storage/index/mbox/mbox-sync-parse.c Sun Sep 05 20:53:45 2004 +0300 @@ -515,7 +515,7 @@ if (ibox->md5hdr_extra_idx == 0) { ibox->md5hdr_extra_idx = mail_index_register_record_extra(ibox->index, - "header-md5", 16); + "header-md5", 0, 16); } if (mail_index_lookup_extra(view, seq, ibox->md5hdr_extra_idx,
--- a/src/lib-storage/index/mbox/mbox-sync.c Sun Sep 05 20:33:19 2004 +0300 +++ b/src/lib-storage/index/mbox/mbox-sync.c Sun Sep 05 20:53:45 2004 +0300 @@ -338,7 +338,7 @@ return -1; } - *offset_r = *((const uint64_t *)data); + *offset_r = data == NULL ? 0 : *((const uint64_t *)data); return 0; } @@ -764,7 +764,7 @@ sync_ctx->ibox->md5hdr_extra_idx = mail_index_register_record_extra( sync_ctx->ibox->index, - "header-md5", 16); + "header-md5", 0, 16); } if (mbox_sync_find_index_md5(sync_ctx,