Mercurial > dovecot > core-2.2
changeset 2240:51719889ffcf HEAD
Record size is allowed to change between index files. This will allow adding
extensions dynamically for existing indexes.
author | Timo Sirainen <tss@iki.fi> |
---|---|
date | Thu, 24 Jun 2004 14:10:41 +0300 |
parents | 1057a5dc0ca7 |
children | c9a27c0875e2 |
files | src/lib-index/mail-index-fsck.c src/lib-index/mail-index-lock.c src/lib-index/mail-index-private.h src/lib-index/mail-index-sync-private.h src/lib-index/mail-index-sync-update.c src/lib-index/mail-index-sync.c src/lib-index/mail-index-transaction-private.h src/lib-index/mail-index-transaction-view.c src/lib-index/mail-index-transaction.c src/lib-index/mail-index-view-sync.c src/lib-index/mail-index-view.c src/lib-index/mail-index.c src/lib-index/mail-index.h src/lib-index/mail-transaction-log.c src/lib-index/mail-transaction-log.h src/lib-index/mail-transaction-util.c src/lib-index/mail-transaction-util.h |
diffstat | 17 files changed, 210 insertions(+), 116 deletions(-) [+] |
line wrap: on
line diff
--- a/src/lib-index/mail-index-fsck.c Wed Jun 23 23:10:34 2004 +0300 +++ b/src/lib-index/mail-index-fsck.c Thu Jun 24 14:10:41 2004 +0300 @@ -75,7 +75,7 @@ hdr.first_deleted_uid_lowwater = rec->uid; last_uid = rec->uid; - rec = CONST_PTR_OFFSET(rec, index->record_size); + rec = CONST_PTR_OFFSET(rec, hdr.record_size); } if (hdr.next_uid <= last_uid) { @@ -100,7 +100,7 @@ CHECK(first_unseen_uid_lowwater, <); CHECK(first_deleted_uid_lowwater, <); - if (mail_index_write_header(index, &hdr) < 0) + if (mail_index_write_base_header(index, &hdr) < 0) return -1; return 1;
--- a/src/lib-index/mail-index-lock.c Wed Jun 23 23:10:34 2004 +0300 +++ b/src/lib-index/mail-index-lock.c Thu Jun 24 14:10:41 2004 +0300 @@ -187,7 +187,7 @@ ret = write_full(fd, index->map->hdr, sizeof(*index->map->hdr)); if (ret < 0 || write_full(fd, index->map->records, index->map->records_count * - index->record_size) < 0) { + index->map->hdr->record_size) < 0) { mail_index_file_set_syscall_error(index, path, "write_full()"); (void)close(fd); (void)unlink(path);
--- a/src/lib-index/mail-index-private.h Wed Jun 23 23:10:34 2004 +0300 +++ b/src/lib-index/mail-index-private.h Thu Jun 24 14:10:41 2004 +0300 @@ -22,9 +22,15 @@ #define MAIL_INDEX_MAP_IS_IN_MEMORY(map) \ ((map)->buffer != NULL) -#define MAIL_INDEX_MAP_IDX(index, map, idx) \ +#define MAIL_INDEX_MAP_IDX(map, idx) \ ((struct mail_index_record *) \ - PTR_OFFSET((map)->records, (idx) * (index)->record_size)) + PTR_OFFSET((map)->records, (idx) * (map)->hdr->record_size)) + +struct mail_index_extra_record_info { + const char *name; + uint16_t offset; + uint16_t size; +}; struct mail_index_map { int refcount; @@ -33,6 +39,9 @@ void *records; /* struct mail_index_record[] */ unsigned int records_count; + struct mail_index_extra_record_info * + extra_record_map[MAIL_INDEX_MAX_EXTRA_RECORDS]; + void *mmap_base; size_t mmap_size, mmap_used_size; @@ -46,12 +55,6 @@ unsigned int write_to_disk:1; }; -struct mail_index_extra_record_info { - const char *name; - uint16_t offset; - uint16_t size; -}; - struct mail_index { char *dir, *prefix; @@ -65,8 +68,7 @@ buffer_t *extra_records_buf; const struct mail_index_extra_record_info *extra_records; unsigned int extra_records_count; - - unsigned int record_size; + unsigned int max_record_size; char *filepath; int fd; @@ -95,8 +97,8 @@ unsigned int fsck:1; }; -int mail_index_write_header(struct mail_index *index, - const struct mail_index_header *hdr); +int mail_index_write_base_header(struct mail_index *index, + const struct mail_index_header *hdr); int mail_index_reopen(struct mail_index *index, int fd); int mail_index_create_tmp_file(struct mail_index *index, const char **path_r); @@ -122,7 +124,7 @@ /* Unreference given mapping and unmap it if it's dropped to zero. */ void mail_index_unmap(struct mail_index *index, struct mail_index_map *map); struct mail_index_map * -mail_index_map_to_memory(struct mail_index *index, struct mail_index_map *map); +mail_index_map_to_memory(struct mail_index_map *map, uint32_t new_record_size); int mail_index_lookup_full(struct mail_index_view *view, uint32_t seq, struct mail_index_map **map_r,
--- a/src/lib-index/mail-index-sync-private.h Wed Jun 23 23:10:34 2004 +0300 +++ b/src/lib-index/mail-index-sync-private.h Thu Jun 24 14:10:41 2004 +0300 @@ -5,12 +5,14 @@ struct mail_index *index; struct mail_index_view *view; - buffer_t *expunges_buf, *updates_buf, *appends_buf; + buffer_t *expunges_buf, *updates_buf; const struct mail_transaction_expunge *expunges; const struct mail_transaction_flag_update *updates; size_t expunges_count, updates_count; + uint32_t append_uid_first, append_uid_last; + const struct mail_transaction_header *hdr; const void *data;
--- a/src/lib-index/mail-index-sync-update.c Wed Jun 23 23:10:34 2004 +0300 +++ b/src/lib-index/mail-index-sync-update.c Thu Jun 24 14:10:41 2004 +0300 @@ -56,7 +56,6 @@ static int sync_expunge(const struct mail_transaction_expunge *e, void *context) { struct mail_index_view *view = context; - struct mail_index *index = view->index; struct mail_index_map *map = view->map; struct mail_index_header *hdr = &map->hdr_copy; struct mail_index_record *rec; @@ -72,15 +71,14 @@ return 1; for (seq = seq1; seq <= seq2; seq++) { - rec = MAIL_INDEX_MAP_IDX(index, map, seq-1); + rec = MAIL_INDEX_MAP_IDX(map, seq-1); mail_index_header_update_counts(hdr, rec->flags, 0); } /* @UNSAFE */ count = seq2 - seq1 + 1; - memmove(MAIL_INDEX_MAP_IDX(index, map, seq1-1), - MAIL_INDEX_MAP_IDX(index, map, seq2), - (map->records_count - seq2) * view->index->record_size); + memmove(MAIL_INDEX_MAP_IDX(map, seq1-1), MAIL_INDEX_MAP_IDX(map, seq2), + (map->records_count - seq2) * map->hdr->record_size); map->records_count -= count; hdr->messages_count -= count; @@ -88,17 +86,20 @@ if (map->buffer != NULL) { buffer_set_used_size(map->buffer, map->records_count * - view->index->record_size); + map->hdr->record_size); map->records = buffer_get_modifyable_data(map->buffer, NULL); } return 1; } -static int sync_append(const struct mail_index_record *rec, void *context) +static int sync_append(const struct mail_transaction_append_header *hdr, + const struct mail_index_record *rec, void *context) { struct mail_index_view *view = context; - struct mail_index *index = view->index; struct mail_index_map *map = view->map; + void *dest; + + i_assert(hdr->record_size <= map->hdr->record_size); if (rec->uid < map->hdr_copy.next_uid) { mail_transaction_log_view_set_corrupted(view->log_view, @@ -108,16 +109,19 @@ } if (MAIL_INDEX_MAP_IS_IN_MEMORY(map)) { - i_assert(map->records_count * index->record_size == + i_assert(map->records_count * map->hdr->record_size == buffer_get_used_size(map->buffer)); - buffer_append(map->buffer, rec, index->record_size); + dest = buffer_append_space_unsafe(map->buffer, + map->hdr->record_size); map->records = buffer_get_modifyable_data(map->buffer, NULL); } else { - i_assert((map->records_count+1) * index->record_size <= + i_assert((map->records_count+1) * map->hdr->record_size <= map->mmap_size); - memcpy(MAIL_INDEX_MAP_IDX(index, map, map->records_count), - rec, index->record_size); + dest = MAIL_INDEX_MAP_IDX(map, map->records_count); } + memcpy(dest, rec, hdr->record_size); + memset(PTR_OFFSET(dest, hdr->record_size), 0, + map->hdr->record_size - hdr->record_size); map->hdr_copy.messages_count++; map->hdr_copy.next_uid = rec->uid+1; @@ -161,7 +165,7 @@ flag_mask = ~u->remove_flags; for (idx = seq1-1; idx < seq2; idx++) { - rec = MAIL_INDEX_MAP_IDX(view->index, view->map, idx); + rec = MAIL_INDEX_MAP_IDX(view->map, idx); old_flags = rec->flags; rec->flags = (rec->flags & flag_mask) | u->add_flags; @@ -187,7 +191,7 @@ view->map->hdr_copy.cache_file_seq = u->new_file_seq; for (i = 0; i < view->messages_count; i++) - MAIL_INDEX_MAP_IDX(view->index, view->map, i)->cache_offset = 0; + MAIL_INDEX_MAP_IDX(view->map, i)->cache_offset = 0; return 1; } @@ -203,8 +207,8 @@ i_assert(ret == 0); if (seq != 0) { - MAIL_INDEX_MAP_IDX(view->index, view->map, seq-1)-> - cache_offset = u->cache_offset; + MAIL_INDEX_MAP_IDX(view->map, seq-1)->cache_offset = + u->cache_offset; } return 1; } @@ -241,7 +245,7 @@ offset = view->index->extra_records[hdr->idx].offset; size = view->index->extra_records[hdr->idx].size; - rec = MAIL_INDEX_MAP_IDX(view->index, view->map, seq-1); + rec = MAIL_INDEX_MAP_IDX(view->map, seq-1); memcpy(PTR_OFFSET(rec, offset), u->data, size); } return 1; @@ -259,7 +263,7 @@ i_assert(map == index->map); size = map->hdr->header_size + - (map->records_count + count) * index->record_size; + (map->records_count + count) * map->hdr->record_size; if (size <= map->mmap_size) return 0; @@ -271,7 +275,7 @@ index->last_grow_count = count; size = map->hdr->header_size + - (map->records_count + count) * index->record_size; + (map->records_count + count) * map->hdr->record_size; if (file_set_size(index->fd, (off_t)size) < 0) return mail_index_set_syscall_error(index, "file_set_size()"); @@ -290,6 +294,18 @@ return 0; } +static void mail_index_sync_replace_map(struct mail_index_view *view, + struct mail_index_map *map) +{ + mail_index_unmap(view->index, view->map); + view->map = map; + view->map->refcount++; + mail_index_unmap(view->index, view->index->map); + view->index->map = map; + view->index->hdr = map->hdr; + map->write_to_disk = TRUE; +} + int mail_index_sync_update_index(struct mail_index_sync_ctx *sync_ctx) { struct mail_index *index = sync_ctx->index; @@ -335,18 +351,23 @@ replace the whole index file. to avoid extra disk I/O we copy the index into memory rather than to temporary file */ - map = mail_index_map_to_memory(index, map); - mail_index_unmap(index, view->map); - view->map = map; - view->map->refcount++; - mail_index_unmap(index, index->map); - index->map = map; - index->hdr = map->hdr; - map->write_to_disk = TRUE; + map = mail_index_map_to_memory(map, + map->hdr->record_size); + mail_index_sync_replace_map(view, map); } if ((hdr->type & MAIL_TRANSACTION_APPEND) != 0) { - count = hdr->size / index->record_size; + const struct mail_transaction_append_header *append_hdr; + + append_hdr = data; + if (append_hdr->record_size > map->hdr->record_size) { + /* we have to grow our record size */ + map = mail_index_map_to_memory(map, + append_hdr->record_size); + mail_index_sync_replace_map(view, map); + } + count = (hdr->size - sizeof(*append_hdr)) / + append_hdr->record_size; if (mail_index_grow(index, view->map, count) < 0) { ret = -1; break; @@ -381,7 +402,7 @@ const struct mail_index_record *rec; for (i = 0; i < map->records_count; i++) { - rec = MAIL_INDEX_MAP_IDX(index, map, i); + rec = MAIL_INDEX_MAP_IDX(map, i); if ((rec->flags & MAIL_INDEX_MAIL_FLAG_DIRTY) != 0) { map->hdr_copy.flags |= MAIL_INDEX_HDR_FLAG_HAVE_DIRTY; @@ -392,7 +413,7 @@ if (!MAIL_INDEX_MAP_IS_IN_MEMORY(map)) { map->mmap_used_size = index->hdr->header_size + - map->records_count * index->record_size; + map->records_count * map->hdr->record_size; memcpy(map->mmap_base, &map->hdr_copy, sizeof(map->hdr_copy)); if (msync(map->mmap_base, map->mmap_used_size, MS_SYNC) < 0) {
--- a/src/lib-index/mail-index-sync.c Wed Jun 23 23:10:34 2004 +0300 +++ b/src/lib-index/mail-index-sync.c Thu Jun 24 14:10:41 2004 +0300 @@ -80,11 +80,23 @@ mail_index_sync_sort_flags(ctx); } break; - case MAIL_TRANSACTION_APPEND: - buffer_append(ctx->appends_buf, ctx->data, ctx->hdr->size); + case MAIL_TRANSACTION_APPEND: { + const struct mail_transaction_append_header *hdr = ctx->data; + const struct mail_index_record *rec = ctx->data; + + if (ctx->append_uid_first == 0 || + rec->uid < ctx->append_uid_first) + ctx->append_uid_first = rec->uid; + + rec = CONST_PTR_OFFSET(ctx->data, + ctx->hdr->size - hdr->record_size); + if (rec->uid > ctx->append_uid_last) + ctx->append_uid_last = rec->uid; + ctx->sync_appends = TRUE; break; } + } } static int mail_index_sync_read_and_sort(struct mail_index_sync_ctx *ctx) @@ -172,8 +184,6 @@ 1024, (size_t)-1); ctx->updates_buf = buffer_create_dynamic(default_pool, 1024, (size_t)-1); - ctx->appends_buf = buffer_create_dynamic(default_pool, - 1024, (size_t)-1); if (mail_index_sync_read_and_sort(ctx) < 0) { mail_index_sync_end(ctx); return -1; @@ -304,12 +314,8 @@ if (ctx->sync_appends) { ctx->sync_appends = FALSE; sync_rec->type = MAIL_INDEX_SYNC_TYPE_APPEND; - sync_rec->appends = buffer_get_data(ctx->appends_buf, - &sync_rec->appends_count); - sync_rec->appends_count /= ctx->index->record_size; - sync_rec->uid1 = sync_rec->appends[0].uid; - sync_rec->uid2 = - sync_rec->appends[sync_rec->appends_count-1].uid; + sync_rec->uid1 = ctx->append_uid_first; + sync_rec->uid2 = ctx->append_uid_last; return 1; } @@ -355,8 +361,6 @@ buffer_free(ctx->expunges_buf); if (ctx->updates_buf != NULL) buffer_free(ctx->updates_buf); - if (ctx->appends_buf != NULL) - buffer_free(ctx->appends_buf); i_free(ctx); return ret; }
--- a/src/lib-index/mail-index-transaction-private.h Wed Jun 23 23:10:34 2004 +0300 +++ b/src/lib-index/mail-index-transaction-private.h Thu Jun 24 14:10:41 2004 +0300 @@ -8,6 +8,7 @@ buffer_t *appends; uint32_t first_new_seq, last_new_seq; + unsigned int append_record_size; buffer_t *expunges;
--- a/src/lib-index/mail-index-transaction-view.c Wed Jun 23 23:10:34 2004 +0300 +++ b/src/lib-index/mail-index-transaction-view.c Thu Jun 24 14:10:41 2004 +0300 @@ -130,7 +130,7 @@ *seq_r = seq; break; } - rec = CONST_PTR_OFFSET(rec, view->index->record_size); + rec = CONST_PTR_OFFSET(rec, view->index->max_record_size); } return 0;
--- a/src/lib-index/mail-index-transaction.c Wed Jun 23 23:10:34 2004 +0300 +++ b/src/lib-index/mail-index-transaction.c Thu Jun 24 14:10:41 2004 +0300 @@ -70,12 +70,10 @@ seq = (uint32_t *)&data[i]; i_assert(seq[0] <= view->map->records_count); - seq[0] = MAIL_INDEX_MAP_IDX(view->index, view->map, - seq[0]-1)->uid; + seq[0] = MAIL_INDEX_MAP_IDX(view->map, seq[0]-1)->uid; if (range) { i_assert(seq[1] <= view->map->records_count); - seq[1] = MAIL_INDEX_MAP_IDX(view->index, view->map, - seq[1]-1)->uid; + seq[1] = MAIL_INDEX_MAP_IDX(view->map, seq[1]-1)->uid; } } } @@ -139,6 +137,40 @@ mail_index_transaction_free(t); } +static void +mail_index_transaction_update_append_size(struct mail_index_transaction *t) +{ + buffer_t *new_buf; + unsigned int new_record_size; + const void *src; + void *dest; + size_t i, size; + + new_record_size = t->view->index->max_record_size; + if (t->append_record_size == new_record_size) + return; + + i_assert(t->append_record_size < new_record_size); + + if (t->append_record_size != 0) { + /* resize the records in buffer */ + src = buffer_get_data(t->appends, &size); + size /= t->append_record_size; + + new_buf = buffer_create_dynamic(default_pool, + size * new_record_size, + (size_t)-1); + for (i = 0; i < size; i++) { + dest = buffer_append_space_unsafe(new_buf, + new_record_size); + memcpy(dest, src, t->append_record_size); + src = CONST_PTR_OFFSET(src, t->append_record_size); + } + } + + t->append_record_size = new_record_size; +} + struct mail_index_record * mail_index_transaction_lookup(struct mail_index_transaction *t, uint32_t seq) { @@ -146,9 +178,10 @@ i_assert(seq >= t->first_new_seq && seq <= t->last_new_seq); - pos = (seq - t->first_new_seq) * t->view->index->record_size; - return buffer_get_space_unsafe(t->appends, pos, - t->view->index->record_size); + mail_index_transaction_update_append_size(t); + + pos = (seq - t->first_new_seq) * t->append_record_size; + return buffer_get_space_unsafe(t->appends, pos, t->append_record_size); } void mail_index_append(struct mail_index_transaction *t, uint32_t uid, @@ -159,7 +192,9 @@ if (t->appends == NULL) { t->appends = buffer_create_dynamic(default_pool, 4096, (size_t)-1); + t->append_record_size = t->view->index->max_record_size; } + mail_index_transaction_update_append_size(t); /* sequence number is visible only inside given view, so let it generate it */ @@ -168,9 +203,8 @@ else *seq_r = t->last_new_seq = t->first_new_seq; - rec = buffer_append_space_unsafe(t->appends, - t->view->index->record_size); - memset(rec, 0, t->view->index->record_size); + rec = buffer_append_space_unsafe(t->appends, t->append_record_size); + memset(rec, 0, t->append_record_size); rec->uid = uid; }
--- a/src/lib-index/mail-index-view-sync.c Wed Jun 23 23:10:34 2004 +0300 +++ b/src/lib-index/mail-index-view-sync.c Thu Jun 24 14:10:41 2004 +0300 @@ -125,7 +125,8 @@ if (view->map != view->index->map) ctx->sync_map_update = TRUE; - map = mail_index_map_to_memory(view->index, view->map); + map = mail_index_map_to_memory(view->map, + view->map->hdr->record_size); mail_index_unmap(view->index, view->map); view->map = map; }
--- a/src/lib-index/mail-index-view.c Wed Jun 23 23:10:34 2004 +0300 +++ b/src/lib-index/mail-index-view.c Thu Jun 24 14:10:41 2004 +0300 @@ -178,7 +178,7 @@ if (mail_index_view_lock(view) < 0) return -1; - rec = MAIL_INDEX_MAP_IDX(view->index, view->map, seq-1); + rec = MAIL_INDEX_MAP_IDX(view->map, seq-1); if (view->map == view->index->map) { *map_r = view->map; *rec_r = rec; @@ -203,7 +203,7 @@ do { // FIXME: we could be skipping more by uid diff seq--; - n_rec = MAIL_INDEX_MAP_IDX(view->index, map, seq); + n_rec = MAIL_INDEX_MAP_IDX(map, seq); if (n_rec->uid <= uid) break; } while (seq > 0); @@ -225,7 +225,7 @@ if (mail_index_view_lock(view) < 0) return -1; - *uid_r = MAIL_INDEX_MAP_IDX(view->index, view->map, seq-1)->uid; + *uid_r = MAIL_INDEX_MAP_IDX(view->map, seq-1)->uid; return 0; } @@ -239,7 +239,7 @@ i_assert(view->messages_count <= view->map->records_count); rec_base = view->map->records; - record_size = view->index->record_size; + record_size = view->map->hdr->record_size; idx = left_idx = *left_idx_p; right_idx = view->messages_count; @@ -301,8 +301,7 @@ left_idx = 0; *first_seq_r = mail_index_bsearch_uid(view, first_uid, &left_idx, 1); if (*first_seq_r == 0 || - MAIL_INDEX_MAP_IDX(view->index, view->map, *first_seq_r-1)->uid > - last_uid) { + MAIL_INDEX_MAP_IDX(view->map, *first_seq_r-1)->uid > last_uid) { *first_seq_r = 0; *last_seq_r = 0; return 0; @@ -351,7 +350,7 @@ } for (; seq <= view->messages_count; seq++) { - rec = MAIL_INDEX_MAP_IDX(view->index, view->map, seq-1); + rec = MAIL_INDEX_MAP_IDX(view->map, seq-1); if ((rec->flags & flags_mask) == (uint8_t)flags) { *seq_r = seq; break;
--- a/src/lib-index/mail-index.c Wed Jun 23 23:10:34 2004 +0300 +++ b/src/lib-index/mail-index.c Thu Jun 24 14:10:41 2004 +0300 @@ -31,7 +31,7 @@ index->extra_records_buf = buffer_create_dynamic(index->extra_records_pool, 64, (size_t)-1); - index->record_size = sizeof(struct mail_index_record); + index->max_record_size = sizeof(struct mail_index_record); index->mode = 0600; index->gid = (gid_t)-1; @@ -66,7 +66,7 @@ i_assert(size % 4 == 0); i_assert(!index->opened); - i_assert(index->record_size + size <= 65535); + i_assert(index->max_record_size + size <= 65535); if (index->extra_records_count >= MAIL_INDEX_MAX_EXTRA_RECORDS) { i_panic("Maximum extra record count reached, " @@ -78,14 +78,14 @@ memset(&info, 0, sizeof(info)); info.name = p_strdup(index->extra_records_pool, name); info.size = size; - info.offset = index->record_size; + info.offset = index->max_record_size; buffer_append(index->extra_records_buf, &info, sizeof(info)); index->extra_records = buffer_get_data(index->extra_records_buf, &buf_size); index->extra_records_count = buf_size / sizeof(info); - index->record_size += size; + index->max_record_size += size; return index->extra_records_count-1; } @@ -132,11 +132,11 @@ return -1; } - if (hdr->record_size != index->record_size) { + if (hdr->record_size < sizeof(struct mail_index_record)) { mail_index_set_error(index, "Corrupted index file %s: " - "record_size mismatch: %d != %d", + "record_size too small: %u < %"PRIuSIZE_T, index->filepath, hdr->record_size, - (int)index->record_size); + sizeof(struct mail_index_record)); return -1; } @@ -227,11 +227,11 @@ hdr = map->mmap_base; map->hdr = hdr; map->mmap_used_size = hdr->header_size + - hdr->messages_count * index->record_size; + hdr->messages_count * hdr->record_size; if (map->mmap_used_size > map->mmap_size) { records_count = (map->mmap_size - hdr->header_size) / - index->record_size; + hdr->record_size; mail_index_set_error(index, "Corrupted index file %s: " "messages_count too large (%u > %u)", index->filepath, hdr->messages_count, @@ -271,7 +271,7 @@ pos += ret; } if (ret >= 0 && pos >= MAIL_INDEX_HEADER_MIN_SIZE) { - records_size = hdr.messages_count * index->record_size; + records_size = hdr.messages_count * hdr.record_size; if (map->buffer == NULL) { map->buffer = buffer_create_dynamic(default_pool, @@ -367,7 +367,7 @@ return -1; used_size = hdr->header_size + - hdr->messages_count * index->record_size; + hdr->messages_count * hdr->record_size; if (map->mmap_size >= used_size && !force) { map->records_count = hdr->messages_count; return 1; @@ -410,27 +410,41 @@ } struct mail_index_map * -mail_index_map_to_memory(struct mail_index *index, struct mail_index_map *map) +mail_index_map_to_memory(struct mail_index_map *map, uint32_t new_record_size) { struct mail_index_map *mem_map; - size_t size; + void *src, *dest; + size_t size, copy_size; + unsigned int i; if (MAIL_INDEX_MAP_IS_IN_MEMORY(map)) { map->refcount++; return map; } - size = map->records_count * index->record_size; + size = map->records_count * new_record_size; mem_map = i_new(struct mail_index_map, 1); mem_map->refcount = 1; mem_map->buffer = buffer_create_dynamic(default_pool, size, (size_t)-1); - buffer_append(mem_map->buffer, map->records, size); + if (map->hdr->record_size == new_record_size) + buffer_append(mem_map->buffer, map->records, size); + else { + copy_size = I_MIN(map->hdr->record_size, new_record_size); + src = map->records; + for (i = 0; i < map->records_count; i++) { + dest = buffer_append_space_unsafe(mem_map->buffer, + new_record_size); + memcpy(dest, src, copy_size); + src = PTR_OFFSET(src, map->hdr->record_size); + } + } mem_map->records = buffer_get_modifyable_data(mem_map->buffer, NULL); mem_map->records_count = map->records_count; mem_map->hdr_copy = *map->hdr; + mem_map->hdr_copy.record_size = new_record_size; mem_map->hdr = &mem_map->hdr_copy; return mem_map; } @@ -497,15 +511,19 @@ return ret; } -int mail_index_write_header(struct mail_index *index, - const struct mail_index_header *hdr) +int mail_index_write_base_header(struct mail_index *index, + const struct mail_index_header *hdr) { + size_t hdr_size; + + hdr_size = I_MIN(sizeof(*hdr), hdr->base_header_size); + if (!MAIL_INDEX_MAP_IS_IN_MEMORY(index->map)) { - memcpy(index->map->mmap_base, hdr, sizeof(*hdr)); - if (msync(index->map->mmap_base, sizeof(*hdr), MS_SYNC) < 0) + memcpy(index->map->mmap_base, hdr, hdr_size); + if (msync(index->map->mmap_base, hdr_size, MS_SYNC) < 0) return mail_index_set_syscall_error(index, "msync()"); } else { - if (pwrite_full(index->fd, hdr, sizeof(*hdr), 0) < 0) { + if (pwrite_full(index->fd, hdr, hdr_size, 0) < 0) { mail_index_set_syscall_error(index, "pwrite_full()"); return -1; } @@ -602,7 +620,7 @@ hdr->minor_version = MAIL_INDEX_MINOR_VERSION; hdr->base_header_size = sizeof(*hdr); hdr->header_size = sizeof(*hdr); - hdr->record_size = index->record_size; + hdr->record_size = index->max_record_size; hdr->keywords_mask_size = sizeof(keywords_mask_t); #ifndef WORDS_BIGENDIAN @@ -890,7 +908,7 @@ hdr = *index->hdr; hdr.flags |= MAIL_INDEX_HDR_FLAG_CORRUPTED; - if (mail_index_write_header(index, &hdr) == 0) { + if (mail_index_write_base_header(index, &hdr) == 0) { if (fsync(index->fd) < 0) mail_index_set_syscall_error(index, "fsync()"); }
--- a/src/lib-index/mail-index.h Wed Jun 23 23:10:34 2004 +0300 +++ b/src/lib-index/mail-index.h Thu Jun 24 14:10:41 2004 +0300 @@ -123,10 +123,6 @@ keywords_mask_t add_keywords; uint8_t remove_flags; keywords_mask_t remove_keywords; - - /* MAIL_INDEX_SYNC_TYPE_APPEND: */ - const struct mail_index_record *appends; - size_t appends_count; }; struct mail_index;
--- a/src/lib-index/mail-transaction-log.c Wed Jun 23 23:10:34 2004 +0300 +++ b/src/lib-index/mail-transaction-log.c Thu Jun 24 14:10:41 2004 +0300 @@ -874,6 +874,7 @@ struct mail_transaction_log_view *sync_view; const struct mail_index_record *old, *old_end; struct mail_index_record *appends, *end, *rec, *dest; + const struct mail_transaction_append_header *append_hdr; const struct mail_transaction_header *hdr; const void *data; size_t size; @@ -883,7 +884,7 @@ if (t->appends == NULL) return 0; - record_size = log->index->record_size; + record_size = t->append_record_size; appends = buffer_get_modifyable_data(t->appends, &size); end = PTR_OFFSET(appends, size); @@ -905,8 +906,10 @@ MAIL_TRANSACTION_APPEND) continue; - old = data; - old_end = CONST_PTR_OFFSET(old, hdr->size); + append_hdr = data; + + old = CONST_PTR_OFFSET(data, sizeof(*append_hdr)); + old_end = CONST_PTR_OFFSET(data, hdr->size); while (old != old_end) { /* appends are sorted */ for (rec = appends; rec != end; ) { @@ -919,7 +922,7 @@ } rec = PTR_OFFSET(rec, record_size); } - old = CONST_PTR_OFFSET(old, record_size); + old = CONST_PTR_OFFSET(old, append_hdr->record_size); } } @@ -1097,7 +1100,15 @@ ret = 0; if (t->appends != NULL) { - ret = log_append_buffer(file, t->appends, NULL, + struct mail_transaction_append_header hdr; + + memset(&hdr, 0, sizeof(hdr)); + hdr.record_size = t->append_record_size; + + hdr_buf = buffer_create_data(pool_datastack_create(), + &hdr, sizeof(hdr)); + buffer_set_used_size(hdr_buf, sizeof(hdr)); + ret = log_append_buffer(file, t->appends, hdr_buf, MAIL_TRANSACTION_APPEND, view->external); }
--- a/src/lib-index/mail-transaction-log.h Wed Jun 23 23:10:34 2004 +0300 +++ b/src/lib-index/mail-transaction-log.h Thu Jun 24 14:10:41 2004 +0300 @@ -75,6 +75,10 @@ unsigned char data[1]; /* variable size */ }; +struct mail_transaction_append_header { + uint32_t record_size; +}; + struct mail_transaction_log * mail_transaction_log_open_or_create(struct mail_index *index); void mail_transaction_log_close(struct mail_transaction_log *log);
--- a/src/lib-index/mail-transaction-util.c Wed Jun 23 23:10:34 2004 +0300 +++ b/src/lib-index/mail-transaction-util.c Thu Jun 24 14:10:41 2004 +0300 @@ -60,23 +60,23 @@ struct mail_transaction_map_functions *map, void *context) { - int ret = 0; switch (hdr->type & MAIL_TRANSACTION_TYPE_MASK) { case MAIL_TRANSACTION_APPEND: { + const struct mail_transaction_append_header *append_hdr = data; const struct mail_index_record *rec, *end; - uint32_t record_size = index->record_size; if (map->append == NULL) break; + rec = CONST_PTR_OFFSET(data, sizeof(*append_hdr)); end = CONST_PTR_OFFSET(data, hdr->size); - for (rec = data; rec != end; ) { - ret = map->append(rec, context); + while (rec != end) { + ret = map->append(append_hdr, rec, context); if (ret <= 0) break; - rec = CONST_PTR_OFFSET(rec, record_size); + rec = CONST_PTR_OFFSET(rec, append_hdr->record_size); } break; }
--- a/src/lib-index/mail-transaction-util.h Wed Jun 23 23:10:34 2004 +0300 +++ b/src/lib-index/mail-transaction-util.h Thu Jun 24 14:10:41 2004 +0300 @@ -10,7 +10,8 @@ struct mail_transaction_map_functions { int (*expunge)(const struct mail_transaction_expunge *e, void *context); - int (*append)(const struct mail_index_record *rec, void *context); + int (*append)(const struct mail_transaction_append_header *hdr, + const struct mail_index_record *rec, void *context); int (*flag_update)(const struct mail_transaction_flag_update *u, void *context); int (*cache_reset)(const struct mail_transaction_cache_reset *u,