Mercurial > dovecot > original-hg > dovecot-1.2
changeset 2884:173a0296e81d HEAD
Update view's header only after syncing the view. This and other changes fix
several view syncing problems and crashes.
author | Timo Sirainen <tss@iki.fi> |
---|---|
date | Sun, 14 Nov 2004 01:08:07 +0200 |
parents | 1f5201e660c1 |
children | 1880f191278a |
files | src/imap/imap-sync.c src/lib-index/mail-index-lock.c src/lib-index/mail-index-private.h src/lib-index/mail-index-sync-update.c src/lib-index/mail-index-sync.c src/lib-index/mail-index-transaction-private.h src/lib-index/mail-index-transaction-view.c src/lib-index/mail-index-transaction.c src/lib-index/mail-index-view-private.h src/lib-index/mail-index-view-sync.c src/lib-index/mail-index-view.c src/lib-index/mail-index.c src/lib-index/mail-transaction-log.c |
diffstat | 13 files changed, 224 insertions(+), 218 deletions(-) [+] |
line wrap: on
line diff
--- a/src/imap/imap-sync.c Sun Nov 14 01:06:16 2004 +0200 +++ b/src/imap/imap-sync.c Sun Nov 14 01:08:07 2004 +0200 @@ -39,8 +39,8 @@ ctx->client = client; ctx->box = box; + ctx->sync_ctx = mailbox_sync_init(box, flags); ctx->t = mailbox_transaction_begin(box, FALSE); - ctx->sync_ctx = mailbox_sync_init(box, flags); ctx->messages_count = client->messages_count; return ctx; }
--- a/src/lib-index/mail-index-lock.c Sun Nov 14 01:06:16 2004 +0200 +++ b/src/lib-index/mail-index-lock.c Sun Nov 14 01:08:07 2004 +0200 @@ -229,29 +229,23 @@ if (fd == -1) return -1; - if (index->map->hdr->base_header_size >= sizeof(*index->map->hdr)) { - /* header size is what's expected */ - ret = write_full(fd, index->map->hdr, - index->map->hdr->header_size); - } else { - /* write base header */ - ret = write_full(fd, index->map->hdr, - index->map->hdr->base_header_size); - if (ret == 0) { - /* write extended headers */ - const struct mail_index_header *hdr; - const void *hdr_ext; + /* write base header */ + ret = write_full(fd, &index->map->hdr, + index->map->hdr.base_header_size); + if (ret == 0) { + /* write extended headers */ + const struct mail_index_header *hdr; + const void *hdr_ext; - hdr = index->map->hdr_base; - hdr_ext = CONST_PTR_OFFSET(hdr, hdr->base_header_size); - ret = write_full(fd, hdr_ext, hdr->header_size - - hdr->base_header_size); - } + hdr = index->map->hdr_base; + hdr_ext = CONST_PTR_OFFSET(hdr, hdr->base_header_size); + ret = write_full(fd, hdr_ext, hdr->header_size - + hdr->base_header_size); } if (ret < 0 || write_full(fd, index->map->records, index->map->records_count * - index->map->hdr->record_size) < 0) { + index->map->hdr.record_size) < 0) { mail_index_file_set_syscall_error(index, path, "write_full()"); (void)close(fd); (void)unlink(path); @@ -316,8 +310,7 @@ /* if header size is smaller than what we have, we'll have to recreate the index to grow it. so don't even try regular locking. */ - if ((index->map->hdr != index->map->hdr_copy_buf->data && - index->map->base_header_size == sizeof(*index->hdr)) || + if (index->map->hdr.base_header_size >= sizeof(*index->hdr) || index->excl_lock_count > 0) { /* wait two seconds for exclusive lock */ ret = mail_index_lock(index, F_WRLCK, 2, TRUE, lock_id_r);
--- a/src/lib-index/mail-index-private.h Sun Nov 14 01:06:16 2004 +0200 +++ b/src/lib-index/mail-index-private.h Sun Nov 14 01:08:07 2004 +0200 @@ -26,7 +26,7 @@ #define MAIL_INDEX_MAP_IDX(map, idx) \ ((struct mail_index_record *) \ - PTR_OFFSET((map)->records, (idx) * (map)->hdr->record_size)) + PTR_OFFSET((map)->records, (idx) * (map)->hdr.record_size)) typedef int mail_index_expunge_handler_t(struct mail_index_sync_map_ctx *ctx, uint32_t seq, const void *data, @@ -61,7 +61,7 @@ struct mail_index_map { int refcount; - const struct mail_index_header *hdr; + struct mail_index_header hdr; const void *hdr_base; void *records; /* struct mail_index_record[] */ unsigned int records_count; @@ -74,13 +74,7 @@ size_t mmap_size, mmap_used_size; buffer_t *buffer; - - uint32_t log_file_seq; - uoff_t log_file_int_offset; - uoff_t log_file_ext_offset; - buffer_t *hdr_copy_buf; - uint32_t base_header_size; /* so we don't need lock to access it */ unsigned int write_to_disk:1; };
--- a/src/lib-index/mail-index-sync-update.c Sun Nov 14 01:06:16 2004 +0200 +++ b/src/lib-index/mail-index-sync-update.c Sun Nov 14 01:08:07 2004 +0200 @@ -20,7 +20,7 @@ view->map->refcount++; mail_index_unmap(view->index, view->index->map); view->index->map = map; - view->index->hdr = map->hdr; + view->index->hdr = &map->hdr; map->write_to_disk = TRUE; } @@ -183,7 +183,6 @@ struct mail_index_sync_map_ctx *ctx = context; struct mail_index_view *view = ctx->view; struct mail_index_map *map = view->map; - struct mail_index_header *hdr; struct mail_index_record *rec; uint32_t count, seq, seq1, seq2; struct mail_index_expunge_handler *expunge_handlers, *eh; @@ -202,8 +201,7 @@ replace the whole index file. to avoid extra disk I/O we copy the index into memory rather than to temporary file */ - map = mail_index_map_to_memory(map, - map->hdr->record_size); + map = mail_index_map_to_memory(map, map->hdr.record_size); mail_index_sync_replace_map(view, map); } i_assert(MAIL_INDEX_MAP_IS_IN_MEMORY(map)); @@ -229,10 +227,9 @@ expunge_handlers_count = 0; } - hdr = buffer_get_modifyable_data(map->hdr_copy_buf, NULL); for (seq = seq1; seq <= seq2; seq++) { rec = MAIL_INDEX_MAP_IDX(map, seq-1); - mail_index_header_update_counts(hdr, rec->flags, 0); + mail_index_header_update_counts(&map->hdr, rec->flags, 0); } for (i = 0; i < expunge_handlers_count; i++) { @@ -249,15 +246,15 @@ /* @UNSAFE */ count = seq2 - seq1 + 1; memmove(MAIL_INDEX_MAP_IDX(map, seq1-1), MAIL_INDEX_MAP_IDX(map, seq2), - (map->records_count - seq2) * map->hdr->record_size); + (map->records_count - seq2) * map->hdr.record_size); map->records_count -= count; - hdr->messages_count -= count; + map->hdr.messages_count -= count; view->messages_count -= count; if (map->buffer != NULL) { buffer_set_used_size(map->buffer, map->records_count * - map->hdr->record_size); + map->hdr.record_size); map->records = buffer_get_modifyable_data(map->buffer, NULL); } return 1; @@ -268,42 +265,40 @@ struct mail_index_sync_map_ctx *ctx = context; struct mail_index_view *view = ctx->view; struct mail_index_map *map = view->map; - struct mail_index_header *hdr; void *dest; - hdr = buffer_get_modifyable_data(map->hdr_copy_buf, NULL); - if (rec->uid < hdr->next_uid) { + if (rec->uid < map->hdr.next_uid) { mail_transaction_log_view_set_corrupted(view->log_view, "Append with UID %u, but next_uid = %u", - rec->uid, hdr->next_uid); + rec->uid, map->hdr.next_uid); return -1; } if (MAIL_INDEX_MAP_IS_IN_MEMORY(map)) { - i_assert(map->records_count * map->hdr->record_size == + i_assert(map->records_count * map->hdr.record_size == buffer_get_used_size(map->buffer)); dest = buffer_append_space_unsafe(map->buffer, - map->hdr->record_size); + map->hdr.record_size); map->records = buffer_get_modifyable_data(map->buffer, NULL); } else { - i_assert((map->records_count+1) * map->hdr->record_size <= + i_assert((map->records_count+1) * map->hdr.record_size <= map->mmap_size); dest = MAIL_INDEX_MAP_IDX(map, map->records_count); } memcpy(dest, rec, sizeof(*rec)); memset(PTR_OFFSET(dest, sizeof(*rec)), 0, - map->hdr->record_size - sizeof(*rec)); + map->hdr.record_size - sizeof(*rec)); - hdr->messages_count++; - hdr->next_uid = rec->uid+1; + map->hdr.messages_count++; + map->hdr.next_uid = rec->uid+1; view->messages_count++; map->records_count++; if ((rec->flags & MAIL_INDEX_MAIL_FLAG_DIRTY) != 0) - hdr->flags |= MAIL_INDEX_HDR_FLAG_HAVE_DIRTY; + map->hdr.flags |= MAIL_INDEX_HDR_FLAG_HAVE_DIRTY; - mail_index_header_update_counts(hdr, 0, rec->flags); - mail_index_header_update_lowwaters(hdr, rec); + mail_index_header_update_counts(&map->hdr, 0, rec->flags); + mail_index_header_update_lowwaters(&map->hdr, rec); return 1; } @@ -333,8 +328,7 @@ if (seq1 == 0) return 1; - hdr = buffer_get_modifyable_data(view->map->hdr_copy_buf, NULL); - + hdr = &view->map->hdr; if ((u->add_flags & MAIL_INDEX_MAIL_FLAG_DIRTY) != 0) hdr->flags |= MAIL_INDEX_HDR_FLAG_HAVE_DIRTY; @@ -369,18 +363,27 @@ void *context) { struct mail_index_sync_map_ctx *ctx = context; - const struct mail_index_header *hdr = ctx->view->map->hdr; + struct mail_index_map *map = ctx->view->map; - if (u->offset >= hdr->base_header_size || - u->offset + u->size > hdr->base_header_size) { + if (u->offset >= map->hdr.base_header_size || + u->offset + u->size > map->hdr.base_header_size) { mail_transaction_log_view_set_corrupted(ctx->view->log_view, "Header update outside range: %u + %u > %u", - u->offset, u->size, hdr->base_header_size); + u->offset, u->size, map->hdr.base_header_size); return -1; } - buffer_write(ctx->view->map->hdr_copy_buf, u->offset, u + 1, u->size); - ctx->view->map->hdr = ctx->view->map->hdr_copy_buf->data; + buffer_write(map->hdr_copy_buf, u->offset, u + 1, u->size); + map->hdr_base = map->hdr_copy_buf->data; + + /* @UNSAFE */ + if ((uint32_t)(u->offset + u->size) <= sizeof(map->hdr)) { + memcpy(PTR_OFFSET(&map->hdr, u->offset), + u + 1, u->size); + } else if (u->offset < sizeof(map->hdr)) { + memcpy(PTR_OFFSET(&map->hdr, u->offset), + u + 1, sizeof(map->hdr) - u->offset); + } return 1; } @@ -389,15 +392,15 @@ { struct mail_index_ext_header *ext_hdr; uint32_t offset; - void *hdr; + void *hdr_base; /* do some kludgy jumping to get to it. */ offset = ext->hdr_offset - MAIL_INDEX_HEADER_SIZE_ALIGN(sizeof(*ext_hdr) + strlen(ext->name)); - hdr = buffer_get_modifyable_data(map->hdr_copy_buf, NULL); - ext_hdr = PTR_OFFSET(hdr, offset); + hdr_base = buffer_get_modifyable_data(map->hdr_copy_buf, NULL); + ext_hdr = PTR_OFFSET(hdr_base, offset); i_assert(memcmp((char *)(ext_hdr + 1), ext->name, strlen(ext->name)) == 0); return ext_hdr; @@ -495,8 +498,8 @@ i == ext_id ? old_size : ext[i].record_size); } - src = CONST_PTR_OFFSET(src, map->hdr->record_size); - offset += new_map->hdr->record_size; + src = CONST_PTR_OFFSET(src, map->hdr.record_size); + offset += new_map->hdr.record_size; } if (ext_id == size-1 && ext[ext_id].record_size != old_size && @@ -526,7 +529,6 @@ struct mail_index_map *map = ctx->view->map; struct mail_index_ext *ext; struct mail_index_ext_header *ext_hdr; - struct mail_index_header *hdr; uint32_t old_size, new_size, old_record_size; int modified = FALSE; @@ -558,9 +560,8 @@ modified = TRUE; if (modified) { - hdr = buffer_get_modifyable_data(map->hdr_copy_buf, NULL); - hdr->header_size = map->hdr_copy_buf->used; - map->hdr = hdr; + map->hdr_base = map->hdr_copy_buf->data; + map->hdr.header_size = map->hdr_copy_buf->used; ext_hdr = get_ext_header(map, ext); ext_hdr->reset_id = ext->reset_id; @@ -583,7 +584,6 @@ struct mail_index_map *map = ctx->view->map; struct mail_index_ext_header ext_hdr; const struct mail_index_ext *ext; - struct mail_index_header *hdr; const char *name; buffer_t *hdr_buf; uint32_t ext_id, hdr_offset; @@ -668,9 +668,8 @@ i_assert(hdr_buf->used == hdr_offset + MAIL_INDEX_HEADER_SIZE_ALIGN(ext->hdr_size)); - hdr = buffer_get_modifyable_data(hdr_buf, NULL); - hdr->header_size = hdr_buf->used; - map->hdr = hdr; + map->hdr.header_size = hdr_buf->used; + map->hdr_base = map->hdr_copy_buf->data; t_pop(); @@ -709,7 +708,7 @@ memset(buffer_get_space_unsafe(map->hdr_copy_buf, ext->hdr_offset, ext->hdr_size), 0, ext->hdr_size); - map->hdr = map->hdr_copy_buf->data; + map->hdr_base = map->hdr_copy_buf->data; for (i = 0; i < view->messages_count; i++) { rec = MAIL_INDEX_MAP_IDX(view->map, i); @@ -743,7 +742,7 @@ buffer_write(map->hdr_copy_buf, ext->hdr_offset + u->offset, u + 1, u->size); - map->hdr = map->hdr_copy_buf->data; + map->hdr_base = map->hdr_copy_buf->data; return 1; } @@ -802,8 +801,8 @@ i_assert(map == index->map); - size = map->hdr->header_size + - (map->records_count + count) * map->hdr->record_size; + size = map->hdr.header_size + + (map->records_count + count) * map->hdr.record_size; if (size <= map->mmap_size) return 0; @@ -814,8 +813,8 @@ count = nearest_power(count); index->last_grow_count = count; - size = map->hdr->header_size + - (map->records_count + count) * map->hdr->record_size; + size = map->hdr.header_size + + (map->records_count + count) * map->hdr.record_size; if (file_set_size(index->fd, (off_t)size) < 0) return mail_index_set_syscall_error(index, "file_set_size()"); @@ -823,9 +822,11 @@ headers as well and may break our modified hdr_copy. so, take a backup of it and put it back afterwards */ t_push(); + i_assert(map->hdr_copy_buf->used == map->hdr.header_size); hdr_copy_size = map->hdr_copy_buf->used; hdr_copy = t_malloc(hdr_copy_size); memcpy(hdr_copy, map->hdr_copy_buf->data, hdr_copy_size); + memcpy(hdr_copy, &map->hdr, sizeof(map->hdr)); if (mail_index_map(index, TRUE) <= 0) { t_pop(); @@ -835,8 +836,10 @@ map = index->map; buffer_reset(map->hdr_copy_buf); buffer_append(map->hdr_copy_buf, hdr_copy, hdr_copy_size); - map->hdr = map->hdr_copy_buf->data; - map->records_count = map->hdr->messages_count; + + map->hdr_base = map->hdr_copy_buf->data; + memcpy(&map->hdr, hdr_copy, sizeof(map->hdr)); + map->records_count = map->hdr.messages_count; i_assert(map->mmap_size >= size); t_pop(); @@ -1046,7 +1049,6 @@ struct mail_index_map *map; struct mail_index_sync_map_ctx sync_map_ctx; const struct mail_transaction_header *thdr; - struct mail_index_header *tmphdr; const void *data; unsigned int count, old_lock_id; uint32_t seq, i, first_append_uid; @@ -1069,22 +1071,21 @@ if (MAIL_INDEX_MAP_IS_IN_MEMORY(map)) map->write_to_disk = TRUE; - if (map->hdr != map->hdr_copy_buf->data) { + if (map->hdr_base != map->hdr_copy_buf->data) { buffer_reset(map->hdr_copy_buf); - buffer_append(map->hdr_copy_buf, map->hdr, - map->hdr->header_size); - map->hdr = map->hdr_copy_buf->data; - index->hdr = map->hdr; + buffer_append(map->hdr_copy_buf, map->hdr_base, + map->hdr.header_size); + map->hdr_base = map->hdr_copy_buf->data; } + i_assert(map->hdr.base_header_size >= sizeof(map->hdr)); mail_index_unmap(index, view->map); view->map = map; view->map->refcount++; - tmphdr = buffer_get_modifyable_data(map->hdr_copy_buf, NULL); - had_dirty = (tmphdr->flags & MAIL_INDEX_HDR_FLAG_HAVE_DIRTY) != 0; + had_dirty = (map->hdr.flags & MAIL_INDEX_HDR_FLAG_HAVE_DIRTY) != 0; if (had_dirty) - tmphdr->flags &= ~MAIL_INDEX_HDR_FLAG_HAVE_DIRTY; + map->hdr.flags &= ~MAIL_INDEX_HDR_FLAG_HAVE_DIRTY; first_append_uid = 0; check_ext_offsets = TRUE; @@ -1102,7 +1103,7 @@ mail_transaction_log_view_get_prev_pos(view->log_view, &prev_seq, &prev_offset); - if (prev_offset < index->hdr->log_file_ext_offset) { + if (prev_offset < map->hdr.log_file_ext_offset) { /* we have already synced this change */ continue; } @@ -1142,22 +1143,20 @@ return -1; } - i_assert(map->records_count == map->hdr->messages_count); - i_assert(view->messages_count == map->hdr->messages_count); + i_assert(map->records_count == map->hdr.messages_count); + i_assert(view->messages_count == map->hdr.messages_count); mail_transaction_log_get_head(index->log, &seq, &offset); - /* hdr pointer may have changed, update it */ - tmphdr = buffer_get_modifyable_data(map->hdr_copy_buf, NULL); - tmphdr->log_file_seq = seq; + map->hdr.log_file_seq = seq; if (!sync_only_external) - tmphdr->log_file_int_offset = offset; - tmphdr->log_file_ext_offset = offset; + map->hdr.log_file_int_offset = offset; + map->hdr.log_file_ext_offset = offset; if (first_append_uid != 0) - mail_index_update_day_headers(tmphdr, first_append_uid); + mail_index_update_day_headers(&map->hdr, first_append_uid); - if ((tmphdr->flags & MAIL_INDEX_HDR_FLAG_HAVE_DIRTY) == 0 && + if ((map->hdr.flags & MAIL_INDEX_HDR_FLAG_HAVE_DIRTY) == 0 && had_dirty) { /* do we have dirty flags anymore? */ const struct mail_index_record *rec; @@ -1165,7 +1164,8 @@ for (i = 0; i < map->records_count; i++) { rec = MAIL_INDEX_MAP_IDX(map, i); if ((rec->flags & MAIL_INDEX_MAIL_FLAG_DIRTY) != 0) { - tmphdr->flags |= MAIL_INDEX_HDR_FLAG_HAVE_DIRTY; + map->hdr.flags |= + MAIL_INDEX_HDR_FLAG_HAVE_DIRTY; break; } } @@ -1173,15 +1173,16 @@ if (!MAIL_INDEX_MAP_IS_IN_MEMORY(map)) { map->mmap_used_size = index->hdr->header_size + - map->records_count * map->hdr->record_size; + map->records_count * map->hdr.record_size; - memcpy(map->mmap_base, tmphdr, tmphdr->header_size); + memcpy(map->mmap_base, &map->hdr, sizeof(map->hdr)); + memcpy(PTR_OFFSET(map->mmap_base, sizeof(map->hdr)), + PTR_OFFSET(map->hdr_base, sizeof(map->hdr)), + map->hdr.header_size - sizeof(map->hdr)); if (msync(map->mmap_base, map->mmap_used_size, MS_SYNC) < 0) { mail_index_set_syscall_error(index, "msync()"); ret = -1; } - map->hdr = map->mmap_base; - index->hdr = map->hdr; } i_assert(view->map == index->map);
--- a/src/lib-index/mail-index-sync.c Sun Nov 14 01:06:16 2004 +0200 +++ b/src/lib-index/mail-index-sync.c Sun Nov 14 01:08:07 2004 +0200 @@ -218,7 +218,7 @@ *seen_external_r = FALSE; - if ((ctx->view->map->hdr->flags & MAIL_INDEX_HDR_FLAG_HAVE_DIRTY) && + if ((ctx->view->map->hdr.flags & MAIL_INDEX_HDR_FLAG_HAVE_DIRTY) && ctx->sync_dirty) { /* show dirty flags as flag updates */ if (mail_index_sync_add_dirty_updates(ctx) < 0)
--- a/src/lib-index/mail-index-transaction-private.h Sun Nov 14 01:06:16 2004 +0200 +++ b/src/lib-index/mail-index-transaction-private.h Sun Nov 14 01:08:07 2004 +0200 @@ -26,6 +26,7 @@ struct mail_cache_transaction_ctx *cache_trans_ctx; unsigned int hide_transaction:1; + unsigned int no_appends:1; unsigned int external:1; unsigned int hdr_changed:1; unsigned int log_updates:1;
--- a/src/lib-index/mail-index-transaction-view.c Sun Nov 14 01:06:16 2004 +0200 +++ b/src/lib-index/mail-index-transaction-view.c Sun Nov 14 01:08:07 2004 +0200 @@ -40,13 +40,7 @@ if (tview->parent->get_header(view, hdr_r) < 0) return -1; - if ((*hdr_r)->messages_count != view->messages_count) { - /* messages_count differs, use a modified copy. - FIXME: same problems as with _view_get_header().. */ - view->tmp_hdr_copy = **hdr_r; - view->tmp_hdr_copy.messages_count = view->messages_count; - *hdr_r = &view->tmp_hdr_copy; - } + /* FIXME: header counters may not be correct */ return 0; } @@ -205,6 +199,14 @@ { struct mail_index_view_transaction *tview; + if (t->view->syncing) { + /* transaction view is being synced. while it's done, it's not + possible to add new messages, but the view itself might + change. so we can't make a copy of the view. */ + mail_index_view_ref(t->view); + return t->view; + } + tview = i_new(struct mail_index_view_transaction, 1); mail_index_view_clone(&tview->view, t->view); tview->view.methods = view_methods;
--- a/src/lib-index/mail-index-transaction.c Sun Nov 14 01:06:16 2004 +0200 +++ b/src/lib-index/mail-index-transaction.c Sun Nov 14 01:08:07 2004 +0200 @@ -30,6 +30,13 @@ t->hide_transaction = hide; t->external = external; t->first_new_seq = mail_index_view_get_message_count(t->view)+1; + + if (view->syncing) { + /* transaction view cannot work if new records are being added + in two places. make sure it doesn't happen. */ + t->no_appends = TRUE; + } + return t; } @@ -197,6 +204,8 @@ { struct mail_index_record *rec; + i_assert(!t->no_appends); + t->log_updates = TRUE; if (t->appends == NULL)
--- a/src/lib-index/mail-index-view-private.h Sun Nov 14 01:06:16 2004 +0200 +++ b/src/lib-index/mail-index-view-private.h Sun Nov 14 01:08:07 2004 +0200 @@ -27,6 +27,8 @@ }; struct mail_index_view { + int refcount; + struct mail_index_view_methods methods; struct mail_index *index; struct mail_transaction_log_view *log_view; @@ -36,7 +38,7 @@ struct mail_index_map *new_map; buffer_t *map_refs; - struct mail_index_header tmp_hdr_copy; + struct mail_index_header hdr; uint32_t messages_count; /* last synced one, map may be different */ uint32_t log_file_seq; @@ -52,6 +54,8 @@ void mail_index_view_clone(struct mail_index_view *dest, const struct mail_index_view *src); +void mail_index_view_ref(struct mail_index_view *view); +void mail_index_view_unref(struct mail_index_view *view); int mail_index_view_lock(struct mail_index_view *view); int mail_index_view_lock_head(struct mail_index_view *view, int update_index); void mail_index_view_unref_maps(struct mail_index_view *view);
--- a/src/lib-index/mail-index-view-sync.c Sun Nov 14 01:06:16 2004 +0200 +++ b/src/lib-index/mail-index-view-sync.c Sun Nov 14 01:08:07 2004 +0200 @@ -135,13 +135,32 @@ /* we need a private copy of the map if we don't want to sync expunges. we need to sync mapping only if we're not using the latest one. */ - if (view->map != view->index->map) + uint32_t old_records_count = view->map->records_count; + + if (view->map != view->index->map) { ctx->sync_map_update = TRUE; + view->map->records_count = view->messages_count; + } map = mail_index_map_to_memory(view->map, - view->map->hdr->record_size); + view->map->hdr.record_size); + view->map->records_count = old_records_count; mail_index_unmap(view->index, view->map); view->map = map; + + if (ctx->sync_map_update) { + if (map->hdr_base != map->hdr_copy_buf->data) { + buffer_reset(map->hdr_copy_buf); + buffer_append(map->hdr_copy_buf, map->hdr_base, + map->hdr.header_size); + map->hdr_base = map->hdr_copy_buf->data; + } + + /* start from our old view's header. */ + buffer_write(map->hdr_copy_buf, 0, + &view->hdr, sizeof(view->hdr)); + map->hdr = view->hdr; + } } mail_index_view_unref_maps(view); @@ -163,7 +182,7 @@ data = buffer_get_data(view->log_syncs, &size); end = data + size; - for (; data < end; ) { + while (data < end) { if (*((const uoff_t *)data) == offset && *((const uint32_t *)(data + sizeof(uoff_t))) == seq) return 1; @@ -338,8 +357,12 @@ view->new_map = NULL; } - if ((ctx->trans_sync_mask & MAIL_TRANSACTION_APPEND) != 0) + if ((ctx->trans_sync_mask & MAIL_TRANSACTION_APPEND) != 0) { + i_assert(view->messages_count == view->map->records_count || + !ctx->sync_map_update); view->messages_count = view->map->records_count; + } + view->hdr = view->map->hdr; (void)mail_transaction_log_view_set(view->log_view, view->log_file_seq,
--- a/src/lib-index/mail-index-view.c Sun Nov 14 01:06:16 2004 +0200 +++ b/src/lib-index/mail-index-view.c Sun Nov 14 01:08:07 2004 +0200 @@ -9,6 +9,7 @@ const struct mail_index_view *src) { memset(dest, 0, sizeof(dest)); + dest->refcount = 1; dest->methods = src->methods; dest->index = src->index; dest->log_view = mail_transaction_log_view_open(src->index->log); @@ -16,14 +17,22 @@ dest->indexid = src->indexid; dest->map = src->map; dest->map->refcount++; + dest->hdr = src->hdr; dest->messages_count = src->messages_count; dest->log_file_seq = src->log_file_seq; dest->log_file_offset = src->log_file_offset; } +void mail_index_view_ref(struct mail_index_view *view) +{ + view->refcount++; +} + static void _view_close(struct mail_index_view *view) { + i_assert(view->refcount == 0); + mail_index_view_unlock(view); mail_transaction_log_view_close(view->log_view); @@ -158,20 +167,7 @@ static int _view_get_header(struct mail_index_view *view, const struct mail_index_header **hdr_r) { - if (mail_index_view_lock(view) < 0) - return -1; - - if (view->map->hdr->messages_count == view->messages_count) - *hdr_r = view->map->hdr; - else { - /* messages_count differs, use a modified copy. - FIXME: so might seen_messages_count, etc. and they're - more difficult to fix. maybe grab a copy of the header - when opening the view initially?.. */ - view->tmp_hdr_copy = *view->map->hdr; - view->tmp_hdr_copy.messages_count = view->messages_count; - *hdr_r = &view->tmp_hdr_copy; - } + *hdr_r = &view->hdr; return 0; } @@ -253,7 +249,7 @@ i_assert(view->messages_count <= view->map->records_count); rec_base = view->map->records; - record_size = view->map->hdr->record_size; + record_size = view->map->hdr.record_size; idx = left_idx = *left_idx_p; right_idx = view->messages_count; @@ -303,8 +299,8 @@ if (mail_index_view_lock(view) < 0) return -1; - if (last_uid >= view->map->hdr->next_uid) { - last_uid = view->map->hdr->next_uid-1; + if (last_uid >= view->map->hdr.next_uid) { + last_uid = view->map->hdr.next_uid-1; if (first_uid > last_uid) { *first_seq_r = 0; *last_seq_r = 0; @@ -346,11 +342,11 @@ return -1; if ((flags_mask & MAIL_RECENT) != 0 && (flags & MAIL_RECENT) != 0) - LOW_UPDATE(view->map->hdr->first_recent_uid_lowwater); + LOW_UPDATE(view->map->hdr.first_recent_uid_lowwater); if ((flags_mask & MAIL_SEEN) != 0 && (flags & MAIL_SEEN) == 0) - LOW_UPDATE(view->map->hdr->first_unseen_uid_lowwater); + LOW_UPDATE(view->map->hdr.first_unseen_uid_lowwater); if ((flags_mask & MAIL_DELETED) != 0 && (flags & MAIL_DELETED) != 0) - LOW_UPDATE(view->map->hdr->first_deleted_uid_lowwater); + LOW_UPDATE(view->map->hdr.first_deleted_uid_lowwater); if (low_uid == 1) seq = 1; @@ -418,13 +414,16 @@ ext = map->extensions->data; ext += ext_id; - *data_r = CONST_PTR_OFFSET(map->hdr, ext->hdr_offset); + *data_r = CONST_PTR_OFFSET(map->hdr_base, ext->hdr_offset); *data_size_r = ext->hdr_size; return 0; } void mail_index_view_close(struct mail_index_view *view) { + if (--view->refcount > 0) + return; + view->methods.close(view); } @@ -521,6 +520,7 @@ struct mail_index_view *view; view = i_new(struct mail_index_view, 1); + view->refcount = 1; view->methods = view_methods; view->index = index; view->log_view = mail_transaction_log_view_open(index->log); @@ -528,12 +528,14 @@ view->indexid = index->indexid; view->map = index->map; view->map->refcount++; + + view->hdr = view->map->hdr; view->messages_count = view->map->records_count; - view->log_file_seq = view->map->log_file_seq; + view->log_file_seq = view->map->hdr.log_file_seq; view->log_file_offset = - I_MIN(view->map->log_file_int_offset, - view->map->log_file_ext_offset); + I_MIN(view->map->hdr.log_file_int_offset, + view->map->hdr.log_file_ext_offset); return view; }
--- a/src/lib-index/mail-index.c Sun Nov 14 01:06:16 2004 +0200 +++ b/src/lib-index/mail-index.c Sun Nov 14 01:08:07 2004 +0200 @@ -212,8 +212,8 @@ /* extension headers always start from 64bit offsets, so if base header doesn't happen to be 64bit aligned we'll skip some bytes */ - offset = MAIL_INDEX_HEADER_SIZE_ALIGN(map->hdr->base_header_size); - if (offset >= map->hdr->header_size && map->extension_pool == NULL) { + offset = MAIL_INDEX_HEADER_SIZE_ALIGN(map->hdr.base_header_size); + if (offset >= map->hdr.header_size && map->extension_pool == NULL) { /* nothing to do, skip allocatations and all */ return 1; } @@ -225,10 +225,10 @@ for (i = 0; i < old_count; i++) buffer_append(map->ext_id_map, &ext_id, sizeof(ext_id)); - while (offset < map->hdr->header_size) { + while (offset < map->hdr.header_size) { ext_hdr = CONST_PTR_OFFSET(map->hdr_base, offset); - size_left = map->hdr->header_size - offset; + size_left = map->hdr.header_size - offset; if (!size_check(&size_left, sizeof(*ext_hdr)) || !size_check(&size_left, ext_hdr->name_size) || !size_check(&size_left, get_align(ext_hdr->name_size)) || @@ -261,7 +261,7 @@ static int mail_index_check_header(struct mail_index *index, struct mail_index_map *map) { - const struct mail_index_header *hdr = map->hdr; + const struct mail_index_header *hdr = &map->hdr; unsigned char compat_data[sizeof(hdr->compat_data)]; memset(compat_data, 0, sizeof(compat_data)); @@ -280,7 +280,7 @@ return -1; } - if ((map->hdr->flags & MAIL_INDEX_HDR_FLAG_CORRUPTED) != 0) { + if ((map->hdr.flags & MAIL_INDEX_HDR_FLAG_CORRUPTED) != 0) { /* we've already complained about it */ return -1; } @@ -339,9 +339,9 @@ } if (map->refcount > 0) { + memset(&map->hdr, 0, sizeof(map->hdr)); map->mmap_size = 0; map->mmap_used_size = 0; - map->hdr = NULL; map->records = NULL; map->records_count = 0; } @@ -370,7 +370,6 @@ static int mail_index_mmap(struct mail_index *index, struct mail_index_map *map) { const struct mail_index_header *hdr; - struct mail_index_header *mhdr; unsigned int records_count; i_assert(!map->write_to_disk); @@ -405,7 +404,6 @@ return 0; } - map->hdr = hdr; map->mmap_used_size = hdr->header_size + hdr->messages_count * hdr->record_size; @@ -419,32 +417,25 @@ return 0; } - if (map->hdr->base_header_size < sizeof(*map->hdr)) { + if (hdr->base_header_size < sizeof(map->hdr)) { /* header smaller than ours, make a copy so our newer headers won't have garbage in them */ - buffer_reset(map->hdr_copy_buf); - buffer_append(map->hdr_copy_buf, - map->hdr, map->hdr->base_header_size); - buffer_append_zero(map->hdr_copy_buf, sizeof(*map->hdr) - - map->hdr->base_header_size); - - mhdr = buffer_get_modifyable_data(map->hdr_copy_buf, NULL); - mhdr->base_header_size = sizeof(*map->hdr); - mhdr->header_size = map->hdr_copy_buf->used; - - map->hdr = mhdr; - } + memset(&map->hdr, 0, sizeof(map->hdr)); + memcpy(&map->hdr, hdr, hdr->base_header_size); + } else { + map->hdr = *hdr; + } map->hdr_base = map->mmap_base; - map->records = PTR_OFFSET(map->mmap_base, map->hdr->header_size); - map->records_count = map->hdr->messages_count; + map->records = PTR_OFFSET(map->mmap_base, map->hdr.header_size); + map->records_count = map->hdr.messages_count; return 1; } static int mail_index_read_map(struct mail_index *index, struct mail_index_map *map, int *retry_r) { - struct mail_index_header hdr, *hdrp; + struct mail_index_header hdr; void *data = NULL; ssize_t ret; size_t pos, records_size; @@ -480,34 +471,15 @@ } buffer_reset(map->hdr_copy_buf); - if (hdr.base_header_size < sizeof(hdr)) { - buffer_append_zero(map->hdr_copy_buf, sizeof(hdr) + - hdr.header_size - - hdr.base_header_size); - buffer_write(map->hdr_copy_buf, 0, - &hdr, hdr.base_header_size); - - /* @UNSAFE */ - ret = pread_full(index->fd, - PTR_OFFSET(map->hdr_copy_buf->data, - sizeof(hdr)), - hdr.header_size - hdr.base_header_size, - hdr.base_header_size); + buffer_append(map->hdr_copy_buf, &hdr, hdr.base_header_size); - hdrp = buffer_get_modifyable_data(map->hdr_copy_buf, - NULL); - hdrp->base_header_size = sizeof(hdr); - hdrp->header_size = map->hdr_copy_buf->used; - } else { - buffer_append(map->hdr_copy_buf, &hdr, pos); - buffer_append_zero(map->hdr_copy_buf, - hdr.header_size - pos); - /* @UNSAFE */ - ret = pread_full(index->fd, - PTR_OFFSET(map->hdr_copy_buf->data, - pos), - hdr.header_size - pos, pos); - } + /* @UNSAFE */ + data = buffer_append_space_unsafe(map->hdr_copy_buf, + hdr.header_size - + hdr.base_header_size); + ret = pread_full(index->fd, data, + hdr.header_size - hdr.base_header_size, + hdr.base_header_size); } if (ret > 0) { @@ -544,7 +516,12 @@ map->records = data; map->records_count = hdr.messages_count; - map->hdr = map->hdr_copy_buf->data; + if (hdr.base_header_size >= sizeof(map->hdr)) + map->hdr = hdr; + else { + memset(&map->hdr, 0, sizeof(map->hdr)); + memcpy(&map->hdr, &hdr, hdr.base_header_size); + } map->hdr_base = map->hdr_copy_buf->data; return 1; } @@ -595,7 +572,7 @@ return -1; used_size = hdr->header_size + hdr->messages_count * hdr->record_size; - if (map->mmap_size >= used_size && map->hdr == hdr) { + if (map->mmap_size >= used_size && map->hdr_base == hdr) { map->records_count = hdr->messages_count; return 1; } @@ -630,7 +607,7 @@ map = i_new(struct mail_index_map, 1); map->refcount = 1; map->hdr_copy_buf = - buffer_create_dynamic(default_pool, sizeof(*map->hdr)); + buffer_create_dynamic(default_pool, sizeof(map->hdr)); } else if (MAIL_INDEX_MAP_IS_IN_MEMORY(map)) { if (map->write_to_disk) { /* we have modified this mapping and it's waiting to @@ -666,12 +643,7 @@ if (ret == 0) index->fsck = TRUE; - map->log_file_seq = map->hdr->log_file_seq; - map->log_file_int_offset = map->hdr->log_file_int_offset; - map->log_file_ext_offset = map->hdr->log_file_ext_offset; - map->base_header_size = map->hdr->base_header_size; - - index->hdr = map->hdr; + index->hdr = &map->hdr; index->map = map; return 1; } @@ -687,7 +659,7 @@ unsigned int i, count; if (MAIL_INDEX_MAP_IS_IN_MEMORY(map) && - map->hdr->record_size == new_record_size) { + map->hdr.record_size == new_record_size) { map->refcount++; return map; } @@ -697,16 +669,16 @@ mem_map = i_new(struct mail_index_map, 1); mem_map->refcount = 1; mem_map->buffer = buffer_create_dynamic(default_pool, size); - if (map->hdr->record_size == new_record_size) + if (map->hdr.record_size == new_record_size) buffer_append(mem_map->buffer, map->records, size); else { - copy_size = I_MIN(map->hdr->record_size, new_record_size); + copy_size = I_MIN(map->hdr.record_size, new_record_size); src = map->records; for (i = 0; i < map->records_count; i++) { dest = buffer_append_space_unsafe(mem_map->buffer, new_record_size); memcpy(dest, src, copy_size); - src = PTR_OFFSET(src, map->hdr->record_size); + src = PTR_OFFSET(src, map->hdr.record_size); } } @@ -714,12 +686,22 @@ mem_map->records_count = map->records_count; mem_map->hdr_copy_buf = - buffer_create_dynamic(default_pool, map->hdr->header_size); - buffer_append(mem_map->hdr_copy_buf, map->hdr, map->hdr->header_size); + buffer_create_dynamic(default_pool, map->hdr.header_size); + if (map->hdr.base_header_size < sizeof(*hdr)) + buffer_append_zero(mem_map->hdr_copy_buf, sizeof(*hdr)); + buffer_write(mem_map->hdr_copy_buf, 0, + &map->hdr, map->hdr.base_header_size); + buffer_append(mem_map->hdr_copy_buf, + CONST_PTR_OFFSET(map->hdr_base, + map->hdr.base_header_size), + map->hdr.header_size - map->hdr.base_header_size); hdr = buffer_get_modifyable_data(mem_map->hdr_copy_buf, NULL); + if (hdr->base_header_size < sizeof(*hdr)) + hdr->base_header_size = sizeof(*hdr); hdr->record_size = new_record_size; - mem_map->hdr = hdr; + mem_map->hdr = *hdr; + mem_map->hdr_base = hdr; /* copy extensions */ if (map->ext_id_map != NULL) { @@ -740,11 +722,6 @@ } } - mem_map->log_file_seq = mem_map->hdr->log_file_seq; - mem_map->log_file_int_offset = mem_map->hdr->log_file_int_offset; - mem_map->log_file_ext_offset = mem_map->hdr->log_file_ext_offset; - mem_map->base_header_size = mem_map->hdr->base_header_size; - return mem_map; } @@ -1141,7 +1118,7 @@ } index->map = old_map; - index->hdr = index->map->hdr; + index->hdr = &index->map->hdr; index->fd = old_fd; index->lock_type = old_lock_type; index->lock_id = old_lock_id;
--- a/src/lib-index/mail-transaction-log.c Sun Nov 14 01:06:16 2004 +0200 +++ b/src/lib-index/mail-transaction-log.c Sun Nov 14 01:08:07 2004 +0200 @@ -482,12 +482,12 @@ } if (log->index->map != NULL && - file->hdr.file_seq == log->index->map->log_file_seq && - log->index->map->log_file_int_offset != 0) { + file->hdr.file_seq == log->index->map->hdr.log_file_seq && + log->index->map->hdr.log_file_int_offset != 0) { /* we can get a valid log offset from index file. initialize sync_offset from it so we don't have to read the whole log file from beginning. */ - file->sync_offset = log->index->map->log_file_int_offset; + file->sync_offset = log->index->map->hdr.log_file_int_offset; } for (p = &log->tail; *p != NULL; p = &(*p)->next) {