Mercurial > dovecot > original-hg > dovecot-1.2
changeset 2050:ee1095ccfd23 HEAD
Index header changes now go through transaction log. Removed the kludgy
parameters for mail_index_sync_end(). Removed code duplication of syncing
index root mapping and view mapping. Some fixes to handling uidvalidity and
nextuid in syncing.
line wrap: on
line diff
--- a/src/lib-index/mail-index-lock.c Mon May 24 04:01:42 2004 +0300 +++ b/src/lib-index/mail-index-lock.c Mon May 24 04:50:16 2004 +0300 @@ -126,6 +126,7 @@ int prot; if (!MAIL_INDEX_MAP_IS_IN_MEMORY(map)) { + i_assert(map->mmap_size != 0); prot = lock_type == F_UNLCK ? PROT_NONE : lock_type == F_WRLCK ? (PROT_READ|PROT_WRITE) : PROT_READ;
--- a/src/lib-index/mail-index-sync-private.h Mon May 24 04:01:42 2004 +0300 +++ b/src/lib-index/mail-index-sync-private.h Mon May 24 04:50:16 2004 +0300 @@ -22,13 +22,12 @@ unsigned int sync_appends:1; }; -int mail_index_sync_update_index(struct mail_index_sync_ctx *sync_ctx, - uint32_t sync_stamp, uint64_t sync_size); +extern struct mail_transaction_map_functions mail_index_map_sync_funcs; -void mail_index_header_update_counts(struct mail_index_header *hdr, - uint8_t old_flags, uint8_t new_flags); -void mail_index_header_update_lowwaters(struct mail_index_header *hdr, - const struct mail_index_record *rec); +int mail_index_sync_update_index(struct mail_index_sync_ctx *sync_ctx); + +void mail_index_sync_expunge(struct mail_index_view *view, + const struct mail_transaction_expunge *e); void mail_index_sync_get_expunge(struct mail_index_sync_rec *rec,
--- a/src/lib-index/mail-index-sync-update.c Mon May 24 04:01:42 2004 +0300 +++ b/src/lib-index/mail-index-sync-update.c Mon May 24 04:50:16 2004 +0300 @@ -7,18 +7,11 @@ #include "mail-index-view-private.h" #include "mail-index-sync-private.h" #include "mail-transaction-log.h" +#include "mail-transaction-util.h" -struct mail_index_update_ctx { - struct mail_index *index; - struct mail_index_view *view; - struct mail_index_header hdr; - struct mail_transaction_log_view *log_view; - - unsigned int have_dirty:1; -}; - -void mail_index_header_update_counts(struct mail_index_header *hdr, - uint8_t old_flags, uint8_t new_flags) +static void +mail_index_header_update_counts(struct mail_index_header *hdr, + uint8_t old_flags, uint8_t new_flags) { if (((old_flags ^ new_flags) & MAIL_RECENT) != 0) { /* different recent-flag */ @@ -45,8 +38,9 @@ } } -void mail_index_header_update_lowwaters(struct mail_index_header *hdr, - const struct mail_index_record *rec) +static void +mail_index_header_update_lowwaters(struct mail_index_header *hdr, + const struct mail_index_record *rec) { if ((rec->flags & MAIL_RECENT) != 0 && rec->uid < hdr->first_recent_uid_lowwater) @@ -59,70 +53,161 @@ hdr->first_deleted_uid_lowwater = rec->uid; } -static void mail_index_sync_update_expunges(struct mail_index_update_ctx *ctx, - uint32_t seq1, uint32_t seq2) +void mail_index_sync_expunge(struct mail_index_view *view, + const struct mail_transaction_expunge *e) { + struct mail_index_map *map = view->map; + struct mail_index_header *hdr = &map->hdr_copy; struct mail_index_record *rec; - - rec = &ctx->index->map->records[seq1-1]; - for (; seq1 <= seq2; seq1++, rec++) - mail_index_header_update_counts(&ctx->hdr, rec->flags, 0); -} + uint32_t count, seq, seq1, seq2; + int ret; -static void mail_index_sync_update_flags(struct mail_index_update_ctx *ctx, - struct mail_index_sync_rec *syncrec) -{ - struct mail_index_record *rec, *end; - uint8_t flag_mask, old_flags; - keywords_mask_t keyword_mask; - uint32_t seq1, seq2; - int i, update_keywords, ret; + i_assert(MAIL_INDEX_MAP_IS_IN_MEMORY(map)); - ret = mail_index_lookup_uid_range(ctx->view, syncrec->uid1, - syncrec->uid2, &seq1, &seq2); + ret = mail_index_lookup_uid_range(view, e->uid1, e->uid2, &seq1, &seq2); i_assert(ret == 0); if (seq1 == 0) return; - if ((syncrec->add_flags & MAIL_INDEX_MAIL_FLAG_DIRTY) != 0) { - ctx->hdr.flags |= MAIL_INDEX_HDR_FLAG_HAVE_DIRTY; - ctx->have_dirty = TRUE; + rec = &map->records[seq1-1]; + for (seq = seq1; seq <= seq2; seq++, rec++) + mail_index_header_update_counts(hdr, rec->flags, 0); + + /* @UNSAFE */ + count = seq2 - seq1 + 1; + memcpy(map->records + (seq1-1), map->records + seq2, + (map->records_count - seq2) * sizeof(*map->records)); + + map->records_count -= count; + hdr->messages_count -= count; + + if (map->buffer != NULL) { + buffer_set_used_size(map->buffer, map->records_count); + map->records = buffer_get_modifyable_data(map->buffer, NULL); + } +} + +static int sync_expunge(const struct mail_transaction_expunge *e, void *context) +{ + struct mail_index_view *view = context; + + mail_index_sync_expunge(view, e); + return 1; +} + +static int sync_append(const struct mail_index_record *rec, void *context) +{ + struct mail_index_view *view = context; + struct mail_index_map *map = view->map; + + if (rec->uid < map->hdr_copy.next_uid) { + mail_transaction_log_view_set_corrupted(view->log_view, + "Append with UID %u, but next_uid = %u", + rec->uid, map->hdr_copy.next_uid); + return -1; } + if (MAIL_INDEX_MAP_IS_IN_MEMORY(map)) { + if (map->records_count * sizeof(*rec) > + buffer_get_used_size(map->buffer)) { + (void)buffer_append_space_unsafe(map->buffer, + sizeof(*rec)); + map->records = + buffer_get_modifyable_data(map->buffer, NULL); + } + } else { + i_assert(map->records_count * sizeof(*rec) <= map->mmap_size); + } + + map->records[map->records_count++] = *rec; + map->hdr_copy.messages_count++; + map->hdr_copy.next_uid = rec->uid+1; + + mail_index_header_update_counts(&map->hdr_copy, 0, rec->flags); + mail_index_header_update_lowwaters(&map->hdr_copy, rec); + return 1; +} + +static int sync_flag_update(const struct mail_transaction_flag_update *u, + void *context) +{ + struct mail_index_view *view = context; + struct mail_index_record *rec, *end; + struct mail_index_header *hdr; + uint8_t flag_mask, old_flags; + keywords_mask_t keyword_mask; + uint32_t seq1, seq2; + int i, update_keywords, ret; + + ret = mail_index_lookup_uid_range(view, u->uid1, u->uid2, &seq1, &seq2); + i_assert(ret == 0); + + if (seq1 == 0) + return 1; + + hdr = &view->map->hdr_copy; + + if ((u->add_flags & MAIL_INDEX_MAIL_FLAG_DIRTY) != 0) + hdr->flags |= MAIL_INDEX_HDR_FLAG_HAVE_DIRTY; + update_keywords = FALSE; for (i = 0; i < INDEX_KEYWORDS_BYTE_COUNT; i++) { - if (syncrec->add_keywords[i] != 0) - update_keywords = TRUE; - if (syncrec->remove_keywords[i] != 0) + if (u->add_keywords[i] != 0 || + u->remove_keywords[i] != 0) update_keywords = TRUE; - keyword_mask[i] = ~syncrec->remove_keywords[i]; + keyword_mask[i] = ~u->remove_keywords[i]; } + flag_mask = ~u->remove_flags; - flag_mask = ~syncrec->remove_flags; - rec = &ctx->index->map->records[seq1-1]; + rec = &view->map->records[seq1-1]; end = rec + (seq2 - seq1) + 1; for (; rec != end; rec++) { old_flags = rec->flags; - rec->flags = (rec->flags & flag_mask) | syncrec->add_flags; + rec->flags = (rec->flags & flag_mask) | u->add_flags; if (update_keywords) { for (i = 0; i < INDEX_KEYWORDS_BYTE_COUNT; i++) { - rec->keywords[i] = - (rec->keywords[i] & keyword_mask[i]) | - syncrec->add_keywords[i]; + rec->keywords[i] = u->add_keywords[i] | + (rec->keywords[i] & keyword_mask[i]); } } - mail_index_header_update_counts(&ctx->hdr, - old_flags, rec->flags); - mail_index_header_update_lowwaters(&ctx->hdr, rec); + mail_index_header_update_counts(hdr, old_flags, rec->flags); + mail_index_header_update_lowwaters(hdr, rec); } + return 1; } -static int mail_index_grow(struct mail_index *index, unsigned int count) +static int sync_cache_update(const struct mail_transaction_cache_update *u, + void *context) { - struct mail_index_map *map = index->map; - unsigned int records_count; + struct mail_index_view *view = context; + uint32_t seq; + int ret; + + ret = mail_index_lookup_uid_range(view, u->uid, u->uid, + &seq, &seq); + i_assert(ret == 0); + + if (seq != 0) + view->map->records[seq-1].cache_offset = u->cache_offset; + return 1; +} + +static int sync_header_update(const struct mail_transaction_header_update *u, + void *context) +{ + struct mail_index_view *view = context; + void *data; + + data = PTR_OFFSET(&view->map->hdr_copy, u->offset); + memcpy(data, u->data, u->size); + return 1; +} + +static int mail_index_grow(struct mail_index *index, struct mail_index_map *map, + unsigned int count) +{ size_t size; if (MAIL_INDEX_MAP_IS_IN_MEMORY(map)) { @@ -132,6 +217,8 @@ return 0; } + i_assert(map == index->map); + size = map->hdr->header_size + (map->records_count + count) * sizeof(struct mail_index_record); if (size <= map->mmap_size) @@ -149,192 +236,115 @@ if (file_set_size(index->fd, (off_t)size) < 0) return mail_index_set_syscall_error(index, "file_set_size()"); - records_count = map->records_count; - if (mail_index_map(index, TRUE) <= 0) return -1; i_assert(map->mmap_size >= size); - map->records_count = records_count; - return 0; -} - -static int mail_index_sync_appends(struct mail_index_update_ctx *ctx, - const struct mail_index_record *appends, - unsigned int count) -{ - struct mail_index_map *map = ctx->index->map; - unsigned int i; - uint32_t next_uid; - - if (mail_index_grow(ctx->index, count) < 0) - return -1; - - next_uid = ctx->hdr.next_uid; - for (i = 0; i < count; i++) { - mail_index_header_update_counts(&ctx->hdr, 0, appends[i].flags); - mail_index_header_update_lowwaters(&ctx->hdr, &appends[i]); - - if (appends[i].uid < next_uid) { - mail_transaction_log_view_set_corrupted(ctx->log_view, - "Append with UID %u, but next_uid = %u", - appends[i].uid, next_uid); - return -1; - } - next_uid = appends[i].uid+1; - } - ctx->hdr.next_uid = next_uid; - - memcpy(map->records + map->records_count, appends, - count * sizeof(*appends)); - map->records_count += count; return 0; } -int mail_index_sync_update_index(struct mail_index_sync_ctx *sync_ctx, - uint32_t sync_stamp, uint64_t sync_size) +int mail_index_sync_update_index(struct mail_index_sync_ctx *sync_ctx) { struct mail_index *index = sync_ctx->index; + struct mail_index_view *view = sync_ctx->view; struct mail_index_map *map; - struct mail_index_update_ctx ctx; - struct mail_index_sync_rec rec; - const struct mail_index_record *appends; - unsigned int append_count; - uint32_t count, file_seq, src_idx, dest_idx, i; - uint32_t seq1, seq2; - uoff_t file_offset; - unsigned int lock_id; - int ret, changed; - - /* rewind */ - sync_ctx->update_idx = sync_ctx->expunge_idx = 0; - sync_ctx->sync_appends = - buffer_get_used_size(sync_ctx->appends_buf) != 0; - - changed = mail_index_sync_have_more(sync_ctx); - - memset(&ctx, 0, sizeof(ctx)); - ctx.index = index; - ctx.view = sync_ctx->view; - ctx.hdr = *index->hdr; - ctx.log_view = sync_ctx->view->log_view; - - /* see if we need to update sync headers */ - if (ctx.hdr.sync_stamp != sync_stamp && sync_stamp != 0) { - ctx.hdr.sync_stamp = sync_stamp; - changed = TRUE; - } - if (ctx.hdr.sync_size != sync_size && sync_size != 0) { - ctx.hdr.sync_size = sync_size; - changed = TRUE; - } - - if (!changed) { - /* nothing to sync */ - return 0; - } + const struct mail_transaction_header *hdr; + const void *data; + unsigned int lock_id, count; + uint32_t seq, i; + uoff_t offset; + int ret, had_dirty, skipped; if (mail_index_lock_exclusive(index, &lock_id) < 0) return -1; + /* NOTE: locking may change index->map so make sure assignment + after locking */ map = index->map; if (MAIL_INDEX_MAP_IS_IN_MEMORY(map)) map->write_to_disk = TRUE; - src_idx = dest_idx = 0; - append_count = 0; appends = NULL; - while (mail_index_sync_next(sync_ctx, &rec) > 0) { - switch (rec.type) { - case MAIL_INDEX_SYNC_TYPE_APPEND: - i_assert(appends == NULL); - appends = rec.appends; - append_count = rec.appends_count; - break; - case MAIL_INDEX_SYNC_TYPE_EXPUNGE: - ret = mail_index_lookup_uid_range(sync_ctx->view, - rec.uid1, rec.uid2, - &seq1, &seq2); - i_assert(ret == 0); + map->hdr_copy = *map->hdr; + map->hdr = &map->hdr_copy; - if (seq1 == 0) - break; + mail_index_unmap(index, view->map); + view->map = map; + view->map->refcount++; + + had_dirty = (map->hdr_copy.flags & MAIL_INDEX_HDR_FLAG_HAVE_DIRTY) != 0; + if (had_dirty) + map->hdr_copy.flags &= ~MAIL_INDEX_HDR_FLAG_HAVE_DIRTY; - if (src_idx == 0) { - /* expunges have to be atomic. so we'll have - to copy the mapping, do the changes there - and then finally replace the whole index - file. to avoid extra disk I/O we copy the - index into memory rather than to temporary - file */ - map = mail_index_map_to_memory(map); - mail_index_unmap(index, index->map); - index->map = map; - index->hdr = map->hdr; - map->write_to_disk = TRUE; + while ((ret = mail_transaction_log_view_next(view->log_view, &hdr, + &data, &skipped)) > 0) { + if ((hdr->type & MAIL_TRANSACTION_EXPUNGE) != 0 && + !map->write_to_disk) { + /* expunges have to be atomic. so we'll have to copy + the mapping, do the changes there and then finally + replace the whole index file. to avoid extra disk + I/O we copy the index into memory rather than to + temporary file */ + map = mail_index_map_to_memory(map); + mail_index_unmap(index, view->map); + view->map = map; + view->map->refcount++; + mail_index_unmap(index, index->map); + index->map = map; + index->hdr = map->hdr; + map->write_to_disk = TRUE; + } - dest_idx = seq1-1; - } else { - count = (seq1-1) - src_idx; - memmove(map->records + dest_idx, - map->records + src_idx, - count * sizeof(*map->records)); - dest_idx += count; - } + if ((hdr->type & MAIL_TRANSACTION_APPEND) != 0) { + count = hdr->size / sizeof(struct mail_index_record); + if (mail_index_grow(index, view->map, count) < 0) + return -1; + } - mail_index_sync_update_expunges(&ctx, seq1, seq2); - src_idx = seq2; - break; - case MAIL_INDEX_SYNC_TYPE_FLAGS: - mail_index_sync_update_flags(&ctx, &rec); + if (mail_transaction_map(hdr, data, &mail_index_map_sync_funcs, + view) < 0) { + ret = -1; break; } } - if (src_idx != 0) { - count = map->records_count - src_idx; - memmove(map->records + dest_idx, - map->records + src_idx, - count * sizeof(*map->records)); - dest_idx += count; + if (ret < 0) + return -1; - map->records_count = dest_idx; - } + mail_transaction_log_get_head(index->log, &seq, &offset); - ret = 0; - if (append_count > 0) - ret = mail_index_sync_appends(&ctx, appends, append_count); - - mail_transaction_log_get_head(index->log, &file_seq, &file_offset); + map->hdr_copy.log_file_seq = seq; + map->hdr_copy.log_file_offset = offset; - ctx.hdr.messages_count = map->records_count; - ctx.hdr.log_file_seq = file_seq; - ctx.hdr.log_file_offset = file_offset; - - if ((ctx.hdr.flags & MAIL_INDEX_HDR_FLAG_HAVE_DIRTY) && - !ctx.have_dirty) { + if ((map->hdr_copy.flags & MAIL_INDEX_HDR_FLAG_HAVE_DIRTY) == 0 && + had_dirty) { /* do we have dirty flags anymore? */ for (i = 0; i < map->records_count; i++) { - if (map->records[i].flags & MAIL_INDEX_MAIL_FLAG_DIRTY) + if ((map->records[i].flags & + MAIL_INDEX_MAIL_FLAG_DIRTY) != 0) { + map->hdr_copy.flags |= + MAIL_INDEX_HDR_FLAG_HAVE_DIRTY; break; + } } - if (i == map->records_count) - ctx.hdr.flags &= ~MAIL_INDEX_HDR_FLAG_HAVE_DIRTY; } if (!MAIL_INDEX_MAP_IS_IN_MEMORY(map)) { map->mmap_used_size = index->hdr->header_size + map->records_count * sizeof(struct mail_index_record); - memcpy(map->mmap_base, &ctx.hdr, sizeof(ctx.hdr)); + memcpy(map->mmap_base, &map->hdr_copy, sizeof(map->hdr_copy)); if (msync(map->mmap_base, map->mmap_used_size, MS_SYNC) < 0) { mail_index_set_syscall_error(index, "msync()"); ret = -1; } - } else { - map->hdr_copy = ctx.hdr; - map->hdr = &map->hdr_copy; + map->hdr = map->mmap_base; } mail_index_unlock(index, lock_id); return ret; } + +struct mail_transaction_map_functions mail_index_map_sync_funcs = { + sync_expunge, sync_append, sync_flag_update, + sync_cache_update, sync_header_update +};
--- a/src/lib-index/mail-index-sync.c Mon May 24 04:01:42 2004 +0300 +++ b/src/lib-index/mail-index-sync.c Mon May 24 04:50:16 2004 +0300 @@ -164,7 +164,7 @@ index->hdr->log_file_offset, seq, offset, MAIL_TRANSACTION_TYPE_MASK) < 0) { - mail_index_sync_end(ctx, 0, 0); + mail_index_sync_end(ctx); return -1; } @@ -177,7 +177,7 @@ ctx->appends_buf = buffer_create_dynamic(default_pool, 1024, (size_t)-1); if (mail_index_sync_read_and_sort(ctx, FALSE) < 0) { - mail_index_sync_end(ctx, 0, 0); + mail_index_sync_end(ctx); return -1; } @@ -312,8 +312,7 @@ ctx->sync_appends; } -int mail_index_sync_end(struct mail_index_sync_ctx *ctx, - uint32_t sync_stamp, uint64_t sync_size) +int mail_index_sync_end(struct mail_index_sync_ctx *ctx) { const struct mail_index_header *hdr; uint32_t seq; @@ -337,8 +336,12 @@ if (ret == 0) { mail_index_sync_read_and_sort(ctx, TRUE); - if (mail_index_sync_update_index(ctx, sync_stamp, - sync_size) < 0) + mail_transaction_log_view_unset(ctx->view->log_view); + if (mail_transaction_log_view_set(ctx->view->log_view, + hdr->log_file_seq, hdr->log_file_offset, + seq, offset, MAIL_TRANSACTION_TYPE_MASK) < 0) + ret = -1; + if (mail_index_sync_update_index(ctx) < 0) ret = -1; }
--- a/src/lib-index/mail-index-transaction-private.h Mon May 24 04:01:42 2004 +0300 +++ b/src/lib-index/mail-index-transaction-private.h Mon May 24 04:50:16 2004 +0300 @@ -13,8 +13,12 @@ struct mail_transaction_flag_update last_update; enum modify_type last_update_modify_type; + unsigned char hdr_change[sizeof(struct mail_index_header)]; + unsigned char hdr_mask[sizeof(struct mail_index_header)]; + buffer_t *cache_updates; unsigned int hide_transaction:1; + unsigned int hdr_changed:1; }; #endif
--- a/src/lib-index/mail-index-transaction.c Mon May 24 04:01:42 2004 +0300 +++ b/src/lib-index/mail-index-transaction.c Mon May 24 04:50:16 2004 +0300 @@ -426,3 +426,16 @@ buffer_insert(t->updates, idx * sizeof(update), &update, sizeof(update)); } + +void mail_index_update_header(struct mail_index_transaction *t, + size_t offset, const void *data, size_t size) +{ + i_assert(offset < sizeof(t->hdr_change)); + i_assert(size <= sizeof(t->hdr_change) - offset); + + t->hdr_changed = TRUE; + + memcpy(t->hdr_change + offset, data, size); + for (; size > 0; size--) + t->hdr_mask[offset++] = 1; +}
--- a/src/lib-index/mail-index-view-private.h Mon May 24 04:01:42 2004 +0300 +++ b/src/lib-index/mail-index-view-private.h Mon May 24 04:50:16 2004 +0300 @@ -9,6 +9,7 @@ unsigned int indexid; struct mail_index_map *map; + struct mail_index_map *new_map; struct mail_index_header tmp_hdr_copy; uint32_t messages_count; /* last synced one, map may be different */
--- a/src/lib-index/mail-index-view-sync.c Mon May 24 04:01:42 2004 +0300 +++ b/src/lib-index/mail-index-view-sync.c Mon May 24 04:50:16 2004 +0300 @@ -10,7 +10,6 @@ struct mail_index_view_sync_ctx { struct mail_index_view *view; enum mail_index_sync_type sync_mask; - struct mail_index_map *sync_map; buffer_t *expunges; const struct mail_transaction_header *hdr; @@ -79,6 +78,7 @@ { const struct mail_index_header *hdr; struct mail_index_view_sync_ctx *ctx; + struct mail_index_map *map; enum mail_transaction_type mask; buffer_t *expunges = NULL; @@ -115,14 +115,20 @@ ctx->expunges = expunges; if ((sync_mask & MAIL_INDEX_SYNC_TYPE_EXPUNGE) != 0) { - ctx->sync_map = view->index->map; - ctx->sync_map->refcount++; + view->new_map = view->index->map; + view->new_map->refcount++; + + /* keep the old mapping without expunges until we're + fully synced */ } else { /* we need a private copy of the map if we don't want to sync expunges */ - if (MAIL_INDEX_MAP_IS_IN_MEMORY(view->map)) + if (view->map != view->index->map) ctx->sync_map_update = TRUE; - ctx->sync_map = mail_index_map_to_memory(view->map); + + map = mail_index_map_to_memory(view->map); + mail_index_unmap(view->index, view->map); + view->map = map; } view->syncing = TRUE; @@ -153,112 +159,6 @@ return 0; } -static int sync_expunge(const struct mail_transaction_expunge *e, void *context) -{ - struct mail_index_view_sync_ctx *ctx = context; - struct mail_index_map *map = ctx->sync_map; - uint32_t idx, count, seq1, seq2; - int ret; - - ret = mail_index_lookup_uid_range(ctx->view, e->uid1, e->uid2, - &seq1, &seq2); - i_assert(ret == 0); - - if (seq1 == 0) - return 1; - - for (idx = seq1-1; idx < seq2; idx++) { - mail_index_header_update_counts(&map->hdr_copy, - map->records[idx].flags, 0); - } - - count = seq2 - seq1 + 1; - buffer_delete(map->buffer, - (seq1-1) * sizeof(struct mail_index_record), - count * sizeof(struct mail_index_record)); - map->records = buffer_get_modifyable_data(map->buffer, NULL); - - map->records_count -= count; - map->hdr_copy.messages_count -= count; - return 1; -} - -static int sync_append(const struct mail_index_record *rec, void *context) -{ - struct mail_index_view_sync_ctx *ctx = context; - struct mail_index_map *map = ctx->sync_map; - - buffer_append(map->buffer, rec, sizeof(*rec)); - map->records = buffer_get_modifyable_data(map->buffer, NULL); - - map->records_count++; - map->hdr_copy.messages_count++; - map->hdr_copy.next_uid = rec->uid+1; - - mail_index_header_update_counts(&map->hdr_copy, 0, rec->flags); - mail_index_header_update_lowwaters(&map->hdr_copy, rec); - return 1; -} - -static int sync_flag_update(const struct mail_transaction_flag_update *u, - void *context) -{ - struct mail_index_view_sync_ctx *ctx = context; - struct mail_index_map *map = ctx->sync_map; - struct mail_index_record *rec; - uint32_t i, idx, seq1, seq2; - uint8_t old_flags; - int ret; - - ret = mail_index_lookup_uid_range(ctx->view, u->uid1, u->uid2, - &seq1, &seq2); - i_assert(ret == 0); - - if (seq1 == 0) - return 1; - - for (idx = seq1-1; idx < seq2; idx++) { - rec = &map->records[idx]; - - old_flags = rec->flags; - rec->flags = (rec->flags & ~u->remove_flags) | u->add_flags; - for (i = 0; i < INDEX_KEYWORDS_BYTE_COUNT; i++) { - rec->keywords[i] = u->add_keywords[i] | - (rec->keywords[i] & ~u->remove_keywords[i]); - } - - mail_index_header_update_counts(&map->hdr_copy, old_flags, - rec->flags); - mail_index_header_update_lowwaters(&map->hdr_copy, rec); - } - return 1; -} - -static int sync_cache_update(const struct mail_transaction_cache_update *u, - void *context) -{ - struct mail_index_view_sync_ctx *ctx = context; - uint32_t seq; - int ret; - - ret = mail_index_lookup_uid_range(ctx->view, u->uid, u->uid, - &seq, &seq); - i_assert(ret == 0); - - if (seq != 0) - ctx->sync_map->records[seq-1].cache_offset = u->cache_offset; - return 1; -} - -static int mail_index_view_sync_map(struct mail_index_view_sync_ctx *ctx) -{ - static struct mail_transaction_map_functions map_funcs = { - sync_expunge, sync_append, sync_flag_update, sync_cache_update - }; - - return mail_transaction_map(ctx->hdr, ctx->data, &map_funcs, ctx); -} - static int mail_index_view_sync_next_trans(struct mail_index_view_sync_ctx *ctx, uint32_t *seq_r, uoff_t *offset_r) { @@ -285,8 +185,13 @@ if (view_is_transaction_synced(view, *seq_r, *offset_r)) return 0; - if (ctx->sync_map_update) { - if (mail_index_view_sync_map(ctx) < 0) + /* expunges have to be synced afterwards so that caller can still get + information of the messages. otherwise caller most likely wants to + see only updated information. */ + if (ctx->sync_map_update && + (ctx->hdr->type & MAIL_TRANSACTION_EXPUNGE) == 0) { + if (mail_transaction_map(ctx->hdr, ctx->data, + &mail_index_map_sync_funcs, view) < 0) return -1; } @@ -408,9 +313,12 @@ view->inconsistent = TRUE; } - mail_index_unmap(view->index, view->map); - view->map = ctx->sync_map; - view->map_protected = FALSE; + if (view->new_map != NULL) { + mail_index_unmap(view->index, view->map); + view->map = view->new_map; + view->new_map = NULL; + view->map_protected = FALSE; + } if ((ctx->sync_mask & MAIL_INDEX_SYNC_TYPE_APPEND) != 0) view->messages_count = view->map->records_count;
--- a/src/lib-index/mail-index.c Mon May 24 04:01:42 2004 +0300 +++ b/src/lib-index/mail-index.c Mon May 24 04:50:16 2004 +0300 @@ -275,7 +275,10 @@ int ret; map = index->map; - if (map != NULL && MAIL_INDEX_MAP_IS_IN_MEMORY(map)) { + if (map == NULL) { + map = i_new(struct mail_index_map, 1); + map->refcount = 1; + } else if (MAIL_INDEX_MAP_IS_IN_MEMORY(map)) { if (map->write_to_disk) { /* we have modified this mapping and it's waiting to be written to disk once we drop exclusive lock. @@ -283,7 +286,7 @@ return 1; } /* FIXME: we need to re-read header */ - } else if (map != NULL) { + } else if (map->mmap_base != NULL) { /* see if re-mmaping is needed (file has grown) */ hdr = map->mmap_base; used_size = hdr->header_size + @@ -296,9 +299,6 @@ if (munmap(map->mmap_base, map->mmap_size) < 0) mail_index_set_syscall_error(index, "munmap()"); map->mmap_base = NULL; - } else { - map = i_new(struct mail_index_map, 1); - map->refcount = 1; } index->hdr = NULL;
--- a/src/lib-index/mail-index.h Mon May 24 04:01:42 2004 +0300 +++ b/src/lib-index/mail-index.h Mon May 24 04:50:16 2004 +0300 @@ -197,10 +197,8 @@ struct mail_index_sync_rec *sync_rec); /* Returns 1 if there's more to sync, 0 if not. */ int mail_index_sync_have_more(struct mail_index_sync_ctx *ctx); -/* End synchronization by unlocking the index and closing the view. - sync_stamp/sync_size in header is updated to given values. */ -int mail_index_sync_end(struct mail_index_sync_ctx *ctx, - uint32_t sync_stamp, uint64_t sync_size); +/* End synchronization by unlocking the index and closing the view. */ +int mail_index_sync_end(struct mail_index_sync_ctx *ctx); /* Mark index file corrupted. Invalidates all views. */ void mail_index_mark_corrupted(struct mail_index *index); @@ -254,6 +252,9 @@ void mail_index_update_flags(struct mail_index_transaction *t, uint32_t seq, enum modify_type modify_type, enum mail_flags flags, keywords_mask_t keywords); +/* Update field in header. */ +void mail_index_update_header(struct mail_index_transaction *t, + size_t offset, const void *data, size_t size); /* Returns the last error code. */ enum mail_index_error mail_index_get_last_error(struct mail_index *index);
--- a/src/lib-index/mail-transaction-log-view.c Mon May 24 04:01:42 2004 +0300 +++ b/src/lib-index/mail-transaction-log-view.c Mon May 24 04:50:16 2004 +0300 @@ -14,7 +14,6 @@ uoff_t min_file_offset, max_file_offset; enum mail_transaction_type type_mask; - buffer_t *expunges_buf; struct mail_transaction_header tmp_hdr; struct mail_transaction_log_file *file; @@ -34,8 +33,6 @@ view = i_new(struct mail_transaction_log_view, 1); view->log = log; view->broken = TRUE; - view->expunges_buf = - buffer_create_dynamic(default_pool, 512, (size_t)-1); view->next = log->views; log->views = view; @@ -54,7 +51,6 @@ } mail_transaction_log_view_unset(view); - buffer_free(view->expunges_buf); i_free(view); } @@ -150,8 +146,6 @@ file = file->next; } - buffer_set_used_size(view->expunges_buf, 0); - view->prev_file_seq = 0; view->prev_file_offset = 0;
--- a/src/lib-index/mail-transaction-log.c Mon May 24 04:01:42 2004 +0300 +++ b/src/lib-index/mail-transaction-log.c Mon May 24 04:50:16 2004 +0300 @@ -960,6 +960,36 @@ return 0; } +static const buffer_t * +log_get_hdr_update_buffer(struct mail_index_transaction *t) +{ + buffer_t *buf; + struct mail_transaction_header_update u; + uint16_t offset; + int state = 0; + + memset(&u, 0, sizeof(u)); + + buf = buffer_create_dynamic(pool_datastack_create(), 256, (size_t)-1); + for (offset = 0; offset <= sizeof(t->hdr_change); offset++) { + if (offset < sizeof(t->hdr_change) && t->hdr_mask[offset]) { + if (state == 0) { + u.offset = offset; + state++; + } + } else { + if (state > 0) { + u.size = offset - u.offset; + buffer_append(buf, &u, sizeof(uint16_t)*2); + buffer_append(buf, t->hdr_change + u.offset, + u.size); + state = 0; + } + } + } + return buf; +} + int mail_transaction_log_append(struct mail_index_transaction *t, uint32_t *log_file_seq_r, uoff_t *log_file_offset_r) @@ -976,7 +1006,7 @@ log = index->log; if (t->updates == NULL && t->cache_updates == NULL && - t->expunges == NULL && t->appends == NULL) { + t->expunges == NULL && t->appends == NULL && !t->hdr_changed) { /* nothing to append */ *log_file_seq_r = 0; *log_file_offset_r = 0; @@ -1036,6 +1066,11 @@ MAIL_TRANSACTION_EXPUNGE, view->external); } + if (t->hdr_changed && ret == 0) { + ret = log_append_buffer(file, log_get_hdr_update_buffer(t), + MAIL_TRANSACTION_HEADER_UPDATE, + view->external); + } if (ret == 0) { /* rewrite used_size */
--- a/src/lib-index/mail-transaction-log.h Mon May 24 04:01:42 2004 +0300 +++ b/src/lib-index/mail-transaction-log.h Mon May 24 04:50:16 2004 +0300 @@ -18,6 +18,7 @@ MAIL_TRANSACTION_APPEND = 0x00000002, MAIL_TRANSACTION_FLAG_UPDATE = 0x00000004, MAIL_TRANSACTION_CACHE_UPDATE = 0x00000008, + MAIL_TRANSACTION_HEADER_UPDATE = 0x00000010, MAIL_TRANSACTION_TYPE_MASK = 0x0000ffff, @@ -40,11 +41,6 @@ uint32_t uid1, uid2; }; -struct mail_transaction_cache_update { - uint32_t uid; - uint32_t cache_offset; -}; - struct mail_transaction_flag_update { uint32_t uid1, uid2; uint8_t add_flags; @@ -53,6 +49,17 @@ keywords_mask_t remove_keywords; }; +struct mail_transaction_cache_update { + uint32_t uid; + uint32_t cache_offset; +}; + +struct mail_transaction_header_update { + uint16_t offset; + uint16_t size; + unsigned char data[1]; /* variable size */ +}; + struct mail_transaction_log * mail_transaction_log_open_or_create(struct mail_index *index); void mail_transaction_log_close(struct mail_transaction_log *log);
--- a/src/lib-index/mail-transaction-util.c Mon May 24 04:01:42 2004 +0300 +++ b/src/lib-index/mail-transaction-util.c Mon May 24 04:50:16 2004 +0300 @@ -22,6 +22,7 @@ sizeof(struct mail_transaction_flag_update) }, { MAIL_TRANSACTION_CACHE_UPDATE, 0, sizeof(struct mail_transaction_cache_update) }, + { MAIL_TRANSACTION_HEADER_UPDATE, 0, 1 }, /* variable size, use 1 */ { 0, 0, 0 } }; @@ -115,6 +116,23 @@ } break; } + case MAIL_TRANSACTION_HEADER_UPDATE: { + const struct mail_transaction_header_update *rec; + unsigned int i; + + if (map->header_update == NULL) + break; + + for (i = 0; i < hdr->size; ) { + rec = CONST_PTR_OFFSET(data, i); + ret = map->header_update(rec, context); + if (ret <= 0) + break; + + i += sizeof(uint16_t)*2 + rec->size; + } + break; + } default: i_unreached(); }
--- a/src/lib-index/mail-transaction-util.h Mon May 24 04:01:42 2004 +0300 +++ b/src/lib-index/mail-transaction-util.h Mon May 24 04:50:16 2004 +0300 @@ -15,6 +15,8 @@ void *context); int (*cache_update)(const struct mail_transaction_cache_update *u, void *context); + int (*header_update)(const struct mail_transaction_header_update *u, + void *context); }; const struct mail_transaction_type_map *
--- a/src/lib-storage/index/index-sync.c Mon May 24 04:01:42 2004 +0300 +++ b/src/lib-storage/index/index-sync.c Mon May 24 04:50:16 2004 +0300 @@ -51,6 +51,8 @@ if (sc->update_flags == NULL) break; + /* FIXME: hide the flag updates for expunged messages */ + if (mail_index_lookup_uid_range(ibox->view, sync.uid1, sync.uid2, &seq1, &seq2) < 0) { @@ -61,7 +63,6 @@ if (seq1 == 0) break; - /* FIXME: hide the flag updates for expunged messages */ for (seq = seq1; seq <= seq2; seq++) { if (mail_index_lookup(ibox->view, seq, &rec) < 0) {
--- a/src/lib-storage/index/maildir/maildir-storage.c Mon May 24 04:01:42 2004 +0300 +++ b/src/lib-storage/index/maildir/maildir-storage.c Mon May 24 04:50:16 2004 +0300 @@ -407,6 +407,7 @@ { struct index_mailbox *ibox; struct mail_index *index; + const struct mail_index_header *hdr; const char *path, *index_dir, *control_dir; struct stat st; @@ -421,12 +422,17 @@ if (ibox == NULL) return NULL; + if (mail_index_get_header(ibox->view, &hdr) < 0) { + index_storage_mailbox_free(&ibox->box); + return NULL; + } + ibox->path = i_strdup(path); ibox->control_dir = i_strdup(control_dir); ibox->get_recent_count = maildir_get_recent_count; ibox->mail_interface = &maildir_mail; - ibox->uidlist = maildir_uidlist_init(ibox); + ibox->uidlist = maildir_uidlist_init(ibox, hdr->uid_validity); /* for shared mailboxes get the create mode from the permissions of dovecot-shared file */
--- a/src/lib-storage/index/maildir/maildir-sync.c Mon May 24 04:01:42 2004 +0300 +++ b/src/lib-storage/index/maildir/maildir-sync.c Mon May 24 04:50:16 2004 +0300 @@ -179,6 +179,7 @@ #include "maildir-uidlist.h" #include <stdio.h> +#include <stddef.h> #include <unistd.h> #include <dirent.h> #include <sys/stat.h> @@ -357,7 +358,7 @@ } if (mail_index_transaction_commit(ctx.trans, &seq, &offset) < 0) ret = -1; - if (mail_index_sync_end(ctx.sync_ctx, 0, 0) < 0) + if (mail_index_sync_end(ctx.sync_ctx) < 0) ret = -1; } @@ -575,7 +576,7 @@ const char *filename; enum mail_flags flags; keywords_mask_t keywords; - uint32_t sync_stamp; + uint32_t uid_validity, next_uid; int ret; memset(&sync_ctx, 0, sizeof(sync_ctx)); @@ -591,6 +592,18 @@ ret = mail_index_get_header(view, &hdr); i_assert(ret == 0); /* view is locked, can't happen */ + uid_validity = maildir_uidlist_get_uid_validity(ibox->uidlist); + if (uid_validity != hdr->uid_validity && hdr->next_uid != 1) { + /* uidvalidity changed and mailbox isn't being initialized, + index must be rebuilt */ + mail_storage_set_critical(ibox->box.storage, + "Maildir sync: UIDVALIDITY changed (%u -> %u)", + hdr->uid_validity, uid_validity); + mail_index_mark_corrupted(ibox->index); + (void)mail_index_sync_end(sync_ctx.sync_ctx); + return -1; + } + trans = mail_index_transaction_begin(view, FALSE); sync_ctx.trans = trans; @@ -633,7 +646,6 @@ } if (mail_index_lookup(view, seq, &rec) < 0) { - mail_storage_set_index_error(ibox); ret = -1; break; } @@ -717,6 +729,27 @@ } } + if (ibox->dirty_cur_time == 0) { + uint32_t sync_stamp = ibox->last_cur_mtime; + + mail_index_update_header(trans, + offsetof(struct mail_index_header, sync_stamp), + &sync_stamp, sizeof(sync_stamp)); + } + + if (uid_validity != hdr->uid_validity) { + mail_index_update_header(trans, + offsetof(struct mail_index_header, uid_validity), + &uid_validity, sizeof(uid_validity)); + } + + next_uid = maildir_uidlist_get_next_uid(ibox->uidlist); + if (next_uid != 0 && hdr->next_uid != next_uid) { + mail_index_update_header(trans, + offsetof(struct mail_index_header, next_uid), + &next_uid, sizeof(next_uid)); + } + if (ret < 0) mail_index_transaction_rollback(trans); else { @@ -724,15 +757,14 @@ uoff_t offset; if (mail_index_transaction_commit(trans, &seq, &offset) < 0) - mail_storage_set_index_error(ibox); + ret = -1; else if (seq != 0) { ibox->commit_log_file_seq = seq; ibox->commit_log_file_offset = offset; } } - sync_stamp = ibox->dirty_cur_time != 0 ? 0 : ibox->last_cur_mtime; - if (mail_index_sync_end(sync_ctx.sync_ctx, sync_stamp, 0) < 0) + if (mail_index_sync_end(sync_ctx.sync_ctx) < 0) ret = -1; if (ret == 0) {
--- a/src/lib-storage/index/maildir/maildir-uidlist.c Mon May 24 04:01:42 2004 +0300 +++ b/src/lib-storage/index/maildir/maildir-uidlist.c Mon May 24 04:50:16 2004 +0300 @@ -109,7 +109,8 @@ uidlist->lock_fd = -1; } -struct maildir_uidlist *maildir_uidlist_init(struct index_mailbox *ibox) +struct maildir_uidlist * +maildir_uidlist_init(struct index_mailbox *ibox, uint32_t uid_validity) { struct maildir_uidlist *uidlist; @@ -123,7 +124,7 @@ uidlist->files = hash_create(default_pool, default_pool, 4096, maildir_hash, maildir_cmp); - uidlist->uid_validity = ioloop_time; + uidlist->uid_validity = uid_validity; uidlist->next_uid = 1; return uidlist; @@ -416,6 +417,16 @@ return count; } +uint32_t maildir_uidlist_get_uid_validity(struct maildir_uidlist *uidlist) +{ + return uidlist->uid_validity; +} + +uint32_t maildir_uidlist_get_next_uid(struct maildir_uidlist *uidlist) +{ + return !uidlist->initial_read ? 0 : uidlist->next_uid; +} + static int maildir_uidlist_rewrite_fd(struct maildir_uidlist *uidlist, const char *temp_path) {
--- a/src/lib-storage/index/maildir/maildir-uidlist.h Mon May 24 04:01:42 2004 +0300 +++ b/src/lib-storage/index/maildir/maildir-uidlist.h Mon May 24 04:50:16 2004 +0300 @@ -13,7 +13,8 @@ int maildir_uidlist_try_lock(struct maildir_uidlist *uidlist); void maildir_uidlist_unlock(struct maildir_uidlist *uidlist); -struct maildir_uidlist *maildir_uidlist_init(struct index_mailbox *ibox); +struct maildir_uidlist * +maildir_uidlist_init(struct index_mailbox *ibox, uint32_t uid_validity); void maildir_uidlist_deinit(struct maildir_uidlist *uidlist); /* Returns -1 if error, 0 if file is broken or lost, 1 if ok. */ @@ -28,6 +29,9 @@ /* Returns number of recent messages. */ uint32_t maildir_uidlist_get_recent_count(struct maildir_uidlist *uidlist); +uint32_t maildir_uidlist_get_uid_validity(struct maildir_uidlist *uidlist); +uint32_t maildir_uidlist_get_next_uid(struct maildir_uidlist *uidlist); + /* Sync uidlist with what's actually on maildir. */ struct maildir_uidlist_sync_ctx * maildir_uidlist_sync_init(struct maildir_uidlist *uidlist, int partial);
--- a/src/lib-storage/index/mbox/mbox-sync.c Mon May 24 04:01:42 2004 +0300 +++ b/src/lib-storage/index/mbox/mbox-sync.c Mon May 24 04:50:16 2004 +0300 @@ -57,6 +57,7 @@ #include "mbox-lock.h" #include "mbox-sync-private.h" +#include <stddef.h> #include <sys/stat.h> static int mbox_sync_grow_file(struct mbox_sync_context *sync_ctx, @@ -378,6 +379,34 @@ } } + if (ret == 0) { + if (fstat(ibox->mbox_fd, &st) < 0) { + mbox_set_syscall_error(ibox, "fstat()"); + ret = -1; + } + } + if (ret < 0) { + st.st_mtime = 0; + st.st_size = 0; + } + + if (mail_index_get_header(sync_view, &hdr) < 0) + ret = -1; + if ((uint32_t)st.st_mtime != hdr->sync_stamp) { + uint32_t sync_stamp = st.st_mtime; + + mail_index_update_header(t, + offsetof(struct mail_index_header, sync_stamp), + &sync_stamp, sizeof(sync_stamp)); + } + if ((uint64_t)st.st_mtime != hdr->sync_size) { + uint64_t sync_size = st.st_size; + + mail_index_update_header(t, + offsetof(struct mail_index_header, sync_size), + &sync_size, sizeof(sync_size)); + } + if (ret < 0) mail_index_transaction_rollback(t); else { @@ -389,20 +418,8 @@ } } - if (ret == 0) { - if (fstat(ibox->mbox_fd, &st) < 0) { - mbox_set_syscall_error(ibox, "fstat()"); - ret = -1; - } - } - if (ret < 0) { - st.st_mtime = 0; - st.st_size = 0; - } - if (ret != -2) { - if (mail_index_sync_end(index_sync_ctx, - st.st_mtime, st.st_size) < 0) + if (mail_index_sync_end(index_sync_ctx) < 0) ret = -1; } @@ -463,7 +480,7 @@ } } } else { - (void)mail_index_sync_end(index_sync_ctx, 0, 0); + (void)mail_index_sync_end(index_sync_ctx); ret = -1; }