Mercurial > dovecot > original-hg > dovecot-1.2
view src/lib-index/mail-index.c @ 3912:fc0b638330a4 HEAD
Added mbox_min_index_size setting.
author | Timo Sirainen <tss@iki.fi> |
---|---|
date | Thu, 19 Jan 2006 01:14:43 +0200 |
parents | 3a2fe49912f3 |
children | b19ccbaa3802 |
line wrap: on
line source
/* Copyright (C) 2003-2004 Timo Sirainen */ #include "lib.h" #include "array.h" #include "buffer.h" #include "hash.h" #include "mmap-util.h" #include "read-full.h" #include "write-full.h" #include "mail-index-private.h" #include "mail-index-sync-private.h" #include "mail-transaction-log.h" #include "mail-cache.h" #include <stdio.h> #include <stddef.h> #include <time.h> #include <sys/stat.h> static int mail_index_try_open_only(struct mail_index *index); static void mail_index_create_in_memory(struct mail_index *index, const struct mail_index_header *hdr); struct mail_index *mail_index_alloc(const char *dir, const char *prefix) { struct mail_index *index; index = i_new(struct mail_index, 1); index->dir = i_strdup(dir); index->prefix = i_strdup(prefix); index->fd = -1; index->extension_pool = pool_alloconly_create("extension", 512); ARRAY_CREATE(&index->extensions, index->extension_pool, struct mail_index_registered_ext, 5); ARRAY_CREATE(&index->sync_lost_handlers, default_pool, mail_index_sync_lost_handler_t *, 4); index->mode = 0600; index->gid = (gid_t)-1; index->keywords_ext_id = mail_index_ext_register(index, "keywords", 128, 2, 1); index->keywords_pool = pool_alloconly_create("keywords", 512); ARRAY_CREATE(&index->keywords, default_pool, const char *, 16); index->keywords_hash = hash_create(default_pool, index->keywords_pool, 0, strcase_hash, (hash_cmp_callback_t *)strcasecmp); return index; } void mail_index_free(struct mail_index **_index) { struct mail_index *index = *_index; *_index = NULL; mail_index_close(index); hash_destroy(index->keywords_hash); pool_unref(index->extension_pool); pool_unref(index->keywords_pool); array_free(&index->sync_lost_handlers); array_free(&index->keywords); i_free(index->error); i_free(index->dir); i_free(index->prefix); i_free(index); } void mail_index_set_permissions(struct mail_index *index, mode_t mode, gid_t gid) { index->mode = mode & 0666; index->gid = gid; } uint32_t mail_index_ext_register(struct mail_index *index, const char *name, uint32_t default_hdr_size, uint16_t default_record_size, uint16_t default_record_align) { const struct mail_index_registered_ext *extensions; struct mail_index_registered_ext rext; unsigned int i, ext_count; extensions = array_get(&index->extensions, &ext_count); /* see if it's already there */ for (i = 0; i < ext_count; i++) { if (strcmp(extensions[i].name, name) == 0) return i; } memset(&rext, 0, sizeof(rext)); rext.name = p_strdup(index->extension_pool, name); rext.index_idx = ext_count; rext.hdr_size = default_hdr_size; rext.record_size = default_record_size; rext.record_align = default_record_align; array_append(&index->extensions, &rext, 1); return ext_count; } void mail_index_register_expunge_handler(struct mail_index *index, uint32_t ext_id, mail_index_expunge_handler_t *cb) { struct mail_index_registered_ext *rext; rext = array_idx_modifyable(&index->extensions, ext_id); i_assert(rext->expunge_handler == NULL); rext->expunge_handler = cb; } void mail_index_unregister_expunge_handler(struct mail_index *index, uint32_t ext_id) { struct mail_index_registered_ext *rext; rext = array_idx_modifyable(&index->extensions, ext_id); i_assert(rext->expunge_handler != NULL); rext->expunge_handler = NULL; } void mail_index_register_sync_handler(struct mail_index *index, uint32_t ext_id, mail_index_sync_handler_t *cb, enum mail_index_sync_handler_type type) { struct mail_index_registered_ext *rext; rext = array_idx_modifyable(&index->extensions, ext_id); i_assert(rext->sync_handler.callback == NULL); rext->sync_handler.callback = cb; rext->sync_handler.type = type; } void mail_index_unregister_sync_handler(struct mail_index *index, uint32_t ext_id) { struct mail_index_registered_ext *rext; rext = array_idx_modifyable(&index->extensions, ext_id); i_assert(rext->sync_handler.callback != NULL); rext->sync_handler.callback = NULL; rext->sync_handler.type = 0; } void mail_index_register_sync_lost_handler(struct mail_index *index, mail_index_sync_lost_handler_t *cb) { array_append(&index->sync_lost_handlers, &cb, 1); } void mail_index_unregister_sync_lost_handler(struct mail_index *index, mail_index_sync_lost_handler_t *cb) { mail_index_sync_lost_handler_t *const *handlers; unsigned int i, count; handlers = array_get(&index->sync_lost_handlers, &count); for (i = 0; i < count; i++) { if (handlers[i] == cb) { array_delete(&index->sync_lost_handlers, i, 1); break; } } } static void mail_index_map_init_extbufs(struct mail_index_map *map, unsigned int initial_count) { #define EXTENSION_NAME_APPROX_LEN 20 size_t size; if (map->extension_pool == NULL) { size = (sizeof(array_t) + BUFFER_APPROX_SIZE) * 2 + initial_count * (EXTENSION_NAME_APPROX_LEN + sizeof(struct mail_index_ext) + sizeof(uint32_t)); map->extension_pool = pool_alloconly_create("extensions", nearest_power(size)); } else { p_clear(map->extension_pool); } ARRAY_CREATE(&map->extensions, map->extension_pool, struct mail_index_ext, initial_count); ARRAY_CREATE(&map->ext_id_map, map->extension_pool, uint32_t, initial_count); } uint32_t mail_index_map_lookup_ext(struct mail_index_map *map, const char *name) { const struct mail_index_ext *extensions; unsigned int i, size; if (!array_is_created(&map->extensions)) return (uint32_t)-1; extensions = array_get(&map->extensions, &size); for (i = 0; i < size; i++) { if (strcmp(extensions[i].name, name) == 0) return i; } return (uint32_t)-1; } uint32_t mail_index_map_register_ext(struct mail_index *index, struct mail_index_map *map, const char *name, uint32_t hdr_offset, uint32_t hdr_size, uint32_t record_offset, uint32_t record_size, uint32_t record_align, uint32_t reset_id) { struct mail_index_ext *ext; uint32_t idx, empty_idx = (uint32_t)-1; if (!array_is_created(&map->extensions)) { mail_index_map_init_extbufs(map, 5); idx = 0; } else { idx = array_count(&map->extensions); } i_assert(mail_index_map_lookup_ext(map, name) == (uint32_t)-1); ext = array_append_space(&map->extensions); ext->name = p_strdup(map->extension_pool, name); ext->hdr_offset = hdr_offset; ext->hdr_size = hdr_size; ext->record_offset = record_offset; ext->record_size = record_size; ext->record_align = record_align; ext->reset_id = reset_id; ext->index_idx = mail_index_ext_register(index, name, hdr_size, record_size, record_align); /* Update index ext_id -> map ext_id mapping. Fill non-used ext_ids with (uint32_t)-1 */ while (array_count(&map->ext_id_map) < ext->index_idx) array_append(&map->ext_id_map, &empty_idx, 1); array_idx_set(&map->ext_id_map, ext->index_idx, &idx); return idx; } static bool size_check(size_t *size_left, size_t size) { if (size > *size_left) return FALSE; *size_left -= size; return TRUE; } static size_t get_align(size_t name_len) { size_t size = sizeof(struct mail_index_ext_header) + name_len; return MAIL_INDEX_HEADER_SIZE_ALIGN(size) - size; } static int mail_index_read_extensions(struct mail_index *index, struct mail_index_map *map) { const struct mail_index_ext_header *ext_hdr; unsigned int i, old_count; const char *name; uint32_t ext_id, offset, name_offset; size_t size_left; /* extension headers always start from 64bit offsets, so if base header doesn't happen to be 64bit aligned we'll skip some bytes */ offset = MAIL_INDEX_HEADER_SIZE_ALIGN(map->hdr.base_header_size); if (offset >= map->hdr.header_size && map->extension_pool == NULL) { /* nothing to do, skip allocatations and all */ return 1; } old_count = array_count(&index->extensions); mail_index_map_init_extbufs(map, old_count + 5); ext_id = (uint32_t)-1; for (i = 0; i < old_count; i++) array_append(&map->ext_id_map, &ext_id, 1); while (offset < map->hdr.header_size) { ext_hdr = CONST_PTR_OFFSET(map->hdr_base, offset); /* Extension header contains: - struct mail_index_ext_header - name (not 0-terminated) - 64bit alignment padding - extension header contents - 64bit alignment padding */ size_left = map->hdr.header_size - offset; if (!size_check(&size_left, sizeof(*ext_hdr)) || !size_check(&size_left, ext_hdr->name_size) || !size_check(&size_left, get_align(ext_hdr->name_size)) || !size_check(&size_left, ext_hdr->hdr_size)) { mail_index_set_error(index, "Corrupted index file %s: " "Header extension goes outside header", index->filepath); return -1; } offset += sizeof(*ext_hdr); name_offset = offset; offset += ext_hdr->name_size + get_align(ext_hdr->name_size); t_push(); name = t_strndup(CONST_PTR_OFFSET(map->hdr_base, name_offset), ext_hdr->name_size); if (mail_index_map_lookup_ext(map, name) != (uint32_t)-1) { mail_index_set_error(index, "Corrupted index file %s: " "Duplicate header extension %s", index->filepath, name); t_pop(); return -1; } if ((ext_hdr->record_offset % ext_hdr->record_align) != 0 || (map->hdr.record_size % ext_hdr->record_align) != 0) { mail_index_set_error(index, "Corrupted index file %s: " "Record field %s alignmentation %u not used", index->filepath, name, ext_hdr->record_align); t_pop(); return -1; } mail_index_map_register_ext(index, map, name, offset, ext_hdr->hdr_size, ext_hdr->record_offset, ext_hdr->record_size, ext_hdr->record_align, ext_hdr->reset_id); t_pop(); offset += MAIL_INDEX_HEADER_SIZE_ALIGN(ext_hdr->hdr_size); } return 1; } bool mail_index_keyword_lookup(struct mail_index *index, const char *keyword, bool autocreate, unsigned int *idx_r) { char *keyword_dup; void *value; /* keywords_hash keeps a name => index mapping of keywords. Keywords are never removed from it, so the index values are valid for the lifetime of the mail_index. */ if (hash_lookup_full(index->keywords_hash, keyword, NULL, &value)) { *idx_r = POINTER_CAST_TO(value, unsigned int); return TRUE; } if (!autocreate) { *idx_r = (unsigned int)-1; return FALSE; } keyword = keyword_dup = p_strdup(index->keywords_pool, keyword); *idx_r = array_count(&index->keywords); hash_insert(index->keywords_hash, keyword_dup, POINTER_CAST(*idx_r)); array_append(&index->keywords, &keyword, 1); return TRUE; } int mail_index_map_read_keywords(struct mail_index *index, struct mail_index_map *map) { const struct mail_index_ext *ext; const struct mail_index_keyword_header *kw_hdr; const struct mail_index_keyword_header_rec *kw_rec; const char *name; unsigned int i, name_area_end_offset, old_count; uint32_t ext_id; ext_id = mail_index_map_lookup_ext(map, "keywords"); if (ext_id == (uint32_t)-1) { if (array_is_created(&map->keyword_idx_map)) array_clear(&map->keyword_idx_map); return 0; } ext = array_idx(&map->extensions, ext_id); /* Extension header contains: - struct mail_index_keyword_header - struct mail_index_keyword_header_rec * keywords_count - const char names[] * keywords_count */ kw_hdr = CONST_PTR_OFFSET(map->hdr_base, ext->hdr_offset); kw_rec = (const void *)(kw_hdr + 1); name = (const char *)(kw_rec + kw_hdr->keywords_count); old_count = !array_is_created(&map->keyword_idx_map) ? 0 : array_count(&map->keyword_idx_map); /* Keywords can only be added into same mapping. Removing requires a new mapping (recreating the index file) */ if (kw_hdr->keywords_count == old_count) { /* nothing changed */ return 0; } /* make sure the header is valid */ if (kw_hdr->keywords_count < old_count) { mail_index_set_error(index, "Corrupted index file %s: " "Keywords removed unexpectedly", index->filepath); return -1; } if ((size_t)(name - (const char *)kw_hdr) > ext->hdr_size) { mail_index_set_error(index, "Corrupted index file %s: " "keywords_count larger than header size", index->filepath); return -1; } name_area_end_offset = (const char *)kw_hdr + ext->hdr_size - name; for (i = 0; i < kw_hdr->keywords_count; i++) { if (kw_rec[i].name_offset > name_area_end_offset) { mail_index_set_error(index, "Corrupted index file %s: " "name_offset points outside allocated header", index->filepath); return -1; } } if (name[name_area_end_offset-1] != '\0') { mail_index_set_error(index, "Corrupted index file %s: " "Keyword header doesn't end with NUL", index->filepath); return -1; } /* create file -> index mapping */ if (!array_is_created(&map->keyword_idx_map)) { ARRAY_CREATE(&map->keyword_idx_map, default_pool, unsigned int, kw_hdr->keywords_count); } #ifdef DEBUG /* Check that existing headers are still the same. It's behind DEBUG since it's pretty useless waste of CPU normally. */ for (i = 0; i < array_count(&map->keyword_idx_map); i++) { const char *keyword = name + kw_rec[i].name_offset; const unsigned int *old_idx; unsigned int idx; old_idx = array_idx(&map->keyword_idx_map, i); if (!mail_index_keyword_lookup(index, keyword, FALSE, &idx) || idx != *old_idx) { mail_index_set_error(index, "Corrupted index file %s: " "Keywords changed unexpectedly", index->filepath); return -1; } } #endif /* Register the newly seen keywords */ i = array_count(&map->keyword_idx_map); for (; i < kw_hdr->keywords_count; i++) { const char *keyword = name + kw_rec[i].name_offset; unsigned int idx; (void)mail_index_keyword_lookup(index, keyword, TRUE, &idx); array_append(&map->keyword_idx_map, &idx, 1); } return 0; } const array_t *mail_index_get_keywords(struct mail_index *index) { /* Make sure all the keywords are in index->keywords. It's quick to do if nothing has changed. */ (void)mail_index_map_read_keywords(index, index->map); return &index->keywords; } static int mail_index_check_header(struct mail_index *index, struct mail_index_map *map) { const struct mail_index_header *hdr = &map->hdr; enum mail_index_header_compat_flags compat_flags = 0; #ifndef WORDS_BIGENDIAN compat_flags |= MAIL_INDEX_COMPAT_LITTLE_ENDIAN; #endif if (hdr->major_version != MAIL_INDEX_MAJOR_VERSION) { /* major version change - handle silently(?) */ return -1; } if (hdr->compat_flags != compat_flags) { /* architecture change - handle silently(?) */ return -1; } if ((map->hdr.flags & MAIL_INDEX_HDR_FLAG_CORRUPTED) != 0) { /* we've already complained about it */ return -1; } /* following some extra checks that only take a bit of CPU */ if (hdr->uid_validity == 0 && hdr->next_uid != 1) { mail_index_set_error(index, "Corrupted index file %s: " "uid_validity = 0, next_uid = %u", index->filepath, hdr->next_uid); return -1; } if (hdr->record_size < sizeof(struct mail_index_record)) { mail_index_set_error(index, "Corrupted index file %s: " "record_size too small: %u < %"PRIuSIZE_T, index->filepath, hdr->record_size, sizeof(struct mail_index_record)); return -1; } if ((hdr->flags & MAIL_INDEX_HDR_FLAG_FSCK) != 0) return 0; if (hdr->next_uid == 0) return 0; if (hdr->recent_messages_count > hdr->messages_count || hdr->seen_messages_count > hdr->messages_count || hdr->deleted_messages_count > hdr->messages_count) return 0; if (hdr->first_recent_uid_lowwater > hdr->next_uid || hdr->first_unseen_uid_lowwater > hdr->next_uid || hdr->first_deleted_uid_lowwater > hdr->next_uid) return 0; return mail_index_read_extensions(index, map); } static void mail_index_map_clear(struct mail_index *index, struct mail_index_map *map) { if (map->buffer != NULL) { i_assert(map->mmap_base == NULL); buffer_free(map->buffer); map->buffer = NULL; } else if (map->mmap_base != NULL) { i_assert(map->buffer == NULL); if (munmap(map->mmap_base, map->mmap_size) < 0) mail_index_set_syscall_error(index, "munmap()"); map->mmap_base = NULL; } if (map->refcount > 0) { memset(&map->hdr, 0, sizeof(map->hdr)); map->mmap_size = 0; map->mmap_used_size = 0; map->records = NULL; map->records_count = 0; } } void mail_index_unmap(struct mail_index *index, struct mail_index_map **_map) { struct mail_index_map *map = *_map; *_map = NULL; if (--map->refcount > 0) return; i_assert(map->refcount == 0); mail_index_map_clear(index, map); if (map->extension_pool != NULL) pool_unref(map->extension_pool); if (array_is_created(&map->keyword_idx_map)) array_free(&map->keyword_idx_map); buffer_free(map->hdr_copy_buf); i_free(map); } static void mail_index_map_copy_hdr(struct mail_index_map *map, const struct mail_index_header *hdr) { if (hdr->base_header_size < sizeof(map->hdr)) { /* header smaller than ours, make a copy so our newer headers won't have garbage in them */ memset(&map->hdr, 0, sizeof(map->hdr)); memcpy(&map->hdr, hdr, hdr->base_header_size); } else { map->hdr = *hdr; } } static int mail_index_mmap(struct mail_index *index, struct mail_index_map *map) { const struct mail_index_header *hdr; unsigned int records_count; i_assert(!map->write_to_disk); if (map->buffer != NULL) { /* we had temporarily used a buffer, eg. for updating index */ buffer_free(map->buffer); map->buffer = NULL; } map->mmap_base = index->readonly ? mmap_ro_file(index->fd, &map->mmap_size) : mmap_rw_file(index->fd, &map->mmap_size); if (map->mmap_base == MAP_FAILED) { map->mmap_base = NULL; mail_index_set_syscall_error(index, "mmap()"); return -1; } hdr = map->mmap_base; if (map->mmap_size > offsetof(struct mail_index_header, major_version) && hdr->major_version != MAIL_INDEX_MAJOR_VERSION) { /* major version change - handle silently */ return 0; } if (map->mmap_size < MAIL_INDEX_HEADER_MIN_SIZE) { mail_index_set_error(index, "Corrupted index file %s: " "File too small (%"PRIuSIZE_T")", index->filepath, map->mmap_size); return 0; } map->mmap_used_size = hdr->header_size + hdr->messages_count * hdr->record_size; if (map->mmap_used_size > map->mmap_size) { records_count = (map->mmap_size - hdr->header_size) / hdr->record_size; mail_index_set_error(index, "Corrupted index file %s: " "messages_count too large (%u > %u)", index->filepath, hdr->messages_count, records_count); return 0; } mail_index_map_copy_hdr(map, hdr); map->hdr_base = map->mmap_base; map->records = PTR_OFFSET(map->mmap_base, map->hdr.header_size); map->records_count = map->hdr.messages_count; return 1; } static int mail_index_read_header(struct mail_index *index, struct mail_index_header *hdr, size_t *pos_r) { size_t pos; int ret; memset(hdr, 0, sizeof(*hdr)); ret = 1; for (pos = 0; ret > 0 && pos < sizeof(*hdr); ) { ret = pread(index->fd, PTR_OFFSET(hdr, pos), sizeof(*hdr) - pos, pos); if (ret > 0) pos += ret; } *pos_r = pos; return ret; } static int mail_index_read_map(struct mail_index *index, struct mail_index_map *map, bool *retry_r) { struct mail_index_header hdr; void *data = NULL; ssize_t ret; size_t pos, records_size; i_assert(map->mmap_base == NULL); *retry_r = FALSE; ret = mail_index_read_header(index, &hdr, &pos); if (pos > (ssize_t)offsetof(struct mail_index_header, major_version) && hdr.major_version != MAIL_INDEX_MAJOR_VERSION) { /* major version change - handle silently */ return 0; } if (ret >= 0 && pos >= MAIL_INDEX_HEADER_MIN_SIZE && (ret > 0 || pos >= hdr.base_header_size)) { if (hdr.base_header_size < MAIL_INDEX_HEADER_MIN_SIZE || hdr.header_size < hdr.base_header_size) { mail_index_set_error(index, "Corrupted index file %s: " "Corrupted header sizes (base %u, full %u)", index->filepath, hdr.base_header_size, hdr.header_size); return 0; } buffer_reset(map->hdr_copy_buf); buffer_append(map->hdr_copy_buf, &hdr, hdr.base_header_size); /* @UNSAFE */ data = buffer_append_space_unsafe(map->hdr_copy_buf, hdr.header_size - hdr.base_header_size); ret = pread_full(index->fd, data, hdr.header_size - hdr.base_header_size, hdr.base_header_size); } if (ret > 0) { records_size = hdr.messages_count * hdr.record_size; if (map->buffer == NULL) { map->buffer = buffer_create_dynamic(default_pool, records_size); } /* @UNSAFE */ buffer_set_used_size(map->buffer, 0); data = buffer_append_space_unsafe(map->buffer, records_size); ret = pread_full(index->fd, data, records_size, hdr.header_size); } if (ret < 0) { if (errno == ESTALE) { *retry_r = TRUE; return 0; } mail_index_set_syscall_error(index, "pread_full()"); return -1; } if (ret == 0) { mail_index_set_error(index, "Corrupted index file %s: File too small", index->filepath); return 0; } map->records = data; map->records_count = hdr.messages_count; mail_index_map_copy_hdr(map, &hdr); map->hdr_base = map->hdr_copy_buf->data; index->sync_log_file_seq = hdr.log_file_seq; index->sync_log_file_offset = hdr.log_file_int_offset; return 1; } static int mail_index_sync_from_transactions(struct mail_index *index, struct mail_index_map **map, bool sync_to_index) { const struct mail_index_header *map_hdr = &(*map)->hdr; struct mail_index_view *view; struct mail_transaction_log_view *log_view; struct mail_index_sync_map_ctx sync_map_ctx; struct mail_index_header hdr; const struct mail_transaction_header *thdr; const void *tdata; uint32_t prev_seq, max_seq; uoff_t prev_offset, max_offset; size_t pos; int ret; bool skipped; if (sync_to_index) { /* read the real log position where we are supposed to be synced */ ret = mail_index_read_header(index, &hdr, &pos); if (ret < 0 && errno != ESTALE) { mail_index_set_syscall_error(index, "pread()"); return -1; } if (pos < MAIL_INDEX_HEADER_MIN_SIZE) return 0; if (map_hdr->log_file_seq == hdr.log_file_seq && map_hdr->log_file_int_offset == hdr.log_file_int_offset) { /* nothing to do */ return 1; } if (map_hdr->log_file_seq > hdr.log_file_seq || (map_hdr->log_file_seq == hdr.log_file_seq && map_hdr->log_file_int_offset > hdr.log_file_int_offset)) { /* we went too far, have to re-read the file */ return 0; } if (map_hdr->log_file_ext_offset != map_hdr->log_file_int_offset || hdr.log_file_ext_offset != hdr.log_file_int_offset) { /* too much trouble to get this right. */ return 0; } max_seq = hdr.log_file_seq; max_offset = hdr.log_file_int_offset; } else { /* sync everything there is */ max_seq = (uint32_t)-1; max_offset = (uoff_t)-1; } log_view = mail_transaction_log_view_open(index->log); if (mail_transaction_log_view_set(log_view, map_hdr->log_file_seq, map_hdr->log_file_int_offset, max_seq, max_offset, MAIL_TRANSACTION_TYPE_MASK) <= 0) { /* can't use it. sync by re-reading index. */ mail_transaction_log_view_close(&log_view); return 0; } index->map = *map; view = mail_index_view_open(index); mail_index_sync_map_init(&sync_map_ctx, view, MAIL_INDEX_SYNC_HANDLER_VIEW); while ((ret = mail_transaction_log_view_next(log_view, &thdr, &tdata, &skipped)) > 0) { if (mail_index_sync_record(&sync_map_ctx, thdr, tdata) < 0) { ret = -1; break; } } mail_transaction_log_view_get_prev_pos(log_view, &prev_seq, &prev_offset); i_assert(prev_seq <= max_seq && (prev_seq != max_seq || prev_offset <= max_offset)); index->map->hdr.log_file_seq = prev_seq; index->map->hdr.log_file_int_offset = index->map->hdr.log_file_ext_offset = prev_offset; mail_index_sync_map_deinit(&sync_map_ctx); mail_index_view_close(&view); mail_transaction_log_view_close(&log_view); *map = index->map; index->map = NULL; if (sync_to_index) { /* make sure we did everything right. note that although the message counts should be equal, the flag counters may not */ i_assert(hdr.messages_count == (*map)->hdr.messages_count); i_assert(hdr.log_file_seq == (*map)->hdr.log_file_seq); i_assert(hdr.log_file_int_offset == (*map)->hdr.log_file_int_offset); i_assert(hdr.log_file_ext_offset == (*map)->hdr.log_file_ext_offset); } return ret < 0 ? -1 : 1; } static int mail_index_read_map_with_retry(struct mail_index *index, struct mail_index_map **map, bool sync_to_index) { mail_index_sync_lost_handler_t *const *handlers; unsigned int i, count; int ret; bool retry; if (index->log_locked) { /* we're most likely syncing the index and we really don't want to read more than what was synced last time. */ sync_to_index = TRUE; } if ((*map)->hdr.indexid != 0 && index->log != NULL) { /* we're not creating the index, or opening transaction log. sync this as a view from transaction log. */ index->sync_update = TRUE; ret = mail_index_sync_from_transactions(index, map, sync_to_index); index->sync_update = FALSE; if (ret != 0) return ret; /* transaction log lost/broken, fallback to re-reading it */ } /* notify all "sync lost" handlers */ handlers = array_get(&index->sync_lost_handlers, &count); for (i = 0; i < count; i++) (*handlers[i])(index); for (i = 0; i < MAIL_INDEX_ESTALE_RETRY_COUNT; i++) { ret = mail_index_read_map(index, *map, &retry); if (ret != 0 || !retry) return ret; /* ESTALE - reopen index file */ if (close(index->fd) < 0) mail_index_set_syscall_error(index, "close()"); index->fd = -1; ret = mail_index_try_open_only(index); if (ret <= 0) { if (ret == 0) { /* the file was lost */ errno = ENOENT; mail_index_set_syscall_error(index, "open()"); } return -1; } } /* Too many ESTALE retries */ mail_index_set_syscall_error(index, "read_map()"); return -1; } static int mail_index_map_try_existing(struct mail_index_map *map) { const struct mail_index_header *hdr; size_t used_size; if (MAIL_INDEX_MAP_IS_IN_MEMORY(map)) return 0; hdr = map->mmap_base; /* always check corrupted-flag to avoid errors later */ if ((hdr->flags & MAIL_INDEX_HDR_FLAG_CORRUPTED) != 0) return -1; used_size = hdr->header_size + hdr->messages_count * hdr->record_size; if (map->mmap_size >= used_size && map->hdr_base == hdr) { map->records_count = hdr->messages_count; mail_index_map_copy_hdr(map, hdr); return 1; } return 0; } int mail_index_map(struct mail_index *index, bool force) { struct mail_index_map *map; int ret; i_assert(!index->mapping); i_assert(index->map == NULL || index->map->refcount > 0); i_assert(index->lock_type != F_UNLCK); if (MAIL_INDEX_IS_IN_MEMORY(index)) { if (index->map == NULL) mail_index_create_in_memory(index, NULL); return 1; } index->mapping = TRUE; if (!force && index->map != NULL) { i_assert(index->hdr != NULL); ret = mail_index_map_try_existing(index->map); if (ret != 0) { index->mapping = FALSE; return ret; } if (index->lock_type == F_WRLCK) { /* we're syncing, don't break the mapping */ index->mapping = FALSE; return 1; } } if (index->map != NULL && index->map->refcount > 1) { /* this map is already used by some views and they may have pointers into it. leave them and create a new mapping. */ if (!index->mmap_disable) { map = NULL; } else { /* create a copy of the mapping instead so we don't have to re-read it */ map = mail_index_map_clone(index->map, index->map->hdr.record_size); } index->map->refcount--; index->map = NULL; } else { map = index->map; } if (map == NULL) { map = i_new(struct mail_index_map, 1); map->refcount = 1; map->hdr_copy_buf = buffer_create_dynamic(default_pool, sizeof(map->hdr)); } else if (MAIL_INDEX_MAP_IS_IN_MEMORY(map)) { i_assert(!map->write_to_disk); } else if (map->mmap_base != NULL) { i_assert(map->buffer == NULL); if (munmap(map->mmap_base, map->mmap_size) < 0) mail_index_set_syscall_error(index, "munmap()"); map->mmap_base = NULL; } index->hdr = NULL; index->map = NULL; if (!index->mmap_disable) ret = mail_index_mmap(index, map); else ret = mail_index_read_map_with_retry(index, &map, force); i_assert(index->map == NULL); if (ret > 0) { ret = mail_index_check_header(index, map); if (ret < 0) ret = 0; else if (ret == 0) { index->fsck = TRUE; ret = 1; } } if (ret <= 0) { mail_index_map_clear(index, map); mail_index_unmap(index, &map); index->mapping = FALSE; return ret; } index->hdr = &map->hdr; index->map = map; i_assert(map->hdr.messages_count == map->records_count); index->mapping = FALSE; return 1; } int mail_index_get_latest_header(struct mail_index *index, struct mail_index_header *hdr_r) { size_t pos; int ret; if (!index->mmap_disable) { ret = mail_index_map(index, FALSE); if (ret > 0) *hdr_r = *index->hdr; else memset(hdr_r, 0, sizeof(*hdr_r)); } else { ret = mail_index_read_header(index, hdr_r, &pos); } return ret; } struct mail_index_map * mail_index_map_clone(struct mail_index_map *map, uint32_t new_record_size) { struct mail_index_map *mem_map; struct mail_index_header *hdr; struct mail_index_ext *extensions; void *src, *dest; size_t size, copy_size; unsigned int i, count; size = map->records_count * new_record_size; mem_map = i_new(struct mail_index_map, 1); mem_map->refcount = 1; mem_map->buffer = buffer_create_dynamic(default_pool, size); if (map->hdr.record_size == new_record_size) buffer_append(mem_map->buffer, map->records, size); else { copy_size = I_MIN(map->hdr.record_size, new_record_size); src = map->records; for (i = 0; i < map->records_count; i++) { dest = buffer_append_space_unsafe(mem_map->buffer, new_record_size); memcpy(dest, src, copy_size); src = PTR_OFFSET(src, map->hdr.record_size); } } mem_map->records = buffer_get_modifyable_data(mem_map->buffer, NULL); mem_map->records_count = map->records_count; mem_map->hdr_copy_buf = buffer_create_dynamic(default_pool, map->hdr.header_size); if (map->hdr.base_header_size < sizeof(*hdr)) buffer_append_zero(mem_map->hdr_copy_buf, sizeof(*hdr)); buffer_write(mem_map->hdr_copy_buf, 0, &map->hdr, map->hdr.base_header_size); buffer_append(mem_map->hdr_copy_buf, CONST_PTR_OFFSET(map->hdr_base, map->hdr.base_header_size), map->hdr.header_size - map->hdr.base_header_size); hdr = buffer_get_modifyable_data(mem_map->hdr_copy_buf, NULL); if (hdr->base_header_size < sizeof(*hdr)) hdr->base_header_size = sizeof(*hdr); hdr->record_size = new_record_size; mem_map->hdr = *hdr; mem_map->hdr_base = hdr; /* copy extensions */ if (array_is_created(&map->ext_id_map)) { count = array_count(&map->ext_id_map); mail_index_map_init_extbufs(mem_map, count + 2); array_append_array(&mem_map->extensions, &map->extensions); array_append_array(&mem_map->ext_id_map, &map->ext_id_map); /* fix the name pointers to use our own pool */ extensions = array_get_modifyable(&mem_map->extensions, &count); for (i = 0; i < count; i++) { extensions[i].name = p_strdup(mem_map->extension_pool, extensions[i].name); } } return mem_map; } int mail_index_map_get_ext_idx(struct mail_index_map *map, uint32_t ext_id, uint32_t *idx_r) { const uint32_t *id; if (!array_is_created(&map->ext_id_map) || ext_id >= array_count(&map->ext_id_map)) return 0; id = array_idx(&map->ext_id_map, ext_id); *idx_r = *id; return *idx_r != (uint32_t)-1; } static int mail_index_try_open_only(struct mail_index *index) { int i; i_assert(!MAIL_INDEX_IS_IN_MEMORY(index)); for (i = 0; i < 3; i++) { index->fd = open(index->filepath, O_RDWR); if (index->fd == -1 && errno == EACCES) { index->fd = open(index->filepath, O_RDONLY); index->readonly = TRUE; } if (index->fd != -1 || errno != ESTALE) break; /* May happen with some OSes with NFS. Try again, although there's still a race condition with another computer creating the index file again. However, we can't try forever as ESTALE happens also if index directory has been deleted from server.. */ } if (index->fd == -1) { if (errno != ENOENT) return mail_index_set_syscall_error(index, "open()"); /* have to create it */ return 0; } return 1; } static int mail_index_try_open(struct mail_index *index, unsigned int *lock_id_r) { unsigned int lock_id; int ret; if (lock_id_r != NULL) *lock_id_r = 0; if (MAIL_INDEX_IS_IN_MEMORY(index)) return 0; ret = mail_index_try_open_only(index); if (ret <= 0) return ret; if (mail_index_lock_shared(index, FALSE, &lock_id) < 0) { (void)close(index->fd); index->fd = -1; return -1; } ret = mail_index_map(index, FALSE); if (ret == 0) { /* it's corrupted - recreate it */ mail_index_unlock(index, lock_id); if (lock_id_r != NULL) *lock_id_r = 0; (void)close(index->fd); index->fd = -1; } else { if (lock_id_r != NULL) *lock_id_r = lock_id; else mail_index_unlock(index, lock_id); } return ret; } int mail_index_write_base_header(struct mail_index *index, const struct mail_index_header *hdr) { size_t hdr_size; hdr_size = I_MIN(sizeof(*hdr), hdr->base_header_size); if (!MAIL_INDEX_MAP_IS_IN_MEMORY(index->map)) { memcpy(index->map->mmap_base, hdr, hdr_size); if (msync(index->map->mmap_base, hdr_size, MS_SYNC) < 0) return mail_index_set_syscall_error(index, "msync()"); index->map->hdr = *hdr; } else { if (!MAIL_INDEX_IS_IN_MEMORY(index)) { if (pwrite_full(index->fd, hdr, hdr_size, 0) < 0) { mail_index_set_syscall_error(index, "pwrite_full()"); return -1; } } index->map->hdr = *hdr; buffer_write(index->map->hdr_copy_buf, 0, hdr, hdr_size); } return 0; } int mail_index_create_tmp_file(struct mail_index *index, const char **path_r) { mode_t old_mask; const char *path; int fd; i_assert(!MAIL_INDEX_IS_IN_MEMORY(index)); path = *path_r = t_strconcat(index->filepath, ".tmp", NULL); old_mask = umask(0); fd = open(path, O_RDWR|O_CREAT|O_TRUNC, index->mode); umask(old_mask); if (fd == -1) return mail_index_file_set_syscall_error(index, path, "open()"); if (index->gid != (gid_t)-1 && fchown(fd, (uid_t)-1, index->gid) < 0) { mail_index_file_set_syscall_error(index, path, "fchown()"); return -1; } return fd; } static int mail_index_create(struct mail_index *index, struct mail_index_header *hdr) { const char *path; uint32_t seq; uoff_t offset; int ret; i_assert(!MAIL_INDEX_IS_IN_MEMORY(index)); i_assert(index->lock_type == F_UNLCK); /* log file lock protects index creation */ if (mail_transaction_log_sync_lock(index->log, &seq, &offset) < 0) return -1; ret = mail_index_try_open(index, NULL); if (ret != 0) { mail_transaction_log_sync_unlock(index->log); return ret < 0 ? -1 : 0; } /* mark the existing log file as synced */ hdr->log_file_seq = seq; hdr->log_file_int_offset = offset; hdr->log_file_ext_offset = offset; /* create it fully in index.tmp first */ index->fd = mail_index_create_tmp_file(index, &path); if (index->fd == -1) ret = -1; else if (write_full(index->fd, hdr, sizeof(*hdr)) < 0) { mail_index_file_set_syscall_error(index, path, "write_full()"); ret = -1; } else { index->lock_type = F_WRLCK; ret = mail_index_map(index, FALSE); index->lock_type = F_UNLCK; } if (ret == 0) { /* it's corrupted even while we just created it, should never happen unless someone pokes the file directly */ mail_index_set_error(index, "Newly created index file is corrupted: %s", path); ret = -1; } if (ret < 0) { if (unlink(path) < 0 && errno != ENOENT) { mail_index_file_set_syscall_error(index, path, "unlink()"); } } else { /* make it visible to others */ if (rename(path, index->filepath) < 0) { mail_index_set_error(index, "rename(%s, %s) failed: %m", path, index->filepath); ret = -1; } } mail_transaction_log_sync_unlock(index->log); return ret; } static void mail_index_header_init(struct mail_index_header *hdr) { time_t now = time(NULL); i_assert((sizeof(*hdr) % sizeof(uint64_t)) == 0); memset(hdr, 0, sizeof(*hdr)); hdr->major_version = MAIL_INDEX_MAJOR_VERSION; hdr->minor_version = MAIL_INDEX_MINOR_VERSION; hdr->base_header_size = sizeof(*hdr); hdr->header_size = sizeof(*hdr); hdr->record_size = sizeof(struct mail_index_record); #ifndef WORDS_BIGENDIAN hdr->compat_flags |= MAIL_INDEX_COMPAT_LITTLE_ENDIAN; #endif hdr->indexid = now; hdr->next_uid = 1; } static void mail_index_create_in_memory(struct mail_index *index, const struct mail_index_header *hdr) { struct mail_index_header tmp_hdr; struct mail_index_map tmp_map; if (hdr == NULL) { mail_index_header_init(&tmp_hdr); hdr = &tmp_hdr; } memset(&tmp_map, 0, sizeof(tmp_map)); tmp_map.hdr = *hdr; tmp_map.hdr_base = hdr; /* a bit kludgy way to do this, but it initializes everything nicely and correctly */ index->map = mail_index_map_clone(&tmp_map, hdr->record_size); index->hdr = &index->map->hdr; } /* returns -1 = error, 0 = won't create, 1 = ok */ static int mail_index_open_files(struct mail_index *index, enum mail_index_open_flags flags) { struct mail_index_header hdr; unsigned int lock_id = 0; int ret; bool created = FALSE; ret = mail_index_try_open(index, &lock_id); if (ret > 0) hdr = *index->hdr; else if (ret == 0) { /* doesn't exist, or corrupted */ if ((flags & MAIL_INDEX_OPEN_FLAG_CREATE) == 0 && !MAIL_INDEX_IS_IN_MEMORY(index)) return 0; mail_index_header_init(&hdr); index->hdr = &hdr; } else if (ret < 0) return -1; index->indexid = hdr.indexid; index->log = mail_transaction_log_open_or_create(index); if (index->log == NULL) return -1; if (index->fd == -1) { if (index->indexid != hdr.indexid) { /* looks like someone else created the transaction log before we had the chance. use its indexid so we don't try to create conflicting ones. */ hdr.indexid = index->indexid; } if (lock_id != 0) { mail_index_unlock(index, lock_id); lock_id = 0; } if (!MAIL_INDEX_IS_IN_MEMORY(index)) { if (mail_index_create(index, &hdr) < 0) { /* fallback to in-memory index */ mail_index_move_to_memory(index); mail_index_create_in_memory(index, &hdr); } } else { mail_index_create_in_memory(index, &hdr); } created = TRUE; } if (lock_id == 0) { if (mail_index_lock_shared(index, FALSE, &lock_id) < 0) return -1; } index->cache = created ? mail_cache_create(index) : mail_cache_open_or_create(index); mail_index_unlock(index, lock_id); return 1; } int mail_index_open(struct mail_index *index, enum mail_index_open_flags flags, enum mail_index_lock_method lock_method) { int i = 0, ret; if (index->opened) { if (index->hdr != NULL && (index->hdr->flags & MAIL_INDEX_HDR_FLAG_CORRUPTED) != 0) { /* corrupted, reopen files */ mail_index_close(index); } else { return 0; } } index->filepath = MAIL_INDEX_IS_IN_MEMORY(index) ? i_strdup("(in-memory index)") : i_strconcat(index->dir, "/", index->prefix, NULL); for (;;) { index->shared_lock_count = 0; index->excl_lock_count = 0; index->lock_type = F_UNLCK; index->lock_id = 2; index->readonly = FALSE; index->nodiskspace = FALSE; index->index_lock_timeout = FALSE; index->log_locked = FALSE; index->mmap_disable = (flags & MAIL_INDEX_OPEN_FLAG_MMAP_DISABLE) != 0; index->mmap_no_write = (flags & MAIL_INDEX_OPEN_FLAG_MMAP_NO_WRITE) != 0; index->lock_method = lock_method; /* don't even bother to handle dotlocking without mmap being disabled. that combination simply doesn't make any sense */ if (lock_method == MAIL_INDEX_LOCK_DOTLOCK && !index->mmap_disable) { i_fatal("lock_method=dotlock and mmap_disable=no " "combination isn't supported. " "You don't _really_ want it anyway."); } ret = mail_index_open_files(index, flags); if (ret <= 0) break; index->opened = TRUE; if (index->fsck) { index->fsck = FALSE; ret = mail_index_fsck(index); if (ret == 0) { /* completely broken, reopen */ if (i++ < 3) continue; /* too many tries */ ret = -1; } } break; } if (ret <= 0) mail_index_close(index); return ret; } void mail_index_close(struct mail_index *index) { if (index->log != NULL) mail_transaction_log_close(&index->log); if (index->map != NULL) mail_index_unmap(index, &index->map); if (index->cache != NULL) mail_cache_free(&index->cache); if (index->fd != -1) { if (close(index->fd) < 0) mail_index_set_syscall_error(index, "close()"); index->fd = -1; } i_free_and_null(index->copy_lock_path); i_free_and_null(index->filepath); index->indexid = 0; index->opened = FALSE; } int mail_index_reopen(struct mail_index *index, int fd) { struct mail_index_map *old_map; unsigned int old_shared_locks, old_lock_id, lock_id = 0; int ret, old_fd, old_lock_type; i_assert(!MAIL_INDEX_IS_IN_MEMORY(index)); old_map = index->map; old_fd = index->fd; old_map->refcount++; /* new file, new locks. the old fd can keep its locks, they don't matter anymore as no-one's going to modify the file. */ old_lock_type = index->lock_type; old_lock_id = index->lock_id; old_shared_locks = index->shared_lock_count; if (index->lock_type == F_RDLCK) index->lock_type = F_UNLCK; index->lock_id += 2; index->shared_lock_count = 0; if (fd != -1) { index->fd = fd; ret = 0; } else { i_assert(index->excl_lock_count == 0); ret = mail_index_try_open_only(index); if (ret > 0) ret = mail_index_lock_shared(index, FALSE, &lock_id); else if (ret == 0) { /* index file is lost */ ret = -1; } } if (ret == 0) { /* read the new mapping. note that with mmap_disable we want to keep the old mapping in index->map so we can update it by reading transaction log. */ if (mail_index_map(index, TRUE) <= 0) ret = -1; } if (lock_id != 0) mail_index_unlock(index, lock_id); if (ret == 0) { mail_index_unmap(index, &old_map); if (close(old_fd) < 0) mail_index_set_syscall_error(index, "close()"); } else { if (index->map != NULL) mail_index_unmap(index, &index->map); if (index->fd != -1) { if (close(index->fd) < 0) mail_index_set_syscall_error(index, "close()"); } index->map = old_map; index->hdr = &index->map->hdr; index->fd = old_fd; index->lock_type = old_lock_type; index->lock_id = old_lock_id; index->shared_lock_count = old_shared_locks; } return ret; } int mail_index_reopen_if_needed(struct mail_index *index) { struct stat st1, st2; if (MAIL_INDEX_IS_IN_MEMORY(index)) return 0; if (fstat(index->fd, &st1) < 0) return mail_index_set_syscall_error(index, "fstat()"); if (stat(index->filepath, &st2) < 0) { mail_index_set_syscall_error(index, "stat()"); if (errno != ENOENT) return -1; /* lost it? recreate */ (void)mail_index_mark_corrupted(index); return -1; } if (st1.st_ino != st2.st_ino || !CMP_DEV_T(st1.st_dev, st2.st_dev)) { if (mail_index_reopen(index, -1) < 0) return -1; return 1; } else { return 0; } } int mail_index_refresh(struct mail_index *index) { unsigned int lock_id; int ret; if (MAIL_INDEX_IS_IN_MEMORY(index)) return 0; if (index->excl_lock_count > 0) { /* we have index exclusively locked, nothing could have changed. */ return 0; } if (!index->mmap_disable) { /* reopening is all we need */ return mail_index_reopen_if_needed(index); } i_assert(!index->mapping); /* mail_index_map() simply reads latest changes from transaction log, which makes us fully refreshed. */ if (mail_index_lock_shared(index, TRUE, &lock_id) < 0) return -1; ret = mail_index_map(index, FALSE); mail_index_unlock(index, lock_id); return ret <= 0 ? -1 : 0; } struct mail_cache *mail_index_get_cache(struct mail_index *index) { return index->cache; } int mail_index_set_error(struct mail_index *index, const char *fmt, ...) { va_list va; i_free(index->error); if (fmt == NULL) index->error = NULL; else { va_start(va, fmt); index->error = i_strdup_vprintf(fmt, va); va_end(va); i_error("%s", index->error); } return -1; } void mail_index_set_inconsistent(struct mail_index *index) { index->indexid = 0; } int mail_index_move_to_memory(struct mail_index *index) { struct mail_index_map *map; int ret = 0; /* set the index as being into memory */ i_free_and_null(index->dir); if (index->map == NULL) { /* mbox file was never even opened. just mark it as being in memory and let the caller re-open the index. */ i_assert(index->fd == -1); return -1; } /* move index map to memory */ map = mail_index_map_clone(index->map, index->map->hdr.record_size); mail_index_unmap(index, &index->map); index->map = map; index->hdr = &map->hdr; /* move transaction log to memory */ if (mail_transaction_log_move_to_memory(index->log) < 0) ret = -1; /* close the index file. */ if (close(index->fd) < 0) mail_index_set_syscall_error(index, "close()"); index->fd = -1; return ret; } void mail_index_mark_corrupted(struct mail_index *index) { struct mail_index_header hdr; mail_index_set_inconsistent(index); if (index->readonly) return; hdr = *index->hdr; hdr.flags |= MAIL_INDEX_HDR_FLAG_CORRUPTED; if (mail_index_write_base_header(index, &hdr) == 0) { if (!MAIL_INDEX_IS_IN_MEMORY(index) && fsync(index->fd) < 0) mail_index_set_syscall_error(index, "fsync()"); } } int mail_index_set_syscall_error(struct mail_index *index, const char *function) { i_assert(function != NULL); if (ENOSPACE(errno)) { index->nodiskspace = TRUE; return -1; } return mail_index_set_error(index, "%s failed with index file %s: %m", function, index->filepath); } int mail_index_file_set_syscall_error(struct mail_index *index, const char *filepath, const char *function) { i_assert(filepath != NULL); i_assert(function != NULL); if (ENOSPACE(errno)) { index->nodiskspace = TRUE; return -1; } return mail_index_set_error(index, "%s failed with file %s: %m", function, filepath); } enum mail_index_error mail_index_get_last_error(struct mail_index *index) { if (index->nodiskspace) return MAIL_INDEX_ERROR_DISKSPACE; if (index->error != NULL) return MAIL_INDEX_ERROR_INTERNAL; return MAIL_INDEX_ERROR_NONE; } const char *mail_index_get_error_message(struct mail_index *index) { return index->error; } void mail_index_reset_error(struct mail_index *index) { if (index->error != NULL) { i_free(index->error); index->error = NULL; } index->nodiskspace = FALSE; index->index_lock_timeout = FALSE; } uint32_t mail_index_uint32_to_offset(uint32_t offset) { unsigned char buf[4]; i_assert(offset < 0x40000000); i_assert((offset & 3) == 0); offset >>= 2; buf[0] = 0x80 | ((offset & 0x0fe00000) >> 21); buf[1] = 0x80 | ((offset & 0x001fc000) >> 14); buf[2] = 0x80 | ((offset & 0x00003f80) >> 7); buf[3] = 0x80 | (offset & 0x0000007f); return *((uint32_t *) buf); } uint32_t mail_index_offset_to_uint32(uint32_t offset) { const unsigned char *buf = (const unsigned char *) &offset; if ((offset & 0x80808080) != 0x80808080) return 0; return (((uint32_t)buf[3] & 0x7f) << 2) | (((uint32_t)buf[2] & 0x7f) << 9) | (((uint32_t)buf[1] & 0x7f) << 16) | (((uint32_t)buf[0] & 0x7f) << 23); }