Mercurial > dovecot > original-hg > dovecot-1.2
changeset 222:cf4d065f2f85 HEAD
lots of cleanups. also index/datafile is now capable of staying in memory,
as long as it's noticed while opening the index.
line wrap: on
line diff
--- a/src/imap/client.c Fri Sep 13 03:01:23 2002 +0300 +++ b/src/imap/client.c Sat Sep 14 14:09:42 2002 +0300 @@ -107,6 +107,8 @@ void client_disconnect(Client *client) { + io_buffer_send_flush(client->outbuf); + io_buffer_close(client->inbuf); io_buffer_close(client->outbuf); }
--- a/src/lib-imap/imap-message-cache.c Fri Sep 13 03:01:23 2002 +0300 +++ b/src/lib-imap/imap-message-cache.c Sat Sep 14 14:09:42 2002 +0300 @@ -156,7 +156,7 @@ } } -static void imap_msgcache_get_inbuf(ImapMessageCache *cache, uoff_t offset) +static int imap_msgcache_get_inbuf(ImapMessageCache *cache, uoff_t offset) { if (cache->open_inbuf == NULL) cache->open_inbuf = cache->iface->open_mail(cache->context); @@ -168,9 +168,12 @@ } if (cache->open_inbuf == NULL) - i_fatal("Can't get input buffer for message"); + return FALSE; + + i_assert(offset >= cache->open_inbuf->offset); io_buffer_skip(cache->open_inbuf, offset - cache->open_inbuf->offset); + return TRUE; } static void msg_get_part(ImapMessageCache *cache) @@ -194,10 +197,9 @@ if ((fields & IMAP_CACHE_BODY) && msg->cached_body == NULL) { value = cache->iface->get_cached_field(IMAP_CACHE_BODY, cache->context); - if (value == NULL) { + if (value == NULL && imap_msgcache_get_inbuf(cache, 0)) { msg_get_part(cache); - imap_msgcache_get_inbuf(cache, 0); value = imap_part_get_bodystructure(msg->pool, &msg->part, cache->open_inbuf, @@ -210,10 +212,9 @@ msg->cached_bodystructure == NULL) { value = cache->iface->get_cached_field(IMAP_CACHE_BODYSTRUCTURE, cache->context); - if (value == NULL) { + if (value == NULL && imap_msgcache_get_inbuf(cache, 0)) { msg_get_part(cache); - imap_msgcache_get_inbuf(cache, 0); value = imap_part_get_bodystructure(msg->pool, &msg->part, cache->open_inbuf, @@ -226,7 +227,8 @@ value = cache->iface->get_cached_field(IMAP_CACHE_ENVELOPE, cache->context); if (value == NULL) { - if (msg->envelope == NULL) { + if (msg->envelope == NULL && + imap_msgcache_get_inbuf(cache, 0)) { /* envelope isn't parsed yet, do it. header size is calculated anyway so save it */ if (msg->hdr_size == NULL) { @@ -234,7 +236,6 @@ MessageSize, 1); } - imap_msgcache_get_inbuf(cache, 0); message_parse_header(NULL, cache->open_inbuf, msg->hdr_size, parse_envelope_header, @@ -261,7 +262,7 @@ if (fields & IMAP_CACHE_MESSAGE_PART) { msg_get_part(cache); - if (msg->part == NULL) { + if (msg->part == NULL && imap_msgcache_get_inbuf(cache, 0)) { /* we need to parse the message */ MessageHeaderFunc func; @@ -274,7 +275,6 @@ func = NULL; } - imap_msgcache_get_inbuf(cache, 0); msg->part = message_parse(msg->pool, cache->open_inbuf, func, msg); } @@ -300,9 +300,10 @@ *msg->hdr_size = msg->part->header_size; } else { /* need to do some light parsing */ - imap_msgcache_get_inbuf(cache, 0); - message_get_header_size(cache->open_inbuf, - msg->hdr_size); + if (imap_msgcache_get_inbuf(cache, 0)) { + message_get_header_size(cache->open_inbuf, + msg->hdr_size); + } } } @@ -401,7 +402,8 @@ if (inbuf != NULL) { offset = hdr_size != NULL ? 0 : msg->hdr_size->physical_size; - imap_msgcache_get_inbuf(cache, offset); + if (!imap_msgcache_get_inbuf(cache, offset)) + return FALSE; *inbuf = cache->open_inbuf; } @@ -468,10 +470,14 @@ i_assert(cache->open_msg != NULL); + *inbuf = NULL; + msg = cache->open_msg; if (msg->hdr_size == NULL) { + if (!imap_msgcache_get_inbuf(cache, 0)) + return FALSE; + msg->hdr_size = p_new(msg->pool, MessageSize, 1); - imap_msgcache_get_inbuf(cache, 0); message_get_header_size(cache->open_inbuf, msg->hdr_size); } @@ -482,7 +488,8 @@ if (virtual_skip == 0) { if (msg->body_size == NULL) { cache_fields(cache, IMAP_CACHE_MESSAGE_BODY_SIZE); - i_assert(msg->body_size != NULL); + if (msg->body_size == NULL) + return FALSE; } if (max_virtual_size >= msg->body_size->virtual_size) { @@ -492,10 +499,12 @@ } if (!size_got) { + if (!imap_msgcache_get_inbuf(cache, + msg->hdr_size->physical_size)) + return FALSE; + if (msg->partial_size == NULL) msg->partial_size = p_new(msg->pool, MessageSize, 1); - - imap_msgcache_get_inbuf(cache, msg->hdr_size->physical_size); get_partial_size(cache->open_inbuf, virtual_skip, max_virtual_size, msg->partial_size, size); @@ -506,7 +515,9 @@ message_size_add(size, msg->hdr_size); /* seek to wanted position */ - imap_msgcache_get_inbuf(cache, physical_skip); + if (!imap_msgcache_get_inbuf(cache, physical_skip)) + return FALSE; + *inbuf = cache->open_inbuf; return TRUE; } @@ -515,7 +526,9 @@ { i_assert(cache->open_msg != NULL); - imap_msgcache_get_inbuf(cache, 0); + if (!imap_msgcache_get_inbuf(cache, 0)) + return FALSE; + *inbuf = cache->open_inbuf; return TRUE; }
--- a/src/lib-index/Makefile.am Fri Sep 13 03:01:23 2002 +0300 +++ b/src/lib-index/Makefile.am Sat Sep 14 14:09:42 2002 +0300 @@ -14,6 +14,7 @@ mail-index-compress.c \ mail-index-fsck.c \ mail-index-data.c \ + mail-index-open.c \ mail-index-update.c \ mail-index-update-cache.c \ mail-index-util.c \
--- a/src/lib-index/mail-custom-flags.c Fri Sep 13 03:01:23 2002 +0300 +++ b/src/lib-index/mail-custom-flags.c Sat Sep 14 14:09:42 2002 +0300 @@ -1,6 +1,7 @@ /* Copyright (C) 2002 Timo Sirainen */ #include "lib.h" +#include "file-lock.h" #include "mmap-util.h" #include "write-full.h" #include "imap-util.h" @@ -182,22 +183,12 @@ static int lock_file(MailCustomFlags *mcf, int type) { - struct flock fl; - if (mcf->lock_type == type) return TRUE; - /* lock whole file */ - fl.l_type = type; - fl.l_whence = SEEK_SET; - fl.l_start = 0; - fl.l_len = 0; - - while (fcntl(mcf->fd, F_SETLKW, &fl) == -1) { - if (errno != EINTR) { - index_cf_set_syscall_error(mcf, "fcntl(F_SETLKW)"); - return FALSE; - } + if (file_wait_lock(mcf->fd, type) < 0) { + index_cf_set_syscall_error(mcf, "file_wait_lock()"); + return FALSE; } mcf->lock_type = type;
--- a/src/lib-index/mail-hash.c Fri Sep 13 03:01:23 2002 +0300 +++ b/src/lib-index/mail-hash.c Sat Sep 14 14:09:42 2002 +0300 @@ -68,9 +68,8 @@ if (hash->mmap_base == MAP_FAILED) { hash->mmap_base = NULL; hash->header = NULL; - index_set_error(hash->index, - "hash: mmap() failed with file %s: %m", - hash->filepath); + index_file_set_syscall_error(hash->index, hash->filepath, + "mmap()"); return FALSE; } @@ -123,6 +122,19 @@ } +static void hash_munmap(MailHash *hash) +{ + if (hash->mmap_base != NULL) { + if (!hash->anon_mmap) + (void)munmap(hash->mmap_base, hash->mmap_length); + else { + (void)munmap_anon(hash->mmap_base, hash->mmap_length); + hash->anon_mmap = FALSE; + } + hash->mmap_base = NULL; + } +} + static MailHash *mail_hash_new(MailIndex *index) { MailHash *hash; @@ -175,8 +187,7 @@ if (failed) { /* recreate it */ - (void)munmap(hash->mmap_base, hash->mmap_length); - hash->mmap_base = NULL; + hash_munmap(hash); hash->dirty_mmap = TRUE; return mail_hash_rebuild(hash); @@ -189,10 +200,7 @@ { hash->index->hash = NULL; - if (hash->mmap_base != NULL) { - (void)munmap(hash->mmap_base, hash->mmap_length); - hash->mmap_base = NULL; - } + hash_munmap(hash); if (hash->fd != -1) (void)close(hash->fd); @@ -210,8 +218,8 @@ msync(hash->mmap_base, hash->mmap_length, MS_SYNC) == 0) return TRUE; else { - index_set_error(hash->index, "msync() failed for %s: %m", - hash->filepath); + index_file_set_syscall_error(hash->index, hash->filepath, + "msync()"); return FALSE; } } @@ -256,6 +264,7 @@ { void *mmap_base; size_t mmap_length, new_size; + int failed; i_assert(hash_size < MAX_HASH_SIZE); @@ -263,39 +272,36 @@ /* fill the file with zeros */ if (file_set_size(fd, (off_t)new_size) < 0) { - index_set_error(index, "Failed to fill temp hash to " - "size %"PRIuSIZE_T": %m", new_size); + index_file_set_syscall_error(index, path, "file_set_size()"); return FALSE; } /* now, mmap() it */ mmap_base = mmap_rw_file(fd, &mmap_length); - if (mmap_base == MAP_FAILED) { - index_set_error(index, - "mmap()ing temp hash failed: %m"); - return FALSE; - } + if (mmap_base == MAP_FAILED) + return index_file_set_syscall_error(index, path, "mmap()"); i_assert(mmap_length == new_size); hash_build(index, mmap_base, hash_size); + failed = FALSE; if (msync(mmap_base, mmap_length, MS_SYNC) < 0) { - index_set_error(index, "msync() failed for temp hash %s: %m", - path); - (void)munmap(mmap_base, mmap_length); - return FALSE; + index_file_set_syscall_error(index, path, "msync()"); + failed = TRUE; } - (void)munmap(mmap_base, mmap_length); - /* we don't want to leave partially written hash around */ - if (fsync(fd) < 0) { - index_set_error(index, "fsync() failed with temp hash %s: %m", - path); - return FALSE; + if (!failed && fsync(fd) < 0) { + index_file_set_syscall_error(index, path, "fsync()"); + failed = TRUE; } - return TRUE; + if (munmap(mmap_base, mmap_length) < 0) { + index_file_set_syscall_error(index, path, "munmap()"); + failed = TRUE; + } + + return !failed; } int mail_hash_rebuild(MailHash *hash) @@ -333,8 +339,8 @@ path, hash_size); if (!failed && rename(path, hash->filepath) < 0) { - index_set_error(hash->index, "rename(%s, %s) failed: %m", - path, hash->filepath); + index_set_error(hash->index, "rename(%s, %s) failed: " + "%m", path, hash->filepath); failed = TRUE; } @@ -356,11 +362,10 @@ if (errno != ENOSPC) return FALSE; - if (hash->mmap_base != NULL) - (void)munmap(hash->mmap_base, hash->mmap_length); + hash_munmap(hash); hash->mmap_length = HASH_FILE_SIZE(hash_size); - hash->mmap_base = mmap_anonymous(hash->mmap_length); + hash->mmap_base = mmap_anon(hash->mmap_length); hash_build(hash->index, hash->mmap_base, hash_size); /* make sure it doesn't exist anymore */ @@ -457,7 +462,8 @@ if (hash->header->used_records > hash->size * FORCED_REBUILD_PERCENTAGE / 100) { /* we really need a rebuild. */ - mail_hash_rebuild(hash); + if (!mail_hash_rebuild(hash)) + return; } /* place the hash into first free record after wanted position */ @@ -468,7 +474,8 @@ at least 1/5 of hash was empty. except, if the header contained invalid record count for some reason. rebuild.. */ i_error("Hash file was 100%% full, rebuilding"); - mail_hash_rebuild(hash); + if (!mail_hash_rebuild(hash)) + return; rec = hash_find_uid_or_free(hash, uid); i_assert(rec != NULL);
--- a/src/lib-index/mail-index-compress.c Fri Sep 13 03:01:23 2002 +0300 +++ b/src/lib-index/mail-index-compress.c Sat Sep 14 14:09:42 2002 +0300 @@ -10,17 +10,48 @@ #include <stdio.h> #include <unistd.h> +int mail_index_truncate(MailIndex *index) +{ + uoff_t empty_space, truncate_threshold; + + i_assert(index->lock_type == MAIL_LOCK_EXCLUSIVE); + + if (index->mmap_full_length <= INDEX_FILE_MIN_SIZE) + return TRUE; + + /* really truncate the file only when it's almost empty */ + empty_space = index->mmap_full_length - index->mmap_used_length; + truncate_threshold = + index->mmap_full_length / 100 * INDEX_TRUNCATE_PERCENTAGE; + + if (empty_space > truncate_threshold) { + index->mmap_full_length = index->mmap_used_length + + (empty_space * INDEX_TRUNCATE_KEEP_PERCENTAGE / 100); + + /* keep the size record-aligned */ + index->mmap_full_length = + (index->mmap_full_length - sizeof(MailIndexHeader)) % + sizeof(MailIndexRecord); + + if (ftruncate(index->fd, (off_t)index->mmap_full_length) < 0) + return index_set_syscall_error(index, "ftruncate()"); + + index->header->sync_id++; + } + + return TRUE; +} + int mail_index_compress(MailIndex *index) { MailIndexRecord *rec, *hole_rec, *end_rec; - size_t fsize; if (!index->set_lock(index, MAIL_LOCK_EXCLUSIVE)) return FALSE; if (index->header->first_hole_position == 0) { /* we don't need to compress after all. shouldn't happen.. */ - index->header->flags &= ~MAIL_INDEX_FLAG_CACHE_FIELDS; + index->header->flags &= ~MAIL_INDEX_FLAG_COMPRESS; return TRUE; } @@ -35,7 +66,7 @@ /* first actually compress the data */ end_rec = (MailIndexRecord *) ((char *) index->mmap_base + - index->mmap_length); + index->mmap_used_length); hole_rec = (MailIndexRecord *) ((char *) index->mmap_base + index->header->first_hole_position); rec = hole_rec + index->header->first_hole_records; @@ -50,24 +81,20 @@ } /* truncate the file to get rid of the extra records */ - fsize = (size_t) ((char *) hole_rec - (char *) index->mmap_base); - if (ftruncate(index->fd, (off_t)fsize) == -1) { - index_set_syscall_error(index, "ftruncate()"); + index->mmap_used_length = (size_t) ((char *) hole_rec - + (char *) index->mmap_base); + if (!mail_index_truncate(index)) return FALSE; - } /* update headers */ index->header->first_hole_position = 0; index->header->first_hole_records = 0; - index->header->sync_id++; - index->dirty_mmap = TRUE; - /* make sure the whole file is synced before removing rebuild-flag */ - if (!mail_index_fmsync(index, fsize)) + if (!mail_index_fmsync(index, index->mmap_used_length)) return FALSE; - index->header->flags &= ~(MAIL_INDEX_FLAG_CACHE_FIELDS | + index->header->flags &= ~(MAIL_INDEX_FLAG_COMPRESS | MAIL_INDEX_FLAG_REBUILD); return TRUE; } @@ -88,6 +115,9 @@ memset(&data_hdr, 0, sizeof(data_hdr)); data_hdr.indexid = index->indexid; if (write_full(fd, &data_hdr, sizeof(data_hdr)) < 0) { + if (errno == ENOSPC) + index->nodiskspace = TRUE; + index_set_error(index, "Error writing to temp index data " "%s: %m", path); return FALSE; @@ -110,6 +140,9 @@ if (write_full(fd, mmap_data + rec->data_position, rec->data_size) < 0) { + if (errno == ENOSPC) + index->nodiskspace = TRUE; + index_set_error(index, "Error writing to temp index " "data %s: %m", path); return FALSE; @@ -127,7 +160,10 @@ int mail_index_compress_data(MailIndex *index) { const char *temppath, *datapath; - int fd; + int fd, failed; + + if (index->anon_mmap) + return TRUE; /* write the data into temporary file updating the offsets in index while doing it. if we fail (especially if out of disk space/quota) @@ -136,32 +172,45 @@ return FALSE; fd = mail_index_create_temp_file(index, &temppath); - if (fd == -1) - return FALSE; - - if (!mail_index_copy_data(index, fd, temppath)) { - (void)close(fd); - (void)unlink(temppath); + if (fd == -1) { + if (errno == ENOSPC) + index->nodiskspace = TRUE; return FALSE; } - /* now, close the old data file and rename the temp file into - new data file */ - mail_index_data_free(index->data); - (void)close(fd); + failed = !mail_index_copy_data(index, fd, temppath); + + if (close(fd) < 0) + index_file_set_syscall_error(index, temppath, "close()"); + + if (!failed) { + /* now, rename the temp file to new data file */ + mail_index_data_free(index->data); - datapath = t_strconcat(index->filepath, DATA_FILE_PREFIX, NULL); - if (rename(temppath, datapath) < 0) { - index_set_error(index, "rename(%s, %s) failed: %m", - temppath, datapath); + datapath = t_strconcat(index->filepath, DATA_FILE_PREFIX, NULL); + if (rename(temppath, datapath) < 0) { + if (errno == ENOSPC) + index->nodiskspace = TRUE; + + index_set_error(index, "rename(%s, %s) failed: %m", + temppath, datapath); + failed = TRUE; + } + } + + if (failed) { + if (unlink(temppath) < 0) { + index_file_set_syscall_error(index, temppath, + "unlink()"); + } return FALSE; } /* make sure the whole file is synced before removing rebuild-flag */ - if (!mail_index_fmsync(index, index->mmap_length)) + if (!mail_index_fmsync(index, index->mmap_used_length)) return FALSE; - index->header->flags &= ~(MAIL_INDEX_FLAG_CACHE_FIELDS | + index->header->flags &= ~(MAIL_INDEX_FLAG_COMPRESS_DATA | MAIL_INDEX_FLAG_REBUILD); return mail_index_data_open(index);
--- a/src/lib-index/mail-index-data.c Fri Sep 13 03:01:23 2002 +0300 +++ b/src/lib-index/mail-index-data.c Sat Sep 14 14:09:42 2002 +0300 @@ -1,6 +1,7 @@ /* Copyright (C) 2002 Timo Sirainen */ #include "lib.h" +#include "file-set-size.h" #include "mmap-util.h" #include "write-full.h" #include "mail-index.h" @@ -13,12 +14,18 @@ #define DATA_FILE_POSITION(data, rec) \ ((uoff_t) ((char *) (rec) - (char *) ((data)->mmap_base))) -/* Never compress the file if it's smaller than this (50kB) */ +/* Never compress the file if it's smaller than this */ #define COMPRESS_MIN_SIZE (1024*50) -/* Compress the file when deleted space reaches 20% of total size */ +/* Compress the file when deleted space reaches n% of total size */ #define COMPRESS_PERCENTAGE 20 +/* Initial size for the file */ +#define INDEX_DATA_INITIAL_SIZE (sizeof(MailIndexDataHeader) + 10240) + +/* When more space is needed, grow the file n% larger than the previous size */ +#define INDEX_DATA_GROW_PERCENTAGE 10 + struct _MailIndexData { MailIndex *index; @@ -26,16 +33,21 @@ char *filepath; void *mmap_base; - size_t mmap_length; + size_t mmap_full_length; + size_t mmap_used_length; + MailIndexDataHeader *header; + + unsigned int anon_mmap:1; unsigned int dirty_mmap:1; }; -void index_data_set_corrupted(MailIndexData *data, const char *fmt, ...) +int index_data_set_corrupted(MailIndexData *data, const char *fmt, ...) { va_list va; INDEX_MARK_CORRUPTED(data->index); + data->index->inconsistent = TRUE; va_start(va, fmt); t_push(); @@ -43,43 +55,73 @@ data->filepath, t_strdup_vprintf(fmt, va)); t_pop(); va_end(va); + + return FALSE; } -static void index_data_set_syscall_error(MailIndexData *data, - const char *function) +static int index_data_set_syscall_error(MailIndexData *data, + const char *function) { i_assert(function != NULL); index_set_error(data->index, "%s failed with index data file %s: %m", function, data->filepath); + return FALSE; } static int mmap_update(MailIndexData *data, uoff_t pos, size_t size) { - if (!data->dirty_mmap && (size != 0 && pos+size <= data->mmap_length)) - return TRUE; + MailIndexDataHeader *hdr; + + if (size != 0) { + if (pos + size <= data->mmap_used_length) + return TRUE; + + if (pos + size <= data->mmap_full_length) { + data->mmap_used_length = data->header->used_file_size; + if (data->mmap_used_length <= data->mmap_full_length) + return TRUE; - if (data->mmap_base != NULL) - (void)munmap(data->mmap_base, data->mmap_length); + /* file size changed, re-mmap() */ + } + } + + data->header = NULL; + data->mmap_used_length = 0; - data->mmap_base = mmap_rw_file(data->fd, &data->mmap_length); + if (data->mmap_base != NULL) { + if (munmap(data->mmap_base, data->mmap_full_length) < 0) + index_data_set_syscall_error(data, "munmap()"); + } + + data->mmap_base = mmap_rw_file(data->fd, &data->mmap_full_length); if (data->mmap_base == MAP_FAILED) { data->mmap_base = NULL; - index_data_set_syscall_error(data, "mmap()"); - return FALSE; - } else if (data->mmap_length < sizeof(MailIndexDataHeader)) { - index_data_set_corrupted(data, "File too small"); + return index_data_set_syscall_error(data, "mmap()"); + } + + if (data->mmap_full_length < sizeof(MailIndexDataHeader)) + return index_data_set_corrupted(data, "File too small"); + + hdr = data->mmap_base; + + if (hdr->used_file_size > data->mmap_full_length) { + index_data_set_corrupted(data, "used_file_size larger than " + "real file size (%"PRIuUOFF_T + " vs %"PRIuSIZE_T")", + hdr->used_file_size, + data->mmap_full_length); return FALSE; - } else { - data->dirty_mmap = FALSE; - return TRUE; } + + data->mmap_used_length = hdr->used_file_size; + data->header = hdr; + return TRUE; } int mail_index_data_open(MailIndex *index) { MailIndexData *data; - MailIndexDataHeader *hdr; const char *path; int fd; @@ -98,7 +140,6 @@ data->index = index; data->fd = fd; data->filepath = i_strdup(path); - data->dirty_mmap = TRUE; index->data = data; @@ -108,8 +149,7 @@ } /* verify that this really is the data file for wanted index */ - hdr = data->mmap_base; - if (hdr->indexid != index->indexid) { + if (data->header->indexid != index->indexid) { INDEX_MARK_CORRUPTED(index); index_set_error(index, "IndexID mismatch for data file %s", path); @@ -120,19 +160,19 @@ return TRUE; } -static const char *init_data_file(MailIndex *index, int fd, - const char *temppath) +static const char *init_data_file(MailIndex *index, MailIndexDataHeader *hdr, + int fd, const char *temppath) { - MailIndexDataHeader hdr; const char *realpath; - /* write header */ - memset(&hdr, 0, sizeof(hdr)); - hdr.indexid = index->indexid; + if (write_full(fd, &hdr, sizeof(hdr)) < 0) { + index_file_set_syscall_error(index, temppath, "write_full()"); + return NULL; + } - if (write_full(fd, &hdr, sizeof(hdr)) < 0) { - index_set_error(index, "Error writing to temp index data " - "%s: %m", temppath); + if (file_set_size(fd, INDEX_DATA_INITIAL_SIZE) < 0) { + index_file_set_syscall_error(index, temppath, + "file_set_size()"); return NULL; } @@ -142,7 +182,6 @@ if (rename(temppath, realpath) < 0) { index_set_error(index, "rename(%s, %s) failed: %m", temppath, realpath); - (void)unlink(temppath); return NULL; } @@ -151,26 +190,56 @@ int mail_index_data_create(MailIndex *index) { + MailIndexDataHeader hdr; MailIndexData *data; const char *temppath, *realpath; int fd; - fd = mail_index_create_temp_file(index, &temppath); - if (fd == -1) - return FALSE; + memset(&hdr, 0, sizeof(hdr)); + hdr.indexid = index->indexid; + hdr.used_file_size = sizeof(hdr); + + realpath = NULL; - realpath = init_data_file(index, fd, temppath); - if (realpath == NULL) { - (void)close(fd); - (void)unlink(temppath); - return FALSE; + /* we'll do anon-mmaping only if initially requested. if we fail + because of out of disk space, we'll just let the main index code + know it and fail. */ + if (index->nodiskspace) { + fd = -1; + } else { + fd = mail_index_create_temp_file(index, &temppath); + if (fd != -1) { + if (errno == ENOSPC) + index->nodiskspace = TRUE; + return FALSE; + } + + realpath = init_data_file(index, &hdr, fd, temppath); + if (realpath == NULL) { + if (errno == ENOSPC) + index->nodiskspace = TRUE; + + (void)close(fd); + (void)unlink(temppath); + return FALSE; + } } data = i_new(MailIndexData, 1); + + if (fd == -1) { + data->mmap_full_length = INDEX_DATA_INITIAL_SIZE; + data->mmap_base = mmap_anon(index->mmap_full_length); + memcpy(data->mmap_base, &hdr, sizeof(hdr)); + + data->anon_mmap = TRUE; + data->filepath = i_strdup("(in-memory index data)"); + } else { + data->filepath = i_strdup(realpath); + } + data->index = index; data->fd = fd; - data->filepath = i_strdup(realpath); - data->dirty_mmap = TRUE; index->data = data; return TRUE; @@ -181,11 +250,22 @@ data->index->data = NULL; if (data->mmap_base != NULL) { - (void)munmap(data->mmap_base, data->mmap_length); + if (data->anon_mmap) { + if (munmap_anon(data->mmap_base, + data->mmap_full_length) < 0) { + index_data_set_syscall_error(data, + "munmap_anon()"); + } + } else { + if (munmap(data->mmap_base, data->mmap_full_length) < 0) + index_data_set_syscall_error(data, "munmap()"); + } + data->mmap_base = NULL; } - (void)close(data->fd); + if (data->fd != -1) + (void)close(data->fd); i_free(data->filepath); i_free(data); } @@ -194,78 +274,119 @@ { MailIndexDataHeader hdr; - if (ftruncate(data->fd, sizeof(MailIndexDataHeader)) < 0) { - index_data_set_syscall_error(data, "ftruncate()"); - return FALSE; - } - memset(&hdr, 0, sizeof(hdr)); hdr.indexid = data->index->indexid; - hdr.deleted_space = 0; + hdr.used_file_size = sizeof(hdr); - if (lseek(data->fd, 0, SEEK_SET) < 0) { - index_data_set_syscall_error(data, "lseek()"); - return FALSE; + if (data->anon_mmap) { + memcpy(data->mmap_base, &hdr, sizeof(hdr)); + return TRUE; } + if (file_set_size(data->fd, INDEX_DATA_INITIAL_SIZE) < 0) { + if (errno == ENOSPC) + data->index->nodiskspace = TRUE; + return index_data_set_syscall_error(data, "file_set_size()"); + } + + if (lseek(data->fd, 0, SEEK_SET) < 0) + return index_data_set_syscall_error(data, "lseek()"); + if (write_full(data->fd, &hdr, sizeof(hdr)) < 0) { - index_data_set_syscall_error(data, "write_full()"); - return FALSE; + if (errno == ENOSPC) + data->index->nodiskspace = TRUE; + return index_data_set_syscall_error(data, "write_full()"); } return TRUE; } -void mail_index_data_new_data_notify(MailIndexData *data) +static int mail_index_data_grow(MailIndexData *data, size_t size) { - data->dirty_mmap = TRUE; + void *base; + uoff_t new_fsize; + off_t pos; + + new_fsize = data->header->used_file_size + size; + new_fsize += new_fsize / 100 * INDEX_DATA_GROW_PERCENTAGE; + + if (data->anon_mmap) { + i_assert(new_fsize < SSIZE_T_MAX); + + base = mremap_anon(data->mmap_base, data->mmap_full_length, + (size_t)new_fsize, MREMAP_MAYMOVE); + if (base == MAP_FAILED) { + index_data_set_syscall_error(data, "mremap_anon()"); + return FALSE; + } + + data->mmap_base = base; + data->mmap_full_length = (size_t)new_fsize; + return TRUE; + } + + pos = lseek(data->fd, 0, SEEK_END); + if (pos < 0) + return index_data_set_syscall_error(data, "lseek()"); + + if (data->header->used_file_size + size <= (uoff_t)pos) { + /* no need to grow, just update mmap */ + if (!mmap_update(data, 0, 0)) + return FALSE; + + i_assert(data->mmap_full_length >= (uoff_t)pos); + return TRUE; + } + + if (pos < (int)sizeof(MailIndexDataHeader)) + return index_data_set_corrupted(data, "Header is missing"); + + if (file_set_size(data->fd, new_fsize) < 0) { + if (errno == ENOSPC) + data->index->nodiskspace = TRUE; + return index_data_set_syscall_error(data, "file_set_size()"); + } + + return mmap_update(data, 0, 0); } uoff_t mail_index_data_append(MailIndexData *data, const void *buffer, size_t size) { - off_t pos; + uoff_t offset; i_assert((size & (MEM_ALIGN_SIZE-1)) == 0); + i_assert(data->index->lock_type == MAIL_LOCK_EXCLUSIVE); - pos = lseek(data->fd, 0, SEEK_END); - if (pos < 0) { - index_data_set_syscall_error(data, "lseek()"); - return 0; + if (size > data->mmap_full_length || + data->mmap_full_length - size < data->header->used_file_size) { + if (!mail_index_data_grow(data, size)) + return 0; } - if (pos < (int)sizeof(MailIndexDataHeader)) { - index_data_set_corrupted(data, "Header is missing"); - return 0; - } + offset = data->header->used_file_size; + i_assert(offset + size <= data->mmap_full_length); - if (write_full(data->fd, buffer, size) < 0) { - index_data_set_syscall_error(data, "write_full()"); - return 0; - } + memcpy((char *) data->mmap_base + offset, buffer, size); + data->header->used_file_size += size; - mail_index_data_new_data_notify(data); - return (uoff_t)pos; + return offset; } int mail_index_data_add_deleted_space(MailIndexData *data, size_t data_size) { - MailIndexDataHeader *hdr; uoff_t max_del_space; i_assert(data->index->lock_type == MAIL_LOCK_EXCLUSIVE); - /* make sure the whole file is mmaped */ - if (!mmap_update(data, 0, 0)) - return FALSE; - - hdr = data->mmap_base; - hdr->deleted_space += data_size; + data->header->deleted_space += data_size; /* see if we've reached the max. deleted space in file */ - if (data->mmap_length >= COMPRESS_MIN_SIZE) { - max_del_space = data->mmap_length / 100 * COMPRESS_PERCENTAGE; - if (hdr->deleted_space >= max_del_space) + if (data->header->used_file_size >= COMPRESS_MIN_SIZE && + (data->index->header->flags & MAIL_INDEX_FLAG_COMPRESS_DATA) == 0) { + max_del_space = data->header->used_file_size / + 100 * COMPRESS_PERCENTAGE; + if (data->header->deleted_space >= max_del_space) data->index->set_flags |= MAIL_INDEX_FLAG_COMPRESS_DATA; } return TRUE; @@ -273,17 +394,16 @@ int mail_index_data_sync_file(MailIndexData *data) { + if (data->anon_mmap) + return TRUE; + if (data->mmap_base != NULL) { - if (msync(data->mmap_base, data->mmap_length, MS_SYNC) < 0) { - index_data_set_syscall_error(data, "msync()"); - return FALSE; - } + if (msync(data->mmap_base, data->mmap_used_length, MS_SYNC) < 0) + return index_data_set_syscall_error(data, "msync()"); } - if (fsync(data->fd) < 0) { - index_data_set_syscall_error(data, "fsync()"); - return FALSE; - } + if (fsync(data->fd) < 0) + return index_data_set_syscall_error(data, "fsync()"); return TRUE; } @@ -296,24 +416,21 @@ uoff_t pos, max_pos; if (index_rec->data_position == 0) { - /* data not yet written to record */ - index_reset_error(data->index); + /* data not yet written to record - FIXME: is this an error? */ return NULL; } if (!mmap_update(data, index_rec->data_position, index_rec->data_size)) return NULL; - if (index_rec->data_position > data->mmap_length || - (data->mmap_length - + if (index_rec->data_position > data->mmap_used_length || + (data->mmap_used_length - index_rec->data_position < index_rec->data_size)) { - index_data_set_corrupted(data, "Given data size larger than " - "file size (%"PRIuUOFF_T - " + %u > %"PRIuSIZE_T") for record %u", - index_rec->data_position, - index_rec->data_size, - data->mmap_length, - index_rec->uid); + index_data_set_corrupted(data, + "Given data size larger than file size " + "(%"PRIuUOFF_T" + %u > %"PRIuSIZE_T") for record %u", + index_rec->data_position, index_rec->data_size, + data->mmap_used_length, index_rec->uid); return NULL; } @@ -328,10 +445,10 @@ this as it won't crash and is quite likely noticed later. */ if (pos + sizeof(MailIndexDataRecord) > max_pos || pos + DATA_RECORD_SIZE(rec) > max_pos) { - index_data_set_corrupted(data, "Field size points " - "outside file (%"PRIuUOFF_T - " / %"PRIuUOFF_T")", - pos, max_pos); + index_data_set_corrupted(data, + "Field %d size points outside file " + "(%"PRIuUOFF_T" / %"PRIuUOFF_T") for record %u", + (int)field, pos, max_pos, index_rec->uid); break; } @@ -385,7 +502,7 @@ int i; if (rec->full_field_size > INT_MAX) { - /* we already check that the full_field_size is within file, + /* we already checked that the full_field_size is within file, so this can happen only if the file really is huge.. */ index_data_set_corrupted(data, "full_field_size (%u) > INT_MAX", rec->full_field_size); @@ -411,6 +528,6 @@ if (!mmap_update(data, 0, 0)) return NULL; - *size = data->mmap_length; + *size = data->mmap_used_length; return data->mmap_base; }
--- a/src/lib-index/mail-index-data.h Fri Sep 13 03:01:23 2002 +0300 +++ b/src/lib-index/mail-index-data.h Sat Sep 14 14:09:42 2002 +0300 @@ -10,10 +10,6 @@ /* Truncate the data file and update it's indexid */ int mail_index_data_reset(MailIndexData *data); -/* Needs to be called whenever new messages are added. File must never - be shrinked while it's open. */ -void mail_index_data_new_data_notify(MailIndexData *data); - /* Append new data at the end of the file. Returns the position in file where the data begins, or 0 if error occured. */ uoff_t mail_index_data_append(MailIndexData *data, const void *buffer, @@ -45,6 +41,6 @@ /* "Error in index data file %s: ...". Also marks the index file as corrupted. */ -void index_data_set_corrupted(MailIndexData *data, const char *fmt, ...); +int index_data_set_corrupted(MailIndexData *data, const char *fmt, ...); #endif
--- a/src/lib-index/mail-index-fsck.c Fri Sep 13 03:01:23 2002 +0300 +++ b/src/lib-index/mail-index-fsck.c Sat Sep 14 14:09:42 2002 +0300 @@ -2,6 +2,7 @@ #include "lib.h" #include "mail-index.h" +#include "mail-index-util.h" int mail_index_fsck(MailIndex *index) { @@ -32,7 +33,7 @@ rec = (MailIndexRecord *) ((char *) index->mmap_base + sizeof(MailIndexHeader)); end_rec = (MailIndexRecord *) ((char *) index->mmap_base + - index->mmap_length); + index->mmap_used_length); max_uid = 0; for (; rec < end_rec; rec++) { @@ -52,9 +53,9 @@ } if (rec->uid < max_uid) { - i_error("fsck %s: UIDs are not ordered (%u < %u)", - index->filepath, rec->uid, max_uid); - return mail_index_rebuild_all(index); + index_set_corrupted(index, "UIDs are not ordered " + "(%u < %u)", rec->uid, max_uid); + return FALSE; } max_uid = rec->uid;
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-index/mail-index-open.c Sat Sep 14 14:09:42 2002 +0300 @@ -0,0 +1,520 @@ +/* Copyright (C) 2002 Timo Sirainen */ + +#include "lib.h" +#include "ioloop.h" +#include "file-lock.h" +#include "file-set-size.h" +#include "hostpid.h" +#include "mmap-util.h" +#include "write-full.h" +#include "mail-index.h" +#include "mail-index-data.h" +#include "mail-index-util.h" +#include "mail-hash.h" +#include "mail-lockdir.h" +#include "mail-modifylog.h" +#include "mail-custom-flags.h" + +#include <stdio.h> +#include <stdlib.h> +#include <unistd.h> +#include <fcntl.h> + +static const char *index_file_prefixes[] = + { "data", "hash", "log", "log.2", NULL }; + +static int delete_index(const char *path) +{ + char tmp[1024]; + int i; + + /* main index */ + if (unlink(path) < 0) + return FALSE; + + for (i = 0; index_file_prefixes[i] != NULL; i++) { + i_snprintf(tmp, sizeof(tmp), "%s.%s", + path, index_file_prefixes[i]); + if (unlink(tmp) < 0) + return FALSE; + i++; + } + + return TRUE; +} + +static int read_and_verify_header(int fd, MailIndexHeader *hdr, + int check_version) +{ + /* read the header */ + if (lseek(fd, 0, SEEK_SET) != 0) + return FALSE; + + if (read(fd, hdr, sizeof(MailIndexHeader)) != sizeof(MailIndexHeader)) + return FALSE; + + /* check the compatibility */ + return hdr->compat_data[1] == MAIL_INDEX_COMPAT_FLAGS && + hdr->compat_data[2] == sizeof(unsigned int) && + hdr->compat_data[3] == sizeof(time_t) && + hdr->compat_data[4] == sizeof(uoff_t) && + hdr->compat_data[5] == MEM_ALIGN_SIZE && + (!check_version || hdr->compat_data[0] == MAIL_INDEX_VERSION); +} + +/* Returns TRUE if we're compatible with given index file. May delete the + file if it's from older version. */ +static int mail_check_compatible_index(MailIndex *index, const char *path) +{ + MailIndexHeader hdr; + int fd, compatible; + + fd = open(path, O_RDONLY); + if (fd == -1) { + if (errno != ENOENT) + index_file_set_syscall_error(index, path, "open()"); + return FALSE; + } + + compatible = read_and_verify_header(fd, &hdr, FALSE); + if (hdr.compat_data[0] != MAIL_INDEX_VERSION) { + /* version mismatch */ + compatible = FALSE; + if (hdr.compat_data[0] < MAIL_INDEX_VERSION) { + /* of older version, we don't need it anymore */ + (void)delete_index(path); + } + } + + (void)close(fd); + return compatible; +} + +/* Returns a file name of compatible index */ +static const char *mail_find_index(MailIndex *index) +{ + const char *name; + char path[1024]; + + hostpid_init(); + + /* first try .imap.index-<hostname> */ + name = t_strconcat(INDEX_FILE_PREFIX "-", my_hostname, NULL); + i_snprintf(path, sizeof(path), "%s/%s", index->dir, name); + if (mail_check_compatible_index(index, path)) + return name; + + /* then try the generic .imap.index */ + name = INDEX_FILE_PREFIX; + i_snprintf(path, sizeof(path), "%s/%s", index->dir, name); + if (mail_check_compatible_index(index, path)) + return name; + + return NULL; +} + +static int mail_index_open_init(MailIndex *index, MailIndexHeader *hdr, + int update_recent) +{ + /* update \Recent message counters */ + if (update_recent && hdr->last_nonrecent_uid != hdr->next_uid-1) { + /* keep last_recent_uid to next_uid-1 */ + if (index->lock_type == MAIL_LOCK_SHARED) { + if (!index->set_lock(index, MAIL_LOCK_UNLOCK)) + return FALSE; + } + + if (!index->set_lock(index, MAIL_LOCK_EXCLUSIVE)) + return FALSE; + + index->first_recent_uid = index->header->last_nonrecent_uid+1; + index->header->last_nonrecent_uid = index->header->next_uid-1; + } else { + index->first_recent_uid = hdr->last_nonrecent_uid+1; + } + + if (hdr->next_uid >= INT_MAX-1024) { + /* UID values are getting too high, rebuild index */ + index->set_flags |= MAIL_INDEX_FLAG_REBUILD; + } + + if (index->lock_type == MAIL_LOCK_EXCLUSIVE) { + /* finally reset the modify log marks, fsck or syncing might + have deleted some messages, and since we're only just + opening the index, there's no need to remember them */ + if (!mail_modifylog_mark_synced(index->modifylog)) + return FALSE; + } + + return TRUE; +} + +static int index_open_and_fix(MailIndex *index, MailIndexHeader *hdr, + int update_recent) +{ + /* open/create the index files */ + if (!mail_index_data_open(index)) { + if ((index->set_flags & MAIL_INDEX_FLAG_REBUILD) == 0) + return FALSE; + + /* data file is corrupted, need to rebuild index */ + hdr->flags |= MAIL_INDEX_FLAG_REBUILD; + index->set_flags = 0; + + if (!mail_index_data_create(index)) + return FALSE; + } + + /* custom flags file needs to be open before + rebuilding index */ + if (!mail_custom_flags_open_or_create(index)) + return FALSE; + + if (hdr->flags & MAIL_INDEX_FLAG_REBUILD) { + /* index is corrupted, rebuild */ + if (!index->rebuild(index)) + return FALSE; + + /* no inconsistency problems while still opening + the index */ + index->inconsistent = FALSE; + } + + if (!mail_hash_open_or_create(index)) + return FALSE; + if (!mail_modifylog_open_or_create(index)) + return FALSE; + + if (hdr->flags & MAIL_INDEX_FLAG_FSCK) { + /* index needs fscking */ + if (!index->fsck(index)) + return FALSE; + } + + if (hdr->flags & MAIL_INDEX_FLAG_COMPRESS) { + /* remove deleted blocks from index file */ + if (!mail_index_compress(index)) + return FALSE; + } + + if (hdr->flags & MAIL_INDEX_FLAG_REBUILD_HASH) { + if (!mail_hash_rebuild(index->hash)) + return FALSE; + } + + if (hdr->flags & MAIL_INDEX_FLAG_CACHE_FIELDS) { + /* need to update cached fields */ + if (!mail_index_update_cache(index)) + return FALSE; + } + + if (hdr->flags & MAIL_INDEX_FLAG_COMPRESS_DATA) { + /* remove unused space from index data file. + keep after cache_fields which may move data + and create unused space.. */ + if (!mail_index_compress_data(index)) + return FALSE; + } + + if (!index->sync(index)) + return FALSE; + + if (!mail_index_open_init(index, hdr, update_recent)) + return FALSE; + + if (!index->set_lock(index, MAIL_LOCK_UNLOCK)) + return FALSE; + + return TRUE; +} + +static int mail_index_open_file(MailIndex *index, const char *path, + int update_recent) +{ + MailIndexHeader hdr; + int fd; + + /* the index file should already be checked that it exists and + we're compatible with it. */ + + fd = open(path, O_RDWR); + if (fd == -1) + return index_file_set_syscall_error(index, path, "open()"); + + /* if index is being created, we'll wait here until it's finished */ + if (file_wait_lock(fd, F_RDLCK) < 0) { + index_file_set_syscall_error(index, path, "file_wait_lock()"); + (void)close(fd); + return FALSE; + } + + /* check the compatibility anyway just to be sure */ + if (!read_and_verify_header(fd, &hdr, TRUE)) { + index_set_error(index, "Non-compatible index file %s", path); + (void)close(fd); + return FALSE; + } + + if (file_wait_lock(fd, F_UNLCK) < 0) { + index_file_set_syscall_error(index, path, "file_wait_lock()"); + (void)close(fd); + return FALSE; + } + + index->fd = fd; + index->filepath = i_strdup(path); + index->indexid = hdr.indexid; + + if (!index_open_and_fix(index, &hdr, update_recent)) { + mail_index_close(index); + return FALSE; + } + + return TRUE; +} + +static int mail_index_init_new_file(MailIndex *index, MailIndexHeader *hdr, + int fd, const char *path, + const char **index_path) +{ + off_t fsize; + + *index_path = NULL; + + if (write_full(fd, hdr, sizeof(MailIndexHeader)) < 0) { + index_file_set_syscall_error(index, path, "write_full()"); + return FALSE; + } + + fsize = sizeof(MailIndexHeader) + + INDEX_MIN_RECORDS_COUNT * sizeof(MailIndexRecord); + if (file_set_size(fd, (off_t)fsize) < 0) { + index_file_set_syscall_error(index, path, "file_set_size()"); + return FALSE; + } + + if (file_wait_lock(fd, F_WRLCK) < 0) { + index_file_set_syscall_error(index, path, "file_wait_lock()"); + return FALSE; + } + + /* move the temp index into the real one. we also need to figure + out what to call ourself on the way. */ + *index_path = t_strconcat(index->dir, "/"INDEX_FILE_PREFIX, NULL); + if (link(path, *index_path) == 0) { + if (unlink(path) < 0) { + /* doesn't really matter, log anyway */ + index_file_set_syscall_error(index, path, "unlink()"); + } + } else { + if (errno != EEXIST) { + /* fatal error */ + index_set_error(index, "link(%s, %s) failed: %m", + path, *index_path); + return FALSE; + } + + if (getenv("OVERWRITE_INCOMPATIBLE_INDEX") != NULL) { + /* don't try to support different architectures, + just overwrite the index if it's already there. */ + } else { + /* fallback to .imap.index-hostname - we require each + system to have a different hostname so it's safe to + override previous index as well */ + hostpid_init(); + + *index_path = t_strconcat(*index_path, "-", + my_hostname, NULL); + } + + if (rename(path, *index_path) < 0) { + index_set_error(index, "rename(%s, %s) failed: %m", + path, *index_path); + return FALSE; + } + } + + return TRUE; +} + +static int mail_index_create(MailIndex *index, int *dir_unlocked, + int update_recent) +{ + MailIndexHeader hdr; + const char *path, *index_path; + int fd; + + *dir_unlocked = FALSE; + index_path = NULL; + + mail_index_init_header(&hdr); + + if (index->nodiskspace) { + /* don't even bother trying to create it */ + fd = -1; + } else { + /* first create the index into temporary file. */ + fd = mail_index_create_temp_file(index, &path); + if (fd != -1) { + if (!mail_index_init_new_file(index, &hdr, fd, + path, &index_path)) { + int old_errno = errno; + + (void)close(fd); + (void)unlink(path); + fd = -1; + + errno = old_errno; + } + } + + if (fd == -1 && errno != ENOSPC) { + /* fatal failure */ + return FALSE; + } + } + + if (fd == -1) { + /* no space for index files, keep it in memory */ + index->mmap_used_length = sizeof(MailIndexHeader); + index->mmap_full_length = INDEX_FILE_MIN_SIZE; + index->mmap_base = mmap_anon(index->mmap_full_length); + + memcpy(index->mmap_base, &hdr, sizeof(hdr)); + + index->anon_mmap = TRUE; + index->filepath = i_strdup("(in-memory index)"); + } else { + index->filepath = i_strdup(index_path); + } + + index->fd = fd; + index->indexid = hdr.indexid; + + /* the fd is actually already locked, now we're just making it + clear to the indexing code. */ + if (!index->set_lock(index, MAIL_LOCK_EXCLUSIVE)) { + mail_index_close(index); + return FALSE; + } + + /* it's not good to keep the directory locked too long. our index file + is locked which is enough. */ + if (!*dir_unlocked && mail_index_lock_dir(index, MAIL_LOCK_UNLOCK)) + *dir_unlocked = TRUE; + + do { + if (!mail_custom_flags_open_or_create(index)) + break; + if (!mail_index_data_create(index)) + break; + + if (!index->rebuild(index)) { + if (!index->anon_mmap && index->nodiskspace) { + /* we're out of disk space, keep it in + memory this time */ + mail_index_close(index); + + index->nodiskspace = TRUE; + return mail_index_create(index, dir_unlocked, + update_recent); + } + break; + } + + if (!mail_hash_create(index)) + break; + if (!mail_modifylog_create(index)) + break; + + index->inconsistent = FALSE; + + if (!mail_index_open_init(index, index->header, update_recent)) + break; + + if (!index->set_lock(index, MAIL_LOCK_UNLOCK)) + break; + + return TRUE; + } while (0); + + mail_index_close(index); + return FALSE; +} + +void mail_index_init_header(MailIndexHeader *hdr) +{ + memset(hdr, 0, sizeof(MailIndexHeader)); + hdr->compat_data[0] = MAIL_INDEX_VERSION; + hdr->compat_data[1] = MAIL_INDEX_COMPAT_FLAGS; + hdr->compat_data[2] = sizeof(unsigned int); + hdr->compat_data[3] = sizeof(time_t); + hdr->compat_data[4] = sizeof(uoff_t); + hdr->compat_data[5] = MEM_ALIGN_SIZE; + hdr->indexid = ioloop_time; + + /* mark the index requiring rebuild - rebuild() removes this flag + when it succeeds */ + hdr->flags = MAIL_INDEX_FLAG_REBUILD; + + /* set the fields we always want to cache */ + hdr->cache_fields |= FIELD_TYPE_LOCATION | FIELD_TYPE_MESSAGEPART; + + hdr->used_file_size = sizeof(MailIndexHeader); + hdr->uid_validity = ioloop_time; + hdr->next_uid = 1; +} + +int mail_index_open(MailIndex *index, int update_recent) +{ + const char *name, *path; + + i_assert(!index->opened); + + /* this isn't initialized anywhere else */ + index->fd = -1; + + name = mail_find_index(index); + if (name == NULL) + return FALSE; + + path = t_strconcat(index->dir, "/", name, NULL); + if (!mail_index_open_file(index, path, update_recent)) + return FALSE; + + index->opened = TRUE; + return TRUE; +} + +int mail_index_open_or_create(MailIndex *index, int update_recent) +{ + int failed, dir_unlocked; + + i_assert(!index->opened); + + if (mail_index_open(index, update_recent)) + return TRUE; + + /* index wasn't found or it was broken. lock the directory and check + again, just to make sure we don't end up having two index files + due to race condition with another process. */ + if (!mail_index_lock_dir(index, MAIL_LOCK_EXCLUSIVE)) + return FALSE; + + if (mail_index_open(index, update_recent)) { + dir_unlocked = FALSE; + failed = FALSE; + } else { + failed = !mail_index_create(index, &dir_unlocked, + update_recent); + } + + if (!dir_unlocked && !mail_index_lock_dir(index, MAIL_LOCK_UNLOCK)) + return FALSE; + + if (failed) + return FALSE; + + index->opened = TRUE; + return TRUE; +}
--- a/src/lib-index/mail-index-util.c Fri Sep 13 03:01:23 2002 +0300 +++ b/src/lib-index/mail-index-util.c Sat Sep 14 14:09:42 2002 +0300 @@ -11,7 +11,7 @@ #include <unistd.h> #include <fcntl.h> -void index_set_error(MailIndex *index, const char *fmt, ...) +int index_set_error(MailIndex *index, const char *fmt, ...) { va_list va; @@ -26,9 +26,11 @@ i_error("%s", index->error); } + + return FALSE; } -void index_set_corrupted(MailIndex *index, const char *fmt, ...) +int index_set_corrupted(MailIndex *index, const char *fmt, ...) { va_list va; @@ -41,14 +43,29 @@ index->filepath, t_strdup_vprintf(fmt, va)); t_pop(); va_end(va); + + return FALSE; } -void index_set_syscall_error(MailIndex *index, const char *function) +int index_set_syscall_error(MailIndex *index, const char *function) { i_assert(function != NULL); index_set_error(index, "%s failed with index file %s: %m", function, index->filepath); + return FALSE; +} + +int index_file_set_syscall_error(MailIndex *index, const char *filepath, + const char *function) +{ + i_assert(filepath != NULL); + i_assert(function != NULL); + + index_set_error(index, "%s failed with file %s: %m", + function, filepath); + + return FALSE; } void index_reset_error(MailIndex *index) @@ -57,6 +74,8 @@ i_free(index->error); index->error = NULL; } + + index->nodiskspace = FALSE; } int mail_index_create_temp_file(MailIndex *index, const char **path)
--- a/src/lib-index/mail-index-util.h Fri Sep 13 03:01:23 2002 +0300 +++ b/src/lib-index/mail-index-util.h Sat Sep 14 14:09:42 2002 +0300 @@ -2,14 +2,18 @@ #define __MAIL_INDEX_UTIL_H /* Set the current error message */ -void index_set_error(MailIndex *index, const char *fmt, ...) +int index_set_error(MailIndex *index, const char *fmt, ...) __attr_format__(2, 3); /* "Error in index file %s: ...". Also marks the index file as corrupted. */ -void index_set_corrupted(MailIndex *index, const char *fmt, ...); +int index_set_corrupted(MailIndex *index, const char *fmt, ...); /* "%s failed with index file %s: %m" */ -void index_set_syscall_error(MailIndex *index, const char *function); +int index_set_syscall_error(MailIndex *index, const char *function); + +/* "%s failed with file %s: %m" */ +int index_file_set_syscall_error(MailIndex *index, const char *filepath, + const char *function); /* Reset the current error */ void index_reset_error(MailIndex *index);
--- a/src/lib-index/mail-index.c Fri Sep 13 03:01:23 2002 +0300 +++ b/src/lib-index/mail-index.c Sat Sep 14 14:09:42 2002 +0300 @@ -2,61 +2,41 @@ #include "lib.h" #include "ioloop.h" -#include "hostpid.h" +#include "file-lock.h" +#include "file-set-size.h" #include "mmap-util.h" -#include "write-full.h" #include "mail-index.h" #include "mail-index-data.h" #include "mail-index-util.h" #include "mail-hash.h" -#include "mail-lockdir.h" #include "mail-modifylog.h" #include "mail-custom-flags.h" -#include <stdio.h> -#include <stdlib.h> #include <unistd.h> #include <fcntl.h> #include <utime.h> -static const char *index_file_prefixes[] = - { "data", "hash", "log", "log.2", NULL }; - -static int mmap_update(MailIndex *index) +static int mmap_verify(MailIndex *index) { unsigned int extra; - if (!index->dirty_mmap) { - index->header = (MailIndexHeader *) index->mmap_base; - - /* make sure file size hasn't changed */ - if (index->header->sync_id == index->sync_id) - return TRUE; - } + index->mmap_used_length = 0; - if (index->mmap_base != NULL) - (void)munmap(index->mmap_base, index->mmap_length); - - index->mmap_base = mmap_rw_file(index->fd, &index->mmap_length); - if (index->mmap_base == MAP_FAILED) { - index->mmap_base = NULL; - index_set_syscall_error(index, "mmap()"); - return FALSE; - } - - if (index->mmap_length < sizeof(MailIndexHeader)) { + if (index->mmap_full_length < sizeof(MailIndexHeader)) { index_set_corrupted(index, "File too small"); return FALSE; } - extra = (index->mmap_length - sizeof(MailIndexHeader)) % + extra = (index->mmap_full_length - sizeof(MailIndexHeader)) % sizeof(MailIndexRecord); if (extra != 0) { /* partial write or corrupted - truncate the file to valid length */ - index->mmap_length -= extra; - (void)ftruncate(index->fd, (off_t)index->mmap_length); + i_assert(!index->anon_mmap); + + index->mmap_full_length -= extra; + (void)ftruncate(index->fd, (off_t)index->mmap_full_length); } index->last_lookup_seq = 0; @@ -64,19 +44,67 @@ index->header = (MailIndexHeader *) index->mmap_base; index->sync_id = index->header->sync_id; - index->dirty_mmap = FALSE; + + if (index->header->used_file_size > index->mmap_full_length) { + index_set_corrupted(index, "used_file_size larger than real " + "file size (%"PRIuUOFF_T" vs %"PRIuSIZE_T + ")", index->header->used_file_size, + index->mmap_full_length); + return FALSE; + } + + if ((index->header->used_file_size - sizeof(MailIndexHeader)) % + sizeof(MailIndexRecord) != 0) { + index_set_corrupted(index, "Invalid used_file_size in header " + "(%"PRIuUOFF_T")", + index->header->used_file_size); + return FALSE; + } + + index->mmap_used_length = index->header->used_file_size; return TRUE; } +static int mmap_update(MailIndex *index) +{ + if (index->anon_mmap) + return mmap_verify(index); + + if (index->mmap_base != NULL) { + index->header = (MailIndexHeader *) index->mmap_base; + + /* make sure file size hasn't changed */ + if (index->header->sync_id == index->sync_id) { + index->mmap_used_length = index->header->used_file_size; + if (index->mmap_used_length > index->mmap_full_length) { + i_panic("Index file size was grown without " + "updating sync_id"); + } + return TRUE; + } + + (void)munmap(index->mmap_base, index->mmap_full_length); + } + + index->mmap_base = mmap_rw_file(index->fd, &index->mmap_full_length); + if (index->mmap_base == MAP_FAILED) { + index->mmap_base = NULL; + index->mmap_used_length = 0; + index_set_syscall_error(index, "mmap()"); + return FALSE; + } + + return mmap_verify(index); +} + void mail_index_close(MailIndex *index) { index->set_flags = 0; index->set_cache_fields = 0; index->opened = FALSE; - index->updating = FALSE; index->inconsistent = FALSE; - index->dirty_mmap = TRUE; + index->nodiskspace = FALSE; index->lock_type = MAIL_LOCK_UNLOCK; index->header = NULL; @@ -92,7 +120,14 @@ } if (index->mmap_base != NULL) { - (void)munmap(index->mmap_base, index->mmap_length); + if (index->anon_mmap) { + (void)munmap_anon(index->mmap_base, + index->mmap_full_length); + index->anon_mmap = FALSE; + } else { + (void)munmap(index->mmap_base, + index->mmap_full_length); + } index->mmap_base = NULL; } @@ -122,40 +157,40 @@ } } -int mail_index_sync_file(MailIndex *index) +static int mail_index_sync_file(MailIndex *index) { struct utimbuf ut; int failed; + if (index->anon_mmap) + return TRUE; + if (!mail_index_data_sync_file(index->data)) return FALSE; - if (index->mmap_base != NULL) { - if (msync(index->mmap_base, index->mmap_length, MS_SYNC) < 0) { - index_set_syscall_error(index, "msync()"); - return FALSE; - } + if (msync(index->mmap_base, index->mmap_used_length, MS_SYNC) < 0) + return index_set_syscall_error(index, "msync()"); + + failed = FALSE; + + if (index->hash != NULL) { + if (!mail_hash_sync_file(index->hash)) + failed = TRUE; } - failed = FALSE; - if (index->hash != NULL && !mail_hash_sync_file(index->hash)) - failed = TRUE; - if (index->modifylog != NULL && - !mail_modifylog_sync_file(index->modifylog)) - failed = TRUE; + if (index->modifylog != NULL) { + if (!mail_modifylog_sync_file(index->modifylog)) + failed = TRUE; + } /* keep index's modify stamp same as the sync file's stamp */ ut.actime = ioloop_time; ut.modtime = index->file_sync_stamp; - if (utime(index->filepath, &ut) < 0) { - index_set_syscall_error(index, "utime()"); - return FALSE; - } + if (utime(index->filepath, &ut) < 0) + return index_set_syscall_error(index, "utime()"); - if (fsync(index->fd) < 0) { - index_set_syscall_error(index, "fsync()"); - return FALSE; - } + if (fsync(index->fd) < 0) + return index_set_syscall_error(index, "fsync()"); return !failed; } @@ -164,33 +199,18 @@ { i_assert(index->lock_type == MAIL_LOCK_EXCLUSIVE); - if (msync(index->mmap_base, size, MS_SYNC) < 0) { - index_set_syscall_error(index, "msync()"); - return FALSE; - } - if (fsync(index->fd) < 0) { - index_set_syscall_error(index, "fsync()"); - return FALSE; + if (!index->anon_mmap) { + if (msync(index->mmap_base, size, MS_SYNC) < 0) + return index_set_syscall_error(index, "msync()"); + if (fsync(index->fd) < 0) + return index_set_syscall_error(index, "fsync()"); } return TRUE; } -int mail_index_rebuild_all(MailIndex *index) -{ - if (!index->rebuild(index)) - return FALSE; - - if (!mail_hash_rebuild(index->hash)) - return FALSE; - - return TRUE; -} - static void mail_index_update_header_changes(MailIndex *index) { - i_assert(index->lock_type == MAIL_LOCK_EXCLUSIVE); - if (index->set_flags != 0) { index->header->flags |= index->set_flags; index->set_flags = 0; @@ -208,123 +228,101 @@ int mail_index_try_lock(MailIndex *index, MailLockType lock_type) { - struct flock fl; - - if (index->lock_type == lock_type) - return TRUE; - - /* lock whole file */ - fl.l_type = MAIL_LOCK_TO_FLOCK(lock_type); - fl.l_whence = SEEK_SET; - fl.l_start = 0; - fl.l_len = 0; - - if (fcntl(index->fd, F_SETLK, &fl) < 0) { - if (errno != EINTR && errno != EACCES) - index_set_syscall_error(index, "fcntl(F_SETLK)"); - return FALSE; - } - - return TRUE; -} - -int mail_index_set_lock(MailIndex *index, MailLockType lock_type) -{ - /* yeah, this function is a bit messy. besides locking, it keeps - the index synced and in a good shape. */ - MailLockType old_lock_type; - struct flock fl; int ret; - if (index->inconsistent) { - /* index is in inconsistent state and nothing else than - free() is allowed for it. FIXME: what about msync()ing.. */ - return FALSE; - } - if (index->lock_type == lock_type) return TRUE; - /* shared -> exclusive isn't allowed */ - i_assert(lock_type != MAIL_LOCK_EXCLUSIVE || - index->lock_type != MAIL_LOCK_SHARED); + if (index->anon_mmap) + return TRUE; - if (index->lock_type == MAIL_LOCK_EXCLUSIVE) { - /* releasing exclusive lock */ - index->header->flags &= ~MAIL_INDEX_FLAG_FSCK; - - mail_index_update_header_changes(index); + ret = file_try_lock(index->fd, MAIL_LOCK_TO_FLOCK(lock_type)); + if (ret < 0) + index_set_syscall_error(index, "file_try_lock()"); - /* sync mmaped memory */ - (void)mail_index_sync_file(index); - } + return ret > 0; +} - if (lock_type != MAIL_LOCK_UNLOCK && - index->lock_type == MAIL_LOCK_UNLOCK && !index->updating) { - /* unlock -> lock */ - index->updating = TRUE; - (void)index->sync(index); +static int mail_index_write_header_changes(MailIndex *index) +{ + int failed; + + /* use our own locking here so we don't mess up with any other + index states, like inconsistency. */ + if (file_wait_lock(index->fd, F_WRLCK) < 0) + return index_set_syscall_error(index, "file_wait_lock()"); - ret = mail_index_set_lock(index, lock_type); - index->updating = FALSE; - return ret; - } + mail_index_update_header_changes(index); - /* lock whole file */ - fl.l_type = MAIL_LOCK_TO_FLOCK(lock_type); - fl.l_whence = SEEK_SET; - fl.l_start = 0; - fl.l_len = 0; + failed = msync(index->mmap_base, sizeof(MailIndexHeader), MS_SYNC) < 0; + if (failed) + index_set_syscall_error(index, "msync()"); + + if (file_wait_lock(index->fd, F_UNLCK) < 0) + return index_set_syscall_error(index, "file_wait_lock()"); - while (fcntl(index->fd, F_SETLKW, &fl) < 0) { - if (errno != EINTR) { - index_set_syscall_error(index, "fcntl(F_SETLKW)"); - return FALSE; - } - } + return !failed; +} - if (lock_type == MAIL_LOCK_UNLOCK) { - /* reset last_lookup so rebuilds don't try to use it */ - index->last_lookup_seq = 0; - index->last_lookup = NULL; - } +static int mail_index_lock_remove(MailIndex *index) +{ + MailLockType old_lock_type; + + if (file_wait_lock(index->fd, F_UNLCK) < 0) + return index_set_syscall_error(index, "file_wait_lock()"); old_lock_type = index->lock_type; - index->lock_type = lock_type; - - if (lock_type != MAIL_LOCK_UNLOCK) { - /* we're always mmap()ed when we're locked */ - if (!mmap_update(index)) { - (void)mail_index_set_lock(index, MAIL_LOCK_UNLOCK); - return FALSE; - } + index->lock_type = MAIL_LOCK_UNLOCK; - if (index->indexid != index->header->indexid) { - /* index was rebuilt, there's no way we can maintain - consistency */ - index_set_error(index, "Warning: Inconsistency - Index " - "%s was rebuilt while we had it open", - index->filepath); - index->inconsistent = TRUE; - return FALSE; - } - } else if (old_lock_type == MAIL_LOCK_SHARED) { - /* releasing shared lock */ + /* reset last_lookup so rebuilds don't try to use it */ + index->last_lookup_seq = 0; + index->last_lookup = NULL; + + if (old_lock_type == MAIL_LOCK_SHARED) { + /* releasing shared lock. we may need to update some + flags in header. */ unsigned int old_flags, old_cache; old_flags = index->header->flags; old_cache = index->header->cache_fields; if ((old_flags | index->set_flags) != old_flags || - (old_cache | index->set_cache_fields) != old_cache) { - /* need to update the header */ - index->updating = TRUE; - if (mail_index_set_lock(index, MAIL_LOCK_EXCLUSIVE)) - mail_index_update_header_changes(index); - index->updating = FALSE; + (old_cache | index->set_cache_fields) != old_cache) + return mail_index_write_header_changes(index); + } + + return TRUE; +} + +static int mail_index_lock_change(MailIndex *index, MailLockType lock_type) +{ + /* shared -> exclusive isn't allowed */ + i_assert(lock_type != MAIL_LOCK_EXCLUSIVE || + index->lock_type != MAIL_LOCK_SHARED); - return mail_index_set_lock(index, MAIL_LOCK_UNLOCK); - } + if (index->inconsistent) { + /* index is in inconsistent state and nothing else than + free() is allowed for it. */ + return FALSE; + } + + if (file_wait_lock(index->fd, MAIL_LOCK_TO_FLOCK(lock_type)) < 0) + return index_set_syscall_error(index, "file_wait_lock()"); + index->lock_type = lock_type; + + if (!mmap_update(index)) { + (void)mail_index_set_lock(index, MAIL_LOCK_UNLOCK); + return FALSE; + } + + if (index->indexid != index->header->indexid) { + /* index was rebuilt, there's no way we can maintain + consistency */ + index_set_error(index, "Warning: Inconsistency - Index " + "%s was rebuilt while we had it open", + index->filepath); + index->inconsistent = TRUE; + return FALSE; } if (lock_type == MAIL_LOCK_EXCLUSIVE) { @@ -338,457 +336,41 @@ } } - if (index->header != NULL && !index->updating && - (index->header->flags & MAIL_INDEX_FLAG_REBUILD) != 0) { - /* index is corrupted, rebuild it */ - index->updating = TRUE; - - if (lock_type == MAIL_LOCK_SHARED) - (void)mail_index_set_lock(index, MAIL_LOCK_UNLOCK); - - if (!mail_index_rebuild_all(index)) - return FALSE; - - ret = mail_index_set_lock(index, lock_type); - index->updating = FALSE; - return ret; - } - - if (lock_type == MAIL_LOCK_UNLOCK) { - /* reset header so it's not used while being unlocked */ - index->last_lookup_seq = 0; - index->last_lookup = NULL; - } - - return TRUE; -} - -static int delete_index(const char *path) -{ - char tmp[1024]; - int i; - - /* main index */ - if (unlink(path) < 0) - return FALSE; - - for (i = 0; index_file_prefixes[i] != NULL; i++) { - i_snprintf(tmp, sizeof(tmp), "%s.%s", - path, index_file_prefixes[i]); - if (unlink(tmp) < 0) - return FALSE; - i++; - } - - return TRUE; -} - -static int read_and_verify_header(int fd, MailIndexHeader *hdr, - int check_version) -{ - /* read the header */ - if (lseek(fd, 0, SEEK_SET) != 0) - return FALSE; - - if (read(fd, hdr, sizeof(MailIndexHeader)) != sizeof(MailIndexHeader)) - return FALSE; - - /* check the compatibility */ - return hdr->compat_data[1] == MAIL_INDEX_COMPAT_FLAGS && - hdr->compat_data[2] == sizeof(unsigned int) && - hdr->compat_data[3] == sizeof(time_t) && - hdr->compat_data[4] == sizeof(uoff_t) && - hdr->compat_data[5] == MEM_ALIGN_SIZE && - (!check_version || hdr->compat_data[0] == MAIL_INDEX_VERSION); -} - -/* Returns TRUE if we're compatible with given index file. May delete the - file if it's from older version. */ -static int mail_check_compatible_index(MailIndex *index, const char *path) -{ - MailIndexHeader hdr; - int fd, compatible; - - fd = open(path, O_RDONLY); - if (fd == -1) { - if (errno != ENOENT) - index_set_error(index, "Can't open index %s: %m", path); - return FALSE; - } - - compatible = read_and_verify_header(fd, &hdr, FALSE); - if (hdr.compat_data[0] != MAIL_INDEX_VERSION) { - /* version mismatch */ - compatible = FALSE; - if (hdr.compat_data[0] < MAIL_INDEX_VERSION) { - /* of older version, we don't need it anymore */ - (void)delete_index(path); - } - } - - (void)close(fd); - return compatible; -} - -/* Returns a file name of compatible index */ -static const char *mail_find_index(MailIndex *index) -{ - const char *name; - char path[1024]; - - hostpid_init(); - - /* first try .imap.index-<hostname> */ - name = t_strconcat(INDEX_FILE_PREFIX "-", my_hostname, NULL); - i_snprintf(path, sizeof(path), "%s/%s", index->dir, name); - if (mail_check_compatible_index(index, path)) - return name; - - /* then try the generic .imap.index */ - name = INDEX_FILE_PREFIX; - i_snprintf(path, sizeof(path), "%s/%s", index->dir, name); - if (mail_check_compatible_index(index, path)) - return name; - - return NULL; -} - -static int mail_index_open_init(MailIndex *index, int update_recent, - MailIndexHeader *hdr) -{ - /* update \Recent message counters */ - if (update_recent && hdr->last_nonrecent_uid != hdr->next_uid-1) { - /* keep last_recent_uid to next_uid-1 */ - if (index->lock_type == MAIL_LOCK_SHARED) { - if (!index->set_lock(index, MAIL_LOCK_UNLOCK)) - return FALSE; - } - - if (!index->set_lock(index, MAIL_LOCK_EXCLUSIVE)) - return FALSE; - - index->first_recent_uid = index->header->last_nonrecent_uid+1; - index->header->last_nonrecent_uid = index->header->next_uid-1; - } else { - index->first_recent_uid = hdr->last_nonrecent_uid+1; - } - - if (hdr->next_uid >= INT_MAX-1024) { - /* UID values are getting too high, rebuild index */ - index->set_flags |= MAIL_INDEX_FLAG_REBUILD; - } - - if (index->lock_type == MAIL_LOCK_EXCLUSIVE) { - /* finally reset the modify log marks, fsck or syncing might - have deleted some messages, and since we're only just - opening the index, there's no need to remember them */ - if (!mail_modifylog_mark_synced(index->modifylog)) - return FALSE; - } - return TRUE; } -static int mail_index_open_file(MailIndex *index, const char *filename, - int update_recent) +int mail_index_set_lock(MailIndex *index, MailLockType lock_type) { - MailIndexHeader hdr; - const char *path; - int fd, failed; - - /* the index file should already be checked that it exists and - we're compatible with it. */ - - path = t_strconcat(index->dir, "/", filename, NULL); - fd = open(path, O_RDWR); - if (fd == -1) { - index_set_error(index, "Can't open index %s: %m", path); - return FALSE; - } - - /* check the compatibility anyway just to be sure */ - if (!read_and_verify_header(fd, &hdr, TRUE)) { - index_set_error(index, "Non-compatible index file %s", path); - return FALSE; - } - - if (index->fd != -1) - mail_index_close(index); - - index->fd = fd; - index->filepath = i_strdup(path); - index->indexid = hdr.indexid; - index->dirty_mmap = TRUE; - index->updating = TRUE; - - failed = TRUE; - do { - /* open/create the index files */ - if (!mail_index_data_open(index)) { - if ((index->set_flags & MAIL_INDEX_FLAG_REBUILD) == 0) - break; - - /* data file is corrupted, need to rebuild index */ - hdr.flags |= MAIL_INDEX_FLAG_REBUILD; - index->set_flags = 0; - - if (!mail_index_data_create(index)) - break; - } - - /* custom flags file needs to be open before - rebuilding index */ - if (!mail_custom_flags_open_or_create(index)) - break; - - if (hdr.flags & MAIL_INDEX_FLAG_REBUILD) { - /* index is corrupted, rebuild */ - if (!index->rebuild(index)) - break; - } - - if (!mail_hash_open_or_create(index)) - break; - if (!mail_modifylog_open_or_create(index)) - break; - - if (hdr.flags & MAIL_INDEX_FLAG_FSCK) { - /* index needs fscking */ - if (!index->fsck(index)) - break; - } - - if (hdr.flags & MAIL_INDEX_FLAG_COMPRESS) { - /* remove deleted blocks from index file */ - if (!mail_index_compress(index)) - break; - } - - if (hdr.flags & MAIL_INDEX_FLAG_REBUILD_HASH) { - if (!mail_hash_rebuild(index->hash)) - break; - } - - if (hdr.flags & MAIL_INDEX_FLAG_CACHE_FIELDS) { - /* need to update cached fields */ - if (!mail_index_update_cache(index)) - break; - } - - if (hdr.flags & MAIL_INDEX_FLAG_COMPRESS_DATA) { - /* remove unused space from index data file. - keep after cache_fields which may move data - and create unused space.. */ - if (!mail_index_compress_data(index)) - break; - } - - if (!index->sync(index)) - break; - if (!mail_index_open_init(index, update_recent, &hdr)) - break; - - failed = FALSE; - } while (FALSE); - - index->updating = FALSE; - - if (!index->set_lock(index, MAIL_LOCK_UNLOCK)) - failed = TRUE; - - if (failed) - mail_index_close(index); - - return !failed; -} - -void mail_index_init_header(MailIndexHeader *hdr) -{ - memset(hdr, 0, sizeof(MailIndexHeader)); - hdr->compat_data[0] = MAIL_INDEX_VERSION; - hdr->compat_data[1] = MAIL_INDEX_COMPAT_FLAGS; - hdr->compat_data[2] = sizeof(unsigned int); - hdr->compat_data[3] = sizeof(time_t); - hdr->compat_data[4] = sizeof(uoff_t); - hdr->compat_data[5] = MEM_ALIGN_SIZE; - hdr->indexid = ioloop_time; - - /* mark the index being rebuilt - rebuild() removes this flag - when it succeeds */ - hdr->flags = MAIL_INDEX_FLAG_REBUILD; - - /* set the fields we always want to cache */ - hdr->cache_fields |= FIELD_TYPE_LOCATION | FIELD_TYPE_MESSAGEPART; - - hdr->uid_validity = ioloop_time; - hdr->next_uid = 1; -} + if (index->lock_type == lock_type) + return TRUE; -static int mail_index_create(MailIndex *index, int *dir_unlocked, - int update_recent) -{ - MailIndexHeader hdr; - const char *path; - char index_path[1024]; - int fd, len; - - *dir_unlocked = FALSE; - - /* first create the index into temporary file. */ - fd = mail_index_create_temp_file(index, &path); - if (fd == -1) - return FALSE; - - /* fill the header */ - mail_index_init_header(&hdr); - - /* write header */ - if (write_full(fd, &hdr, sizeof(hdr)) < 0) { - index_set_error(index, "Error writing to temp index %s: %m", - path); - (void)close(fd); - (void)unlink(path); - return FALSE; - } - - /* move the temp index into the real one. we also need to figure - out what to call ourself on the way. */ - len = i_snprintf(index_path, sizeof(index_path), - "%s/" INDEX_FILE_PREFIX, index->dir); - if (link(path, index_path) == 0) - (void)unlink(path); - else { - if (errno != EEXIST) { - /* fatal error */ - index_set_error(index, "link(%s, %s) failed: %m", - path, index_path); - (void)close(fd); - (void)unlink(path); - return FALSE; - } - - if (getenv("OVERWRITE_INCOMPATIBLE_INDEX") != NULL) { - /* don't try to support different architectures, - just overwrite the index if it's already there. */ - } else { - /* fallback to .imap.index-hostname - we require each - system to have a different hostname so it's safe to - override previous index as well */ - hostpid_init(); - i_snprintf(index_path + len, sizeof(index_path)-len, - "-%s", my_hostname); - } - - if (rename(path, index_path) < 0) { - index_set_error(index, "rename(%s, %s) failed: %m", - path, index_path); - (void)close(fd); - (void)unlink(path); - return FALSE; - } - - /* FIXME: race condition here! index may be opened before - it's rebuilt. maybe set it locked here, and make it require - shared lock when finding the indexes.. */ - } - - if (index->fd != -1) - mail_index_close(index); - - index->fd = fd; - index->filepath = i_strdup(index_path); - index->indexid = hdr.indexid; - index->updating = TRUE; - index->dirty_mmap = TRUE; - - /* lock the index file and unlock the directory */ - if (!index->set_lock(index, MAIL_LOCK_EXCLUSIVE)) { - index->updating = FALSE; - return FALSE; - } - - if (mail_index_lock_dir(index, MAIL_LOCK_UNLOCK)) - *dir_unlocked = TRUE; - - /* create the data file, build the index and hash */ - if (!mail_custom_flags_open_or_create(index) || - !mail_index_data_create(index) || !index->rebuild(index) || - !mail_hash_create(index) || !mail_modifylog_create(index)) { - index->updating = FALSE; - mail_index_close(index); - return FALSE; - } - index->updating = FALSE; - - if (!mail_index_open_init(index, update_recent, index->header)) { - mail_index_close(index); - return FALSE; - } - - /* unlock finally */ - if (!index->set_lock(index, MAIL_LOCK_UNLOCK)) { - mail_index_close(index); - return FALSE; - } - - return TRUE; -} - -int mail_index_open(MailIndex *index, int update_recent) -{ - const char *name; - - i_assert(!index->opened); - - name = mail_find_index(index); - if (name == NULL) - return FALSE; - - if (!mail_index_open_file(index, name, update_recent)) - return FALSE; - - index->opened = TRUE; - return TRUE; -} - -int mail_index_open_or_create(MailIndex *index, int update_recent) -{ - const char *name; - int failed, dir_unlocked; - - i_assert(!index->opened); - - /* first see if it's already there */ - name = mail_find_index(index); - if (name != NULL && mail_index_open_file(index, name, update_recent)) { - index->opened = TRUE; + if (index->anon_mmap) { + /* anonymous mmaps are private and don't need any locking */ + mail_index_update_header_changes(index); + index->lock_type = lock_type; return TRUE; } - /* index wasn't found or it was broken. get exclusive lock and check - again, just to make sure we don't end up having two index files - due to race condition with another process. */ - if (!mail_index_lock_dir(index, MAIL_LOCK_EXCLUSIVE)) - return FALSE; + if (index->lock_type == MAIL_LOCK_EXCLUSIVE) { + /* dropping exclusive lock (either unlock or to shared) */ + mail_index_update_header_changes(index); - name = mail_find_index(index); - if (name == NULL || !mail_index_open_file(index, name, update_recent)) { - /* create/rebuild index */ - failed = !mail_index_create(index, &dir_unlocked, - update_recent); - } else { - dir_unlocked = FALSE; - failed = FALSE; + /* remove the FSCK flag only after successful fsync() */ + if (mail_index_sync_file(index)) { + index->header->flags &= ~MAIL_INDEX_FLAG_FSCK; + if (msync(index->mmap_base, sizeof(MailIndexHeader), + MS_SYNC) < 0) { + /* we only failed to remove the fsck flag, + so this isn't fatal. */ + index_set_syscall_error(index, "msync()"); + } + } } - if (!dir_unlocked && !mail_index_lock_dir(index, MAIL_LOCK_UNLOCK)) - return FALSE; - - if (failed) - return FALSE; - - index->opened = TRUE; - return TRUE; + if (lock_type == MAIL_LOCK_UNLOCK) + return mail_index_lock_remove(index); + else + return mail_index_lock_change(index, lock_type); } int mail_index_verify_hole_range(MailIndex *index) @@ -810,7 +392,7 @@ } /* make sure position is in range.. */ - if (hdr->first_hole_position >= index->mmap_length) { + if (hdr->first_hole_position >= index->mmap_used_length) { index_set_corrupted(index, "first_hole_position points " "outside file"); return FALSE; @@ -830,23 +412,31 @@ return TRUE; } -static MailIndexRecord *mail_index_lookup_mapped(MailIndex *index, - unsigned int lookup_seq) +MailIndexHeader *mail_index_get_header(MailIndex *index) +{ + i_assert(index->lock_type != MAIL_LOCK_UNLOCK); + + return index->header; +} + +MailIndexRecord *mail_index_lookup(MailIndex *index, unsigned int seq) { MailIndexHeader *hdr; MailIndexRecord *rec, *last_rec; - unsigned int seq; + unsigned int rec_seq; uoff_t seekpos; - off_t pos; - if (lookup_seq == index->last_lookup_seq && + i_assert(seq > 0); + i_assert(index->lock_type != MAIL_LOCK_UNLOCK); + + if (seq == index->last_lookup_seq && index->last_lookup != NULL && index->last_lookup->uid != 0) { /* wanted the same record as last time */ return index->last_lookup; } hdr = index->header; - if (lookup_seq > hdr->messages_count) { + if (seq > hdr->messages_count) { /* out of range */ return NULL; } @@ -855,16 +445,13 @@ return NULL; seekpos = sizeof(MailIndexHeader) + - (uoff_t)(lookup_seq-1) * sizeof(MailIndexRecord); - if (seekpos + sizeof(MailIndexRecord) > index->mmap_length) { + (uoff_t)(seq-1) * sizeof(MailIndexRecord); + if (seekpos + sizeof(MailIndexRecord) > index->mmap_used_length) { /* minimum file position for wanted sequence would point ouside file, so it can't exist. however, header said it - should be found.. fsck. */ - pos = lseek(index->fd, 0, SEEK_END); - if (pos >= 0 && (uoff_t)pos > seekpos) { - i_panic("Index lookup failed because whole file " - "isn't mmap()ed (dirty_mmap not properly set)"); - } + should be found.. */ + i_assert(index->header->used_file_size == + index->mmap_used_length); index_set_corrupted(index, "Header contains invalid message count"); @@ -874,13 +461,13 @@ rec = (MailIndexRecord *) ((char *) index->mmap_base + sizeof(MailIndexHeader)); last_rec = (MailIndexRecord *) ((char *) index->mmap_base + - index->mmap_length - + index->mmap_used_length - sizeof(MailIndexRecord)); if (hdr->first_hole_position == 0 || hdr->first_hole_position > seekpos) { /* easy, it's just at the expected index */ - rec += lookup_seq-1; + rec += seq-1; i_assert(rec <= last_rec); if (rec->uid == 0) { @@ -888,56 +475,40 @@ "wasn't updated properly"); return NULL; } + + index->last_lookup = rec; + index->last_lookup_seq = seq; return rec; } /* we need to walk through the index to get to wanted position */ - if (lookup_seq > index->last_lookup_seq && index->last_lookup != NULL) { + if (seq > index->last_lookup_seq && index->last_lookup != NULL) { /* we want to lookup data after last lookup - this helps us some */ rec = index->last_lookup; - seq = index->last_lookup_seq; + rec_seq = index->last_lookup_seq; } else { /* some mails are deleted, jump after the first known hole and start counting non-deleted messages.. */ - seq = INDEX_POSITION_INDEX(hdr->first_hole_position + 1) + 1; - rec += seq-1 + hdr->first_hole_records; + rec_seq = INDEX_POSITION_INDEX(hdr->first_hole_position+1) + 1; + rec += rec_seq-1 + hdr->first_hole_records; } - while (seq < lookup_seq && rec <= last_rec) { + while (rec_seq < seq && rec <= last_rec) { if (rec->uid != 0) - seq++; + rec_seq++; rec++; } - return rec; -} - -MailIndexHeader *mail_index_get_header(MailIndex *index) -{ - i_assert(index->lock_type != MAIL_LOCK_UNLOCK); - - return index->header; -} - -MailIndexRecord *mail_index_lookup(MailIndex *index, unsigned int seq) -{ - i_assert(seq > 0); - i_assert(index->lock_type != MAIL_LOCK_UNLOCK); - - if (!mmap_update(index)) - return NULL; - - index->last_lookup = mail_index_lookup_mapped(index, seq); - index->last_lookup_seq = seq; - return index->last_lookup; + index->last_lookup = rec; + index->last_lookup_seq = rec_seq; + return rec_seq == seq ? rec : NULL; } MailIndexRecord *mail_index_next(MailIndex *index, MailIndexRecord *rec) { MailIndexRecord *end_rec; - i_assert(!index->dirty_mmap); i_assert(index->lock_type != MAIL_LOCK_UNLOCK); i_assert(rec >= (MailIndexRecord *) index->mmap_base); @@ -946,7 +517,7 @@ /* go to the next non-deleted record */ end_rec = (MailIndexRecord *) ((char *) index->mmap_base + - index->mmap_length); + index->mmap_used_length); while (++rec < end_rec) { if (rec->uid != 0) return rec; @@ -967,18 +538,15 @@ i_assert(first_uid > 0 && last_uid > 0); i_assert(first_uid <= last_uid); - if (!mmap_update(index)) - return NULL; - if (!mail_index_verify_hole_range(index)) return NULL; end_rec = (MailIndexRecord *) ((char *) index->mmap_base + - index->mmap_length); + index->mmap_used_length); /* check if first_uid is the first UID in the index, or an UID before that. this is quite common and hash lookup would be - useless to try with those nonexisting old UIDs.. */ + useless to try with those nonexisting old UIDs */ if (index->header->first_hole_position != sizeof(MailIndexHeader)) { rec = (MailIndexRecord *) ((char *) index->mmap_base + sizeof(MailIndexHeader)); @@ -1000,26 +568,28 @@ return last_uid >= rec->uid ? rec : NULL; } + if (first_uid >= index->header->next_uid) { + /* UID doesn't even exist yet */ + return NULL; + } + /* try the few first with hash lookups */ last_try_uid = last_uid - first_uid < 10 ? last_uid : first_uid + 4; for (uid = first_uid; uid <= last_try_uid; uid++) { pos = mail_hash_lookup_uid(index->hash, uid); - if (pos != 0) { - rec = (MailIndexRecord *) - ((char *) index->mmap_base + pos); - if (rec->uid != uid) { - index_set_error(index, - "Corrupted hash for index %s: " - "lookup returned offset to " - "different UID (%u vs %u)", - index->filepath, - rec->uid, uid); - index->set_flags |= - MAIL_INDEX_FLAG_REBUILD_HASH; - rec = NULL; - } - return rec; + if (pos == 0) + continue; + + rec = (MailIndexRecord *) ((char *) index->mmap_base + pos); + if (rec->uid != uid) { + index_set_error(index, "Corrupted hash for index %s: " + "lookup returned offset to different " + "UID (%u vs %u)", index->filepath, + rec->uid, uid); + index->set_flags |= MAIL_INDEX_FLAG_REBUILD_HASH; + rec = NULL; } + return rec; } if (last_try_uid == last_uid) @@ -1046,8 +616,6 @@ static MailIndexDataRecord * index_lookup_data_field(MailIndex *index, MailIndexRecord *rec, MailField field) { - MailIndexDataRecord *datarec; - i_assert(index->lock_type != MAIL_LOCK_UNLOCK); /* first check if the field even could be in the file */ @@ -1067,15 +635,7 @@ return NULL; } - datarec = mail_index_data_lookup(index->data, rec, field); - if (datarec == NULL) { - /* corrupted, the field should have been there */ - index_set_corrupted(index, "Field %u not found from data file " - "for record %u", field, rec->uid); - return NULL; - } - - return datarec; + return mail_index_data_lookup(index->data, rec, field); } const char *mail_index_lookup_field(MailIndex *index, MailIndexRecord *rec, @@ -1206,30 +766,23 @@ index->header->first_hole_position) + index->header->first_hole_records; end_rec = (MailIndexRecord *) ((char *) index->mmap_base + - index->mmap_length); + index->mmap_used_length); while (rec < end_rec && rec->uid == 0) { index->header->first_hole_records++; rec++; } } -static int mail_index_truncate(MailIndex *index) +static int mail_index_truncate_hole(MailIndex *index) { - off_t file_size; - - /* truncate index file */ - file_size = (off_t)index->header->first_hole_position; - if (ftruncate(index->fd, file_size) < 0) { - index_set_syscall_error(index, "ftruncate()"); - return FALSE; - } - - index->mmap_length = (size_t)file_size; - - /* update header */ + index->header->used_file_size = + (size_t)index->header->first_hole_position; index->header->first_hole_position = 0; index->header->first_hole_records = 0; - index->header->sync_id++; + + index->mmap_used_length = index->header->used_file_size; + if (!mail_index_truncate(index)) + return FALSE; if (index->header->messages_count == 0) { /* all mail was deleted, truncate data file */ @@ -1324,7 +877,7 @@ if ((hdr->first_hole_position - sizeof(MailIndexHeader)) / sizeof(MailIndexRecord) == hdr->messages_count) { /* the hole reaches end of file, truncate it */ - (void)mail_index_truncate(index); + (void)mail_index_truncate_hole(index); } else { /* update deleted_space in data file */ (void)mail_index_data_add_deleted_space(index->data, @@ -1352,46 +905,83 @@ rec->uid, external_change); } +static int mail_index_grow(MailIndex *index) +{ + uoff_t pos, grow_size; + void *base; + + grow_size = index->header->messages_count * sizeof(MailIndexRecord) * + INDEX_GROW_PERCENTAGE / 100; + if (grow_size < 16) + grow_size = 16; + + pos = index->mmap_full_length + grow_size; + i_assert(pos < OFF_T_MAX); + + if (index->anon_mmap) { + i_assert(pos < SSIZE_T_MAX); + + base = mremap_anon(index->mmap_base, index->mmap_full_length, + (size_t)pos, MREMAP_MAYMOVE); + if (base == MAP_FAILED) + return index_set_syscall_error(index, "mremap_anon()"); + + index->mmap_base = base; + index->mmap_full_length = (size_t)pos; + return TRUE; + } + + if (file_set_size(index->fd, (off_t)pos) < 0) { + if (errno == ENOSPC) + index->nodiskspace = TRUE; + return index_set_syscall_error(index, "file_set_size()"); + } + + /* file size changed, let others know about it too by changing + sync_id in header. */ + index->header->sync_id++; + + if (msync(index->mmap_base, sizeof(MailIndexHeader), MS_SYNC) < 0) + return index_set_syscall_error(index, "msync()"); + + if (!mmap_update(index)) + return FALSE; + + return TRUE; +} + int mail_index_append_begin(MailIndex *index, MailIndexRecord **rec) { - off_t pos; + MailIndexRecord *destrec; i_assert(index->lock_type == MAIL_LOCK_EXCLUSIVE); i_assert((*rec)->uid == 0); - pos = lseek(index->fd, 0, SEEK_END); - if (pos < 0) { - index_set_syscall_error(index, "lseek()"); - return FALSE; - } - - if (write_full(index->fd, *rec, sizeof(MailIndexRecord)) < 0) { - index_set_syscall_error(index, "write_full()"); - return FALSE; + if (index->mmap_used_length == index->mmap_full_length) { + /* we need more space */ + if (!mail_index_grow(index)) + return FALSE; } - index->header->messages_count++; - mail_index_mark_flag_changes(index, *rec, 0, (*rec)->msg_flags); - - /* file size changed, let others know about it too by changing - sync_id in header. */ - index->header->sync_id++; - index->dirty_mmap = TRUE; + i_assert(index->header->used_file_size == index->mmap_used_length); + i_assert(index->mmap_used_length <= + index->mmap_full_length - sizeof(MailIndexRecord)); - if (msync(index->mmap_base, sizeof(MailIndexHeader), MS_SYNC) < 0) { - index_set_syscall_error(index, "msync()"); - return FALSE; - } + destrec = (MailIndexRecord *) ((char *) index->mmap_base + + index->mmap_used_length); + memcpy(destrec, *rec, sizeof(MailIndexRecord)); + *rec = destrec; - if (!mmap_update(index)) - return FALSE; - - *rec = (MailIndexRecord *) ((char *) index->mmap_base + pos); + index->header->used_file_size += sizeof(MailIndexRecord); + index->mmap_used_length += sizeof(MailIndexRecord); return TRUE; } int mail_index_append_end(MailIndex *index, MailIndexRecord *rec) { + index->header->messages_count++; + mail_index_mark_flag_changes(index, rec, 0, rec->msg_flags); + rec->uid = index->header->next_uid++; if (index->hash != NULL) { @@ -1407,6 +997,11 @@ return index->error; } +int mail_index_is_diskspace_error(MailIndex *index) +{ + return index->nodiskspace; +} + int mail_index_is_inconsistency_error(MailIndex *index) { return index->inconsistent;
--- a/src/lib-index/mail-index.h Fri Sep 13 03:01:23 2002 +0300 +++ b/src/lib-index/mail-index.h Sat Sep 14 14:09:42 2002 +0300 @@ -107,6 +107,8 @@ unsigned int flags; unsigned int cache_fields; + uoff_t used_file_size; + uoff_t first_hole_position; unsigned int first_hole_records; @@ -127,6 +129,7 @@ unsigned int indexid; unsigned int reserved; /* for alignment mostly */ + uoff_t used_file_size; uoff_t deleted_space; }; @@ -189,9 +192,9 @@ /* Rebuild the whole index. Note that this changes the indexid so all the other files must also be rebuilt after this call. - Index MUST NOT have shared lock, exclusive lock or no lock at all - is fine. Note that this function may leave the index exclusively - locked. */ + Index MUST NOT have shared lock, but exclusive lock or no lock at + all is fine. Note that this function may leave the index + exclusively locked, and always sets index->inconsistent = TRUE. */ int (*rebuild)(MailIndex *index); /* Verify that the index is valid. If anything invalid is found, @@ -290,6 +293,10 @@ /* Returns last error message */ const char *(*get_last_error)(MailIndex *index); + /* Returns TRUE if last error was because we ran out of available + disk space. */ + int (*is_diskspace_error)(MailIndex *index); + /* Returns TRUE if index is now in inconsistent state with the previous known state, meaning that the message IDs etc. may have changed - only way to recover this would be to fully close @@ -316,7 +323,8 @@ char *error; /* last error message */ void *mmap_base; - size_t mmap_length; + size_t mmap_used_length; + size_t mmap_full_length; MailLockType lock_type; @@ -333,10 +341,10 @@ unsigned int set_flags; unsigned int set_cache_fields; + unsigned int anon_mmap:1; unsigned int opened:1; - unsigned int updating:1; unsigned int inconsistent:1; - unsigned int dirty_mmap:1; + unsigned int nodiskspace:1; }; /* needed to remove annoying warnings about not initializing all struct @@ -344,7 +352,7 @@ #define MAIL_INDEX_PRIVATE_FILL \ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, \ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, \ - 0, 0, 0, 0, 0, 0, 0, 0 + 0, 0, 0, 0, 0, 0, 0, 0, 0 /* defaults - same as above but prefixed with mail_index_. */ int mail_index_open(MailIndex *index, int update_recent); @@ -378,13 +386,12 @@ void mail_index_update_field_raw(MailIndexUpdate *update, MailField field, const void *value, size_t size); const char *mail_index_get_last_error(MailIndex *index); +int mail_index_is_diskspace_error(MailIndex *index); int mail_index_is_inconsistency_error(MailIndex *index); /* INTERNAL: */ void mail_index_init_header(MailIndexHeader *hdr); void mail_index_close(MailIndex *index); -int mail_index_rebuild_all(MailIndex *index); -int mail_index_sync_file(MailIndex *index); int mail_index_fmsync(MailIndex *index, size_t size); int mail_index_verify_hole_range(MailIndex *index); void mail_index_mark_flag_changes(MailIndex *index, MailIndexRecord *rec, @@ -395,10 +402,20 @@ int mail_index_update_cache(MailIndex *index); int mail_index_compress(MailIndex *index); int mail_index_compress_data(MailIndex *index); +int mail_index_truncate(MailIndex *index); /* Max. mmap()ed size for a message */ -//FIXME:#define MAIL_MMAP_BLOCK_SIZE (1024*256) -#define MAIL_MMAP_BLOCK_SIZE (1024*8) // FIXME: for debugging +#define MAIL_MMAP_BLOCK_SIZE (1024*256) + +/* number of records to always keep allocated in index file, + either used or unused */ +#define INDEX_MIN_RECORDS_COUNT 64 +/* when empty space in index file gets full, grow the file n% larger */ +#define INDEX_GROW_PERCENTAGE 10 +/* ftruncate() the index file when only n% of it is in use */ +#define INDEX_TRUNCATE_PERCENTAGE 30 +/* don't truncate whole file anyway, keep n% of the empty space */ +#define INDEX_TRUNCATE_KEEP_PERCENTAGE 10 /* uoff_t to index file for given record */ #define INDEX_FILE_POSITION(index, ptr) \ @@ -414,7 +431,12 @@ /* get number of records in mmaped index */ #define MAIL_INDEX_RECORD_COUNT(index) \ - ((index->mmap_length - sizeof(MailIndexHeader)) / \ + ((index->mmap_used_length - sizeof(MailIndexHeader)) / \ sizeof(MailIndexRecord)) +/* minimum size for index file */ +#define INDEX_FILE_MIN_SIZE \ + (sizeof(MailIndexHeader) + \ + INDEX_MIN_RECORDS_COUNT * sizeof(MailIndexRecord)) + #endif
--- a/src/lib-index/mail-modifylog.c Fri Sep 13 03:01:23 2002 +0300 +++ b/src/lib-index/mail-modifylog.c Sat Sep 14 14:09:42 2002 +0300 @@ -1,6 +1,7 @@ /* Copyright (C) 2002 Timo Sirainen */ #include "lib.h" +#include "file-lock.h" #include "mmap-util.h" #include "write-full.h" #include "mail-index.h" @@ -34,6 +35,8 @@ unsigned int second_log:1; }; +static const unsigned int no_expunges[] = { 0 }; + static void modifylog_set_syscall_error(MailModifyLog *log, const char *function) { @@ -53,40 +56,22 @@ (void)unlink(log->filepath); } -static int file_lock(int fd, int wait_lock, int lock_type) -{ - struct flock fl; - - fl.l_type = lock_type; - fl.l_whence = SEEK_SET; - fl.l_start = 0; - fl.l_len = 0; - - if (fcntl(fd, wait_lock ? F_SETLKW : F_SETLK, &fl) == -1) { - if (errno == EACCES || errno == EAGAIN) - return 0; - return -1; - } - - return 1; -} - /* Returns 1 = ok, 0 = failed to get the lock, -1 = error */ static int mail_modifylog_try_lock(MailModifyLog *log, int lock_type) { int ret; - ret = file_lock(log->fd, FALSE, lock_type); + ret = file_try_lock(log->fd, lock_type); if (ret == -1) - modifylog_set_syscall_error(log, "fcntl(F_SETLK)"); + modifylog_set_syscall_error(log, "file_try_lock()"); return ret; } static int mail_modifylog_wait_lock(MailModifyLog *log) { - if (file_lock(log->fd, TRUE, F_RDLCK) < 1) - modifylog_set_syscall_error(log, "fcntl(F_SETLKW)"); + if (file_wait_lock(log->fd, F_RDLCK) < 1) + modifylog_set_syscall_error(log, "file_wait_lock()"); return TRUE; } @@ -105,7 +90,7 @@ switch (mail_modifylog_try_lock(log, F_RDLCK)) { case 0: /* shouldn't happen */ - index_set_error(log->index, "fcntl(F_WRLCK -> F_RDLCK) " + index_set_error(log->index, "file_lock(F_WRLCK -> F_RDLCK) " "failed with file %s", log->filepath); /* fall through */ case -1: @@ -221,7 +206,7 @@ return FALSE; } - ret = file_lock(fd, FALSE, F_WRLCK); + ret = file_wait_lock(fd, F_WRLCK); if (ret == -1) { index_set_error(log->index, "Error locking modify log " "file %s: %m", path); @@ -569,9 +554,7 @@ if (rec >= end_rec) { /* none found */ - expunges = t_malloc(sizeof(unsigned int)); - *expunges = 0; - return expunges; + return no_expunges; } /* allocate memory for the returned array. the file size - synced @@ -651,9 +634,7 @@ if (rec >= end_rec) { /* none found */ - expunges = t_malloc(sizeof(unsigned int)); - *expunges = 0; - return expunges; + return no_expunges; } /* allocate memory for the returned array. the file size - synced
--- a/src/lib-index/maildir/maildir-build.c Fri Sep 13 03:01:23 2002 +0300 +++ b/src/lib-index/maildir/maildir-build.c Sat Sep 14 14:09:42 2002 +0300 @@ -38,16 +38,15 @@ i_assert(fname != NULL); /* check that file size is somewhat reasonable */ - if (fstat(fd, &st) == -1) { - index_set_error(index, "fstat() failed for file %s: %m", path); - return FALSE; - } + if (fstat(fd, &st) < 0) + return index_file_set_syscall_error(index, path, "fstat()"); if (st.st_size < 10) { /* This cannot be a mail file - delete it */ index_set_error(index, "Invalid size %"PRIuUOFF_T " with mail in %s - deleted", st.st_size, path); - (void)unlink(path); + if (unlink(path) < 0) + index_file_set_syscall_error(index, path, "unlink()"); return TRUE; } @@ -98,12 +97,12 @@ if (errno == EEXIST) return TRUE; - index_set_error(index, "Error opening mail file %s: %m", path); - return FALSE; + return index_file_set_syscall_error(index, path, "open()"); } ret = maildir_index_append_fd(index, fd, path, fname); - (void)close(fd); + if (close(fd) < 0) + return index_file_set_syscall_error(index, path, "close()"); return ret; } @@ -118,14 +117,12 @@ int failed; i_assert(index->lock_type != MAIL_LOCK_SHARED); - i_assert(source_dir != NULL); dirp = opendir(source_dir); if (dirp == NULL) { - index_set_error(index, "opendir() failed for dir %s: %m", - source_dir); - return FALSE; + return index_file_set_syscall_error(index, source_dir, + "opendir()"); } final_dir = dest_dir != NULL ? dest_dir : source_dir; @@ -153,7 +150,7 @@ /* race condition here - ignore it as the chance of it happening is pretty much zero */ - if (rename(sourcepath, destpath) == -1) { + if (rename(sourcepath, destpath) < 0) { index_set_error(index, "maildir build: " "rename(%s, %s) failed: %m", sourcepath, destpath); @@ -168,6 +165,7 @@ t_pop(); } - (void)closedir(dirp); + if (closedir(dirp) < 0) + index_file_set_syscall_error(index, source_dir, "closedir()"); return !failed; }
--- a/src/lib-index/maildir/maildir-index.c Fri Sep 13 03:01:23 2002 +0300 +++ b/src/lib-index/maildir/maildir-index.c Sat Sep 14 14:09:42 2002 +0300 @@ -2,6 +2,7 @@ #include "lib.h" #include "maildir-index.h" +#include "mail-index-data.h" #include "mail-index-util.h" #include <stdio.h> @@ -166,8 +167,8 @@ /* we need to update the flags in the file name */ old_fname = index->lookup_field(index, rec, FIELD_TYPE_LOCATION); if (old_fname == NULL) { - index_data_set_corrupted(index, "Missing location field for " - "record %u", rec->uid); + index_data_set_corrupted(index->data, "Missing location field " + "for record %u", rec->uid); return FALSE; } @@ -179,6 +180,9 @@ /* minor problem: new_path is overwritten if it exists.. */ if (rename(old_path, new_path) == -1) { + if (errno == ENOSPC) + index->nodiskspace = TRUE; + index_set_error(index, "maildir flags update: " "rename(%s, %s) failed: %m", old_path, new_path); @@ -222,6 +226,7 @@ mail_index_update_field, mail_index_update_field_raw, mail_index_get_last_error, + mail_index_is_diskspace_error, mail_index_is_inconsistency_error, MAIL_INDEX_PRIVATE_FILL
--- a/src/lib-index/maildir/maildir-open.c Fri Sep 13 03:01:23 2002 +0300 +++ b/src/lib-index/maildir/maildir-open.c Sat Sep 14 14:09:42 2002 +0300 @@ -3,6 +3,7 @@ #include "lib.h" #include "iobuffer.h" #include "maildir-index.h" +#include "mail-index-data.h" #include "mail-index-util.h" #include <unistd.h> @@ -13,10 +14,16 @@ const char *fname, *path; int fd; + i_assert(index->lock_type != MAIL_LOCK_UNLOCK); + + /* check for inconsistency here, to avoid extra error messages */ + if (index->inconsistent) + return NULL; + fname = index->lookup_field(index, rec, FIELD_TYPE_LOCATION); if (fname == NULL) { - index_data_set_corrupted(index, "Missing location field for " - "record %u", rec->uid); + index_data_set_corrupted(index->data, "Missing location field " + "for record %u", rec->uid); return NULL; }
--- a/src/lib-index/maildir/maildir-rebuild.c Fri Sep 13 03:01:23 2002 +0300 +++ b/src/lib-index/maildir/maildir-rebuild.c Sat Sep 14 14:09:42 2002 +0300 @@ -23,18 +23,13 @@ /* reset the header */ mail_index_init_header(index->header); - /* update indexid */ + /* update indexid, which also means that our state has completely + changed */ index->indexid = index->header->indexid; - - if (msync(index->mmap_base, sizeof(MailIndexHeader), MS_SYNC) == -1) - return FALSE; + index->inconsistent = TRUE; - /* truncate the file first, so it won't contain - any invalid data even if we crash */ - if (ftruncate(index->fd, sizeof(MailIndexHeader)) == -1) { - index_set_syscall_error(index, "ftruncate()"); + if (msync(index->mmap_base, sizeof(MailIndexHeader), MS_SYNC) < 0) return FALSE; - } /* reset data file */ if (!mail_index_data_reset(index->data)) @@ -51,11 +46,8 @@ return FALSE; /* update sync stamp */ - if (stat(cur_dir, &st) == -1) { - index_set_error(index, "stat() failed for maildir %s: %m", - cur_dir); - return FALSE; - } + if (stat(cur_dir, &st) < 0) + return index_file_set_syscall_error(index, cur_dir, "stat()"); index->file_sync_stamp = st.st_mtime;
--- a/src/lib-index/maildir/maildir-sync.c Fri Sep 13 03:01:23 2002 +0300 +++ b/src/lib-index/maildir/maildir-sync.c Sat Sep 14 14:09:42 2002 +0300 @@ -38,13 +38,15 @@ /* file itself changed - reload the header */ fd = open(path, O_RDONLY); if (fd == -1) { - index_set_error(index, "open() failed for file %s: %m", - path); + index_file_set_syscall_error(index, path, "open()"); failed = TRUE; } else { if (!maildir_record_update(update, fd, path)) failed = TRUE; - (void)close(fd); + if (close(fd) < 0) { + index_file_set_syscall_error(index, path, + "close()"); + } } } @@ -108,9 +110,9 @@ if (!check_content_changes) file_changed = FALSE; else { - if (stat(str, &st) == -1) { - index_set_error(index, "stat() failed for " - "file %s: %m", str); + if (stat(str, &st) < 0) { + index_file_set_syscall_error(index, str, + "stat()"); return FALSE; } @@ -192,11 +194,8 @@ from hash, so finally the hash should contain only the new files which will be added then. */ dirp = opendir(dir); - if (dirp == NULL) { - index_set_error(index, "opendir() failed for dir %s: %m", - dir); - return FALSE; - } + if (dirp == NULL) + return index_file_set_syscall_error(index, dir, "opendir()"); count = index->header->messages_count + 16; pool = pool_create("Maildir sync", nearest_power(count*30), FALSE); @@ -216,7 +215,9 @@ key = p == NULL ? value : p_strdup_until(pool, d->d_name, p); hash_insert(files, key, value); } - (void)closedir(dirp); + + if (closedir(dirp) < 0) + index_file_set_syscall_error(index, dir, "closedir()"); /* Do we want to check changes in file contents? This slows down things as we need to do extra stat() for all files. */ @@ -244,20 +245,15 @@ i_assert(index->lock_type != MAIL_LOCK_SHARED); - if (fstat(index->fd, &sti) == -1) { - index_set_syscall_error(index, "fstat()"); - return FALSE; - } + if (fstat(index->fd, &sti) < 0) + return index_set_syscall_error(index, "fstat()"); /* cur/ and new/ directories can have new mail - sync the cur/ first so it'll be a bit bit faster since we haven't yet added the new mail. */ cur_dir = t_strconcat(index->dir, "/cur", NULL); - if (stat(cur_dir, &std) == -1) { - index_set_error(index, "stat() failed for maildir %s: %m", - cur_dir); - return FALSE; - } + if (stat(cur_dir, &std) < 0) + return index_file_set_syscall_error(index, cur_dir, "stat()"); if (std.st_mtime != sti.st_mtime) { if (!maildir_index_sync_dir(index, cur_dir)) @@ -266,11 +262,8 @@ /* move mail from new/ to cur/ */ new_dir = t_strconcat(index->dir, "/new", NULL); - if (stat(new_dir, &std) == -1) { - index_set_error(index, "stat() failed for maildir " - "%s: %m", new_dir); - return FALSE; - } + if (stat(new_dir, &std) < 0) + return index_file_set_syscall_error(index, new_dir, "stat()"); if (std.st_mtime != sti.st_mtime) { if (!maildir_index_build_dir(index, new_dir, cur_dir)) @@ -280,15 +273,13 @@ make sure if someone adds new mail it the new/ dir's timestamp isn't set to same as cur/ directory's. */ ut.actime = ut.modtime = ioloop_time-60; - if (utime(cur_dir, &ut) == -1) { - index_set_error(index, "utime() failed for %s: %m", - cur_dir); - return FALSE; + if (utime(cur_dir, &ut) < 0) { + return index_file_set_syscall_error(index, cur_dir, + "utime()"); } - if (utime(new_dir, &ut) == -1) { - index_set_error(index, "utime() failed for %s: %m", - new_dir); - return FALSE; + if (utime(new_dir, &ut) < 0) { + return index_file_set_syscall_error(index, new_dir, + "utime()"); } /* it's possible that new mail came in just after we @@ -301,11 +292,8 @@ } /* update sync stamp */ - if (stat(cur_dir, &std) == -1) { - index_set_error(index, "stat() failed for maildir %s: %m", - cur_dir); - return FALSE; - } + if (stat(cur_dir, &std) < 0) + return index_file_set_syscall_error(index, cur_dir, "stat()"); index->file_sync_stamp = std.st_mtime; if (index->lock_type == MAIL_LOCK_UNLOCK) { @@ -313,11 +301,8 @@ ourself to get it changed */ ut.actime = ioloop_time; ut.modtime = index->file_sync_stamp; - if (utime(index->filepath, &ut) == -1) { - index_set_error(index, "utime() failed for %s: %m", - index->filepath); - return FALSE; - } + if (utime(index->filepath, &ut) < 0) + return index_set_syscall_error(index, "utime()"); } return TRUE;
--- a/src/lib-index/maildir/maildir-update.c Fri Sep 13 03:01:23 2002 +0300 +++ b/src/lib-index/maildir/maildir-update.c Sat Sep 14 14:09:42 2002 +0300 @@ -13,5 +13,6 @@ inbuf = io_buffer_create_mmap(fd, default_pool, MAIL_MMAP_BLOCK_SIZE, 0); mail_index_update_headers(update, inbuf, 0, NULL, NULL); + io_buffer_destroy(inbuf); return TRUE; }
--- a/src/lib-index/mbox/mbox-index.c Fri Sep 13 03:01:23 2002 +0300 +++ b/src/lib-index/mbox/mbox-index.c Sat Sep 14 14:09:42 2002 +0300 @@ -10,12 +10,13 @@ static MailIndex mbox_index; -void mbox_set_syscall_error(MailIndex *index, const char *function) +int mbox_set_syscall_error(MailIndex *index, const char *function) { i_assert(function != NULL); index_set_error(index, "%s failed with mbox file %s: %m", function, index->mbox_path); + return FALSE; } void mbox_header_init_context(MboxHeaderContext *ctx, MailIndex *index) @@ -390,6 +391,7 @@ mail_index_update_field, mail_index_update_field_raw, mail_index_get_last_error, + mail_index_is_diskspace_error, mail_index_is_inconsistency_error, MAIL_INDEX_PRIVATE_FILL
--- a/src/lib-index/mbox/mbox-index.h Fri Sep 13 03:01:23 2002 +0300 +++ b/src/lib-index/mbox/mbox-index.h Sat Sep 14 14:09:42 2002 +0300 @@ -12,7 +12,7 @@ int received; } MboxHeaderContext; -void mbox_set_syscall_error(MailIndex *index, const char *function);; +int mbox_set_syscall_error(MailIndex *index, const char *function);; void mbox_header_init_context(MboxHeaderContext *ctx, MailIndex *index); void mbox_header_free_context(MboxHeaderContext *ctx);
--- a/src/lib-index/mbox/mbox-open.c Fri Sep 13 03:01:23 2002 +0300 +++ b/src/lib-index/mbox/mbox-open.c Sat Sep 14 14:09:42 2002 +0300 @@ -18,6 +18,10 @@ i_assert(index->lock_type != MAIL_LOCK_UNLOCK); + /* check for inconsistency here, to avoid extra error messages */ + if (index->inconsistent) + return NULL; + if (!mbox_mail_get_start_offset(index, rec, &offset)) return NULL;
--- a/src/lib-index/mbox/mbox-rebuild.c Fri Sep 13 03:01:23 2002 +0300 +++ b/src/lib-index/mbox/mbox-rebuild.c Sat Sep 14 14:09:42 2002 +0300 @@ -17,7 +17,7 @@ { IOBuffer *inbuf; struct stat st; - int fd; + int fd, failed; i_assert(index->lock_type != MAIL_LOCK_SHARED); @@ -30,20 +30,18 @@ /* we require MD5 to be cached */ index->header->cache_fields |= FIELD_TYPE_MD5; - /* update indexid */ + /* update indexid, which also means that our state has completely + changed */ index->indexid = index->header->indexid; + index->inconsistent = TRUE; - if (msync(index->mmap_base, sizeof(MailIndexHeader), MS_SYNC) < 0) { - index_set_syscall_error(index, "msync()"); - return FALSE; - } + if (msync(index->mmap_base, sizeof(MailIndexHeader), MS_SYNC) < 0) + return index_set_syscall_error(index, "msync()"); /* truncate the file first, so it won't contain any invalid data even if we crash */ - if (ftruncate(index->fd, sizeof(MailIndexHeader)) < 0) { - index_set_syscall_error(index, "ftruncate()"); - return FALSE; - } + if (ftruncate(index->fd, sizeof(MailIndexHeader)) < 0) + return index_set_syscall_error(index, "ftruncate()"); /* reset data file */ if (!mail_index_data_reset(index->data)) @@ -52,10 +50,8 @@ /* open the mbox file. we don't really need to open it read-write, but fcntl() locking requires it. */ fd = open(index->mbox_path, O_RDWR); - if (fd == -1) { - mbox_set_syscall_error(index, "open()"); - return FALSE; - } + if (fd == -1) + return mbox_set_syscall_error(index, "open()"); /* lock the mailbox so we can be sure no-one interrupts us. */ if (!mbox_lock(index, index->mbox_path, fd)) { @@ -65,21 +61,18 @@ inbuf = io_buffer_create_mmap(fd, default_pool, MAIL_MMAP_BLOCK_SIZE, 0); - if (!mbox_index_append(index, inbuf)) { - (void)mbox_unlock(index, index->mbox_path, fd); - (void)close(fd); - return FALSE; - } + failed = !mbox_index_append(index, inbuf); (void)mbox_unlock(index, index->mbox_path, fd); (void)close(fd); io_buffer_destroy(inbuf); + if (failed) + return FALSE; + /* update sync stamp */ - if (stat(index->mbox_path, &st) == -1) { - mbox_set_syscall_error(index, "fstat()"); - return FALSE; - } + if (stat(index->mbox_path, &st) == -1) + return mbox_set_syscall_error(index, "fstat()"); index->file_sync_stamp = st.st_mtime;
--- a/src/lib-index/mbox/mbox-rewrite.c Fri Sep 13 03:01:23 2002 +0300 +++ b/src/lib-index/mbox/mbox-rewrite.c Sat Sep 14 14:09:42 2002 +0300 @@ -283,16 +283,16 @@ return FALSE; in_fd = open(index->mbox_path, O_RDWR); - if (in_fd == -1) { - mbox_set_syscall_error(index, "open()"); - return FALSE; - } + if (in_fd == -1) + return mbox_set_syscall_error(index, "open()"); + inbuf = io_buffer_create_mmap(in_fd, default_pool, MAIL_MMAP_BLOCK_SIZE, 0); out_fd = mail_index_create_temp_file(index, &path); if (out_fd == -1) { (void)close(in_fd); + io_buffer_destroy(inbuf); return FALSE; } outbuf = io_buffer_create_file(out_fd, default_pool, 8192);
--- a/src/lib-storage/index/index-copy.c Fri Sep 13 03:01:23 2002 +0300 +++ b/src/lib-storage/index/index-copy.c Sat Sep 14 14:09:42 2002 +0300 @@ -46,6 +46,9 @@ return FALSE; } + if (!ibox->index->sync(ibox->index)) + return mail_storage_set_index_error(ibox); + if (!ibox->index->set_lock(ibox->index, MAIL_LOCK_SHARED)) return mail_storage_set_index_error(ibox);
--- a/src/lib-storage/index/index-fetch-section.c Fri Sep 13 03:01:23 2002 +0300 +++ b/src/lib-storage/index/index-fetch-section.c Sat Sep 14 14:09:42 2002 +0300 @@ -32,7 +32,7 @@ /* fetch BODY[] or BODY[TEXT] */ static int fetch_body(MailIndexRecord *rec, MailFetchBodyData *sect, - FetchContext *ctx, int fetch_header) + FetchContext *ctx, const char *prefix, int fetch_header) { MessageSize size; IOBuffer *inbuf; @@ -46,11 +46,12 @@ return FALSE; } - str = t_strdup_printf("{%"PRIuUOFF_T"}\r\n", size.virtual_size); - (void)io_buffer_send(ctx->outbuf, str, strlen(str)); + str = t_strdup_printf("%s {%"PRIuUOFF_T"}\r\n", + prefix, size.virtual_size); + if (io_buffer_send(ctx->outbuf, str, strlen(str)) < 0) + return FALSE; - (void)message_send(ctx->outbuf, inbuf, &size, 0, sect->max_size); - return TRUE; + return message_send(ctx->outbuf, inbuf, &size, 0, sect->max_size); } static char *const *get_fields_array(const char *fields) @@ -191,32 +192,33 @@ } /* fetch wanted headers from given data */ -static void fetch_header_from(IOBuffer *inbuf, MessageSize *size, - const char *section, MailFetchBodyData *sect, - FetchContext *ctx) +static int fetch_header_from(IOBuffer *inbuf, MessageSize *size, + const char *section, MailFetchBodyData *sect, + FetchContext *ctx, const char *prefix) { const char *str; char *dest; size_t len; + int failed; /* HEADER, MIME, HEADER.FIELDS (list), HEADER.FIELDS.NOT (list) */ if (strcasecmp(section, "HEADER") == 0) { /* all headers */ - str = t_strdup_printf("{%"PRIuUOFF_T"}\r\n", - size->virtual_size); - (void)io_buffer_send(ctx->outbuf, str, strlen(str)); - (void)message_send(ctx->outbuf, inbuf, size, - sect->skip, sect->max_size); - return; + str = t_strdup_printf("%s {%"PRIuUOFF_T"}\r\n", + prefix, size->virtual_size); + if (io_buffer_send(ctx->outbuf, str, strlen(str)) < 0) + return FALSE; + return message_send(ctx->outbuf, inbuf, size, + sect->skip, sect->max_size); } /* partial headers - copy the wanted fields into temporary memory. Insert missing CRs on the way. FIXME: not a good idea with huge headers. */ if (size->virtual_size > SSIZE_T_MAX) { - io_buffer_send(ctx->outbuf, "{0}\r\n", 5); - return; + i_error("Message header too large"); + return FALSE; } t_push(); @@ -250,15 +252,17 @@ len = sect->max_size; } - str = t_strdup_printf("{%"PRIuSIZE_T"}\r\n", len); - io_buffer_send(ctx->outbuf, str, strlen(str)); - if (len > 0) io_buffer_send(ctx->outbuf, dest, len); + str = t_strdup_printf("%s {%"PRIuSIZE_T"}\r\n", prefix, len); + failed = io_buffer_send(ctx->outbuf, str, strlen(str)) < 0 || + io_buffer_send(ctx->outbuf, dest, len); t_pop(); + return !failed; } /* fetch BODY[HEADER...] */ -static int fetch_header(MailFetchBodyData *sect, FetchContext *ctx) +static int fetch_header(MailFetchBodyData *sect, FetchContext *ctx, + const char *prefix) { MessageSize hdr_size; IOBuffer *inbuf; @@ -266,8 +270,8 @@ if (!imap_msgcache_get_rfc822(ctx->cache, &inbuf, &hdr_size, NULL)) return FALSE; - fetch_header_from(inbuf, &hdr_size, sect->section, sect, ctx); - return TRUE; + return fetch_header_from(inbuf, &hdr_size, sect->section, + sect, ctx, prefix); } /* Find MessagePart for section (eg. 1.3.4) */ @@ -312,7 +316,7 @@ /* fetch BODY[1.2] or BODY[1.2.TEXT] */ static int fetch_part_body(MessagePart *part, MailFetchBodyData *sect, - FetchContext *ctx) + FetchContext *ctx, const char *prefix) { IOBuffer *inbuf; const char *str; @@ -325,20 +329,21 @@ skip_pos = part->physical_pos + part->header_size.physical_size; io_buffer_skip(inbuf, skip_pos); - str = t_strdup_printf("{%"PRIuUOFF_T"}\r\n", - part->body_size.virtual_size); - (void)io_buffer_send(ctx->outbuf, str, strlen(str)); + str = t_strdup_printf("%s {%"PRIuUOFF_T"}\r\n", + prefix, part->body_size.virtual_size); + if (io_buffer_send(ctx->outbuf, str, strlen(str)) < 0) + return FALSE; /* FIXME: potential performance problem with big messages: FETCH BODY[1]<100000..1024>, hopefully no clients do this */ - (void)message_send(ctx->outbuf, inbuf, &part->body_size, - sect->skip, sect->max_size); - return TRUE; + return message_send(ctx->outbuf, inbuf, &part->body_size, + sect->skip, sect->max_size); } /* fetch BODY[1.2.MIME|HEADER...] */ static int fetch_part_header(MessagePart *part, const char *section, - MailFetchBodyData *sect, FetchContext *ctx) + MailFetchBodyData *sect, FetchContext *ctx, + const char *prefix) { IOBuffer *inbuf; @@ -346,11 +351,12 @@ return FALSE; io_buffer_skip(inbuf, part->physical_pos); - fetch_header_from(inbuf, &part->header_size, section, sect, ctx); - return TRUE; + return fetch_header_from(inbuf, &part->header_size, section, + sect, ctx, prefix); } -static int fetch_part(MailFetchBodyData *sect, FetchContext *ctx) +static int fetch_part(MailFetchBodyData *sect, FetchContext *ctx, + const char *prefix) { MessagePart *part; const char *section; @@ -360,45 +366,39 @@ return FALSE; if (*section == '\0' || strcasecmp(section, "TEXT") == 0) - return fetch_part_body(part, sect, ctx); + return fetch_part_body(part, sect, ctx, prefix); if (strncasecmp(section, "HEADER", 6) == 0) - return fetch_part_header(part, section, sect, ctx); + return fetch_part_header(part, section, sect, ctx, prefix); if (strcasecmp(section, "MIME") == 0) - return fetch_part_header(part, section, sect, ctx); + return fetch_part_header(part, section, sect, ctx, prefix); return FALSE; } -void index_fetch_body_section(MailIndexRecord *rec, - unsigned int seq __attr_unused__, - MailFetchBodyData *sect, FetchContext *ctx) +int index_fetch_body_section(MailIndexRecord *rec, + unsigned int seq __attr_unused__, + MailFetchBodyData *sect, FetchContext *ctx) { - const char *str; - int fetch_ok; + const char *prefix; - str = !sect->skip_set ? + prefix = !sect->skip_set ? t_strdup_printf(" BODY[%s] ", sect->section) : t_strdup_printf(" BODY[%s]<%"PRIuUOFF_T"> ", sect->section, sect->skip); - if (ctx->first) str++; else ctx->first = FALSE; - - (void)io_buffer_send(ctx->outbuf, str, strlen(str)); - - if (*sect->section == '\0') { - fetch_ok = fetch_body(rec, sect, ctx, TRUE); - } else if (strcasecmp(sect->section, "TEXT") == 0) { - fetch_ok = fetch_body(rec, sect, ctx, FALSE); - } else if (strncasecmp(sect->section, "HEADER", 6) == 0) { - fetch_ok = fetch_header(sect, ctx); - } else if (*sect->section >= '0' && *sect->section <= '9') { - fetch_ok = fetch_part(sect, ctx); - } else { - fetch_ok = FALSE; + if (ctx->first) { + prefix++; ctx->first = FALSE; } - if (!fetch_ok) { - /* error */ - (void)io_buffer_send(ctx->outbuf, "{0}\r\n", 5); - } + if (*sect->section == '\0') + return fetch_body(rec, sect, ctx, prefix, TRUE); + if (strcasecmp(sect->section, "TEXT") == 0) + return fetch_body(rec, sect, ctx, prefix, FALSE); + if (strncasecmp(sect->section, "HEADER", 6) == 0) + return fetch_header(sect, ctx, prefix); + if (*sect->section >= '0' && *sect->section <= '9') + return fetch_part(sect, ctx, prefix); + + /* FIXME: point the error to user */ + return FALSE; }
--- a/src/lib-storage/index/index-fetch.c Fri Sep 13 03:01:23 2002 +0300 +++ b/src/lib-storage/index/index-fetch.c Sat Sep 14 14:09:42 2002 +0300 @@ -14,47 +14,53 @@ #include <unistd.h> -static void index_fetch_body(MailIndexRecord *rec, FetchContext *ctx) +static int index_fetch_body(MailIndexRecord *rec, FetchContext *ctx) { const char *body; body = imap_msgcache_get(ctx->cache, IMAP_CACHE_BODY); - if (body != NULL) - t_string_printfa(ctx->str, " BODY %s", body); - else { + if (body != NULL) { + t_string_printfa(ctx->str, "BODY %s ", body); + return TRUE; + } else { i_error("Couldn't generate BODY for UID %u (index %s)", rec->uid, ctx->index->filepath); + return FALSE; } } -static void index_fetch_bodystructure(MailIndexRecord *rec, FetchContext *ctx) +static int index_fetch_bodystructure(MailIndexRecord *rec, FetchContext *ctx) { const char *bodystructure; bodystructure = imap_msgcache_get(ctx->cache, IMAP_CACHE_BODYSTRUCTURE); if (bodystructure != NULL) { - t_string_printfa(ctx->str, " BODYSTRUCTURE %s", + t_string_printfa(ctx->str, "BODYSTRUCTURE %s ", bodystructure); + return TRUE; } else { i_error("Couldn't generate BODYSTRUCTURE for UID %u (index %s)", rec->uid, ctx->index->filepath); + return FALSE; } } -static void index_fetch_envelope(MailIndexRecord *rec, FetchContext *ctx) +static int index_fetch_envelope(MailIndexRecord *rec, FetchContext *ctx) { const char *envelope; envelope = imap_msgcache_get(ctx->cache, IMAP_CACHE_ENVELOPE); - if (envelope != NULL) - t_string_printfa(ctx->str, " ENVELOPE (%s)", envelope); - else { + if (envelope != NULL) { + t_string_printfa(ctx->str, "ENVELOPE (%s) ", envelope); + return TRUE; + } else { i_error("Couldn't generate ENVELOPE for UID %u (index %s)", rec->uid, ctx->index->filepath); + return FALSE; } } -static void index_fetch_rfc822_size(MailIndexRecord *rec, FetchContext *ctx) +static int index_fetch_rfc822_size(MailIndexRecord *rec, FetchContext *ctx) { MessageSize hdr_size, body_size; @@ -62,11 +68,12 @@ &hdr_size, &body_size)) { i_error("Couldn't get RFC822.SIZE for UID %u (index %s)", rec->uid, ctx->index->filepath); - return; + return FALSE; } - t_string_printfa(ctx->str, " RFC822.SIZE %"PRIuUOFF_T, + t_string_printfa(ctx->str, "RFC822.SIZE %"PRIuUOFF_T" ", hdr_size.virtual_size + body_size.virtual_size); + return TRUE; } static void index_fetch_flags(MailIndexRecord *rec, FetchContext *ctx) @@ -79,22 +86,22 @@ if (ctx->update_seen) flags |= MAIL_SEEN; - t_string_printfa(ctx->str, " FLAGS (%s)", + t_string_printfa(ctx->str, "FLAGS (%s) ", imap_write_flags(flags, ctx->custom_flags)); } static void index_fetch_internaldate(MailIndexRecord *rec, FetchContext *ctx) { - t_string_printfa(ctx->str, " INTERNALDATE \"%s\"", - imap_to_datetime(rec->internal_date)); + t_string_printfa(ctx->str, "INTERNALDATE \"%s\" ", + imap_to_datetime(rec->internal_date)); } static void index_fetch_uid(MailIndexRecord *rec, FetchContext *ctx) { - t_string_printfa(ctx->str, " UID %u", rec->uid); + t_string_printfa(ctx->str, "UID %u ", rec->uid); } -static void index_fetch_rfc822(MailIndexRecord *rec, FetchContext *ctx) +static int index_fetch_send_rfc822(MailIndexRecord *rec, FetchContext *ctx) { MessageSize hdr_size, body_size; IOBuffer *inbuf; @@ -104,20 +111,24 @@ &hdr_size, &body_size)) { i_error("Couldn't get RFC822 for UID %u (index %s)", rec->uid, ctx->index->filepath); - return; + return FALSE; } str = t_strdup_printf(" RFC822 {%"PRIuUOFF_T"}\r\n", hdr_size.virtual_size + body_size.virtual_size); - if (ctx->first) str++; else ctx->first = FALSE; - (void)io_buffer_send(ctx->outbuf, str, strlen(str)); + if (ctx->first) { + str++; ctx->first = FALSE; + } + if (io_buffer_send(ctx->outbuf, str, strlen(str)) < 0) + return FALSE; body_size.physical_size += hdr_size.physical_size; body_size.virtual_size += hdr_size.virtual_size; - (void)message_send(ctx->outbuf, inbuf, &body_size, 0, (uoff_t)-1); + return message_send(ctx->outbuf, inbuf, &body_size, 0, (uoff_t)-1); } -static void index_fetch_rfc822_header(MailIndexRecord *rec, FetchContext *ctx) +static int index_fetch_send_rfc822_header(MailIndexRecord *rec, + FetchContext *ctx) { MessageSize hdr_size; IOBuffer *inbuf; @@ -126,17 +137,21 @@ if (!imap_msgcache_get_rfc822(ctx->cache, &inbuf, &hdr_size, NULL)) { i_error("Couldn't get RFC822.HEADER for UID %u (index %s)", rec->uid, ctx->index->filepath); - return; + return FALSE; } str = t_strdup_printf(" RFC822.HEADER {%"PRIuUOFF_T"}\r\n", hdr_size.virtual_size); - if (ctx->first) str++; else ctx->first = FALSE; - (void)io_buffer_send(ctx->outbuf, str, strlen(str)); - (void)message_send(ctx->outbuf, inbuf, &hdr_size, 0, (uoff_t)-1); + if (ctx->first) { + str++; ctx->first = FALSE; + } + if (io_buffer_send(ctx->outbuf, str, strlen(str)) < 0) + return FALSE; + + return message_send(ctx->outbuf, inbuf, &hdr_size, 0, (uoff_t)-1); } -static void index_fetch_rfc822_text(MailIndexRecord *rec, FetchContext *ctx) +static int index_fetch_send_rfc822_text(MailIndexRecord *rec, FetchContext *ctx) { MessageSize body_size; IOBuffer *inbuf; @@ -145,14 +160,18 @@ if (!imap_msgcache_get_rfc822(ctx->cache, &inbuf, NULL, &body_size)) { i_error("Couldn't get RFC822.TEXT for UID %u (index %s)", rec->uid, ctx->index->filepath); - return; + return FALSE; } str = t_strdup_printf(" RFC822.TEXT {%"PRIuUOFF_T"}\r\n", body_size.virtual_size); - if (ctx->first) str++; else ctx->first = FALSE; - (void)io_buffer_send(ctx->outbuf, str, strlen(str)); - (void)message_send(ctx->outbuf, inbuf, &body_size, 0, (uoff_t)-1); + if (ctx->first) { + str++; ctx->first = FALSE; + } + if (io_buffer_send(ctx->outbuf, str, strlen(str)) < 0) + return FALSE; + + return message_send(ctx->outbuf, inbuf, &body_size, 0, (uoff_t)-1); } static ImapCacheField index_get_cache(MailFetchData *fetch_data) @@ -218,57 +237,85 @@ { FetchContext *ctx = context; MailFetchBodyData *sect; + unsigned int orig_len; + int failed, data_written; ctx->str = t_string_new(2048); t_string_printfa(ctx->str, "* %u FETCH (", seq); - (void)io_buffer_send(ctx->outbuf, ctx->str->str, ctx->str->len); - t_string_truncate(ctx->str, 0); + orig_len = ctx->str->len; /* first see what we need to do. this way we don't first do some light parsing and later notice that we need to do heavier parsing anyway */ index_msgcache_open(ctx, rec); - if (ctx->fetch_data->uid) - index_fetch_uid(rec, ctx); - if (ctx->fetch_data->flags) - index_fetch_flags(rec, ctx); - if (ctx->fetch_data->internaldate) - index_fetch_internaldate(rec, ctx); + failed = TRUE; + data_written = FALSE; + do { + /* these can't fail */ + if (ctx->fetch_data->uid) + index_fetch_uid(rec, ctx); + if (ctx->fetch_data->flags) + index_fetch_flags(rec, ctx); + if (ctx->fetch_data->internaldate) + index_fetch_internaldate(rec, ctx); - if (ctx->fetch_data->body) - index_fetch_body(rec, ctx); - if (ctx->fetch_data->bodystructure) - index_fetch_bodystructure(rec, ctx); - if (ctx->fetch_data->envelope) - index_fetch_envelope(rec, ctx); - if (ctx->fetch_data->rfc822_size) - index_fetch_rfc822_size(rec, ctx); + /* rest can */ + if (ctx->fetch_data->body) + if (!index_fetch_body(rec, ctx)) + break; + if (ctx->fetch_data->bodystructure) + if (!index_fetch_bodystructure(rec, ctx)) + break; + if (ctx->fetch_data->envelope) + if (!index_fetch_envelope(rec, ctx)) + break; + if (ctx->fetch_data->rfc822_size) + if (!index_fetch_rfc822_size(rec, ctx)) + break; + + /* send the data written into temp string, + not including the trailing zero */ + ctx->first = ctx->str->len == orig_len; + if (ctx->str->len > 0) { + if (!ctx->first) + ctx->str->len--; - /* send the data written into temp string, skipping the initial space */ - if (ctx->str->len > 0) { - (void)io_buffer_send(ctx->outbuf, ctx->str->str+1, - ctx->str->len-1); - } - ctx->first = ctx->str->len == 0; + if (io_buffer_send(ctx->outbuf, ctx->str->str, + ctx->str->len) < 0) + break; + } + + data_written = TRUE; - /* large data */ - if (ctx->fetch_data->rfc822) - index_fetch_rfc822(rec, ctx); - if (ctx->fetch_data->rfc822_text) - index_fetch_rfc822_text(rec, ctx); - if (ctx->fetch_data->rfc822_header) - index_fetch_rfc822_header(rec, ctx); + /* large data */ + if (ctx->fetch_data->rfc822) + if (!index_fetch_send_rfc822(rec, ctx)) + break; + if (ctx->fetch_data->rfc822_text) + if (!index_fetch_send_rfc822_text(rec, ctx)) + break; + if (ctx->fetch_data->rfc822_header) + if (!index_fetch_send_rfc822_header(rec, ctx)) + break; - sect = ctx->fetch_data->body_sections; - for (; sect != NULL; sect = sect->next) - index_fetch_body_section(rec, seq, sect, ctx); + sect = ctx->fetch_data->body_sections; + for (; sect != NULL; sect = sect->next) { + if (!index_fetch_body_section(rec, seq, sect, ctx)) + break; + } - (void)io_buffer_send(ctx->outbuf, ")\r\n", 3); + failed = FALSE; + } while (0); + + if (data_written) { + if (io_buffer_send(ctx->outbuf, ")\r\n", 3) < 0) + failed = TRUE; + } imap_msgcache_close(ctx->cache); - return TRUE; + return !failed; } int index_storage_fetch(Mailbox *box, MailFetchData *fetch_data, @@ -279,6 +326,9 @@ MailFetchBodyData *sect; int ret; + if (!ibox->index->sync(ibox->index)) + return mail_storage_set_index_error(ibox); + if (!ibox->index->set_lock(ibox->index, MAIL_LOCK_SHARED)) return mail_storage_set_index_error(ibox);
--- a/src/lib-storage/index/index-fetch.h Fri Sep 13 03:01:23 2002 +0300 +++ b/src/lib-storage/index/index-fetch.h Sat Sep 14 14:09:42 2002 +0300 @@ -15,7 +15,7 @@ } FetchContext; ImapCacheField index_fetch_body_get_cache(const char *section); -void index_fetch_body_section(MailIndexRecord *rec, unsigned int seq, - MailFetchBodyData *sect, FetchContext *data); +int index_fetch_body_section(MailIndexRecord *rec, unsigned int seq, + MailFetchBodyData *sect, FetchContext *data); #endif
--- a/src/lib-storage/index/index-messageset.c Fri Sep 13 03:01:23 2002 +0300 +++ b/src/lib-storage/index/index-messageset.c Sat Sep 14 14:09:42 2002 +0300 @@ -246,18 +246,18 @@ i_assert(index->lock_type != MAIL_LOCK_UNLOCK); *error = NULL; - if (messages_count == 0) { - /* no messages in mailbox */ - return 1; - } all_found = TRUE; input = uidset; while (*input != '\0') { if (*input == '*') { /* last message */ - rec = index->lookup(index, messages_count); - uid = rec == NULL ? 0 : rec->uid; + if (messages_count == 0) + uid = 0; + else { + rec = index->lookup(index, messages_count); + uid = rec == NULL ? 0 : rec->uid; + } input++; } else { uid = get_next_number(&input); @@ -283,6 +283,12 @@ } } else { uid2 = index->header->next_uid-1; + if (uid2 < uid) { + /* allow requesting "n:*" where n is + larger than the actual (synced) + messages count */ + uid2 = uid; + } input++; } }
--- a/src/lib-storage/index/index-search.c Fri Sep 13 03:01:23 2002 +0300 +++ b/src/lib-storage/index/index-search.c Sat Sep 14 14:09:42 2002 +0300 @@ -703,6 +703,9 @@ { IndexMailbox *ibox = (IndexMailbox *) box; + if (!ibox->index->sync(ibox->index)) + return mail_storage_set_index_error(ibox); + if (!ibox->index->set_lock(ibox->index, MAIL_LOCK_SHARED)) return mail_storage_set_index_error(ibox);
--- a/src/lib-storage/index/index-status.c Fri Sep 13 03:01:23 2002 +0300 +++ b/src/lib-storage/index/index-status.c Sat Sep 14 14:09:42 2002 +0300 @@ -101,6 +101,9 @@ memset(status, 0, sizeof(MailboxStatus)); + if (!ibox->index->sync(ibox->index)) + return mail_storage_set_index_error(ibox); + if (!ibox->index->set_lock(ibox->index, MAIL_LOCK_SHARED)) return mail_storage_set_index_error(ibox);
--- a/src/lib-storage/index/index-storage.c Fri Sep 13 03:01:23 2002 +0300 +++ b/src/lib-storage/index/index-storage.c Sat Sep 14 14:09:42 2002 +0300 @@ -48,7 +48,11 @@ { ibox->box.inconsistent = ibox->index->is_inconsistency_error(ibox->index); - mail_storage_set_internal_error(ibox->box.storage); + + if (ibox->index->is_diskspace_error(ibox->index)) + mail_storage_set_error(ibox->box.storage, "Out of disk space"); + else + mail_storage_set_internal_error(ibox->box.storage); index_reset_error(ibox->index); return FALSE; }
--- a/src/lib-storage/index/index-sync.c Fri Sep 13 03:01:23 2002 +0300 +++ b/src/lib-storage/index/index-sync.c Sat Sep 14 14:09:42 2002 +0300 @@ -25,6 +25,9 @@ *messages = 0; + if (!ibox->index->sync(ibox->index)) + return mail_storage_set_index_error(ibox); + if (!ibox->index->set_lock(ibox->index, expunge ? MAIL_LOCK_EXCLUSIVE : MAIL_LOCK_SHARED)) return mail_storage_set_index_error(ibox);
--- a/src/lib-storage/index/maildir/maildir-copy.c Fri Sep 13 03:01:23 2002 +0300 +++ b/src/lib-storage/index/maildir/maildir-copy.c Sat Sep 14 14:09:42 2002 +0300 @@ -20,6 +20,9 @@ const char *fname; char src[1024], dest[1024]; + /* FIXME: this is buggy */ + if (1) return FALSE; + /* link the file */ fname = index->lookup_field(index, rec, FIELD_TYPE_LOCATION); i_snprintf(src, sizeof(src), "%s/cur/%s", index->dir, fname); @@ -44,6 +47,9 @@ CopyHardContext ctx; int ret; + if (!src->index->sync(src->index)) + return mail_storage_set_index_error(src); + if (!src->index->set_lock(src->index, MAIL_LOCK_SHARED)) return mail_storage_set_index_error(src); if (!dest->index->set_lock(dest->index, MAIL_LOCK_EXCLUSIVE)) {
--- a/src/lib-storage/subscription-file/subscription-file.c Fri Sep 13 03:01:23 2002 +0300 +++ b/src/lib-storage/subscription-file/subscription-file.c Sat Sep 14 14:09:42 2002 +0300 @@ -3,6 +3,7 @@ /* ugly code here - text files are annoying to manage */ #include "lib.h" +#include "file-lock.h" #include "mmap-util.h" #include "write-full.h" #include "imap-match.h" @@ -14,24 +15,6 @@ #define SUBSCRIPTION_FILE_NAME ".subscriptions" -static int lock_file(int fd, int type) -{ - struct flock fl; - - /* lock whole file */ - fl.l_type = type; - fl.l_whence = SEEK_SET; - fl.l_start = 0; - fl.l_len = 0; - - while (fcntl(fd, F_SETLKW, &fl) == -1) { - if (errno != EINTR) - return FALSE; - } - - return TRUE; -} - static int subscription_open(MailStorage *storage, int update, const char **path, void **mmap_base, size_t *mmap_length) @@ -51,9 +34,10 @@ return -1; } - if (!lock_file(fd, update ? F_WRLCK : F_RDLCK)) { - mail_storage_set_critical(storage, "fcntl() failed for " - "subscription file %s: %m", *path); + if (!file_wait_lock(fd, update ? F_WRLCK : F_RDLCK)) { + mail_storage_set_critical(storage, "file_wait_lock() failed " + "for subscription file %s: %m", + *path); (void)close(fd); return -1; } @@ -171,11 +155,7 @@ failed = TRUE; } - if (close(fd) == -1) { - mail_storage_set_critical(storage, "close() failed for " - "subscription file %s: %m", path); - failed = TRUE; - } + (void)close(fd); return !failed; }
--- a/src/lib/Makefile.am Fri Sep 13 03:01:23 2002 +0300 +++ b/src/lib/Makefile.am Sat Sep 14 14:09:42 2002 +0300 @@ -15,6 +15,7 @@ compat.c \ failures.c \ fdpass.c \ + file-lock.c \ file-set-size.c \ gmtoff.c \ hash.c \ @@ -30,6 +31,7 @@ mempool.c \ mempool-alloconly.c \ mempool-system.c \ + mmap-anon.c \ mmap-util.c \ network.c \ primes.c \ @@ -48,6 +50,7 @@ compat.h \ failures.h \ fdpass.h \ + file-lock.h \ file-set-size.h \ gmtoff.h \ hash.h \
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib/file-lock.c Sat Sep 14 14:09:42 2002 +0300 @@ -0,0 +1,57 @@ +/* + file-lock.c - Simple way to lock whole file descriptor + + Copyright (c) 2002 Timo Sirainen + + Permission is hereby granted, free of charge, to any person obtaining + a copy of this software and associated documentation files (the + "Software"), to deal in the Software without restriction, including + without limitation the rights to use, copy, modify, merge, publish, + distribute, sublicense, and/or sell copies of the Software, and to + permit persons to whom the Software is furnished to do so, subject to + the following conditions: + + The above copyright notice and this permission notice shall be + included in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS + OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY + CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, + TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +*/ + +#include "lib.h" +#include "file-lock.h" + +static int file_lock(int fd, int wait_lock, int lock_type) +{ + struct flock fl; + + fl.l_type = lock_type; + fl.l_whence = SEEK_SET; + fl.l_start = 0; + fl.l_len = 0; + + while (fcntl(fd, wait_lock ? F_SETLKW : F_SETLK, &fl) < 0) { + if (!wait_lock && (errno == EACCES || errno == EAGAIN)) + return 0; + + if (errno != EINTR) + return -1; + } + + return 1; +} + +int file_try_lock(int fd, int lock_type) +{ + return file_lock(fd, FALSE, lock_type); +} + +int file_wait_lock(int fd, int lock_type) +{ + return file_lock(fd, TRUE, lock_type); +}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib/file-lock.h Sat Sep 14 14:09:42 2002 +0300 @@ -0,0 +1,14 @@ +#ifndef __FILE_LOCK_H +#define __FILE_LOCK_H + +#include <unistd.h> +#include <fcntl.h> + +/* Lock whole file descriptor. Returns 1 if successful, 0 if lock failed, + or -1 if error. lock_type is F_WRLCK, F_RDLCK or F_UNLCK. */ +int file_try_lock(int fd, int lock_type); + +/* Lock whole file descriptor. Returns 1 if successful, or -1 if error. */ +int file_wait_lock(int fd, int lock_type); + +#endif
--- a/src/lib/file-set-size.c Fri Sep 13 03:01:23 2002 +0300 +++ b/src/lib/file-set-size.c Sat Sep 14 14:09:42 2002 +0300 @@ -25,6 +25,7 @@ #include "lib.h" #include "write-full.h" +#include "file-set-size.h" #include <unistd.h> @@ -36,15 +37,18 @@ i_assert(size >= 0); + /* FIXME: this may not be good idea, since mmap()ing and writing + to it creates fragmentation. */ + /* try truncating it to the size we want. if this succeeds, the written area is full of zeros - exactly what we want. however, this may not work at all, in which case we fallback to write()ing the zeros. */ - ret = ftruncate(fd, size); + ret = -1;/*ftruncate(fd, size)*/; old_errno = errno; pos = lseek(fd, 0, SEEK_END); if (ret != -1 && pos == size) - return lseek(fd, 0, SEEK_SET) < 0 ? -1 : 0; + return 0; if (pos < 0) return -1; @@ -55,8 +59,9 @@ return -1; } + size -= pos; + /* start growing the file */ - size -= pos; memset(block, 0, sizeof(block)); while ((uoff_t)size > sizeof(block)) {
--- a/src/lib/file-set-size.h Fri Sep 13 03:01:23 2002 +0300 +++ b/src/lib/file-set-size.h Sat Sep 14 14:09:42 2002 +0300 @@ -2,7 +2,8 @@ #define __FILE_SET_SIZE_H /* Shrink/grow file. If file is grown, the new data is guaranteed to - be zeros. Returns -1 if failed, 0 if successful. */ + be zeros. The file offset may be anywhere after this call. + Returns -1 if failed, 0 if successful. */ int file_set_size(int fd, off_t size); #endif
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib/mmap-anon.c Sat Sep 14 14:09:42 2002 +0300 @@ -0,0 +1,312 @@ +/* + Copyright (c) 2002 Timo Sirainen + + Permission is hereby granted, free of charge, to any person obtaining + a copy of this software and associated documentation files (the + "Software"), to deal in the Software without restriction, including + without limitation the rights to use, copy, modify, merge, publish, + distribute, sublicense, and/or sell copies of the Software, and to + permit persons to whom the Software is furnished to do so, subject to + the following conditions: + + The above copyright notice and this permission notice shall be + included in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS + OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY + CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, + TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +*/ + +#include "lib.h" +#include "mmap-util.h" + +#include <fcntl.h> + +#if !defined(MAP_ANONYMOUS) && defined(MAP_ANON) +# define MAP_ANONYMOUS MAP_ANON +#endif + +#undef HAVE_LINUX_MREMAP +#undef MAP_ANONYMOUS + +#ifndef HAVE_LINUX_MREMAP + +#include <sys/mman.h> + +/* MMAP_BASE_MOVE may be negative as well */ +#if SSIZE_T_MAX >= LLONG_MAX + /* 64bit or more */ +# define MMAP_BASE_MOVE (1024ULL*1024ULL*1024ULL*128ULL) /* 128GB */ +#else + /* 32bit most likely */ +# define MMAP_BASE_MOVE (1024UL*1024UL*128UL) /* 128M */ +#endif + +/* get it near 4kB which is the most common page size */ +#define MAX_CHUNKS (4096 / 2 / sizeof(size_t) - 3) + +#define MMAP_SIGNATURE 0xdeadbeef + +#define PAGE_ALIGN(size) \ + (((size) + page_size) & ~(page_size-1)) + +struct movable_header { + unsigned int signature; + int chunks; + size_t size; + + void *chunk_ptr[MAX_CHUNKS]; + size_t chunk_size[MAX_CHUNKS]; +}; + +static int page_size = 0; +static int header_size = 0; +static void *movable_mmap_base = NULL; + +static void movable_mmap_init(void) +{ + page_size = getpagesize(); + + if (page_size >= header_size) + header_size = page_size; + else { + header_size = sizeof(struct movable_header) + page_size - + sizeof(struct movable_header) % page_size; + } +} + +static int anon_mmap_fixed(void *address, size_t length) +{ + void *base; + +#ifdef MAP_ANONYMOUS + base = mmap(address, length, PROT_READ | PROT_WRITE, + MAP_FIXED | MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); +#else + int fd; + + /* mmap()ing /dev/zero should be the same with some platforms */ + fd = open("/dev/zero", O_RDWR); + if (fd == -1) + i_fatal("Can't open /dev/zero for creating anonymous mmap"); + + base = mmap(address, length, PROT_READ | PROT_WRITE, + MAP_FIXED | MAP_PRIVATE, fd, 0); + + (void)close(fd); +#endif + + if (base != MAP_FAILED && base != address) { + /* shouldn't happen with MAP_FIXED, but who knows.. */ + if (munmap(base, length) < 0) + i_panic("munmap() failed: %m"); + base = MAP_FAILED; + errno = EINVAL; + } + + return base == MAP_FAILED ? -1 : 0; +} + +void *mmap_anon(size_t length) +{ + struct movable_header *hdr; + int ret; + + if (header_size == 0) + movable_mmap_init(); + + /* we need extra page to store the pieces which construct + the full mmap. also allocate only page-aligned mmap sizes. */ + length += header_size; + length = PAGE_ALIGN(length); + + if (movable_mmap_base == NULL) { + /* this is fully guessing */ + movable_mmap_base = ((char *) mmap_anon) + MMAP_BASE_MOVE; + movable_mmap_base = NULL + PAGE_ALIGN(movable_mmap_base - NULL); + } + + do { + ret = anon_mmap_fixed(movable_mmap_base, length); + hdr = movable_mmap_base; + + movable_mmap_base = (char *) movable_mmap_base + + MMAP_BASE_MOVE; + + } while (ret == -1 && errno == EINVAL); + + if (ret == -1) + return MAP_FAILED; + + /* initialize the header */ + hdr->signature = MMAP_SIGNATURE; + hdr->chunks = 1; + hdr->size = length; + hdr->chunk_ptr[0] = hdr; + hdr->chunk_size[0] = length; + + return (char *) hdr + header_size; +} + +static void *remove_chunks(struct movable_header *hdr, size_t new_size) +{ + int i; + + for (i = hdr->chunks-1; i > 0; i--) { + if (hdr->size - hdr->chunk_size[i] < new_size) + break; + + if (munmap(hdr->chunk_ptr[i], hdr->chunk_size[i]) < 0) + i_panic("munmap() failed: %m"); + + hdr->size -= hdr->chunk_size[i]; + hdr->chunks--; + } + + return (char *) hdr + header_size; +} + +static void *move_chunks(struct movable_header *hdr, size_t new_size, + int maymove) +{ + unsigned char *new_base, *p; + int i; + + if (!maymove) { + errno = ENOMEM; + return MAP_FAILED; + } + + new_base = mmap_anon(new_size - header_size); + if (new_base == MAP_FAILED) + return MAP_FAILED; + + /* copy first chunk without header */ + memcpy(new_base, (char *) hdr->chunk_ptr[0] + header_size, + hdr->chunk_size[0] - header_size); + p = new_base + (hdr->chunk_size[0] - header_size); + + for (i = 1; i < hdr->chunks; i++) { + memcpy(p, hdr->chunk_ptr[i], hdr->chunk_size[i]); + p += hdr->chunk_size[i]; + + if (munmap(hdr->chunk_ptr[i], hdr->chunk_size[i]) < 0) + i_panic("munmap() failed: %m"); + } + + if (munmap(hdr->chunk_ptr[0], hdr->chunk_size[0]) < 0) + i_panic("munmap() failed: %m"); + return new_base; +} + +static void *add_chunks(struct movable_header *hdr, size_t new_size, + int maymove) +{ + void *base; + size_t chunk_size; + + new_size = PAGE_ALIGN(new_size); + if (hdr->chunks == MAX_CHUNKS) + return move_chunks(hdr, new_size, maymove); + + /* get our new address, make sure we're not over/underflowing + our address */ +#if MMAP_BASE_MOVE > 0 + base = (char *) hdr->chunk_ptr[hdr->chunks-1] + + hdr->chunk_size[hdr->chunks-1]; + if (base < hdr->chunk_ptr[hdr->chunks-1]) + return move_chunks(hdr, new_size, maymove); +#else + base = (char *) hdr->chunk_ptr[hdr->chunks-1] - + hdr->chunk_size[hdr->chunks-1]; + if (base > hdr->chunk_ptr[hdr->chunks-1]) + return move_chunks(hdr, new_size, maymove); +#endif + + chunk_size = new_size - hdr->size; + if (anon_mmap_fixed(base, chunk_size) < 0) { + if (errno == EINVAL) { + /* can't grow, the memory address is used */ + return move_chunks(hdr, new_size, maymove); + } + + return MAP_FAILED; + } + + hdr->chunk_ptr[hdr->chunks] = base; + hdr->chunk_size[hdr->chunks] = chunk_size; + hdr->size += chunk_size; + hdr->chunks++; + + return (char *) hdr + header_size; +} + +void *mremap_anon(void *old_address, size_t old_size __attr_unused__, + size_t new_size, unsigned long flags) +{ + struct movable_header *hdr; + + if (old_address == NULL || old_address == MAP_FAILED) { + errno = EINVAL; + return MAP_FAILED; + } + + hdr = (struct movable_header *) ((char *) old_address - header_size); + if (hdr->signature != MMAP_SIGNATURE) + i_panic("movable_mremap(): Invalid old_address"); + + new_size += header_size; + + if (hdr->size > new_size) + return remove_chunks(hdr, new_size); + else + return add_chunks(hdr, new_size, (flags & MREMAP_MAYMOVE) != 0); +} + +int munmap_anon(void *start, size_t length __attr_unused__) +{ + struct movable_header *hdr; + int i; + + if (start == NULL || start == MAP_FAILED) { + errno = EINVAL; + return -1; + } + + hdr = (struct movable_header *) ((char *) start - header_size); + if (hdr->signature != MMAP_SIGNATURE) + i_panic("movable_munmap(): Invalid address"); + + /* [0] chunk must be free'd last since it contains the header */ + for (i = hdr->chunks-1; i >= 0; i--) { + if (munmap(hdr->chunk_ptr[i], hdr->chunk_size[i]) < 0) + i_panic("munmap() failed: %m"); + } + + return 0; +} + +#else + +void *mmap_anon(size_t length) +{ + return mmap(NULL, length, PROT_READ | PROT_WRITE, + MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); +} + +void *mremap_anon(void *old_address, size_t old_size, size_t new_size, + unsigned long flags) +{ + return mremap(old_address, old_size, new_size, flags); +} + +int munmap_anon(void *start, size_t length) +{ + return munmap(start, length); +} + +#endif
--- a/src/lib/mmap-util.c Fri Sep 13 03:01:23 2002 +0300 +++ b/src/lib/mmap-util.c Sat Sep 14 14:09:42 2002 +0300 @@ -26,10 +26,6 @@ #include "lib.h" #include "mmap-util.h" -#if !defined(MAP_ANONYMOUS) && defined(MAP_ANON) -# define MAP_ANONYMOUS MAP_ANON -#endif - static void *mmap_file(int fd, size_t *length, int access) { off_t size; @@ -85,12 +81,6 @@ return mmap_base; } -void *mmap_anonymous(size_t length) -{ - return mmap(NULL, length, PROT_READ | PROT_WRITE, - MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); -} - #ifndef HAVE_MADVISE int madvise(void *start, size_t length, int advice) {
--- a/src/lib/mmap-util.h Fri Sep 13 03:01:23 2002 +0300 +++ b/src/lib/mmap-util.h Sat Sep 14 14:09:42 2002 +0300 @@ -2,7 +2,17 @@ #define __MMAP_UTIL_H #include <unistd.h> + +#ifdef HAVE_LINUX_MREMAP +# define __USE_GNU /* for MREMAP_MAYMOVE */ +#endif + #include <sys/mman.h> +#undef __USE_GNU + +#if !defined (MREMAP_MAYMOVE) && !defined (HAVE_LINUX_MREMAP) +# define MREMAP_MAYMOVE 1 +#endif #ifndef HAVE_MADVISE int madvise(void *start, size_t length, int advice); @@ -21,6 +31,11 @@ void *mmap_aligned(int fd, int access, off_t offset, size_t length, void **data_start, size_t *mmap_length); -void *mmap_anonymous(size_t length); +/* for allocating anonymous mmap()s, with portable mremap(). these must not + be mixed with any standard mmap calls. */ +void *mmap_anon(size_t length); +void *mremap_anon(void *old_address, size_t old_size, size_t new_size, + unsigned long flags); +int munmap_anon(void *start, size_t length); #endif