Mercurial > dovecot > original-hg > dovecot-1.2
changeset 1622:d22e03714d36 HEAD
index_mmap_invalidate = yes now invalidate memory maps before accessing
them. Setting this on should fix some problems with OpenBSD. It should also
make it possible to use index files over NFS as long as lock daemon is used.
It might be such a good idea however.
| author | Timo Sirainen <tss@iki.fi> |
|---|---|
| date | Tue, 15 Jul 2003 21:26:42 +0300 |
| parents | 401f0c3a3aa2 |
| children | e31dbc7ca0da |
| files | configure.in dovecot-example.conf src/imap/cmd-append.c src/imap/cmd-copy.c src/imap/cmd-select.c src/imap/cmd-status.c src/imap/common.h src/imap/main.c src/lib-index/mail-index-data.c src/lib-index/mail-index.c src/lib-index/mail-index.h src/lib-index/mail-modifylog.c src/lib-index/mail-tree.c src/lib-storage/index/index-storage.c src/lib-storage/index/index-storage.h src/lib-storage/index/maildir/maildir-storage.c src/lib-storage/index/mbox/mbox-storage.c src/lib-storage/mail-storage.h src/master/mail-process.c src/master/master-settings.c src/master/master-settings.h src/pop3/client.c |
| diffstat | 22 files changed, 144 insertions(+), 33 deletions(-) [+] |
line wrap: on
line diff
--- a/configure.in Tue Jul 15 15:38:05 2003 +0300 +++ b/configure.in Tue Jul 15 21:26:42 2003 +0300 @@ -593,6 +593,44 @@ AC_MSG_RESULT(no) ]) +dnl * If mmap() plays nicely with write() +AC_MSG_CHECKING([whether we need to use MS_INVALIDATE with mmaps]) +AC_TRY_RUN([ + #include <stdio.h> + #include <sys/types.h> + #include <sys/stat.h> + #include <unistd.h> + #include <fcntl.h> + #include <sys/mman.h> + int main() { + /* return 0 if we're signed */ + int f = open("conftest.mmap", O_RDWR|O_CREAT|O_TRUNC); + void *mem; + if (f == -1) { + perror("open()"); + return 1; + } + write(f, "1", 2); + mem = mmap(NULL, 2, PROT_READ|PROT_WRITE, MAP_SHARED, f, 0); + if (mem == MAP_FAILED) { + perror("mmap()"); + return 1; + } + strcpy(mem, "2"); + msync(mem, 2, MS_SYNC); + lseek(f, 0, SEEK_SET); + write(f, "3", 2); + + return strcmp(mem, "3") == 0 ? 0 : 1; + } +], + AC_MSG_RESULT(no) +], [ + AC_MSG_RESULT(yes) + AC_DEFINE(NEED_MS_INVALIDATE,, Define if your mmap() implementation requires use of MS_INVALIDATE to work with write()) +]) + + dnl * Solaris compatible sendfilev() AC_CHECK_LIB(sendfile, sendfilev, [ LIBS="$LIBS -lsendfile"
--- a/dovecot-example.conf Tue Jul 15 15:38:05 2003 +0300 +++ b/dovecot-example.conf Tue Jul 15 21:26:42 2003 +0300 @@ -234,6 +234,7 @@ # Use mmap() instead of read() to read mail files. read() seems to be a bit # faster with my Linux/x86 and it's better with NFS, so that's the default. +# Note that OpenBSD 3.3 and older don't work right with mail_read_mmaped = yes. #mail_read_mmaped = no # Copy mail to another folders using hard links. This is much faster than @@ -286,6 +287,11 @@ # for multiple users, as the users could ptrace() each others processes then. #mail_drop_priv_before_exec = no +# Call mmap() with MS_INVALIDATE flag each time before accessing message +# indexes. This is currently required with OpenBSD 3.3 and older versions. +# It's also required if you insist on saving index files over NFS. +#index_mmap_invalidate = no + # Set max. process size in megabytes. Most of the memory goes to mmap()ing # files, so it shouldn't harm much even if this limit is set pretty high. #mail_process_size = 256
--- a/src/imap/cmd-append.c Tue Jul 15 15:38:05 2003 +0300 +++ b/src/imap/cmd-append.c Tue Jul 15 21:26:42 2003 +0300 @@ -67,7 +67,8 @@ return TRUE; box = client->storage->open_mailbox(client->storage, - mailbox, FALSE, TRUE); + mailbox, mailbox_open_flags | + MAILBOX_OPEN_FAST); if (box == NULL) { client_send_storage_error(client); return TRUE;
--- a/src/imap/cmd-copy.c Tue Jul 15 15:38:05 2003 +0300 +++ b/src/imap/cmd-copy.c Tue Jul 15 21:26:42 2003 +0300 @@ -21,7 +21,8 @@ return TRUE; destbox = client->storage->open_mailbox(client->storage, - mailbox, FALSE, TRUE); + mailbox, mailbox_open_flags | + MAILBOX_OPEN_FAST); if (destbox == NULL) { client_send_storage_error(client); return TRUE;
--- a/src/imap/cmd-select.c Tue Jul 15 15:38:05 2003 +0300 +++ b/src/imap/cmd-select.c Tue Jul 15 21:26:42 2003 +0300 @@ -7,6 +7,7 @@ { struct mailbox *box; struct mailbox_status status; + enum mailbox_open_flags flags; const char *mailbox; /* <mailbox> */ @@ -20,8 +21,10 @@ client_send_untagged_storage_error(client); } - box = client->storage->open_mailbox(client->storage, mailbox, - readonly, FALSE); + flags = mailbox_open_flags; + if (readonly) + flags |= MAILBOX_OPEN_READONLY; + box = client->storage->open_mailbox(client->storage, mailbox, flags); if (box == NULL) { client_send_storage_error(client); return TRUE;
--- a/src/imap/cmd-status.c Tue Jul 15 15:38:05 2003 +0300 +++ b/src/imap/cmd-status.c Tue Jul 15 21:26:42 2003 +0300 @@ -65,7 +65,10 @@ } else { /* open the mailbox */ box = client->storage->open_mailbox(client->storage, - mailbox, TRUE, TRUE); + mailbox, + mailbox_open_flags | + MAILBOX_OPEN_FAST | + MAILBOX_OPEN_READONLY); if (box == NULL) return FALSE; }
--- a/src/imap/common.h Tue Jul 15 15:38:05 2003 +0300 +++ b/src/imap/common.h Tue Jul 15 21:26:42 2003 +0300 @@ -17,6 +17,7 @@ extern struct ioloop *ioloop; extern unsigned int max_custom_flag_length, mailbox_check_interval; extern unsigned int imap_max_line_length; +extern enum mailbox_open_flags mailbox_open_flags; extern string_t *capability_string;
--- a/src/imap/main.c Tue Jul 15 15:38:05 2003 +0300 +++ b/src/imap/main.c Tue Jul 15 21:26:42 2003 +0300 @@ -24,6 +24,7 @@ struct ioloop *ioloop; unsigned int max_custom_flag_length, mailbox_check_interval; unsigned int imap_max_line_length; +enum mailbox_open_flags mailbox_open_flags; static struct module *modules; static char log_prefix[128]; /* syslog() needs this to be permanent */ @@ -147,6 +148,9 @@ mailbox_check_interval = str == NULL ? 0 : (unsigned int)strtoul(str, NULL, 10); + mailbox_open_flags = getenv("MMAP_INVALIDATE") != NULL ? + MAILBOX_OPEN_MMAP_INVALIDATE : 0; + client = client_create(hin, hout, storage); o_stream_cork(client->output);
--- a/src/lib-index/mail-index-data.c Tue Jul 15 15:38:05 2003 +0300 +++ b/src/lib-index/mail-index-data.c Tue Jul 15 21:26:42 2003 +0300 @@ -133,6 +133,12 @@ { struct mail_index_data_header *hdr; + if (data->index->mmap_invalidate && data->mmap_base != NULL) { + if (msync(data->mmap_base, data->mmap_used_length, + MS_SYNC | MS_INVALIDATE) < 0) + return index_data_set_syscall_error(data, "msync()"); + } + if (data->header != NULL && data->header->indexid != data->index->indexid) { if (data->header->indexid != 0) {
--- a/src/lib-index/mail-index.c Tue Jul 15 15:38:05 2003 +0300 +++ b/src/lib-index/mail-index.c Tue Jul 15 21:26:42 2003 +0300 @@ -89,6 +89,15 @@ if (index->mmap_base != NULL) { index->header = (struct mail_index_header *) index->mmap_base; + if (index->mmap_invalidate) { + if (msync(index->mmap_base, + index->mmap_used_length, + MS_SYNC | MS_INVALIDATE) < 0) { + index_set_syscall_error(index, "msync()"); + return FALSE; + } + } + /* make sure file size hasn't changed */ if (index->header->sync_id == index->sync_id) { index->mmap_used_length = index->header->used_file_size; @@ -99,9 +108,13 @@ return TRUE; } - if (msync(index->mmap_base, - index->mmap_used_length, MS_SYNC) < 0) - return index_set_syscall_error(index, "msync()"); + if (!index->mmap_invalidate) { + if (msync(index->mmap_base, + index->mmap_used_length, MS_SYNC) < 0) { + index_set_syscall_error(index, "msync()"); + return FALSE; + } + } if (munmap(index->mmap_base, index->mmap_full_length) < 0) return index_set_syscall_error(index, "munmap()");
--- a/src/lib-index/mail-index.h Tue Jul 15 15:38:05 2003 +0300 +++ b/src/lib-index/mail-index.h Tue Jul 15 21:26:42 2003 +0300 @@ -16,9 +16,11 @@ MAIL_INDEX_OPEN_FLAG_UPDATE_RECENT = 0x02, /* Compressing and cache updates are not performed */ MAIL_INDEX_OPEN_FLAG_FAST = 0x04, + /* Invalidate memory maps before accessing them */ + MAIL_INDEX_OPEN_FLAG_MMAP_INVALIDATE = 0x08, /* internal: we're creating the index */ - _MAIL_INDEX_OPEN_FLAG_CREATING = 0x10 + _MAIL_INDEX_OPEN_FLAG_CREATING = 0x100 }; enum mail_index_header_compat { @@ -433,6 +435,7 @@ enum mail_data_field set_cache_fields; unsigned int anon_mmap:1; + unsigned int mmap_invalidate:1; unsigned int opened:1; unsigned int rebuilding:1; unsigned int mail_read_mmaped:1; @@ -460,7 +463,7 @@ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, \ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, \ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, \ - 0, 0, 0, 0, 0 + 0, 0, 0, 0, 0, 0 #endif /* defaults - same as above but prefixed with mail_index_. */
--- a/src/lib-index/mail-modifylog.c Tue Jul 15 15:38:05 2003 +0300 +++ b/src/lib-index/mail-modifylog.c Tue Jul 15 21:26:42 2003 +0300 @@ -176,6 +176,12 @@ struct modify_log_header *hdr; unsigned int extra; + if (file->log->index->mmap_invalidate && file->mmap_base != NULL) { + if (msync(file->mmap_base, file->mmap_used_length, + MS_SYNC | MS_INVALIDATE) < 0) + return modifylog_set_syscall_error(file, "msync()"); + } + if (!forced && file->header != NULL && file->mmap_full_length >= file->header->used_file_size) { file->mmap_used_length = file->header->used_file_size;
--- a/src/lib-index/mail-tree.c Tue Jul 15 15:38:05 2003 +0300 +++ b/src/lib-index/mail-tree.c Tue Jul 15 21:26:42 2003 +0300 @@ -131,6 +131,12 @@ int _mail_tree_mmap_update(struct mail_tree *tree, int forced) { + if (tree->index->mmap_invalidate && tree->mmap_base != NULL) { + if (msync(tree->mmap_base, tree->mmap_used_length, + MS_SYNC | MS_INVALIDATE) < 0) + return tree_set_syscall_error(tree, "msync()"); + } + debug_mprotect(tree->mmap_base, tree->mmap_full_length, tree->index);
--- a/src/lib-storage/index/index-storage.c Tue Jul 15 15:38:05 2003 +0300 +++ b/src/lib-storage/index/index-storage.c Tue Jul 15 21:26:42 2003 +0300 @@ -299,20 +299,28 @@ struct index_mailbox * index_storage_mailbox_init(struct mail_storage *storage, struct mailbox *box, struct mail_index *index, const char *name, - int readonly, int fast) + enum mailbox_open_flags flags) { struct index_mailbox *ibox; - enum mail_index_open_flags flags; + enum mail_index_open_flags index_flags; i_assert(name != NULL); + index_flags = MAIL_INDEX_OPEN_FLAG_CREATE; + if ((flags & MAILBOX_OPEN_FAST) != 0) + index_flags |= MAIL_INDEX_OPEN_FLAG_FAST; + if ((flags & MAILBOX_OPEN_READONLY) != 0) + index_flags |= MAIL_INDEX_OPEN_FLAG_UPDATE_RECENT; + if ((flags & MAILBOX_OPEN_MMAP_INVALIDATE) != 0) + index_flags |= MAIL_INDEX_OPEN_FLAG_MMAP_INVALIDATE; + do { ibox = i_new(struct index_mailbox, 1); ibox->box = *box; ibox->box.storage = storage; ibox->box.name = i_strdup(name); - ibox->box.readonly = readonly; + ibox->box.readonly = (flags & MAILBOX_OPEN_READONLY) != 0; ibox->index = index; @@ -326,13 +334,7 @@ index->never_cache_fields = get_never_cache_fields(); - flags = MAIL_INDEX_OPEN_FLAG_CREATE; - if (fast) - flags |= MAIL_INDEX_OPEN_FLAG_FAST; - if (!readonly) - flags |= MAIL_INDEX_OPEN_FLAG_UPDATE_RECENT; - - if (!index->open(index, flags)) + if (!index->open(index, index_flags)) break; if (INDEX_IS_IN_MEMORY(index) &&
--- a/src/lib-storage/index/index-storage.h Tue Jul 15 15:38:05 2003 +0300 +++ b/src/lib-storage/index/index-storage.h Tue Jul 15 21:26:42 2003 +0300 @@ -52,7 +52,7 @@ struct index_mailbox * index_storage_mailbox_init(struct mail_storage *storage, struct mailbox *box, struct mail_index *index, const char *name, - int readonly, int fast); + enum mailbox_open_flags flags); int index_storage_mailbox_free(struct mailbox *box); int index_storage_sync_and_lock(struct index_mailbox *ibox,
--- a/src/lib-storage/index/maildir/maildir-storage.c Tue Jul 15 15:38:05 2003 +0300 +++ b/src/lib-storage/index/maildir/maildir-storage.c Tue Jul 15 21:26:42 2003 +0300 @@ -321,7 +321,7 @@ static struct mailbox * maildir_open(struct mail_storage *storage, const char *name, - int readonly, int fast) + enum mailbox_open_flags flags) { struct index_mailbox *ibox; struct mail_index *index; @@ -338,7 +338,7 @@ } ibox = index_storage_mailbox_init(storage, &maildir_mailbox, - index, name, readonly, fast); + index, name, flags); if (ibox != NULL) ibox->expunge_locked = maildir_expunge_locked; return (struct mailbox *) ibox; @@ -359,7 +359,7 @@ static struct mailbox * maildir_open_mailbox(struct mail_storage *storage, - const char *name, int readonly, int fast) + const char *name, enum mailbox_open_flags flags) { const char *path; struct stat st; @@ -370,7 +370,7 @@ if (strcmp(name, "INBOX") == 0) { if (!verify_inbox(storage)) return NULL; - return maildir_open(storage, "INBOX", readonly, fast); + return maildir_open(storage, "INBOX", flags); } if (!maildir_is_valid_existing_name(name)) { @@ -386,7 +386,7 @@ !create_control_dir(storage, name)) return FALSE; - return maildir_open(storage, name, readonly, fast); + return maildir_open(storage, name, flags); } else if (errno == ENOENT) { mail_storage_set_error(storage, "Mailbox doesn't exist: %s", name);
--- a/src/lib-storage/index/mbox/mbox-storage.c Tue Jul 15 15:38:05 2003 +0300 +++ b/src/lib-storage/index/mbox/mbox-storage.c Tue Jul 15 21:26:42 2003 +0300 @@ -320,7 +320,7 @@ } static struct mailbox *mbox_open(struct mail_storage *storage, const char *name, - int readonly, int fast) + enum mailbox_open_flags flags) { struct index_mailbox *ibox; struct mail_index *index; @@ -347,7 +347,7 @@ } ibox = index_storage_mailbox_init(storage, &mbox_mailbox, index, - name, readonly, fast); + name, flags); if (ibox != NULL) ibox->expunge_locked = mbox_expunge_locked; return (struct mailbox *) ibox; @@ -355,7 +355,7 @@ static struct mailbox * mbox_open_mailbox(struct mail_storage *storage, - const char *name, int readonly, int fast) + const char *name, enum mailbox_open_flags flags) { const char *path; struct stat st; @@ -367,7 +367,7 @@ /* make sure inbox exists */ if (!verify_inbox(storage)) return FALSE; - return mbox_open(storage, "INBOX", readonly, fast); + return mbox_open(storage, "INBOX", flags); } if (!mbox_is_valid_existing_name(name)) { @@ -387,7 +387,7 @@ if (!create_mbox_index_dirs(storage, name)) return NULL; - return mbox_open(storage, name, readonly, fast); + return mbox_open(storage, name, flags); } if (ENOTFOUND(errno)) {
--- a/src/lib-storage/mail-storage.h Tue Jul 15 15:38:05 2003 +0300 +++ b/src/lib-storage/mail-storage.h Tue Jul 15 21:26:42 2003 +0300 @@ -5,6 +5,12 @@ #include "imap-util.h" +enum mailbox_open_flags { + MAILBOX_OPEN_READONLY = 0x01, + MAILBOX_OPEN_FAST = 0x02, + MAILBOX_OPEN_MMAP_INVALIDATE = 0x04 +}; + enum mailbox_list_flags { MAILBOX_LIST_SUBSCRIBED = 0x01, MAILBOX_LIST_FAST_FLAGS = 0x02, @@ -148,7 +154,7 @@ with possibly different readonly-state. */ struct mailbox *(*open_mailbox)(struct mail_storage *storage, const char *name, - int readonly, int fast); + enum mailbox_open_flags flags); /* name is allowed to contain multiple new hierarchy levels. If only_hierarchy is TRUE, the mailbox itself isn't created, just
--- a/src/master/mail-process.c Tue Jul 15 15:38:05 2003 +0300 +++ b/src/master/mail-process.c Tue Jul 15 21:26:42 2003 +0300 @@ -214,6 +214,8 @@ env_put("MAILDIR_CHECK_CONTENT_CHANGES=1"); if (set->mail_full_filesystem_access) env_put("FULL_FILESYSTEM_ACCESS=1"); + if (set->index_mmap_invalidate) + env_put("MMAP_INVALIDATE=1"); (void)umask(set->umask); env_put(t_strconcat("MBOX_LOCKS=", set->mbox_locks, NULL));
--- a/src/master/master-settings.c Tue Jul 15 15:38:05 2003 +0300 +++ b/src/master/master-settings.c Tue Jul 15 21:26:42 2003 +0300 @@ -92,6 +92,7 @@ DEF(SET_INT, mbox_dotlock_change_timeout), DEF(SET_INT, umask), DEF(SET_BOOL, mail_drop_priv_before_exec), + DEF(SET_BOOL, index_mmap_invalidate), DEF(SET_STR, mail_executable), DEF(SET_INT, mail_process_size), @@ -193,6 +194,11 @@ MEMBER(mbox_dotlock_change_timeout) 30, MEMBER(umask) 0077, MEMBER(mail_drop_priv_before_exec) FALSE, +#ifdef NEED_MS_INVALIDATE + MEMBER(index_mmap_invalidate) TRUE, +#else + MEMBER(index_mmap_invalidate) FALSE, +#endif MEMBER(mail_executable) PKG_LIBEXECDIR"/imap", MEMBER(mail_process_size) 256,
--- a/src/master/master-settings.h Tue Jul 15 15:38:05 2003 +0300 +++ b/src/master/master-settings.h Tue Jul 15 21:26:42 2003 +0300 @@ -79,6 +79,7 @@ unsigned int mbox_dotlock_change_timeout; unsigned int umask; int mail_drop_priv_before_exec; + int index_mmap_invalidate; const char *mail_executable; unsigned int mail_process_size;
--- a/src/pop3/client.c Tue Jul 15 15:38:05 2003 +0300 +++ b/src/pop3/client.c Tue Jul 15 21:26:42 2003 +0300 @@ -106,6 +106,7 @@ struct client *client_create(int hin, int hout, struct mail_storage *storage) { struct client *client; + enum mailbox_open_flags flags; client = i_new(struct client, 1); client->input = i_stream_create_file(hin, default_pool, @@ -122,7 +123,9 @@ storage->set_callbacks(storage, &mail_storage_callbacks, client); - client->mailbox = storage->open_mailbox(storage, "INBOX", FALSE, FALSE); + flags = getenv("MMAP_INVALIDATE") != NULL ? + MAILBOX_OPEN_MMAP_INVALIDATE : 0; + client->mailbox = storage->open_mailbox(storage, "INBOX", flags); if (client->mailbox == NULL) { client_send_line(client, "-ERR No INBOX for user."); return NULL;