Mercurial > dovecot > original-hg > dovecot-1.2
changeset 8983:fdaf0bda70d5 HEAD
struct mail_private now contains all kinds of statistics about file accesses.
author | Timo Sirainen <tss@iki.fi> |
---|---|
date | Tue, 28 Apr 2009 19:55:51 -0400 |
parents | 947fab5a8992 |
children | e7bfb3c134f9 |
files | src/lib-storage/index/Makefile.am src/lib-storage/index/cydir/cydir-mail.c src/lib-storage/index/dbox/dbox-file.h src/lib-storage/index/dbox/dbox-mail.c src/lib-storage/index/index-mail-headers.c src/lib-storage/index/index-mail.c src/lib-storage/index/index-mail.h src/lib-storage/index/istream-mail-stats.c src/lib-storage/index/istream-mail-stats.h src/lib-storage/index/maildir/maildir-mail.c src/lib-storage/index/raw/raw-mail.c src/lib-storage/mail-storage-private.h |
diffstat | 12 files changed, 199 insertions(+), 16 deletions(-) [+] |
line wrap: on
line diff
--- a/src/lib-storage/index/Makefile.am Tue Apr 28 17:21:14 2009 -0400 +++ b/src/lib-storage/index/Makefile.am Tue Apr 28 19:55:51 2009 -0400 @@ -10,6 +10,7 @@ -I$(top_srcdir)/src/lib-storage libstorage_index_a_SOURCES = \ + istream-mail-stats.c \ index-fetch.c \ index-mail.c \ index-mail-headers.c \ @@ -29,6 +30,7 @@ index-transaction.c headers = \ + istream-mail-stats.h \ index-mail.h \ index-search-result.h \ index-sort.h \
--- a/src/lib-storage/index/cydir/cydir-mail.c Tue Apr 28 17:21:14 2009 -0400 +++ b/src/lib-storage/index/cydir/cydir-mail.c Tue Apr 28 19:55:51 2009 -0400 @@ -20,11 +20,13 @@ static int cydir_mail_stat(struct mail *mail, struct stat *st_r) { + struct mail_private *p = (struct mail_private *)mail; const char *path; if (mail->lookup_abort == MAIL_LOOKUP_ABORT_NOT_IN_CACHE) return mail_set_aborted(mail); + p->stats_dentry_lookup_count++; path = cydir_mail_get_path(mail); if (stat(path, st_r) < 0) { if (errno == ENOENT) @@ -98,6 +100,7 @@ int fd; if (mail->data.stream == NULL) { + mail->mail.stats_dentry_lookup_count++; path = cydir_mail_get_path(_mail); fd = open(path, O_RDONLY); if (fd == -1) {
--- a/src/lib-storage/index/dbox/dbox-file.h Tue Apr 28 17:21:14 2009 -0400 +++ b/src/lib-storage/index/dbox/dbox-file.h Tue Apr 28 19:55:51 2009 -0400 @@ -139,6 +139,8 @@ unsigned int deleted:1; }; +#define dbox_file_is_open(file) ((file)->input != NULL) + extern enum mail_flags dbox_mail_flags_map[DBOX_METADATA_FLAGS_COUNT]; extern char dbox_mail_flag_chars[DBOX_METADATA_FLAGS_COUNT];
--- a/src/lib-storage/index/dbox/dbox-mail.c Tue Apr 28 17:21:14 2009 -0400 +++ b/src/lib-storage/index/dbox/dbox-mail.c Tue Apr 28 19:55:51 2009 -0400 @@ -78,6 +78,9 @@ if (dbox_mail_lookup(mail, &offset, file_r) < 0) return -1; + if (!dbox_file_is_open(*file_r)) + mail->imail.mail.stats_dentry_lookup_count++; + ret = dbox_file_metadata_seek_mail_offset(*file_r, offset, &expunged); if (ret <= 0) { if (ret < 0) @@ -251,6 +254,8 @@ if (dbox_mail_lookup(mail, &offset, &mail->open_file) < 0) return -1; + if (!dbox_file_is_open(mail->open_file)) + mail->imail.mail.stats_dentry_lookup_count++; ret = dbox_file_get_mail_stream(mail->open_file, offset, &uid, &size, &input, &expunged); if (ret < 0)
--- a/src/lib-storage/index/index-mail-headers.c Tue Apr 28 17:21:14 2009 -0400 +++ b/src/lib-storage/index/index-mail-headers.c Tue Apr 28 19:55:51 2009 -0400 @@ -471,8 +471,8 @@ string_t *str; str = str_new(mail->data_pool, 256); - if (mail_cache_lookup_field(mail->trans->cache_view, str, - mail->data.seq, cache_field_envelope) > 0) { + if (index_mail_cache_lookup_field(mail, str, + cache_field_envelope) > 0) { mail->data.envelope = str_c(str); return 0; } @@ -637,6 +637,7 @@ *value_r = index_mail_get_parsed_header(mail, field_idx); return 0; } + mail->mail.stats_cache_hit_count++; data = buffer_get_modifiable_data(dest, &len); if (len == 0) { @@ -819,6 +820,7 @@ if (mail_cache_lookup_headers(mail->trans->cache_view, dest, mail->data.seq, headers->idx, headers->count) > 0) { + mail->mail.stats_cache_hit_count++; if (mail->data.filter_stream != NULL) i_stream_destroy(&mail->data.filter_stream); mail->data.filter_stream =
--- a/src/lib-storage/index/index-mail.c Tue Apr 28 17:21:14 2009 -0400 +++ b/src/lib-storage/index/index-mail.c Tue Apr 28 19:55:51 2009 -0400 @@ -15,6 +15,7 @@ #include "mail-cache.h" #include "mail-index-modseq.h" #include "index-storage.h" +#include "istream-mail-stats.h" #include "index-mail.h" struct mail_cache_field global_cache_fields[MAIL_INDEX_CACHE_FIELD_COUNT] = { @@ -40,6 +41,18 @@ static int index_mail_parse_body(struct index_mail *mail, enum index_cache_field field); +int index_mail_cache_lookup_field(struct index_mail *mail, buffer_t *buf, + unsigned int field_idx) +{ + int ret; + + ret = mail_cache_lookup_field(mail->trans->cache_view, buf, + mail->data.seq, field_idx); + if (ret > 0) + mail->mail.stats_cache_hit_count++; + return ret; +} + static struct message_part *get_unserialized_parts(struct index_mail *mail) { unsigned int field_idx = @@ -50,8 +63,7 @@ int ret; part_buf = buffer_create_dynamic(pool_datastack_create(), 128); - ret = mail_cache_lookup_field(mail->trans->cache_view, part_buf, - mail->data.seq, field_idx); + ret = index_mail_cache_lookup_field(mail, part_buf, field_idx); if (ret <= 0) return NULL; @@ -100,8 +112,7 @@ buf = buffer_create_data(pool_datastack_create(), data, data_size); - if (mail_cache_lookup_field(mail->trans->cache_view, buf, - mail->data.seq, field_idx) <= 0) + if (index_mail_cache_lookup_field(mail, buf, field_idx) <= 0) ret = FALSE; else { i_assert(buf->used == data_size); @@ -802,8 +813,17 @@ struct istream **stream_r) { struct index_mail_data *data = &mail->data; + struct istream *input; int ret; + if (!data->initialized_wrapper_stream && mail->mail.stats_track) { + input = i_stream_create_mail_stats_counter(&mail->mail, + data->stream); + i_stream_unref(&data->stream); + data->stream = input; + data->initialized_wrapper_stream = TRUE; + } + i_stream_set_destroy_callback(data->stream, index_mail_stream_destroy_callback, mail); @@ -953,11 +973,10 @@ get_cached_parts(mail)) { index_mail_get_plain_bodystructure(mail, str, FALSE); data->body = str_c(str); - } else if (mail_cache_lookup_field(mail->trans->cache_view, str, - mail->data.seq, body_cache_field) > 0) + } else if (index_mail_cache_lookup_field(mail, str, + body_cache_field) > 0) data->body = str_c(str); - else if (mail_cache_lookup_field(mail->trans->cache_view, str, - mail->data.seq, + else if (index_mail_cache_lookup_field(mail, str, bodystructure_cache_field) > 0) { data->bodystructure = p_strdup(mail->data_pool, str_c(str)); @@ -998,8 +1017,7 @@ get_cached_parts(mail)) { index_mail_get_plain_bodystructure(mail, str, TRUE); data->bodystructure = str_c(str); - } else if (mail_cache_lookup_field(mail->trans->cache_view, str, - mail->data.seq, + } else if (index_mail_cache_lookup_field(mail, str, bodystructure_cache_field) > 0) { data->bodystructure = str_c(str); } else {
--- a/src/lib-storage/index/index-mail.h Tue Apr 28 17:21:14 2009 -0400 +++ b/src/lib-storage/index/index-mail.h Tue Apr 28 19:55:51 2009 -0400 @@ -108,6 +108,7 @@ unsigned int header_parsed:1; unsigned int no_caching:1; unsigned int destroying_stream:1; + unsigned int initialized_wrapper_stream:1; }; struct index_mail { @@ -215,4 +216,7 @@ void index_mail_cache_parse_deinit(struct mail *mail, time_t received_date, bool success); +int index_mail_cache_lookup_field(struct index_mail *mail, buffer_t *buf, + unsigned int field_idx); + #endif
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-storage/index/istream-mail-stats.c Tue Apr 28 19:55:51 2009 -0400 @@ -0,0 +1,117 @@ +/* Copyright (c) 2009 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "mail-storage-private.h" +#include "istream-internal.h" +#include "istream-mail-stats.h" + +struct mail_stats_istream { + struct istream_private istream; + + struct mail_private *mail; + unsigned int files_read_increased:1; +}; + +static void i_stream_mail_stats_destroy(struct iostream_private *stream) +{ + struct mail_stats_istream *mstream = + (struct mail_stats_istream *)stream; + + i_stream_unref(&mstream->istream.parent); +} + +static void +i_stream_mail_stats_set_max_buffer_size(struct iostream_private *stream, + size_t max_size) +{ + struct mail_stats_istream *mstream = + (struct mail_stats_istream *)stream; + + mstream->istream.max_buffer_size = max_size; + i_stream_set_max_buffer_size(mstream->istream.parent, max_size); +} + +static ssize_t +i_stream_mail_stats_read_mail_stats(struct istream_private *stream) +{ + struct mail_stats_istream *mstream = + (struct mail_stats_istream *)stream; + size_t pos; + ssize_t ret; + + if (stream->parent->v_offset != + stream->parent_start_offset + stream->istream.v_offset) { + i_stream_seek(stream->parent, stream->parent_start_offset + + stream->istream.v_offset); + } + + stream->buffer = i_stream_get_data(stream->parent, &pos); + if (pos <= stream->pos) { + if ((ret = i_stream_read(stream->parent)) == -2) + return -2; + + if (ret > 0) { + mstream->mail->stats_files_read_bytes+= ret; + if (!mstream->files_read_increased) { + mstream->files_read_increased = TRUE; + mstream->mail->stats_files_read_count++; + } + } + + stream->istream.stream_errno = stream->parent->stream_errno; + stream->istream.eof = stream->parent->eof; + stream->buffer = i_stream_get_data(stream->parent, &pos); + } else { + ret = 0; + } + + stream->pos -= stream->skip; + stream->skip = 0; + + ret = pos > stream->pos ? (ssize_t)(pos - stream->pos) : + (ret == 0 ? 0 : -1); + stream->pos = pos; + i_assert(ret != -1 || stream->istream.eof || + stream->istream.stream_errno != 0); + return ret; +} + +static void +i_stream_mail_stats_seek(struct istream_private *stream, + uoff_t v_offset, bool mark ATTR_UNUSED) +{ + stream->istream.v_offset = v_offset; + stream->skip = stream->pos = 0; +} + +static const struct stat * +i_stream_mail_stats_stat(struct istream_private *stream, bool exact) +{ + return i_stream_stat(stream->parent, exact); +} + +struct istream *i_stream_create_mail_stats_counter(struct mail_private *mail, + struct istream *input) +{ + struct mail_stats_istream *mstream; + + i_stream_ref(input); + + mstream = i_new(struct mail_stats_istream, 1); + mstream->mail = mail; + mstream->istream.max_buffer_size = input->real_stream->max_buffer_size; + + mstream->istream.iostream.destroy = i_stream_mail_stats_destroy; + mstream->istream.iostream.set_max_buffer_size = + i_stream_mail_stats_set_max_buffer_size; + + mstream->istream.parent = input; + mstream->istream.read = i_stream_mail_stats_read_mail_stats; + mstream->istream.seek = i_stream_mail_stats_seek; + mstream->istream.stat = i_stream_mail_stats_stat; + + mstream->istream.istream.blocking = input->blocking; + mstream->istream.istream.seekable = input->seekable; + return i_stream_create(&mstream->istream, input, + i_stream_get_fd(input)); +}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-storage/index/istream-mail-stats.h Tue Apr 28 19:55:51 2009 -0400 @@ -0,0 +1,7 @@ +#ifndef ISTREAM_MAIL_STATS_H +#define ISTREAM_MAIL_STATS_H + +struct istream *i_stream_create_mail_stats_counter(struct mail_private *mail, + struct istream *input); + +#endif
--- a/src/lib-storage/index/maildir/maildir-mail.c Tue Apr 28 17:21:14 2009 -0400 +++ b/src/lib-storage/index/maildir/maildir-mail.c Tue Apr 28 19:55:51 2009 -0400 @@ -44,11 +44,13 @@ maildir_open_mail(struct maildir_mailbox *mbox, struct mail *mail, bool *deleted_r) { + struct mail_private *p = (struct mail_private *)mail; const char *path; int fd = -1; *deleted_r = FALSE; + p->stats_dentry_lookup_count++; if (mail->uid != 0) { if (maildir_file_do(mbox, mail->uid, do_open, &fd) < 0) return NULL; @@ -69,30 +71,32 @@ static int maildir_mail_stat(struct mail *mail, struct stat *st) { struct maildir_mailbox *mbox = (struct maildir_mailbox *)mail->box; - struct index_mail_data *data = &((struct index_mail *)mail)->data; + struct index_mail *imail = (struct index_mail *)mail; const char *path; int fd, ret; if (mail->lookup_abort == MAIL_LOOKUP_ABORT_NOT_IN_CACHE) return mail_set_aborted(mail); - if (data->access_part != 0 && data->stream == NULL) { + if (imail->data.access_part != 0 && imail->data.stream == NULL) { /* we're going to open the mail anyway */ struct istream *input; (void)mail_get_stream(mail, NULL, NULL, &input); } - if (data->stream != NULL) { - fd = i_stream_get_fd(data->stream); + if (imail->data.stream != NULL) { + fd = i_stream_get_fd(imail->data.stream); i_assert(fd != -1); + imail->mail.stats_attr_lookup_count++; if (fstat(fd, st) < 0) { mail_storage_set_critical(&mbox->storage->storage, "fstat(maildir) failed: %m"); return -1; } } else if (mail->uid != 0) { + imail->mail.stats_dentry_lookup_count++; ret = maildir_file_do(mbox, mail->uid, do_stat, st); if (ret <= 0) { if (ret == 0) @@ -100,6 +104,7 @@ return -1; } } else { + imail->mail.stats_dentry_lookup_count++; path = maildir_save_file_get_path(mail->transaction, mail->seq); if (stat(path, st) < 0) { mail_storage_set_critical(mail->box->storage,
--- a/src/lib-storage/index/raw/raw-mail.c Tue Apr 28 17:21:14 2009 -0400 +++ b/src/lib-storage/index/raw/raw-mail.c Tue Apr 28 19:55:51 2009 -0400 @@ -12,11 +12,13 @@ static int raw_mail_stat(struct mail *mail) { struct raw_mailbox *mbox = (struct raw_mailbox *)mail->box; + struct mail_private *p = (struct mail_private *)mail; const struct stat *st; if (mail->lookup_abort == MAIL_LOOKUP_ABORT_NOT_IN_CACHE) return mail_set_aborted(mail); + p->stats_attr_lookup_count++; st = i_stream_stat(mbox->input, TRUE); if (st == NULL) { mail_storage_set_critical(mail->box->storage,
--- a/src/lib-storage/mail-storage-private.h Tue Apr 28 17:21:14 2009 -0400 +++ b/src/lib-storage/mail-storage-private.h Tue Apr 28 19:55:51 2009 -0400 @@ -283,6 +283,22 @@ pool_t pool; ARRAY_DEFINE(module_contexts, union mail_module_context *); + + /* these statistics are never reset by mail-storage API: */ + + /* open(), stat(), .. */ + unsigned long stats_dentry_lookup_count; + /* fstat() mostly */ + unsigned long stats_attr_lookup_count; + /* number of files we've opened and read */ + unsigned long stats_files_read_count; + /* number of bytes we've had to read from files */ + unsigned long long stats_files_read_bytes; + /* number of cache lookup hits */ + unsigned long stats_cache_hit_count; + + /* Set to TRUE to update stats_* fields */ + unsigned int stats_track:1; }; struct mailbox_list_context {