Mercurial > dovecot > core-2.2
changeset 18292:a9952ceeac61
stats process/plugin redesign to be more modular.
The visible functionality doesn't actually change with this patch yet, but
it allows other plugins/services to add their own fields to stats process.
For example auth process could send auth success/failures or auth cache
hits/misses.
line wrap: on
line diff
--- a/configure.ac Thu Mar 05 22:19:02 2015 +0200 +++ b/configure.ac Thu Mar 05 23:02:48 2015 +0200 @@ -2549,7 +2549,7 @@ LIBDOVECOT_COMPRESS='$(top_builddir)/src/lib-compression/libcompression.la' LIBDOVECOT_LDA='$(top_builddir)/src/lib-lda/libdovecot-lda.la' else - LIBDOVECOT_DEPS='$(top_builddir)/src/lib-master/libmaster.la $(top_builddir)/src/lib-settings/libsettings.la $(top_builddir)/src/lib-http/libhttp.la $(top_builddir)/src/lib-dict/libdict.la $(top_builddir)/src/lib-dns/libdns.la $(top_builddir)/src/lib-fs/libfs.la $(top_builddir)/src/lib-imap/libimap.la $(top_builddir)/src/lib-mail/libmail.la $(top_builddir)/src/lib-sasl/libsasl.la $(top_builddir)/src/lib-auth/libauth.la $(top_builddir)/src/lib-charset/libcharset.la $(top_builddir)/src/lib-ssl-iostream/libssl_iostream.la $(top_builddir)/src/lib-test/libtest.la $(top_builddir)/src/lib/liblib.la' + LIBDOVECOT_DEPS='$(top_builddir)/src/lib-master/libmaster.la $(top_builddir)/src/lib-settings/libsettings.la $(top_builddir)/src/lib-stats/libstats.la $(top_builddir)/src/lib-http/libhttp.la $(top_builddir)/src/lib-dict/libdict.la $(top_builddir)/src/lib-dns/libdns.la $(top_builddir)/src/lib-fs/libfs.la $(top_builddir)/src/lib-imap/libimap.la $(top_builddir)/src/lib-mail/libmail.la $(top_builddir)/src/lib-sasl/libsasl.la $(top_builddir)/src/lib-auth/libauth.la $(top_builddir)/src/lib-charset/libcharset.la $(top_builddir)/src/lib-ssl-iostream/libssl_iostream.la $(top_builddir)/src/lib-test/libtest.la $(top_builddir)/src/lib/liblib.la' LIBDOVECOT="$LIBDOVECOT_DEPS \$(LIBICONV) \$(MODULE_LIBS)" LIBDOVECOT_STORAGE_DEPS='$(top_builddir)/src/lib-storage/libstorage.la' LIBDOVECOT_LOGIN='$(top_builddir)/src/login-common/liblogin.la' @@ -2849,6 +2849,7 @@ src/lib-sasl/Makefile src/lib-settings/Makefile src/lib-ssl-iostream/Makefile +src/lib-stats/Makefile src/lib-test/Makefile src/lib-storage/Makefile src/lib-storage/list/Makefile
--- a/src/Makefile.am Thu Mar 05 22:19:02 2015 +0200 +++ b/src/Makefile.am Thu Mar 05 23:02:48 2015 +0200 @@ -9,6 +9,7 @@ lib-dict \ lib-sasl \ lib-ssl-iostream \ + lib-stats \ lib-http \ lib-fs \ lib-mail \
--- a/src/lib-dovecot/Makefile.am Thu Mar 05 22:19:02 2015 +0200 +++ b/src/lib-dovecot/Makefile.am Thu Mar 05 23:02:48 2015 +0200 @@ -3,6 +3,7 @@ ../lib-master/libmaster.la \ ../lib-fs/libfs.la \ ../lib-settings/libsettings.la \ + ../lib-stats/libstats.la \ ../lib-http/libhttp.la \ ../lib-dict/libdict.la \ ../lib-imap/libimap.la \
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-stats/Makefile.am Thu Mar 05 23:02:48 2015 +0200 @@ -0,0 +1,15 @@ +noinst_LTLIBRARIES = libstats.la + +AM_CPPFLAGS = \ + -I$(top_srcdir)/src/lib + +libstats_la_SOURCES = \ + stats.c \ + stats-parser.c + +headers = \ + stats.h \ + stats-parser.h + +pkginc_libdir = $(pkgincludedir) +pkginc_lib_HEADERS = $(headers)
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-stats/stats-parser.c Thu Mar 05 23:02:48 2015 +0200 @@ -0,0 +1,180 @@ +/* Copyright (c) 2011-2015 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "str.h" +#include "time-util.h" +#include "stats-parser.h" + +#define USECS_PER_SEC 1000000 + +static bool stats_diff_timeval(struct timeval *dest, + const struct timeval *src1, + const struct timeval *src2) +{ + long long diff_usecs; + + diff_usecs = timeval_diff_usecs(src2, src1); + if (diff_usecs < 0) + return FALSE; + dest->tv_sec = diff_usecs / USECS_PER_SEC; + dest->tv_usec = diff_usecs % USECS_PER_SEC; + return TRUE; +} + +static bool +stats_diff_uint32(uint32_t *dest, const uint32_t *src1, const uint32_t *src2) +{ + if (*src1 > *src2) + return FALSE; + *dest = *src2 - *src1; + return TRUE; +} + +static bool +stats_diff_uint64(uint64_t *dest, const uint64_t *src1, const uint64_t *src2) +{ + if (*src1 > *src2) + return FALSE; + *dest = *src2 - *src1; + return TRUE; +} + +bool stats_parser_diff(const struct stats_parser_field *fields, + unsigned int fields_count, + const struct stats *stats1, const struct stats *stats2, + struct stats *diff_stats_r, const char **error_r) +{ + unsigned int i; + + for (i = 0; i < fields_count; i++) { + unsigned int offset = fields[i].offset; + void *dest = PTR_OFFSET(diff_stats_r, offset); + const void *src1 = CONST_PTR_OFFSET(stats1, offset); + const void *src2 = CONST_PTR_OFFSET(stats2, offset); + + switch (fields[i].type) { + case STATS_PARSER_TYPE_UINT: + switch (fields[i].size) { + case sizeof(uint32_t): + if (!stats_diff_uint32(dest, src1, src2)) { + *error_r = t_strdup_printf("%s %u < %u", + fields[i].name, + *(const uint32_t *)src2, + *(const uint32_t *)src1); + return FALSE; + } + break; + case sizeof(uint64_t): + if (!stats_diff_uint64(dest, src1, src2)) { + const uint64_t *n1 = src1, *n2 = src2; + + *error_r = t_strdup_printf("%s %llu < %llu", + fields[i].name, + (unsigned long long)*n2, + (unsigned long long)*n1); + return FALSE; + } + break; + default: + i_unreached(); + } + break; + case STATS_PARSER_TYPE_TIMEVAL: + if (!stats_diff_timeval(dest, src1, src2)) { + const struct timeval *tv1 = src1, *tv2 = src2; + + *error_r = t_strdup_printf("%s %ld.%d < %ld.%d", + fields[i].name, + (long)tv2->tv_sec, (int)tv2->tv_usec, + (long)tv1->tv_sec, (int)tv1->tv_usec); + return FALSE; + } + break; + } + } + return TRUE; +} + +static void stats_timeval_add(struct timeval *dest, const struct timeval *src) +{ + dest->tv_sec += src->tv_sec; + dest->tv_usec += src->tv_usec; + if (dest->tv_usec > USECS_PER_SEC) { + dest->tv_usec -= USECS_PER_SEC; + dest->tv_sec++; + } +} + +void stats_parser_add(const struct stats_parser_field *fields, + unsigned int fields_count, + struct stats *dest, const struct stats *src) +{ + unsigned int i; + + for (i = 0; i < fields_count; i++) { + unsigned int offset = fields[i].offset; + void *f_dest = PTR_OFFSET(dest, offset); + const void *f_src = CONST_PTR_OFFSET(src, offset); + + switch (fields[i].type) { + case STATS_PARSER_TYPE_UINT: + switch (fields[i].size) { + case sizeof(uint32_t): { + uint32_t *n_dest = f_dest; + const uint32_t *n_src = f_src; + + *n_dest += *n_src; + break; + } + case sizeof(uint64_t): { + uint64_t *n_dest = f_dest; + const uint64_t *n_src = f_src; + + *n_dest += *n_src; + break; + } + default: + i_unreached(); + } + break; + case STATS_PARSER_TYPE_TIMEVAL: + stats_timeval_add(f_dest, f_src); + break; + } + } +} + +void stats_parser_value(string_t *str, + const struct stats_parser_field *field, + const void *data) +{ + const void *ptr = CONST_PTR_OFFSET(data, field->offset); + + switch (field->type) { + case STATS_PARSER_TYPE_UINT: + switch (field->size) { + case sizeof(uint32_t): { + const uint32_t *n = ptr; + + str_printfa(str, "%u", *n); + break; + } + case sizeof(uint64_t): { + const uint64_t *n = ptr; + + str_printfa(str, "%llu", (unsigned long long)*n); + break; + } + default: + i_unreached(); + } + break; + case STATS_PARSER_TYPE_TIMEVAL: { + const struct timeval *tv = ptr; + + str_printfa(str, "%lu.%u", (unsigned long)tv->tv_sec, + (unsigned int)tv->tv_usec); + break; + } + } +}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-stats/stats-parser.h Thu Mar 05 23:02:48 2015 +0200 @@ -0,0 +1,29 @@ +#ifndef STATS_PARSER_H +#define STATS_PARSER_H + +struct stats; + +enum stats_parser_type { + STATS_PARSER_TYPE_UINT, + STATS_PARSER_TYPE_TIMEVAL +}; + +struct stats_parser_field { + const char *name; + unsigned int offset; + unsigned int size; + enum stats_parser_type type; +}; + +bool stats_parser_diff(const struct stats_parser_field *fields, + unsigned int fields_count, + const struct stats *stats1, const struct stats *stats2, + struct stats *diff_stats_r, const char **error_r); +void stats_parser_add(const struct stats_parser_field *fields, + unsigned int fields_count, + struct stats *dest, const struct stats *src); +void stats_parser_value(string_t *str, + const struct stats_parser_field *field, + const void *data); + +#endif
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-stats/stats.c Thu Mar 05 23:02:48 2015 +0200 @@ -0,0 +1,220 @@ +/* Copyright (c) 2015 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "stats.h" + +struct stats_item { + struct stats_vfuncs v; + size_t pos; +}; + +static ARRAY(struct stats_item *) stats_items = ARRAY_INIT; +static unsigned int stats_total_size = 0; +static bool stats_allocated = FALSE; + +struct stats_item *stats_register(const struct stats_vfuncs *vfuncs) +{ + struct stats_item *item; + + if (stats_allocated) + i_panic("stats_register() called after stats_alloc_size() was already called - this will break existing allocations"); + + if (!array_is_created(&stats_items)) + i_array_init(&stats_items, 8); + + item = i_new(struct stats_item, 1); + item->v = *vfuncs; + item->pos = stats_total_size; + array_append(&stats_items, &item, 1); + + stats_total_size += vfuncs->alloc_size(); + return item; +} + +static bool stats_item_find(struct stats_item *item, unsigned int *idx_r) +{ + struct stats_item *const *itemp; + + array_foreach(&stats_items, itemp) { + if (*itemp == item) { + *idx_r = array_foreach_idx(&stats_items, itemp); + return TRUE; + } + } + return FALSE; +} + +static struct stats_item *stats_item_find_by_name(const char *name) +{ + struct stats_item *const *itemp; + + array_foreach(&stats_items, itemp) { + if (strcmp((*itemp)->v.short_name, name) == 0) + return *itemp; + } + return NULL; +} + +void stats_unregister(struct stats_item **_item) +{ + struct stats_item *item = *_item; + unsigned int idx; + + *_item = NULL; + + if (!stats_item_find(item, &idx)) + i_unreached(); + array_delete(&stats_items, idx, 1); + + i_free(item); + if (array_count(&stats_items) == 0) + array_free(&stats_items); +} + +struct stats *stats_alloc(pool_t pool) +{ + return p_malloc(pool, stats_alloc_size()); +} + +size_t stats_alloc_size(void) +{ + stats_allocated = TRUE; + return stats_total_size; +} + +void stats_copy(struct stats *dest, const struct stats *src) +{ + memcpy(dest, src, stats_total_size); +} + +unsigned int stats_field_count(void) +{ + struct stats_item *const *itemp; + unsigned int count = 0; + + array_foreach(&stats_items, itemp) + count += (*itemp)->v.field_count(); + return count; +} + +const char *stats_field_name(unsigned int n) +{ + struct stats_item *const *itemp; + unsigned int i = 0, count; + + array_foreach(&stats_items, itemp) { + count = (*itemp)->v.field_count(); + if (i + count > n) + return (*itemp)->v.field_name(n - i); + i += count; + } + i_unreached(); +} + +void stats_field_value(string_t *str, const struct stats *stats, + unsigned int n) +{ + struct stats_item *const *itemp; + unsigned int i = 0, count; + + array_foreach(&stats_items, itemp) { + count = (*itemp)->v.field_count(); + if (i + count > n) { + const void *item_stats + = CONST_PTR_OFFSET(stats, (*itemp)->pos); + (*itemp)->v.field_value(str, item_stats, n - i); + return; + } + i += count; + } + i_unreached(); +} + +bool stats_diff(const struct stats *stats1, const struct stats *stats2, + struct stats *diff_stats_r, const char **error_r) +{ + struct stats_item *const *itemp; + bool ret = TRUE; + + array_foreach(&stats_items, itemp) { + if (!(*itemp)->v.diff(CONST_PTR_OFFSET(stats1, (*itemp)->pos), + CONST_PTR_OFFSET(stats2, (*itemp)->pos), + PTR_OFFSET(diff_stats_r, (*itemp)->pos), + error_r)) + ret = FALSE; + } + return ret; +} + +void stats_add(struct stats *dest, const struct stats *src) +{ + struct stats_item *const *itemp; + + array_foreach(&stats_items, itemp) { + (*itemp)->v.add(PTR_OFFSET(dest, (*itemp)->pos), + CONST_PTR_OFFSET(src, (*itemp)->pos)); + } +} + +bool stats_have_changed(const struct stats *prev, const struct stats *cur) +{ + struct stats_item *const *itemp; + + array_foreach(&stats_items, itemp) { + if ((*itemp)->v.have_changed(CONST_PTR_OFFSET(prev, (*itemp)->pos), + CONST_PTR_OFFSET(cur, (*itemp)->pos))) + return TRUE; + } + return FALSE; +} + +void stats_export(buffer_t *buf, const struct stats *stats) +{ + struct stats_item *const *itemp; + + array_foreach(&stats_items, itemp) { + buffer_append(buf, (*itemp)->v.short_name, + strlen((*itemp)->v.short_name)+1); + (*itemp)->v.export(buf, CONST_PTR_OFFSET(stats, (*itemp)->pos)); + } +} + +bool stats_import(const unsigned char *data, size_t size, + const struct stats *old_stats, struct stats *stats, + const char **error_r) +{ + struct stats_item *item; + const unsigned char *p; + size_t pos; + + memcpy(stats, old_stats, stats_total_size); + while (size > 0) { + const char *next_name = (const void *)data; + + p = memchr(data, '\0', size); + if (p == NULL) { + *error_r = "Expected name, but NUL is missing"; + return -1; + } + item = stats_item_find_by_name(next_name); + if (item == NULL) { + *error_r = t_strdup_printf("Unknown stats name: '%s'", next_name); + return -1; + } + size -= (p+1) - data; + data = p+1; + if (!item->v.import(data, size, &pos, + PTR_OFFSET(stats, item->pos), error_r)) + return FALSE; + i_assert(pos <= size); + data += pos; + size -= pos; + } + return TRUE; +} + +void *stats_fill_ptr(struct stats *stats, struct stats_item *item) +{ + return PTR_OFFSET(stats, item->pos); +}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-stats/stats.h Thu Mar 05 23:02:48 2015 +0200 @@ -0,0 +1,69 @@ +#ifndef STATS_H +#define STATS_H + +struct stats; +struct stats_item; + +struct stats_vfuncs { + const char *short_name; + + size_t (*alloc_size)(void); + unsigned int (*field_count)(void); + const char *(*field_name)(unsigned int n); + void (*field_value)(string_t *str, const struct stats *stats, + unsigned int n); + + bool (*diff)(const struct stats *stats1, const struct stats *stats2, + struct stats *diff_stats_r, const char **error_r); + void (*add)(struct stats *dest, const struct stats *src); + bool (*have_changed)(const struct stats *prev, const struct stats *cur); + + void (*export)(buffer_t *buf, const struct stats *stats); + bool (*import)(const unsigned char *data, size_t size, size_t *pos_r, + struct stats *stats, const char **error_r); +}; + +struct stats_item *stats_register(const struct stats_vfuncs *vfuncs); +void stats_unregister(struct stats_item **item); + +/* Allocate struct stats from a given pool. */ +struct stats *stats_alloc(pool_t pool); +/* Returns the number of bytes allocated to stats. */ +size_t stats_alloc_size(void); +/* Copy all stats from src to dest. */ +void stats_copy(struct stats *dest, const struct stats *src); + +/* Returns the number of stats fields. */ +unsigned int stats_field_count(void); +/* Returns the name of a stats field (exported to doveadm). */ +const char *stats_field_name(unsigned int n); +/* Returns the value of a stats field as a string (exported to doveadm). */ +void stats_field_value(string_t *str, const struct stats *stats, + unsigned int n); + +/* Return diff_stats_r->field = stats2->field - stats1->field. + diff1 is supposed to have smaller values than diff2. Returns TRUE if this + is so, FALSE if not */ +bool stats_diff(const struct stats *stats1, const struct stats *stats2, + struct stats *diff_stats_r, const char **error_r); +/* dest->field += src->field */ +void stats_add(struct stats *dest, const struct stats *src); +/* Returns TRUE if any fields have changed in cur since prev in a way that + a plugin should send the updated statistics to the stats process. Not all + fields necessarily require sending an update. */ +bool stats_have_changed(const struct stats *prev, const struct stats *cur); + +/* Export stats into a buffer in binary format. */ +void stats_export(buffer_t *buf, const struct stats *stats); +/* Import stats from a buffer. The buffer doesn't need to contain an update to + all the stats items - old_stats are used for that item in such case. + Currently it's not allowed to have unknown items in the buffer. */ +bool stats_import(const unsigned char *data, size_t size, + const struct stats *old_stats, struct stats *stats, + const char **error_r); +/* Return a pointer to stats where the specified item starts. The returned + pointer can be used to fill up the item-specific stats (up to its + alloc_size() number of bytes). */ +void *stats_fill_ptr(struct stats *stats, struct stats_item *item); + +#endif
--- a/src/lib-storage/mail-user.c Thu Mar 05 22:19:02 2015 +0200 +++ b/src/lib-storage/mail-user.c Thu Mar 05 23:02:48 2015 +0200 @@ -40,6 +40,11 @@ mountpoint_list_deinit(&user->mountpoints); } +static void mail_user_stats_fill_base(struct mail_user *user ATTR_UNUSED, + struct stats *stats ATTR_UNUSED) +{ +} + struct mail_user *mail_user_alloc(const char *username, const struct setting_parser_info *set_info, const struct mail_user_settings *set) @@ -68,6 +73,7 @@ i_panic("Settings check unexpectedly failed: %s", error); user->v.deinit = mail_user_deinit_base; + user->v.stats_fill = mail_user_stats_fill_base; p_array_init(&user->module_contexts, user->pool, 5); return user; } @@ -556,3 +562,8 @@ ssl_set->ca_dir = mail_set->ssl_client_ca_dir; ssl_set->ca_file = mail_set->ssl_client_ca_file; } + +void mail_user_stats_fill(struct mail_user *user, struct stats *stats) +{ + user->v.stats_fill(user, stats); +}
--- a/src/lib-storage/mail-user.h Thu Mar 05 22:19:02 2015 +0200 +++ b/src/lib-storage/mail-user.h Thu Mar 05 23:02:48 2015 +0200 @@ -5,12 +5,14 @@ #include "mail-storage-settings.h" struct module; -struct mail_user; +struct stats; struct fs_settings; struct ssl_iostream_settings; +struct mail_user; struct mail_user_vfuncs { void (*deinit)(struct mail_user *user); + void (*stats_fill)(struct mail_user *user, struct stats *stats); }; struct mail_user { @@ -166,4 +168,8 @@ struct fs_settings *fs_set, struct ssl_iostream_settings *ssl_set); +/* Fill statistics for user. By default there are no statistics, so stats + plugin must be loaded to have anything filled. */ +void mail_user_stats_fill(struct mail_user *user, struct stats *stats); + #endif
--- a/src/plugins/imap-stats/Makefile.am Thu Mar 05 22:19:02 2015 +0200 +++ b/src/plugins/imap-stats/Makefile.am Thu Mar 05 23:02:48 2015 +0200 @@ -1,5 +1,6 @@ AM_CPPFLAGS = \ -I$(top_srcdir)/src/lib \ + -I$(top_srcdir)/src/lib-stats \ -I$(top_srcdir)/src/lib-mail \ -I$(top_srcdir)/src/lib-imap \ -I$(top_srcdir)/src/lib-index \
--- a/src/plugins/imap-stats/imap-stats-plugin.c Thu Mar 05 22:19:02 2015 +0200 +++ b/src/plugins/imap-stats/imap-stats-plugin.c Thu Mar 05 23:02:48 2015 +0200 @@ -1,8 +1,10 @@ /* Copyright (c) 2011-2015 Dovecot authors, see the included COPYING file */ #include "imap-common.h" +#include "base64.h" #include "str.h" #include "imap-commands.h" +#include "stats.h" #include "stats-plugin.h" #include "stats-connection.h" #include "imap-stats-plugin.h" @@ -15,8 +17,7 @@ unsigned int id; bool continued; - struct mail_stats stats, pre_stats; - struct mailbox_transaction_stats pre_trans_stats; + struct stats *stats, *pre_stats; }; static MODULE_CONTEXT_DEFINE_INIT(imap_stats_imap_module, @@ -43,33 +44,34 @@ if (scmd == NULL) { scmd = p_new(cmd->pool, struct stats_client_command, 1); scmd->id = ++stats_cmd_id_counter; + scmd->stats = stats_alloc(cmd->pool); + scmd->pre_stats = stats_alloc(cmd->pool); MODULE_CONTEXT_SET(cmd, imap_stats_imap_module, scmd); } - mail_stats_get(suser, &scmd->pre_stats); - scmd->pre_trans_stats = suser->session_stats.trans_stats; + + mail_user_stats_fill(cmd->client->user, scmd->pre_stats); } static void stats_command_post(struct client_command_context *cmd) { struct stats_user *suser = STATS_USER_CONTEXT(cmd->client->user); struct stats_client_command *scmd = IMAP_STATS_IMAP_CONTEXT(cmd); - struct mail_stats stats, pre_trans_stats, trans_stats; + struct stats *new_stats, *diff_stats; + const char *error; unsigned int args_pos = 0, args_len = 0; string_t *str; + buffer_t *buf; if (scmd == NULL) return; - mail_stats_get(suser, &stats); - mail_stats_add_diff(&scmd->stats, &scmd->pre_stats, &stats); + new_stats = stats_alloc(pool_datastack_create()); + diff_stats = stats_alloc(pool_datastack_create()); - /* mail_stats_get() can't see the transactions that already went - away, so we'll need to use the session's stats difference */ - memset(&pre_trans_stats, 0, sizeof(pre_trans_stats)); - memset(&trans_stats, 0, sizeof(trans_stats)); - pre_trans_stats.trans_stats = scmd->pre_trans_stats; - trans_stats.trans_stats = suser->session_stats.trans_stats; - mail_stats_add_diff(&scmd->stats, &pre_trans_stats, &trans_stats); + mail_user_stats_fill(cmd->client->user, new_stats); + if (!stats_diff(scmd->pre_stats, new_stats, diff_stats, &error)) + i_error("stats: command stats shrank: %s", error); + stats_add(scmd->stats, diff_stats); str = t_str_new(128); str_append(str, "UPDATE-CMD\t"); @@ -91,7 +93,11 @@ scmd->continued = TRUE; } - mail_stats_export(str, &scmd->stats); + buf = buffer_create_dynamic(pool_datastack_create(), 128); + stats_export(buf, scmd->stats); + str_append_c(str, '\t'); + base64_encode(buf->data, buf->used, str); + str_append_c(str, '\n'); if (str_len(str) > PIPE_BUF) {
--- a/src/plugins/stats/Makefile.am Thu Mar 05 22:19:02 2015 +0200 +++ b/src/plugins/stats/Makefile.am Thu Mar 05 23:02:48 2015 +0200 @@ -3,6 +3,7 @@ -I$(top_srcdir)/src/lib-settings \ -I$(top_srcdir)/src/lib-mail \ -I$(top_srcdir)/src/lib-master \ + -I$(top_srcdir)/src/lib-stats \ -I$(top_srcdir)/src/lib-index \ -I$(top_srcdir)/src/lib-storage @@ -13,9 +14,20 @@ lib90_stats_plugin.la lib90_stats_plugin_la_SOURCES = \ + mail-stats.c \ + mail-stats-fill.c \ stats-connection.c \ stats-plugin.c noinst_HEADERS = \ + mail-stats.h \ stats-connection.h \ stats-plugin.h + +stats_moduledir = $(moduledir)/stats +stats_module_LTLIBRARIES = libstats_mail.la + +libstats_mail_la_LDFLAGS = -module -avoid-version +libstats_mail_la_LIBADD = mail-stats.lo $(LIBDOVECOT) +libstats_mail_la_DEPENDENCIES = mail-stats.lo +libstats_mail_la_SOURCES =
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/plugins/stats/mail-stats-fill.c Thu Mar 05 23:02:48 2015 +0200 @@ -0,0 +1,130 @@ +/* Copyright (c) 2011-2015 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "stats-plugin.h" +#include "mail-stats.h" + +#include <sys/resource.h> + +#define PROC_IO_PATH "/proc/self/io" + +static bool proc_io_disabled = FALSE; +static int proc_io_fd = -1; + +static int +process_io_buffer_parse(const char *buf, struct mail_stats *stats) +{ + const char *const *tmp; + + tmp = t_strsplit(buf, "\n"); + for (; *tmp != NULL; tmp++) { + if (strncmp(*tmp, "rchar: ", 7) == 0) { + if (str_to_uint64(*tmp + 7, &stats->read_bytes) < 0) + return -1; + } else if (strncmp(*tmp, "wchar: ", 7) == 0) { + if (str_to_uint64(*tmp + 7, &stats->write_bytes) < 0) + return -1; + } else if (strncmp(*tmp, "syscr: ", 7) == 0) { + if (str_to_uint32(*tmp + 7, &stats->read_count) < 0) + return -1; + } else if (strncmp(*tmp, "syscw: ", 7) == 0) { + if (str_to_uint32(*tmp + 7, &stats->write_count) < 0) + return -1; + } + } + return 0; +} + +static int process_io_open(void) +{ + uid_t uid; + + if (proc_io_fd != -1) + return proc_io_fd; + + if (proc_io_disabled) + return -1; + proc_io_fd = open(PROC_IO_PATH, O_RDONLY); + if (proc_io_fd == -1 && errno == EACCES) { + /* kludge: if we're running with permissions temporarily + dropped, get them temporarily back so we can open + /proc/self/io. */ + uid = geteuid(); + if (seteuid(0) == 0) { + proc_io_fd = open(PROC_IO_PATH, O_RDONLY); + if (seteuid(uid) < 0) { + /* oops, this is bad */ + i_fatal("stats: seteuid(%s) failed", dec2str(uid)); + } + } + errno = EACCES; + } + if (proc_io_fd == -1) { + if (errno != ENOENT) + i_error("open(%s) failed: %m", PROC_IO_PATH); + proc_io_disabled = TRUE; + return -1; + } + return proc_io_fd; +} + +static void process_read_io_stats(struct mail_stats *stats) +{ + char buf[1024]; + int fd, ret; + + if ((fd = process_io_open()) == -1) + return; + + ret = pread(fd, buf, sizeof(buf), 0); + if (ret <= 0) { + if (ret == -1) + i_error("read(%s) failed: %m", PROC_IO_PATH); + else + i_error("read(%s) returned EOF", PROC_IO_PATH); + } else if (ret == sizeof(buf)) { + /* just shouldn't happen.. */ + i_error("%s is larger than expected", PROC_IO_PATH); + proc_io_disabled = TRUE; + } else { + buf[ret] = '\0'; + T_BEGIN { + if (process_io_buffer_parse(buf, stats) < 0) { + i_error("Invalid input in file %s", + PROC_IO_PATH); + proc_io_disabled = TRUE; + } + } T_END; + } +} + +static void +user_trans_stats_get(struct stats_user *suser, struct mail_stats *dest_r) +{ + struct stats_transaction_context *strans; + + mail_stats_add_transaction(dest_r, &suser->finished_transaction_stats); + for (strans = suser->transactions; strans != NULL; strans = strans->next) + mail_stats_add_transaction(dest_r, &strans->trans->stats); +} + +void mail_stats_fill(struct stats_user *suser, struct mail_stats *stats_r) +{ + struct rusage usage; + + memset(stats_r, 0, sizeof(*stats_r)); + /* cputime */ + if (getrusage(RUSAGE_SELF, &usage) < 0) + memset(&usage, 0, sizeof(usage)); + stats_r->user_cpu = usage.ru_utime; + stats_r->sys_cpu = usage.ru_stime; + stats_r->min_faults = usage.ru_minflt; + stats_r->maj_faults = usage.ru_majflt; + stats_r->vol_cs = usage.ru_nvcsw; + stats_r->invol_cs = usage.ru_nivcsw; + stats_r->disk_input = (unsigned long long)usage.ru_inblock * 512ULL; + stats_r->disk_output = (unsigned long long)usage.ru_oublock * 512ULL; + (void)gettimeofday(&stats_r->clock_time, NULL); + process_read_io_stats(stats_r); + user_trans_stats_get(suser, stats_r); +}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/plugins/stats/mail-stats.c Thu Mar 05 23:02:48 2015 +0200 @@ -0,0 +1,168 @@ +/* Copyright (c) 2011-2015 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "time-util.h" +#include "stats.h" +#include "stats-parser.h" +#include "mail-stats.h" + +static struct stats_parser_field mail_stats_fields[] = { +#define E(parsename, name, type) { parsename, offsetof(struct mail_stats, name), sizeof(((struct mail_stats *)0)->name), type } +#define EN(parsename, name) E(parsename, name, STATS_PARSER_TYPE_UINT) + E("user_cpu", user_cpu, STATS_PARSER_TYPE_TIMEVAL), + E("sys_cpu", sys_cpu, STATS_PARSER_TYPE_TIMEVAL), + E("clock_time", clock_time, STATS_PARSER_TYPE_TIMEVAL), + EN("min_faults", min_faults), + EN("maj_faults", maj_faults), + EN("vol_cs", vol_cs), + EN("invol_cs", invol_cs), + EN("disk_input", disk_input), + EN("disk_output", disk_output), + + EN("read_count", read_count), + EN("read_bytes", read_bytes), + EN("write_count", write_count), + EN("write_bytes", write_bytes), + + /*EN("mopen", trans_stats.open_lookup_count), + EN("mstat", trans_stats.stat_lookup_count), + EN("mfstat", trans_stats.fstat_lookup_count),*/ + EN("mail_lookup_path", trans_lookup_path), + EN("mail_lookup_attr", trans_lookup_attr), + EN("mail_read_count", trans_files_read_count), + EN("mail_read_bytes", trans_files_read_bytes), + EN("mail_cache_hits", trans_cache_hit_count) +}; + +static size_t mail_stats_alloc_size(void) +{ + return sizeof(struct mail_stats); +} + +static unsigned int mail_stats_field_count(void) +{ + return N_ELEMENTS(mail_stats_fields); +} + +static const char *mail_stats_field_name(unsigned int n) +{ + i_assert(n < N_ELEMENTS(mail_stats_fields)); + + return mail_stats_fields[n].name; +} + +static void +mail_stats_field_value(string_t *str, const struct stats *stats, + unsigned int n) +{ + i_assert(n < N_ELEMENTS(mail_stats_fields)); + + stats_parser_value(str, &mail_stats_fields[n], stats); +} + +static bool +mail_stats_diff(const struct stats *stats1, const struct stats *stats2, + struct stats *diff_stats_r, const char **error_r) +{ + return stats_parser_diff(mail_stats_fields, N_ELEMENTS(mail_stats_fields), + stats1, stats2, diff_stats_r, error_r); +} + +static void mail_stats_add(struct stats *dest, const struct stats *src) +{ + stats_parser_add(mail_stats_fields, N_ELEMENTS(mail_stats_fields), + dest, src); +} + +static bool +mail_stats_have_changed(const struct stats *_prev, const struct stats *_cur) +{ + const struct mail_stats *prev = (const struct mail_stats *)_prev; + const struct mail_stats *cur = (const struct mail_stats *)_cur; + + if (cur->disk_input != prev->disk_input || + cur->disk_output != prev->disk_output || + cur->trans_lookup_path != prev->trans_lookup_path || + cur->trans_lookup_attr != prev->trans_lookup_attr || + cur->trans_files_read_count != prev->trans_files_read_count || + cur->trans_files_read_bytes != prev->trans_files_read_bytes || + cur->trans_cache_hit_count != prev->trans_cache_hit_count) + return TRUE; + + /* allow a tiny bit of changes that are caused by this + timeout handling */ + if (timeval_diff_msecs(&cur->user_cpu, &prev->user_cpu) != 0) + return TRUE; + if (timeval_diff_msecs(&cur->sys_cpu, &prev->sys_cpu) != 0) + return TRUE; + + if (cur->maj_faults > prev->maj_faults+10) + return TRUE; + if (cur->invol_cs > prev->invol_cs+10) + return TRUE; + /* don't check for read/write count/bytes changes, since they get + changed by stats checking itself */ + return FALSE; +} + +static void mail_stats_export(buffer_t *buf, const struct stats *_stats) +{ + const struct mail_stats *stats = (const struct mail_stats *)_stats; + + buffer_append(buf, stats, sizeof(*stats)); +} + +static bool +mail_stats_import(const unsigned char *data, size_t size, size_t *pos_r, + struct stats *_stats, const char **error_r) +{ + struct mail_stats *stats = (struct mail_stats *)_stats; + + if (size < sizeof(*stats)) { + *error_r = "mail_stats too small"; + return FALSE; + } + memcpy(stats, data, sizeof(*stats)); + *pos_r = sizeof(*stats); + return TRUE; +} + +void mail_stats_add_transaction(struct mail_stats *stats, + const struct mailbox_transaction_stats *trans_stats) +{ + stats->trans_lookup_path += trans_stats->open_lookup_count; + stats->trans_lookup_attr += trans_stats->stat_lookup_count + + trans_stats->fstat_lookup_count; + stats->trans_files_read_count += trans_stats->files_read_count; + stats->trans_files_read_bytes += trans_stats->files_read_bytes; + stats->trans_cache_hit_count += trans_stats->cache_hit_count; +} + +const struct stats_vfuncs mail_stats_vfuncs = { + "mail", + mail_stats_alloc_size, + mail_stats_field_count, + mail_stats_field_name, + mail_stats_field_value, + mail_stats_diff, + mail_stats_add, + mail_stats_have_changed, + mail_stats_export, + mail_stats_import +}; + +/* for the stats_mail plugin: */ +void stats_mail_init(void); +void stats_mail_deinit(void); + +static struct stats_item *mail_stats_item; + +void stats_mail_init(void) +{ + mail_stats_item = stats_register(&mail_stats_vfuncs); +} + +void stats_mail_deinit(void) +{ + stats_unregister(&mail_stats_item); +}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/plugins/stats/mail-stats.h Thu Mar 05 23:02:48 2015 +0200 @@ -0,0 +1,38 @@ +#ifndef MAIL_STATS_H +#define MAIL_STATS_H + +#include <sys/time.h> +#include "mail-storage-private.h" + +struct stats_user; + +struct mail_stats { + /* user/system CPU time used */ + struct timeval user_cpu, sys_cpu; + /* clock time used (not counting the time in ioloop wait) */ + struct timeval clock_time; + /* minor / major page faults */ + uint32_t min_faults, maj_faults; + /* voluntary / involuntary context switches */ + uint32_t vol_cs, invol_cs; + /* disk input/output bytes */ + uint64_t disk_input, disk_output; + /* read()/write() syscall count and number of bytes */ + uint32_t read_count, write_count; + uint64_t read_bytes, write_bytes; + + /* based on struct mailbox_transaction_stats: */ + uint32_t trans_lookup_path; + uint32_t trans_lookup_attr; + uint32_t trans_files_read_count; + uint64_t trans_files_read_bytes; + uint64_t trans_cache_hit_count; +}; + +extern const struct stats_vfuncs mail_stats_vfuncs; + +void mail_stats_fill(struct stats_user *suser, struct mail_stats *mail_stats); +void mail_stats_add_transaction(struct mail_stats *stats, + const struct mailbox_transaction_stats *trans_stats); + +#endif
--- a/src/plugins/stats/stats-connection.c Thu Mar 05 22:19:02 2015 +0200 +++ b/src/plugins/stats/stats-connection.c Thu Mar 05 23:02:48 2015 +0200 @@ -1,12 +1,14 @@ /* Copyright (c) 2011-2015 Dovecot authors, see the included COPYING file */ #include "lib.h" +#include "base64.h" #include "hostpid.h" #include "net.h" #include "str.h" #include "strescape.h" #include "master-service.h" #include "mail-storage.h" +#include "stats.h" #include "stats-plugin.h" #include "stats-connection.h" @@ -147,15 +149,19 @@ void stats_connection_send_session(struct stats_connection *conn, struct mail_user *user, - const struct mail_stats *stats) + const struct stats *stats) { struct stats_user *suser = STATS_USER_CONTEXT(user); - string_t *str = t_str_new(128); + string_t *str = t_str_new(256); + buffer_t *buf; + + buf = buffer_create_dynamic(pool_datastack_create(), 128); + stats_export(buf, stats); str_append(str, "UPDATE-SESSION\t"); str_append(str, suser->stats_session_id); - - mail_stats_export(str, stats); + str_append_c(str, '\t'); + base64_encode(buf->data, buf->used, str); str_append_c(str, '\n'); stats_connection_send(conn, str);
--- a/src/plugins/stats/stats-connection.h Thu Mar 05 22:19:02 2015 +0200 +++ b/src/plugins/stats/stats-connection.h Thu Mar 05 23:02:48 2015 +0200 @@ -15,7 +15,7 @@ void stats_connection_send_session(struct stats_connection *conn, struct mail_user *user, - const struct mail_stats *stats); + const struct stats *stats); void stats_connection_send(struct stats_connection *conn, const string_t *str); #endif
--- a/src/plugins/stats/stats-plugin.c Thu Mar 05 22:19:02 2015 +0200 +++ b/src/plugins/stats/stats-plugin.c Thu Mar 05 23:02:48 2015 +0200 @@ -6,12 +6,11 @@ #include "str.h" #include "time-util.h" #include "settings-parser.h" +#include "mail-stats.h" +#include "stats.h" #include "stats-connection.h" #include "stats-plugin.h" -#include <sys/time.h> -#include <sys/resource.h> - #define STATS_CONTEXT(obj) \ MODULE_CONTEXT(obj, stats_storage_module) @@ -20,18 +19,6 @@ #define SESSION_STATS_FORCE_REFRESH_SECS (5*60) #define REFRESH_CHECK_INTERVAL 100 #define MAIL_STATS_SOCKET_NAME "stats-mail" -#define PROC_IO_PATH "/proc/self/io" - -#define USECS_PER_SEC 1000000 - -struct stats_transaction_context { - union mailbox_transaction_module_context module_ctx; - - struct stats_transaction_context *prev, *next; - struct mailbox_transaction_context *trans; - - struct mailbox_transaction_stats prev_stats; -}; struct stats_storage { union mail_storage_module_context module_ctx; @@ -48,162 +35,20 @@ struct stats_user_module stats_user_module = MODULE_CONTEXT_INIT(&mail_user_module_register); -static MODULE_CONTEXT_DEFINE_INIT(stats_storage_module, - &mail_storage_module_register); +struct stats_storage_module stats_storage_module = + MODULE_CONTEXT_INIT(&mail_storage_module_register); -static bool proc_io_disabled = FALSE; -static int proc_io_fd = -1; - +static struct stats_item *mail_stats_item; static struct stats_connection *global_stats_conn = NULL; static struct mail_user *stats_global_user = NULL; static unsigned int stats_user_count = 0; static void session_stats_refresh_timeout(struct mail_user *user); -static void trans_stats_dec(struct mailbox_transaction_stats *dest, - const struct mailbox_transaction_stats *src) -{ - dest->open_lookup_count -= src->open_lookup_count; - dest->stat_lookup_count -= src->stat_lookup_count; - dest->fstat_lookup_count -= src->fstat_lookup_count; - dest->files_read_count -= src->files_read_count; - dest->files_read_bytes -= src->files_read_bytes; - dest->cache_hit_count -= src->cache_hit_count; -} - -static void trans_stats_add(struct mailbox_transaction_stats *dest, - const struct mailbox_transaction_stats *src) -{ - dest->open_lookup_count += src->open_lookup_count; - dest->stat_lookup_count += src->stat_lookup_count; - dest->fstat_lookup_count += src->fstat_lookup_count; - dest->files_read_count += src->files_read_count; - dest->files_read_bytes += src->files_read_bytes; - dest->cache_hit_count += src->cache_hit_count; -} - -static void user_trans_stats_get(struct stats_user *suser, - struct mailbox_transaction_stats *dest_r) -{ - struct stats_transaction_context *strans; - - memset(dest_r, 0, sizeof(*dest_r)); - strans = suser->transactions; - for (; strans != NULL; strans = strans->next) - trans_stats_add(dest_r, &strans->trans->stats); -} - -static int -process_io_buffer_parse(const char *buf, struct mail_stats *stats) -{ - const char *const *tmp; - - tmp = t_strsplit(buf, "\n"); - for (; *tmp != NULL; tmp++) { - if (strncmp(*tmp, "rchar: ", 7) == 0) { - if (str_to_uint64(*tmp + 7, &stats->read_bytes) < 0) - return -1; - } else if (strncmp(*tmp, "wchar: ", 7) == 0) { - if (str_to_uint64(*tmp + 7, &stats->write_bytes) < 0) - return -1; - } else if (strncmp(*tmp, "syscr: ", 7) == 0) { - if (str_to_uint32(*tmp + 7, &stats->read_count) < 0) - return -1; - } else if (strncmp(*tmp, "syscw: ", 7) == 0) { - if (str_to_uint32(*tmp + 7, &stats->write_count) < 0) - return -1; - } - } - return 0; -} - -static int process_io_open(void) -{ - uid_t uid; - - if (proc_io_fd != -1) - return proc_io_fd; - - if (proc_io_disabled) - return -1; - proc_io_fd = open(PROC_IO_PATH, O_RDONLY); - if (proc_io_fd == -1 && errno == EACCES) { - /* kludge: if we're running with permissions temporarily - dropped, get them temporarily back so we can open - /proc/self/io. */ - uid = geteuid(); - if (seteuid(0) == 0) { - proc_io_fd = open(PROC_IO_PATH, O_RDONLY); - if (seteuid(uid) < 0) { - /* oops, this is bad */ - i_fatal("stats: seteuid(%s) failed", dec2str(uid)); - } - } - errno = EACCES; - } - if (proc_io_fd == -1) { - if (errno != ENOENT) - i_error("open(%s) failed: %m", PROC_IO_PATH); - proc_io_disabled = TRUE; - return -1; - } - return proc_io_fd; -} - -static void process_read_io_stats(struct mail_stats *stats) -{ - char buf[1024]; - int fd, ret; - - if ((fd = process_io_open()) == -1) - return; - - ret = pread(fd, buf, sizeof(buf), 0); - if (ret <= 0) { - if (ret == -1) - i_error("read(%s) failed: %m", PROC_IO_PATH); - else - i_error("read(%s) returned EOF", PROC_IO_PATH); - } else if (ret == sizeof(buf)) { - /* just shouldn't happen.. */ - i_error("%s is larger than expected", PROC_IO_PATH); - proc_io_disabled = TRUE; - } else { - buf[ret] = '\0'; - T_BEGIN { - if (process_io_buffer_parse(buf, stats) < 0) { - i_error("Invalid input in file %s", - PROC_IO_PATH); - proc_io_disabled = TRUE; - } - } T_END; - } -} - -void mail_stats_get(struct stats_user *suser, struct mail_stats *stats_r) -{ - struct rusage usage; - - memset(stats_r, 0, sizeof(*stats_r)); - /* cputime */ - if (getrusage(RUSAGE_SELF, &usage) < 0) - memset(&usage, 0, sizeof(usage)); - stats_r->user_cpu = usage.ru_utime; - stats_r->sys_cpu = usage.ru_stime; - stats_r->min_faults = usage.ru_minflt; - stats_r->maj_faults = usage.ru_majflt; - stats_r->vol_cs = usage.ru_nvcsw; - stats_r->invol_cs = usage.ru_nivcsw; - stats_r->disk_input = (unsigned long long)usage.ru_inblock * 512ULL; - stats_r->disk_output = (unsigned long long)usage.ru_oublock * 512ULL; - (void)gettimeofday(&stats_r->clock_time, NULL); - process_read_io_stats(stats_r); - user_trans_stats_get(suser, &stats_r->trans_stats); -} - static void stats_io_activate(struct mail_user *user) { struct stats_user *suser = STATS_USER_CONTEXT(user); + struct mail_stats *mail_stats; if (stats_user_count == 1) { /* the first user sets the global user. the second user sets @@ -211,121 +56,35 @@ the global user again somewhere. do it here. */ stats_global_user = user; /* skip time spent waiting in ioloop */ - suser->pre_io_stats.clock_time = ioloop_timeval; + mail_stats = stats_fill_ptr(suser->pre_io_stats, mail_stats_item); + mail_stats->clock_time = ioloop_timeval; } else { i_assert(stats_global_user == NULL); - mail_stats_get(suser, &suser->pre_io_stats); - } -} - -static void timeval_add_diff(struct timeval *dest, - const struct timeval *newsrc, - const struct timeval *oldsrc) -{ - long long usecs; - - usecs = timeval_diff_usecs(newsrc, oldsrc); - dest->tv_sec += usecs / USECS_PER_SEC; - dest->tv_usec += usecs % USECS_PER_SEC; - if (dest->tv_usec > USECS_PER_SEC) { - dest->tv_usec -= USECS_PER_SEC; - dest->tv_sec++; + mail_user_stats_fill(user, suser->pre_io_stats); } } -void mail_stats_add_diff(struct mail_stats *dest, - const struct mail_stats *old_stats, - const struct mail_stats *new_stats) -{ - dest->disk_input += new_stats->disk_input - old_stats->disk_input; - dest->disk_output += new_stats->disk_output - old_stats->disk_output; - dest->min_faults += new_stats->min_faults - old_stats->min_faults; - dest->maj_faults += new_stats->maj_faults - old_stats->maj_faults; - dest->vol_cs += new_stats->vol_cs - old_stats->vol_cs; - dest->invol_cs += new_stats->invol_cs - old_stats->invol_cs; - dest->read_count += new_stats->read_count - old_stats->read_count; - dest->write_count += new_stats->write_count - old_stats->write_count; - dest->read_bytes += new_stats->read_bytes - old_stats->read_bytes; - dest->write_bytes += new_stats->write_bytes - old_stats->write_bytes; - - timeval_add_diff(&dest->user_cpu, &new_stats->user_cpu, - &old_stats->user_cpu); - timeval_add_diff(&dest->sys_cpu, &new_stats->sys_cpu, - &old_stats->sys_cpu); - timeval_add_diff(&dest->clock_time, &new_stats->clock_time, - &old_stats->clock_time); - trans_stats_dec(&dest->trans_stats, &old_stats->trans_stats); - trans_stats_add(&dest->trans_stats, &new_stats->trans_stats); -} - -void mail_stats_export(string_t *str, const struct mail_stats *stats) -{ - const struct mailbox_transaction_stats *tstats = &stats->trans_stats; - - str_printfa(str, "\tucpu=%ld.%ld", (long)stats->user_cpu.tv_sec, - (long)stats->user_cpu.tv_usec); - str_printfa(str, "\tscpu=%ld.%ld", (long)stats->sys_cpu.tv_sec, - (long)stats->sys_cpu.tv_usec); - str_printfa(str, "\ttime=%ld.%ld", (long)stats->clock_time.tv_sec, - (long)stats->clock_time.tv_usec); - str_printfa(str, "\tminflt=%u", stats->min_faults); - str_printfa(str, "\tmajflt=%u", stats->maj_faults); - str_printfa(str, "\tvolcs=%u", stats->vol_cs); - str_printfa(str, "\tinvolcs=%u", stats->invol_cs); - str_printfa(str, "\tdiskin=%llu", - (unsigned long long)stats->disk_input); - str_printfa(str, "\tdiskout=%llu", - (unsigned long long)stats->disk_output); - str_printfa(str, "\trchar=%llu", - (unsigned long long)stats->read_bytes); - str_printfa(str, "\twchar=%llu", - (unsigned long long)stats->write_bytes); - str_printfa(str, "\tsyscr=%u", stats->read_count); - str_printfa(str, "\tsyscw=%u", stats->write_count); - str_printfa(str, "\tmlpath=%lu", - tstats->open_lookup_count + tstats->stat_lookup_count); - str_printfa(str, "\tmlattr=%lu", - tstats->fstat_lookup_count + tstats->stat_lookup_count); - str_printfa(str, "\tmrcount=%lu", tstats->files_read_count); - str_printfa(str, "\tmrbytes=%llu", tstats->files_read_bytes); - str_printfa(str, "\tmcache=%lu", tstats->cache_hit_count); -} - static void stats_add_session(struct mail_user *user) { struct stats_user *suser = STATS_USER_CONTEXT(user); - struct mail_stats new_stats; + struct stats *new_stats, *diff_stats; + const char *error; - mail_stats_get(suser, &new_stats); - mail_stats_add_diff(&suser->session_stats, &suser->pre_io_stats, - &new_stats); - suser->pre_io_stats = new_stats; -} + new_stats = stats_alloc(pool_datastack_create()); + diff_stats = stats_alloc(pool_datastack_create()); -static bool session_has_changed(const struct mail_stats *prev, - const struct mail_stats *cur) -{ - if (cur->disk_input != prev->disk_input || - cur->disk_output != prev->disk_output || - memcmp(&cur->trans_stats, &prev->trans_stats, - sizeof(cur->trans_stats)) != 0) - return TRUE; - - /* allow a tiny bit of changes that are caused by this - timeout handling */ - if (timeval_diff_msecs(&cur->user_cpu, &prev->user_cpu) != 0) - return TRUE; - if (timeval_diff_msecs(&cur->sys_cpu, &prev->sys_cpu) != 0) - return TRUE; - - if (cur->maj_faults > prev->maj_faults+10) - return TRUE; - if (cur->invol_cs > prev->invol_cs+10) - return TRUE; - /* don't check for read/write count/bytes changes, since they get - changed by stats checking itself */ - return FALSE; + mail_user_stats_fill(user, new_stats); + /* we'll count new_stats-pre_io_stats and add the changes to + session_stats. the new_stats can't be directly copied to + session_stats because there are some fields that don't start from + zero, like clock_time. (actually with stats_global_user code we're + requiring that clock_time is the only such field..) */ + if (!stats_diff(suser->pre_io_stats, new_stats, diff_stats, &error)) + i_error("stats: session stats shrank: %s", error); + stats_add(suser->session_stats, diff_stats); + /* copying is only needed if stats_global_user=NULL */ + stats_copy(suser->pre_io_stats, new_stats); } static bool @@ -336,8 +95,8 @@ *to_next_secs_r = SESSION_STATS_FORCE_REFRESH_SECS; - if (session_has_changed(&suser->last_sent_session_stats, - &suser->session_stats)) { + if (stats_have_changed(suser->last_sent_session_stats, + suser->session_stats)) { *to_next_secs_r = suser->refresh_secs; *changed_r = TRUE; return TRUE; @@ -372,9 +131,9 @@ if (session_stats_need_send(suser, now, &changed, &to_next_secs)) { suser->session_sent_duplicate = !changed; suser->last_session_update = now; - suser->last_sent_session_stats = suser->session_stats; + stats_copy(suser->last_sent_session_stats, suser->session_stats); stats_connection_send_session(suser->stats_conn, user, - &suser->session_stats); + suser->session_stats); } if (suser->to_stats_timeout != NULL) @@ -407,10 +166,18 @@ static void stats_transaction_free(struct stats_user *suser, struct stats_transaction_context *strans) { + const struct mailbox_transaction_stats *src = &strans->trans->stats; + struct mailbox_transaction_stats *dest = + &suser->finished_transaction_stats; + DLLIST_REMOVE(&suser->transactions, strans); - trans_stats_add(&suser->session_stats.trans_stats, - &strans->trans->stats); + dest->open_lookup_count += src->open_lookup_count; + dest->stat_lookup_count += src->stat_lookup_count; + dest->fstat_lookup_count += src->fstat_lookup_count; + dest->files_read_count += src->files_read_count; + dest->files_read_bytes += src->files_read_bytes; + dest->cache_hit_count += src->cache_hit_count; i_free(strans); } @@ -538,6 +305,17 @@ } } +static void stats_user_stats_fill(struct mail_user *user, struct stats *stats) +{ + struct stats_user *suser = STATS_USER_CONTEXT(user); + struct mail_stats *mail_stats; + + mail_stats = stats_fill_ptr(stats, mail_stats_item); + mail_stats_fill(suser, mail_stats); + + suser->module_ctx.super.stats_fill(user, stats); +} + static void stats_user_deinit(struct mail_user *user) { struct stats_user *suser = STATS_USER_CONTEXT(user); @@ -624,6 +402,7 @@ suser->module_ctx.super = *v; user->vlast = &suser->module_ctx.super; v->deinit = stats_user_deinit; + v->stats_fill = stats_user_stats_fill; suser->refresh_secs = refresh_secs; str = mail_user_plugin_getenv(user, "stats_track_cmds"); @@ -647,6 +426,10 @@ stats_io_activate, stats_io_deactivate, user); + suser->pre_io_stats = stats_alloc(user->pool); + suser->session_stats = stats_alloc(user->pool); + suser->last_sent_session_stats = stats_alloc(user->pool); + MODULE_CONTEXT_SET(user, stats_user_module, suser); stats_connection_connect(suser->stats_conn, user); } @@ -658,6 +441,7 @@ void stats_plugin_init(struct module *module) { + mail_stats_item = stats_register(&mail_stats_vfuncs); mail_storage_hooks_add(module, &stats_mail_storage_hooks); } @@ -666,4 +450,5 @@ if (global_stats_conn != NULL) stats_connection_unref(&global_stats_conn); mail_storage_hooks_remove(&stats_mail_storage_hooks); + stats_unregister(&mail_stats_item); }
--- a/src/plugins/stats/stats-plugin.h Thu Mar 05 22:19:02 2015 +0200 +++ b/src/plugins/stats/stats-plugin.h Thu Mar 05 23:02:48 2015 +0200 @@ -5,28 +5,9 @@ #include "mail-user.h" #include "mail-storage-private.h" -#include <sys/time.h> - #define STATS_USER_CONTEXT(obj) \ MODULE_CONTEXT(obj, stats_user_module) -struct mail_stats { - /* user/system CPU time used */ - struct timeval user_cpu, sys_cpu; - /* clock time used (not counting the time in ioloop wait) */ - struct timeval clock_time; - /* minor / major page faults */ - uint32_t min_faults, maj_faults; - /* voluntary / involuntary context switches */ - uint32_t vol_cs, invol_cs; - /* disk input/output bytes */ - uint64_t disk_input, disk_output; - /* read()/write() syscall count and number of bytes */ - uint32_t read_count, write_count; - uint64_t read_bytes, write_bytes; - struct mailbox_transaction_stats trans_stats; -}; - struct stats_user { union mail_user_module_context module_ctx; @@ -39,28 +20,34 @@ unsigned int refresh_check_counter; /* current session statistics */ - struct mail_stats session_stats; + struct stats *session_stats; + /* cumulative trans_stats for all already freed transactions. */ + struct mailbox_transaction_stats finished_transaction_stats; /* stats before calling IO callback. after IO callback this value is compared to current stats to see the difference */ - struct mail_stats pre_io_stats; + struct stats *pre_io_stats; time_t last_session_update; struct timeout *to_stats_timeout; /* stats that were last sent to stats server */ - struct mail_stats last_sent_session_stats; + struct stats *last_sent_session_stats; bool session_sent_duplicate; /* list of all currently existing transactions for this user */ struct stats_transaction_context *transactions; }; -extern MODULE_CONTEXT_DEFINE(stats_user_module, &mail_user_module_register); +struct stats_transaction_context { + union mailbox_transaction_module_context module_ctx; + + struct stats_transaction_context *prev, *next; + struct mailbox_transaction_context *trans; -void mail_stats_get(struct stats_user *suser, struct mail_stats *stats_r); -void mail_stats_add_diff(struct mail_stats *dest, - const struct mail_stats *old_stats, - const struct mail_stats *new_stats); -void mail_stats_export(string_t *str, const struct mail_stats *stats); + struct mailbox_transaction_stats prev_stats; +}; + +extern MODULE_CONTEXT_DEFINE(stats_user_module, &mail_user_module_register); +extern MODULE_CONTEXT_DEFINE(stats_storage_module, &mail_storage_module_register); void stats_plugin_init(struct module *module); void stats_plugin_deinit(void);
--- a/src/stats/Makefile.am Thu Mar 05 22:19:02 2015 +0200 +++ b/src/stats/Makefile.am Thu Mar 05 23:02:48 2015 +0200 @@ -1,11 +1,14 @@ +stats_moduledir = $(moduledir)/stats pkglibexecdir = $(libexecdir)/dovecot pkglibexec_PROGRAMS = stats AM_CPPFLAGS = \ + -DSTATS_MODULE_DIR=\""$(stats_moduledir)"\" \ -I$(top_srcdir)/src/lib \ -I$(top_srcdir)/src/lib-settings \ - -I$(top_srcdir)/src/lib-master + -I$(top_srcdir)/src/lib-master \ + -I$(top_srcdir)/src/lib-stats stats_LDADD = $(LIBDOVECOT) stats_DEPENDENCIES = $(LIBDOVECOT_DEPS)
--- a/src/stats/client-export.c Thu Mar 05 22:19:02 2015 +0200 +++ b/src/stats/client-export.c Thu Mar 05 23:02:48 2015 +0200 @@ -96,34 +96,34 @@ } static void -client_export_mail_stats(string_t *str, const struct mail_stats *stats) +client_export_stats_headers(struct client *client) { -#define MAIL_STATS_HEADER "\tuser_cpu\tsys_cpu\tclock_time" \ - "\tmin_faults\tmaj_faults\tvol_cs\tinvol_cs" \ - "\tdisk_input\tdisk_output" \ - "\tread_count\tread_bytes\twrite_count\twrite_bytes" \ - "\tmail_lookup_path\tmail_lookup_attr" \ - "\tmail_read_count\tmail_read_bytes\tmail_cache_hits\n" + unsigned int i, count = stats_field_count(); + string_t *str = t_str_new(128); + + i_assert(count > 0); - str_printfa(str, "\t%ld.%06u", (long)stats->user_cpu.tv_sec, - (unsigned int)stats->user_cpu.tv_usec); - str_printfa(str, "\t%ld.%06u", (long)stats->sys_cpu.tv_sec, - (unsigned int)stats->sys_cpu.tv_usec); - str_printfa(str, "\t%ld.%06u", (long)stats->clock_time.tv_sec, - (unsigned int)stats->clock_time.tv_usec); - str_printfa(str, "\t%u\t%u", stats->min_faults, stats->maj_faults); - str_printfa(str, "\t%u\t%u", stats->vol_cs, stats->invol_cs); - str_printfa(str, "\t%llu\t%llu", - (unsigned long long)stats->disk_input, - (unsigned long long)stats->disk_output); - str_printfa(str, "\t%u\t%llu\t%u\t%llu", - stats->read_count, (unsigned long long)stats->read_bytes, - stats->write_count, (unsigned long long)stats->write_bytes); - str_printfa(str, "\t%u\t%u\t%u\t%llu\t%u", - stats->mail_lookup_path, stats->mail_lookup_attr, - stats->mail_read_count, - (unsigned long long)stats->mail_read_bytes, - stats->mail_cache_hits); + str_append(str, stats_field_name(0)); + for (i = 1; i < count; i++) { + str_append_c(str, '\t'); + str_append(str, stats_field_name(i)); + } + str_append_c(str, '\n'); + o_stream_send(client->output, str_data(str), str_len(str)); +} + +static void +client_export_stats(string_t *str, const struct stats *stats) +{ + unsigned int i, count = stats_field_count(); + + i_assert(count > 0); + + stats_field_value(str, stats, 0); + for (i = 1; i < count; i++) { + str_append_c(str, '\t'); + stats_field_value(str, stats, i); + } } static bool @@ -259,7 +259,8 @@ if (!cmd->header_sent) { o_stream_nsend_str(client->output, - "cmd\targs\tsession\tuser\tlast_update"MAIL_STATS_HEADER); + "cmd\targs\tsession\tuser\tlast_update\t"); + client_export_stats_headers(client); cmd->header_sent = TRUE; } @@ -275,14 +276,13 @@ str_append_c(cmd->str, '\t'); str_append_tabescaped(cmd->str, command->args); str_append_c(cmd->str, '\t'); - T_BEGIN { - str_append(cmd->str, command->session->id); - str_append_c(cmd->str, '\t'); - str_append_tabescaped(cmd->str, - command->session->user->name); - } T_END; + str_append(cmd->str, command->session->id); + str_append_c(cmd->str, '\t'); + str_append_tabescaped(cmd->str, + command->session->user->name); client_export_timeval(cmd->str, &command->last_update); - client_export_mail_stats(cmd->str, &command->stats); + str_append_c(cmd->str, '\t'); + client_export_stats(cmd->str, command->stats); str_append_c(cmd->str, '\n'); o_stream_nsend(client->output, str_data(cmd->str), str_len(cmd->str)); @@ -307,8 +307,8 @@ if (!cmd->header_sent) { o_stream_nsend_str(client->output, "session\tuser\tip\tservice\tpid\tconnected" - "\tlast_update\tnum_cmds" - MAIL_STATS_HEADER); + "\tlast_update\tnum_cmds\t"); + client_export_stats_headers(client); cmd->header_sent = TRUE; } @@ -319,23 +319,20 @@ continue; str_truncate(cmd->str, 0); - T_BEGIN { - str_append(cmd->str, session->id); - str_append_c(cmd->str, '\t'); - str_append_tabescaped(cmd->str, session->user->name); - str_append_c(cmd->str, '\t'); - if (session->ip != NULL) { - str_append(cmd->str, - net_ip2addr(&session->ip->ip)); - } - str_append_c(cmd->str, '\t'); - str_append_tabescaped(cmd->str, session->service); + str_append(cmd->str, session->id); + str_append_c(cmd->str, '\t'); + str_append_tabescaped(cmd->str, session->user->name); + str_append_c(cmd->str, '\t'); + if (session->ip != NULL) T_BEGIN { + str_append(cmd->str, net_ip2addr(&session->ip->ip)); } T_END; + str_append_c(cmd->str, '\t'); + str_append_tabescaped(cmd->str, session->service); str_printfa(cmd->str, "\t%ld", (long)session->pid); str_printfa(cmd->str, "\t%d", !session->disconnected); client_export_timeval(cmd->str, &session->last_update); - str_printfa(cmd->str, "\t%u", session->num_cmds); - client_export_mail_stats(cmd->str, &session->stats); + str_printfa(cmd->str, "\t%u\t", session->num_cmds); + client_export_stats(cmd->str, session->stats); str_append_c(cmd->str, '\n'); o_stream_nsend(client->output, str_data(cmd->str), str_len(cmd->str)); @@ -360,7 +357,8 @@ if (!cmd->header_sent) { o_stream_nsend_str(client->output, "user\treset_timestamp\tlast_update" - "\tnum_logins\tnum_cmds"MAIL_STATS_HEADER); + "\tnum_logins\tnum_cmds\t"); + client_export_stats_headers(client); cmd->header_sent = TRUE; } @@ -374,9 +372,9 @@ str_append_tabescaped(cmd->str, user->name); str_printfa(cmd->str, "\t%ld", (long)user->reset_timestamp); client_export_timeval(cmd->str, &user->last_update); - str_printfa(cmd->str, "\t%u\t%u", + str_printfa(cmd->str, "\t%u\t%u\t", user->num_logins, user->num_cmds); - client_export_mail_stats(cmd->str, &user->stats); + client_export_stats(cmd->str, user->stats); str_append_c(cmd->str, '\n'); o_stream_nsend(client->output, str_data(cmd->str), str_len(cmd->str)); @@ -401,7 +399,8 @@ if (!cmd->header_sent) { o_stream_nsend_str(client->output, "domain\treset_timestamp\tlast_update" - "\tnum_logins\tnum_cmds\tnum_connected_sessions"MAIL_STATS_HEADER); + "\tnum_logins\tnum_cmds\tnum_connected_sessions\t"); + client_export_stats_headers(client); cmd->header_sent = TRUE; } @@ -415,10 +414,10 @@ str_append_tabescaped(cmd->str, domain->name); str_printfa(cmd->str, "\t%ld", (long)domain->reset_timestamp); client_export_timeval(cmd->str, &domain->last_update); - str_printfa(cmd->str, "\t%u\t%u\t%u", + str_printfa(cmd->str, "\t%u\t%u\t%u\t", domain->num_logins, domain->num_cmds, domain->num_connected_sessions); - client_export_mail_stats(cmd->str, &domain->stats); + client_export_stats(cmd->str, domain->stats); str_append_c(cmd->str, '\n'); o_stream_nsend(client->output, str_data(cmd->str), str_len(cmd->str)); @@ -443,7 +442,8 @@ if (!cmd->header_sent) { o_stream_nsend_str(client->output, "ip\treset_timestamp\tlast_update" - "\tnum_logins\tnum_cmds\tnum_connected_sessions"MAIL_STATS_HEADER); + "\tnum_logins\tnum_cmds\tnum_connected_sessions\t"); + client_export_stats_headers(client); cmd->header_sent = TRUE; } @@ -459,9 +459,9 @@ } T_END; str_printfa(cmd->str, "\t%ld", (long)ip->reset_timestamp); client_export_timeval(cmd->str, &ip->last_update); - str_printfa(cmd->str, "\t%u\t%u\t%u", + str_printfa(cmd->str, "\t%u\t%u\t%u\t", ip->num_logins, ip->num_cmds, ip->num_connected_sessions); - client_export_mail_stats(cmd->str, &ip->stats); + client_export_stats(cmd->str, ip->stats); str_append_c(cmd->str, '\n'); o_stream_nsend(client->output, str_data(cmd->str), str_len(cmd->str)); @@ -485,16 +485,17 @@ if (!cmd->header_sent) { o_stream_nsend_str(client->output, "reset_timestamp\tlast_update" - "\tnum_logins\tnum_cmds\tnum_connected_sessions"MAIL_STATS_HEADER); + "\tnum_logins\tnum_cmds\tnum_connected_sessions\t"); + client_export_stats_headers(client); cmd->header_sent = TRUE; } str_truncate(cmd->str, 0); str_printfa(cmd->str, "%ld", (long)g->reset_timestamp); client_export_timeval(cmd->str, &g->last_update); - str_printfa(cmd->str, "\t%u\t%u\t%u", + str_printfa(cmd->str, "\t%u\t%u\t%u\t", g->num_logins, g->num_cmds, g->num_connected_sessions); - client_export_mail_stats(cmd->str, &g->stats); + client_export_stats(cmd->str, g->stats); str_append_c(cmd->str, '\n'); o_stream_nsend(client->output, str_data(cmd->str), str_len(cmd->str));
--- a/src/stats/mail-command.c Thu Mar 05 22:19:02 2015 +0200 +++ b/src/stats/mail-command.c Thu Mar 05 23:02:48 2015 +0200 @@ -1,6 +1,8 @@ /* Copyright (c) 2011-2015 Dovecot authors, see the included COPYING file */ #include "lib.h" +#include "buffer.h" +#include "base64.h" #include "ioloop.h" #include "llist.h" #include "global-memory.h" @@ -45,7 +47,8 @@ { struct mail_command *cmd; - cmd = i_new(struct mail_command, 1); + cmd = i_malloc(sizeof(struct mail_command) + stats_alloc_size()); + cmd->stats = (void *)(cmd + 1); cmd->refcount = 1; /* unrefed at "done" */ cmd->session = session; cmd->name = i_strdup(name); @@ -98,13 +101,14 @@ { struct mail_session *session; struct mail_command *cmd; - struct mail_stats stats, diff_stats; + struct stats *new_stats, *diff_stats; + buffer_t *buf; const char *error; unsigned int i, cmd_id; bool done = FALSE, continued = FALSE; - /* <session guid> <cmd id> [d] <name> <args> [key=value ..] - <session guid> <cmd id> c[d] [key=value ..] */ + /* <session guid> <cmd id> [d] <name> <args> <stats> + <session guid> <cmd id> c[d] <stats> */ if (str_array_length(args) < 3) { *error_r = "UPDATE-CMD: Too few parameters"; return -1; @@ -161,16 +165,26 @@ args += 3; cmd->last_update = ioloop_timeval; } - if (mail_stats_parse(args, &stats, error_r) < 0) { - *error_r = t_strconcat("UPDATE-CMD: ", *error_r, NULL); + buf = buffer_create_dynamic(pool_datastack_create(), 256); + if (args[0] == NULL || + base64_decode(args[0], strlen(args[0]), NULL, buf) < 0) { + *error_r = t_strdup_printf("UPDATE-CMD: Invalid base64 input"); return -1; } - if (!mail_stats_diff(&cmd->stats, &stats, &diff_stats, &error)) { - *error_r = t_strconcat("UPDATE-CMD: stats shrank: ", - error, NULL); + + new_stats = stats_alloc(pool_datastack_create()); + diff_stats = stats_alloc(pool_datastack_create()); + + if (!stats_import(buf->data, buf->used, cmd->stats, new_stats, &error)) { + *error_r = t_strdup_printf("UPDATE-CMD: %s", error); return -1; } - mail_stats_add(&cmd->stats, &diff_stats); + + if (!stats_diff(cmd->stats, new_stats, diff_stats, &error)) { + *error_r = t_strdup_printf("UPDATE-CMD: stats shrank: %s", error); + return -1; + } + stats_add(cmd->stats, diff_stats); if (done) { cmd->id = 0;
--- a/src/stats/mail-domain.c Thu Mar 05 22:19:02 2015 +0200 +++ b/src/stats/mail-domain.c Thu Mar 05 23:02:48 2015 +0200 @@ -29,7 +29,8 @@ return domain; } - domain = i_new(struct mail_domain, 1); + domain = i_malloc(sizeof(struct mail_domain) + stats_alloc_size()); + domain->stats = (void *)(domain + 1); domain->name = i_strdup(name); domain->reset_timestamp = ioloop_time; @@ -93,10 +94,10 @@ } void mail_domain_refresh(struct mail_domain *domain, - const struct mail_stats *diff_stats) + const struct stats *diff_stats) { if (diff_stats != NULL) - mail_stats_add(&domain->stats, diff_stats); + stats_add(domain->stats, diff_stats); domain->last_update = ioloop_timeval; DLLIST2_REMOVE_FULL(&mail_domains_head, &mail_domains_tail, domain, sorted_prev, sorted_next);
--- a/src/stats/mail-domain.h Thu Mar 05 22:19:02 2015 +0200 +++ b/src/stats/mail-domain.h Thu Mar 05 23:02:48 2015 +0200 @@ -1,7 +1,7 @@ #ifndef MAIL_DOMAIN_H #define MAIL_DOMAIN_H -struct mail_stats; +struct stats; extern struct mail_domain *stable_mail_domains; @@ -10,7 +10,7 @@ void mail_domain_disconnected(struct mail_domain *domain); struct mail_domain *mail_domain_lookup(const char *name); void mail_domain_refresh(struct mail_domain *domain, - const struct mail_stats *diff_stats) ATTR_NULL(2); + const struct stats *diff_stats) ATTR_NULL(2); void mail_domain_ref(struct mail_domain *domain); void mail_domain_unref(struct mail_domain **domain);
--- a/src/stats/mail-ip.c Thu Mar 05 22:19:02 2015 +0200 +++ b/src/stats/mail-ip.c Thu Mar 05 23:02:48 2015 +0200 @@ -31,7 +31,8 @@ return ip; } - ip = i_new(struct mail_ip, 1); + ip = i_malloc(sizeof(struct mail_ip) + stats_alloc_size()); + ip->stats = (void *)(ip + 1); ip->ip = *ip_addr; ip->reset_timestamp = ioloop_time; @@ -86,10 +87,10 @@ i_free(ip); } -void mail_ip_refresh(struct mail_ip *ip, const struct mail_stats *diff_stats) +void mail_ip_refresh(struct mail_ip *ip, const struct stats *diff_stats) { if (diff_stats != NULL) - mail_stats_add(&ip->stats, diff_stats); + stats_add(ip->stats, diff_stats); ip->last_update = ioloop_timeval; DLLIST2_REMOVE_FULL(&mail_ips_head, &mail_ips_tail, ip, sorted_prev, sorted_next);
--- a/src/stats/mail-ip.h Thu Mar 05 22:19:02 2015 +0200 +++ b/src/stats/mail-ip.h Thu Mar 05 23:02:48 2015 +0200 @@ -6,7 +6,7 @@ struct mail_ip *mail_ip_login(const struct ip_addr *ip_addr); void mail_ip_disconnected(struct mail_ip *ip); struct mail_ip *mail_ip_lookup(const struct ip_addr *ip_addr); -void mail_ip_refresh(struct mail_ip *ip, const struct mail_stats *diff_stats) +void mail_ip_refresh(struct mail_ip *ip, const struct stats *diff_stats) ATTR_NULL(2); void mail_ip_ref(struct mail_ip *ip);
--- a/src/stats/mail-server-connection.c Thu Mar 05 22:19:02 2015 +0200 +++ b/src/stats/mail-server-connection.c Thu Mar 05 23:02:48 2015 +0200 @@ -19,23 +19,6 @@ struct io *io; }; -static const char *const* -mail_server_connection_next_line(struct mail_server_connection *conn) -{ - const char *line; - char **args; - unsigned int i; - - line = i_stream_next_line(conn->input); - if (line == NULL) - return NULL; - - args = p_strsplit(pool_datastack_create(), line, "\t"); - for (i = 0; args[i] != NULL; i++) - args[i] = str_tabunescape(args[i]); - return (void *)args; -} - static int mail_server_connection_request(const char *const *args, const char **error_r) { @@ -62,7 +45,7 @@ static void mail_server_connection_input(struct mail_server_connection *conn) { - const char *const *args, *error; + const char *line, *const *args, *error; switch (i_stream_read(conn->input)) { case -2: @@ -74,10 +57,11 @@ return; } - while ((args = mail_server_connection_next_line(conn)) != NULL) { + while ((line = i_stream_next_line(conn->input)) != NULL) T_BEGIN { + args = t_strsplit_tabescaped(line); if (mail_server_connection_request(args, &error) < 0) i_error("Mail server input error: %s", error); - } + } T_END; } struct mail_server_connection *mail_server_connection_create(int fd)
--- a/src/stats/mail-session.c Thu Mar 05 22:19:02 2015 +0200 +++ b/src/stats/mail-session.c Thu Mar 05 23:02:48 2015 +0200 @@ -1,6 +1,8 @@ /* Copyright (c) 2011-2015 Dovecot authors, see the included COPYING file */ #include "lib.h" +#include "buffer.h" +#include "base64.h" #include "ioloop.h" #include "hash.h" #include "llist.h" @@ -91,7 +93,8 @@ *error_r = "CONNECT: Duplicate session ID"; return -1; } - session = i_new(struct mail_session, 1); + session = i_malloc(sizeof(struct mail_session) + stats_alloc_size()); + session->stats = (void *)(session + 1); session->refcount = 1; /* unrefed at disconnect */ session->id = i_strdup(session_id); session->service = str_table_ref(services, args[2]); @@ -233,12 +236,12 @@ } void mail_session_refresh(struct mail_session *session, - const struct mail_stats *diff_stats) + const struct stats *diff_stats) { timeout_reset(session->to_idle); if (diff_stats != NULL) - mail_stats_add(&session->stats, diff_stats); + stats_add(session->stats, diff_stats); session->last_update = ioloop_timeval; DLLIST2_REMOVE_FULL(&mail_sessions_head, &mail_sessions_tail, session, sorted_prev, sorted_next); @@ -253,27 +256,40 @@ int mail_session_update_parse(const char *const *args, const char **error_r) { struct mail_session *session; - struct mail_stats stats, diff_stats; + struct stats *new_stats, *diff_stats; + buffer_t *buf; const char *error; - /* <session id> [key=value ..] */ + /* <session id> <stats> */ if (mail_session_get(args[0], &session, error_r) < 0) return -1; - if (mail_stats_parse(args+1, &stats, error_r) < 0) { - *error_r = t_strdup_printf("UPDATE-SESSION %s %s: %s", + buf = buffer_create_dynamic(pool_datastack_create(), 256); + if (args[1] == NULL || + base64_decode(args[1], strlen(args[1]), NULL, buf) < 0) { + *error_r = t_strdup_printf("UPDATE-SESSION %s %s: Invalid base64 input", session->user->name, - session->service, *error_r); + session->service); return -1; } - if (!mail_stats_diff(&session->stats, &stats, &diff_stats, &error)) { + new_stats = stats_alloc(pool_datastack_create()); + diff_stats = stats_alloc(pool_datastack_create()); + + if (!stats_import(buf->data, buf->used, session->stats, new_stats, &error)) { + *error_r = t_strdup_printf("UPDATE-SESSION %s %s: %s", + session->user->name, + session->service, error); + return -1; + } + + if (!stats_diff(session->stats, new_stats, diff_stats, &error)) { *error_r = t_strdup_printf("UPDATE-SESSION %s %s: stats shrank: %s", session->user->name, session->service, error); return -1; } - mail_session_refresh(session, &diff_stats); + mail_session_refresh(session, diff_stats); return 0; }
--- a/src/stats/mail-session.h Thu Mar 05 22:19:02 2015 +0200 +++ b/src/stats/mail-session.h Thu Mar 05 23:02:48 2015 +0200 @@ -1,7 +1,7 @@ #ifndef MAIL_SESSION_H #define MAIL_SESSION_H -struct mail_stats; +struct stats; struct mail_session; extern struct mail_session *stable_mail_sessions; @@ -19,7 +19,7 @@ int mail_session_get(const char *guid, struct mail_session **session_r, const char **error_r); void mail_session_refresh(struct mail_session *session, - const struct mail_stats *diff_stats) ATTR_NULL(2); + const struct stats *diff_stats) ATTR_NULL(2); void mail_sessions_free_memory(void); void mail_sessions_init(void);
--- a/src/stats/mail-stats.c Thu Mar 05 22:19:02 2015 +0200 +++ b/src/stats/mail-stats.c Thu Mar 05 23:02:48 2015 +0200 @@ -5,278 +5,17 @@ #include "time-util.h" #include "mail-stats.h" -enum mail_stats_type { - TYPE_NUM, - TYPE_TIMEVAL -}; - -static struct mail_stats_parse_map { - const char *name; - unsigned int offset; - unsigned int size; - enum mail_stats_type type; -} parse_map[] = { -#define E(parsename, name, type) { parsename, offsetof(struct mail_stats, name), sizeof(((struct mail_stats *)0)->name), type } -#define EN(parsename, name) E(parsename, name, TYPE_NUM) - E("ucpu", user_cpu, TYPE_TIMEVAL), - E("scpu", sys_cpu, TYPE_TIMEVAL), - E("time", clock_time, TYPE_TIMEVAL), - EN("minflt", min_faults), - EN("majflt", maj_faults), - EN("volcs", vol_cs), - EN("involcs", invol_cs), - EN("diskin", disk_input), - EN("diskout", disk_output), - - EN("rchar", read_bytes), - EN("wchar", write_bytes), - EN("syscr", read_count), - EN("syscw", write_count), - - EN("mlpath", mail_lookup_path), - EN("mlattr", mail_lookup_attr), - EN("mrcount", mail_read_count), - EN("mrbytes", mail_read_bytes), - EN("mcache", mail_cache_hits) -}; - struct mail_global mail_global_stats; -static int mail_stats_parse_timeval(const char *value, struct timeval *tv) -{ - const char *p, *secs_str; - unsigned long secs, usecs; - - p = strchr(value, '.'); - if (p == NULL) - return -1; - - secs_str = t_strdup_until(value, p++); - if (str_to_ulong(secs_str, &secs) < 0 || - str_to_ulong(p, &usecs) < 0 || - usecs > 1000000) - return -1; - - tv->tv_sec = secs; - tv->tv_usec = usecs; - return 0; -} - -static struct mail_stats_parse_map * -parse_map_find(const char *name) -{ - unsigned int i; - - for (i = 0; i < N_ELEMENTS(parse_map); i++) { - if (strcmp(parse_map[i].name, name) == 0) - return &parse_map[i]; - } - return NULL; -} - -static int -mail_stats_parse_one(const char *key, const char *value, - struct mail_stats *stats, const char **error_r) -{ - struct mail_stats_parse_map *map; - void *dest; - - map = parse_map_find(key); - if (map == NULL) - return 0; - - dest = PTR_OFFSET(stats, map->offset); - switch (map->type) { - case TYPE_NUM: - switch (map->size) { - case sizeof(uint32_t): - if (str_to_uint32(value, dest) < 0) { - *error_r = "invalid number"; - return -1; - } - break; - case sizeof(uint64_t): - if (str_to_uint64(value, dest) < 0) { - *error_r = "invalid number"; - return -1; - } - break; - default: - i_unreached(); - } - break; - case TYPE_TIMEVAL: - if (mail_stats_parse_timeval(value, dest) < 0) { - *error_r = "invalid cpu parameter"; - return -1; - } - break; - } - return 0; -} - -int mail_stats_parse(const char *const *args, struct mail_stats *stats_r, - const char **error_r) -{ - const char *p, *key, *value; - unsigned int i; - - memset(stats_r, 0, sizeof(*stats_r)); - for (i = 0; args[i] != NULL; i++) { - p = strchr(args[i], '='); - if (p == NULL) { - *error_r = "mail stats parameter missing '='"; - return -1; - } - key = t_strdup_until(args[i], p); - value = p + 1; - if (mail_stats_parse_one(key, value, stats_r, error_r) < 0) - return -1; - } - return 0; -} - -static bool mail_stats_diff_timeval(struct timeval *dest, - const struct timeval *src1, - const struct timeval *src2) -{ - long long diff_usecs; - - diff_usecs = timeval_diff_usecs(src2, src1); - if (diff_usecs < 0) - return FALSE; - dest->tv_sec = diff_usecs / 1000000; - dest->tv_usec = diff_usecs % 1000000; - return TRUE; -} - -static bool -mail_stats_diff_uint32(uint32_t *dest, const uint32_t *src1, - const uint32_t *src2) -{ - if (*src1 > *src2) - return FALSE; - *dest = *src2 - *src1; - return TRUE; -} - -static bool -mail_stats_diff_uint64(uint64_t *dest, const uint64_t *src1, - const uint64_t *src2) -{ - if (*src1 > *src2) - return FALSE; - *dest = *src2 - *src1; - return TRUE; -} - -bool mail_stats_diff(const struct mail_stats *stats1, - const struct mail_stats *stats2, - struct mail_stats *diff_stats_r, const char **error_r) -{ - unsigned int i; - - memset(diff_stats_r, 0, sizeof(*diff_stats_r)); - - for (i = 0; i < N_ELEMENTS(parse_map); i++) { - unsigned int offset = parse_map[i].offset; - void *dest = PTR_OFFSET(diff_stats_r, offset); - const void *src1 = CONST_PTR_OFFSET(stats1, offset); - const void *src2 = CONST_PTR_OFFSET(stats2, offset); - - switch (parse_map[i].type) { - case TYPE_NUM: - switch (parse_map[i].size) { - case sizeof(uint32_t): - if (!mail_stats_diff_uint32(dest, src1, src2)) { - *error_r = t_strdup_printf("%s %u < %u", - parse_map[i].name, - *(const uint32_t *)src2, - *(const uint32_t *)src1); - return FALSE; - } - break; - case sizeof(uint64_t): - if (!mail_stats_diff_uint64(dest, src1, src2)) { - const uint64_t *n1 = src1, *n2 = src2; - - *error_r = t_strdup_printf("%s %llu < %llu", - parse_map[i].name, - (unsigned long long)*n2, - (unsigned long long)*n1); - return FALSE; - } - break; - default: - i_unreached(); - } - break; - case TYPE_TIMEVAL: - if (!mail_stats_diff_timeval(dest, src1, src2)) { - const struct timeval *tv1 = src1, *tv2 = src2; - - *error_r = t_strdup_printf("%s %ld.%d < %ld.%d", - parse_map[i].name, - (long)tv2->tv_sec, (int)tv2->tv_usec, - (long)tv1->tv_sec, (int)tv1->tv_usec); - return FALSE; - } - break; - } - } - return TRUE; -} - -static void timeval_add(struct timeval *dest, const struct timeval *src) -{ - dest->tv_sec += src->tv_sec; - dest->tv_usec += src->tv_usec; - if (dest->tv_usec > 1000000) { - dest->tv_usec -= 1000000; - dest->tv_sec++; - } -} - -void mail_stats_add(struct mail_stats *dest, const struct mail_stats *src) -{ - unsigned int i; - - for (i = 0; i < N_ELEMENTS(parse_map); i++) { - unsigned int offset = parse_map[i].offset; - void *f_dest = PTR_OFFSET(dest, offset); - const void *f_src = CONST_PTR_OFFSET(src, offset); - - switch (parse_map[i].type) { - case TYPE_NUM: - switch (parse_map[i].size) { - case sizeof(uint32_t): { - uint32_t *n_dest = f_dest; - const uint32_t *n_src = f_src; - - *n_dest += *n_src; - break; - } - case sizeof(uint64_t): { - uint64_t *n_dest = f_dest; - const uint64_t *n_src = f_src; - - *n_dest += *n_src; - break; - } - default: - i_unreached(); - } - break; - case TYPE_TIMEVAL: - timeval_add(f_dest, f_src); - break; - } - } -} - void mail_global_init(void) { mail_global_stats.reset_timestamp = ioloop_time; + mail_global_stats.stats = stats_alloc(default_pool); +} + +void mail_global_deinit(void) +{ + i_free(mail_global_stats.stats); } void mail_global_login(void) @@ -291,9 +30,9 @@ mail_global_stats.num_connected_sessions--; } -void mail_global_refresh(const struct mail_stats *diff_stats) +void mail_global_refresh(const struct stats *diff_stats) { if (diff_stats != NULL) - mail_stats_add(&mail_global_stats.stats, diff_stats); + stats_add(mail_global_stats.stats, diff_stats); mail_global_stats.last_update = ioloop_timeval; }
--- a/src/stats/mail-stats.h Thu Mar 05 22:19:02 2015 +0200 +++ b/src/stats/mail-stats.h Thu Mar 05 23:02:48 2015 +0200 @@ -3,20 +3,7 @@ #include "net.h" #include "guid.h" - -struct mail_stats { - struct timeval user_cpu, sys_cpu, clock_time; - uint32_t min_faults, maj_faults; - uint32_t vol_cs, invol_cs; - uint64_t disk_input, disk_output; - - uint32_t read_count, write_count; - uint64_t read_bytes, write_bytes; - - uint32_t mail_lookup_path, mail_lookup_attr, mail_read_count; - uint32_t mail_cache_hits; - uint64_t mail_read_bytes; -}; +#include "stats.h" struct mail_command { struct mail_command *stable_prev, *stable_next; @@ -28,7 +15,7 @@ unsigned int id; struct timeval last_update; - struct mail_stats stats; + struct stats *stats; int refcount; }; @@ -48,7 +35,7 @@ struct mail_ip *ip; struct timeout *to_idle; - struct mail_stats stats; + struct stats *stats; struct timeval last_update; unsigned int num_cmds; @@ -67,7 +54,7 @@ time_t reset_timestamp; struct timeval last_update; - struct mail_stats stats; + struct stats *stats; unsigned int num_logins; unsigned int num_cmds; @@ -82,7 +69,7 @@ time_t reset_timestamp; struct timeval last_update; - struct mail_stats stats; + struct stats *stats; unsigned int num_logins; unsigned int num_cmds; unsigned int num_connected_sessions; @@ -98,7 +85,7 @@ time_t reset_timestamp; struct timeval last_update; - struct mail_stats stats; + struct stats *stats; unsigned int num_logins; unsigned int num_cmds; unsigned int num_connected_sessions; @@ -111,7 +98,7 @@ time_t reset_timestamp; struct timeval last_update; - struct mail_stats stats; + struct stats *stats; unsigned int num_logins; unsigned int num_cmds; unsigned int num_connected_sessions; @@ -119,18 +106,11 @@ extern struct mail_global mail_global_stats; -int mail_stats_parse(const char *const *args, struct mail_stats *stats_r, - const char **error_r); -/* diff1 is supposed to have smaller values than diff2. Returns TRUE if this - is so, FALSE if not */ -bool mail_stats_diff(const struct mail_stats *stats1, - const struct mail_stats *stats2, - struct mail_stats *diff_stats_r, const char **error_r); -void mail_stats_add(struct mail_stats *dest, const struct mail_stats *src); +void mail_global_init(void); +void mail_global_deinit(void); -void mail_global_init(void); void mail_global_login(void); void mail_global_disconnected(void); -void mail_global_refresh(const struct mail_stats *diff_stats); +void mail_global_refresh(const struct stats *diff_stats); #endif
--- a/src/stats/mail-user.c Thu Mar 05 22:19:02 2015 +0200 +++ b/src/stats/mail-user.c Thu Mar 05 23:02:48 2015 +0200 @@ -39,7 +39,8 @@ else domain = ""; - user = i_new(struct mail_user, 1); + user = i_malloc(sizeof(struct mail_user) + stats_alloc_size()); + user->stats = (void *)(user + 1); user->name = i_strdup(username); user->reset_timestamp = ioloop_time; user->domain = mail_domain_login_create(domain); @@ -104,10 +105,10 @@ } void mail_user_refresh(struct mail_user *user, - const struct mail_stats *diff_stats) + const struct stats *diff_stats) { if (diff_stats != NULL) - mail_stats_add(&user->stats, diff_stats); + stats_add(user->stats, diff_stats); user->last_update = ioloop_timeval; DLLIST2_REMOVE_FULL(&mail_users_head, &mail_users_tail, user, sorted_prev, sorted_next);
--- a/src/stats/mail-user.h Thu Mar 05 22:19:02 2015 +0200 +++ b/src/stats/mail-user.h Thu Mar 05 23:02:48 2015 +0200 @@ -1,7 +1,7 @@ #ifndef MAIL_USER_H #define MAIL_USER_H -struct mail_stats; +struct stats; extern struct mail_user *stable_mail_users; @@ -10,7 +10,7 @@ struct mail_user *mail_user_lookup(const char *username); void mail_user_refresh(struct mail_user *user, - const struct mail_stats *diff_stats) ATTR_NULL(2); + const struct stats *diff_stats) ATTR_NULL(2); void mail_user_ref(struct mail_user *user); void mail_user_unref(struct mail_user **user);
--- a/src/stats/main.c Thu Mar 05 22:19:02 2015 +0200 +++ b/src/stats/main.c Thu Mar 05 23:02:48 2015 +0200 @@ -1,6 +1,7 @@ /* Copyright (c) 2011-2015 Dovecot authors, see the included COPYING file */ #include "lib.h" +#include "module-dir.h" #include "restrict-access.h" #include "master-service.h" #include "master-service-settings.h" @@ -16,6 +17,7 @@ #include "client.h" static struct mail_server_connection *mail_server_conn = NULL; +static struct module *modules = NULL; static void client_connected(struct master_service_connection *conn) { @@ -31,6 +33,21 @@ master_service_client_connection_accept(conn); } +static void main_preinit(void) +{ + struct module_dir_load_settings mod_set; + + memset(&mod_set, 0, sizeof(mod_set)); + mod_set.abi_version = DOVECOT_ABI_VERSION; + mod_set.require_init_funcs = TRUE; + + modules = module_dir_load(STATS_MODULE_DIR, NULL, &mod_set); + module_dir_init(modules); + + restrict_access_by_env(NULL, FALSE); + restrict_access_allow_coredumps(TRUE); +} + int main(int argc, char *argv[]) { const struct setting_parser_info *set_roots[] = { @@ -52,8 +69,7 @@ i_fatal("Error reading configuration: %s", error); master_service_init_log(master_service, "stats: "); - restrict_access_by_env(NULL, FALSE); - restrict_access_allow_coredumps(TRUE); + main_preinit(); sets = master_service_settings_get_others(master_service); stats_settings = sets[0]; @@ -74,10 +90,12 @@ mail_users_deinit(); mail_domains_deinit(); mail_ips_deinit(); + mail_global_deinit(); if (mail_server_conn != NULL) mail_server_connection_destroy(&mail_server_conn); + module_dir_unload(&modules); i_assert(global_used_memory == 0); master_service_deinit(&master_service); return 0;