Mercurial > dovecot > original-hg > dovecot-2.1
changeset 14957:9c23d1fb8969
zlib: Keep the last read mail cached uncompressed in a temp file.
This fixes performance problems with partial IMAP FETCH commands.
author | Timo Sirainen <tss@iki.fi> |
---|---|
date | Mon, 22 Apr 2013 19:03:53 +0300 |
parents | 143bd7f3d4a2 |
children | d044963bf65c |
files | src/plugins/zlib/zlib-plugin.c |
diffstat | 1 files changed, 124 insertions(+), 0 deletions(-) [+] |
line wrap: on
line diff
--- a/src/plugins/zlib/zlib-plugin.c Sat Apr 20 20:58:06 2013 +0300 +++ b/src/plugins/zlib/zlib-plugin.c Mon Apr 22 19:03:53 2013 +0300 @@ -3,7 +3,11 @@ #include "lib.h" #include "array.h" #include "istream.h" +#include "istream-seekable.h" #include "ostream.h" +#include "close-keep-errno.h" +#include "safe-mkstemp.h" +#include "str.h" #include "mail-user.h" #include "dbox-single/sdbox-storage.h" #include "dbox-multi/mdbox-storage.h" @@ -36,6 +40,7 @@ #endif #define MAX_INBUF_SIZE (1024*1024) +#define ZLIB_MAIL_CACHE_EXPIRE_MSECS (60*1000) struct zlib_transaction_context { union mailbox_transaction_module_context module_ctx; @@ -43,9 +48,19 @@ struct mail *tmp_mail; }; +struct zlib_mail_cache { + struct timeout *to; + struct mailbox *box; + uint32_t uid; + + struct istream *input; +}; + struct zlib_user { union mail_user_module_context module_ctx; + struct zlib_mail_cache cache; + const struct zlib_handler *save_handler; unsigned int save_level; }; @@ -129,9 +144,84 @@ return NULL; } +static void zlib_mail_cache_close(struct zlib_user *zuser) +{ + struct zlib_mail_cache *cache = &zuser->cache; + + if (cache->to != NULL) + timeout_remove(&cache->to); + if (cache->input != NULL) + i_stream_unref(&cache->input); + memset(cache, 0, sizeof(*cache)); +} + +static int seekable_fd_callback(const char **path_r, void *context) +{ + struct mail_user *user = context; + string_t *path; + int fd; + + path = t_str_new(128); + mail_user_set_get_temp_prefix(path, user->set); + fd = safe_mkstemp(path, 0600, (uid_t)-1, (gid_t)-1); + if (fd == -1) { + i_error("safe_mkstemp(%s) failed: %m", str_c(path)); + return -1; + } + + /* we just want the fd, unlink it */ + if (unlink(str_c(path)) < 0) { + /* shouldn't happen.. */ + i_error("unlink(%s) failed: %m", str_c(path)); + close_keep_errno(fd); + return -1; + } + + *path_r = str_c(path); + return fd; +} + +static struct istream * +zlib_mail_cache_open(struct zlib_user *zuser, struct mail *mail, + struct istream *input) +{ + struct zlib_mail_cache *cache = &zuser->cache; + struct istream *inputs[2]; + string_t *temp_prefix = t_str_new(128); + + zlib_mail_cache_close(zuser); + + /* zlib istream is seekable, but very slow. create a seekable istream + which we can use to quickly seek around in the stream that's been + read so far. usually the partial IMAP FETCHes continue from where + the previous left off, so this isn't strictly necessary, but with + the way lib-imap-storage's CRLF-cache works it has to seek backwards + somewhat, which causes a zlib stream reset. And the CRLF-cache isn't + easy to fix.. */ + input->seekable = FALSE; + inputs[0] = input; + inputs[1] = NULL; + mail_user_set_get_temp_prefix(temp_prefix, mail->box->storage->user->set); + input = i_stream_create_seekable(inputs, + i_stream_get_max_buffer_size(inputs[0]), + seekable_fd_callback, mail->box->storage->user); + i_stream_unref(&inputs[0]); + + cache->to = timeout_add(ZLIB_MAIL_CACHE_EXPIRE_MSECS, + zlib_mail_cache_close, zuser); + cache->box = mail->box; + cache->uid = mail->uid; + cache->input = input; + + /* index-mail wants the stream to be destroyed at close, so create + a new stream instead of just increasing reference. */ + return i_stream_create_limit(cache->input, (uoff_t)-1); +} + static int zlib_istream_opened(struct mail *_mail, struct istream **stream) { struct zlib_user *zuser = ZLIB_USER_CONTEXT(_mail->box->storage->user); + struct zlib_mail_cache *cache = &zuser->cache; struct mail_private *mail = (struct mail_private *)_mail; union mail_module_context *zmail = ZLIB_MAIL_CONTEXT(mail); struct istream *input; @@ -144,6 +234,15 @@ if (_mail->saving && zuser->save_handler == NULL) return zmail->super.istream_opened(_mail, stream); + if (cache->uid == _mail->uid && cache->box == _mail->box) { + /* use the cached stream. when doing partial reads it should + already be seeked into the wanted offset. */ + i_stream_unref(stream); + i_stream_seek(cache->input, 0); + *stream = i_stream_create_limit(cache->input, (uoff_t)-1); + return zmail->super.istream_opened(_mail, stream); + } + handler = zlib_get_zlib_handler(*stream); if (handler != NULL) { if (handler->create_istream == NULL) { @@ -156,6 +255,8 @@ input = *stream; *stream = handler->create_istream(input, TRUE); i_stream_unref(&input); + + *stream = zlib_mail_cache_open(zuser, _mail, *stream); } return zmail->super.istream_opened(_mail, stream); } @@ -346,6 +447,16 @@ return zbox->super.open(box); } +static void zlib_mailbox_close(struct mailbox *box) +{ + union mailbox_module_context *zbox = ZLIB_CONTEXT(box); + struct zlib_user *zuser = ZLIB_USER_CONTEXT(box->storage->user); + + if (zuser->cache.box == box) + zlib_mail_cache_close(zuser); + zbox->super.close(box); +} + static void zlib_mailbox_allocated(struct mailbox *box) { struct mailbox_vfuncs *v = box->vlast; @@ -355,6 +466,7 @@ zbox->super = *v; box->vlast = &zbox->super; v->open = zlib_mailbox_open; + v->close = zlib_mailbox_close; MODULE_CONTEXT_SET_SELF(box, zlib_storage_module, zbox); @@ -364,12 +476,24 @@ zlib_permail_alloc_init(box, v); } +static void zlib_mail_user_deinit(struct mail_user *user) +{ + struct zlib_user *zuser = ZLIB_USER_CONTEXT(user); + + zlib_mail_cache_close(zuser); + zuser->module_ctx.super.deinit(user); +} + static void zlib_mail_user_created(struct mail_user *user) { + struct mail_user_vfuncs *v = user->vlast; struct zlib_user *zuser; const char *name; zuser = p_new(user->pool, struct zlib_user, 1); + zuser->module_ctx.super = *v; + user->vlast = &zuser->module_ctx.super; + v->deinit = zlib_mail_user_deinit; name = mail_user_plugin_getenv(user, "zlib_save"); if (name != NULL && *name != '\0') {