# HG changeset patch # User Timo Sirainen # Date 1296439324 -7200 # Node ID 3dde816d945dab965dacdde3d087e6b54bb2c3c9 # Parent 1e88287fc72179279cd6cdd78a9b6880765cb112 imapc: Write large message bodies to temp files rather than keeping in memory. diff -r 1e88287fc721 -r 3dde816d945d src/lib-storage/index/imapc/imapc-client.c --- a/src/lib-storage/index/imapc/imapc-client.c Mon Jan 31 04:00:44 2011 +0200 +++ b/src/lib-storage/index/imapc/imapc-client.c Mon Jan 31 04:02:04 2011 +0200 @@ -2,11 +2,15 @@ #include "lib.h" #include "array.h" +#include "str.h" #include "ioloop.h" +#include "safe-mkstemp.h" #include "imapc-seqmap.h" #include "imapc-connection.h" #include "imapc-client-private.h" +#include + struct imapc_client_command_context { struct imapc_client_mailbox *box; @@ -42,6 +46,8 @@ client->set.password = p_strdup(pool, set->password); client->set.dns_client_socket_path = p_strdup(pool, set->dns_client_socket_path); + client->set.temp_path_prefix = + p_strdup(pool, set->temp_path_prefix); p_array_init(&client->conns, pool, 8); return client; } @@ -271,3 +277,28 @@ connp = array_idx(&client->conns, 0); return imapc_connection_get_capabilities((*connp)->conn); } + +int imapc_client_create_temp_fd(struct imapc_client *client, + const char **path_r) +{ + string_t *path; + int fd; + + path = t_str_new(128); + str_append(path, client->set.temp_path_prefix); + fd = safe_mkstemp(path, 0600, (uid_t)-1, (gid_t)-1); + if (fd == -1) { + i_error("safe_mkstemp(%s) failed: %m", str_c(path)); + return -1; + } + + /* we just want the fd, unlink it */ + if (unlink(str_c(path)) < 0) { + /* shouldn't happen.. */ + i_error("unlink(%s) failed: %m", str_c(path)); + (void)close(fd); + return -1; + } + *path_r = str_c(path); + return fd; +} diff -r 1e88287fc721 -r 3dde816d945d src/lib-storage/index/imapc/imapc-client.h --- a/src/lib-storage/index/imapc/imapc-client.h Mon Jan 31 04:00:44 2011 +0200 +++ b/src/lib-storage/index/imapc/imapc-client.h Mon Jan 31 04:02:04 2011 +0200 @@ -32,6 +32,7 @@ const char *password; const char *dns_client_socket_path; + const char *temp_path_prefix; }; struct imapc_command_reply { @@ -46,6 +47,16 @@ const char *text_without_resp; }; +struct imapc_arg_file { + /* file descriptor containing the value */ + int fd; + + /* parent_arg.list[list_idx] points to the IMAP_ARG_LITERAL_SIZE + argument */ + const struct imap_arg *parent_arg; + unsigned int list_idx; +}; + struct imapc_untagged_reply { /* name of the untagged reply, e.g. EXISTS */ const char *name; @@ -54,6 +65,10 @@ uint32_t num; /* the rest of the reply can be read from these args. */ const struct imap_arg *args; + /* arguments whose contents are stored into files. only + "FETCH (BODY[" arguments can be here. */ + const struct imapc_arg_file *file_args; + unsigned int file_args_count; /* "* OK [RESP TEXT]" produces key=RESP, value=TEXT. "* OK [RESP]" produces key=RESP, value=NULL @@ -111,4 +126,7 @@ enum imapc_capability imapc_client_get_capabilities(struct imapc_client *client); +int imapc_client_create_temp_fd(struct imapc_client *client, + const char **path_r); + #endif diff -r 1e88287fc721 -r 3dde816d945d src/lib-storage/index/imapc/imapc-connection.c --- a/src/lib-storage/index/imapc/imapc-connection.c Mon Jan 31 04:00:44 2011 +0200 +++ b/src/lib-storage/index/imapc/imapc-connection.c Mon Jan 31 04:02:04 2011 +0200 @@ -6,6 +6,7 @@ #include "istream.h" #include "ostream.h" #include "base64.h" +#include "write-full.h" #include "str.h" #include "dns-lookup.h" #include "imap-quote.h" @@ -15,10 +16,12 @@ #include "imapc-seqmap.h" #include "imapc-connection.h" +#include #include #define IMAPC_DNS_LOOKUP_TIMEOUT_MSECS (1000*30) #define IMAPC_CONNECT_TIMEOUT_MSECS (1000*30) +#define IMAPC_MAX_INLINE_LITERAL_SIZE (1024*32) enum imapc_input_state { IMAPC_INPUT_STATE_NONE = 0, @@ -45,6 +48,15 @@ void *context; }; +struct imapc_connection_literal { + char *temp_path; + int fd; + uoff_t bytes_left; + + const struct imap_arg *parent_arg; + unsigned int list_idx; +}; + struct imapc_connection { struct imapc_client *client; char *name; @@ -73,6 +85,9 @@ unsigned int ips_count, prev_connect_idx; struct ip_addr *ips; + struct imapc_connection_literal literal; + ARRAY_DEFINE(literal_files, struct imapc_arg_file); + unsigned int idling:1; unsigned int idle_stopping:1; unsigned int idle_plus_waiting:1; @@ -94,8 +109,10 @@ conn->fd = -1; conn->name = i_strdup_printf("%s:%u", client->set.host, client->set.port); + conn->literal.fd = -1; i_array_init(&conn->cmd_send_queue, 8); i_array_init(&conn->cmd_wait_list, 32); + i_array_init(&conn->literal_files, 4); return conn; } @@ -109,6 +126,7 @@ p_strsplit_free(default_pool, conn->capabilities_list); array_free(&conn->cmd_send_queue); array_free(&conn->cmd_wait_list); + array_free(&conn->literal_files); i_free(conn->ips); i_free(conn->name); i_free(conn); @@ -162,11 +180,37 @@ conn->state = state; } +static void imapc_connection_lfiles_free(struct imapc_connection *conn) +{ + struct imapc_arg_file *lfile; + + array_foreach_modifiable(&conn->literal_files, lfile) { + if (close(lfile->fd) < 0) + i_error("imapc: close(literal file) failed: %m"); + } + array_clear(&conn->literal_files); +} + +static void +imapc_connection_literal_reset(struct imapc_connection_literal *literal) +{ + if (literal->fd != -1) { + if (close(literal->fd) < 0) + i_error("close(%s) failed: %m", literal->temp_path); + } + i_free_and_null(literal->temp_path); + + memset(literal, 0, sizeof(*literal)); + literal->fd = -1; +} + static void imapc_connection_disconnect(struct imapc_connection *conn) { if (conn->fd == -1) return; + imapc_connection_lfiles_free(conn); + imapc_connection_literal_reset(&conn->literal); if (conn->to != NULL) timeout_remove(&conn->to); imap_parser_destroy(&conn->parser); @@ -192,14 +236,104 @@ va_end(va); } +static bool last_arg_is_fetch_body(const struct imap_arg *args, + const struct imap_arg **parent_arg_r, + unsigned int *idx_r) +{ + const struct imap_arg *list; + const char *name; + unsigned int count; + + if (args[0].type == IMAP_ARG_ATOM && + imap_arg_atom_equals(&args[1], "FETCH") && + imap_arg_get_list_full(&args[2], &list, &count) && count >= 2 && + list[count].type == IMAP_ARG_LITERAL_SIZE && + imap_arg_get_atom(&list[count-1], &name) && + strncasecmp(name, "BODY[", 5) == 0) { + *parent_arg_r = &args[2]; + *idx_r = count; + return TRUE; + } + return FALSE; +} + static int -imapc_connection_read_line(struct imapc_connection *conn, - const struct imap_arg **imap_args_r) +imapc_connection_read_literal_init(struct imapc_connection *conn, uoff_t size, + const struct imap_arg *args) { - int ret; + const char *path; + const struct imap_arg *parent_arg; + unsigned int idx; + + i_assert(size > 0); + i_assert(conn->literal.fd == -1); + + if (size <= IMAPC_MAX_INLINE_LITERAL_SIZE || + !last_arg_is_fetch_body(args, &parent_arg, &idx)) { + /* read the literal directly into parser */ + return 0; + } + + conn->literal.fd = imapc_client_create_temp_fd(conn->client, &path); + if (conn->literal.fd == -1) + return -1; + conn->literal.temp_path = i_strdup(path); + conn->literal.bytes_left = size; + conn->literal.parent_arg = parent_arg; + conn->literal.list_idx = idx; + return 1; +} + +static int imapc_connection_read_literal(struct imapc_connection *conn) +{ + struct imapc_arg_file *lfile; + const unsigned char *data; + size_t size; + + if (conn->literal.bytes_left == 0) + return 1; + + data = i_stream_get_data(conn->input, &size); + if (size > conn->literal.bytes_left) + size = conn->literal.bytes_left; + if (size > 0) { + if (write_full(conn->literal.fd, data, size) < 0) { + i_error("imapc(%s): write(%s) failed: %m", + conn->name, conn->literal.temp_path); + imapc_connection_disconnect(conn); + return -1; + } + i_stream_skip(conn->input, size); + conn->literal.bytes_left -= size; + } + if (conn->literal.bytes_left > 0) + return 0; + + /* finished */ + lfile = array_append_space(&conn->literal_files); + lfile->fd = conn->literal.fd; + lfile->parent_arg = conn->literal.parent_arg; + lfile->list_idx = conn->literal.list_idx; + + conn->literal.fd = -1; + imapc_connection_literal_reset(&conn->literal); + return 1; +} + +static int +imapc_connection_read_line_more(struct imapc_connection *conn, + const struct imap_arg **imap_args_r) +{ + uoff_t literal_size; bool fatal; + int ret; - ret = imap_parser_read_args(conn->parser, 0, 0, imap_args_r); + if ((ret = imapc_connection_read_literal(conn)) <= 0) + return ret; + + ret = imap_parser_read_args(conn->parser, 0, + IMAP_PARSE_FLAG_LITERAL_SIZE | + IMAP_PARSE_FLAG_ATOM_ALLCHARS, imap_args_r); if (ret == -2) { /* need more data */ return 0; @@ -209,10 +343,30 @@ imap_parser_get_error(conn->parser, &fatal)); return -1; } + + if (imap_parser_get_literal_size(conn->parser, &literal_size)) { + if (imapc_connection_read_literal_init(conn, literal_size, + *imap_args_r) <= 0) { + imap_parser_read_last_literal(conn->parser); + return 2; + } + return imapc_connection_read_line_more(conn, imap_args_r); + } return 1; } static int +imapc_connection_read_line(struct imapc_connection *conn, + const struct imap_arg **imap_args_r) +{ + int ret; + + while ((ret = imapc_connection_read_line_more(conn, imap_args_r)) == 2) + ; + return ret; +} + +static int imapc_connection_parse_capability(struct imapc_connection *conn, const char *value) { @@ -332,6 +486,7 @@ conn->cur_tag = 0; conn->cur_num = 0; imap_parser_reset(conn->parser); + imapc_connection_lfiles_free(conn); } static int imapc_connection_skip_line(struct imapc_connection *conn) @@ -507,6 +662,9 @@ reply.name = name; reply.num = conn->cur_num; reply.args = imap_args; + reply.file_args = array_get(&conn->literal_files, + &reply.file_args_count); + if (conn->selected_box != NULL) { reply.untagged_box_context = conn->selected_box->untagged_box_context; diff -r 1e88287fc721 -r 3dde816d945d src/lib-storage/index/imapc/imapc-mail.c --- a/src/lib-storage/index/imapc/imapc-mail.c Mon Jan 31 04:00:44 2011 +0200 +++ b/src/lib-storage/index/imapc/imapc-mail.c Mon Jan 31 04:02:04 2011 +0200 @@ -23,6 +23,15 @@ return &mail->imail.mail.mail; } +static void imapc_mail_free(struct mail *_mail) +{ + struct imapc_mail *mail = (struct imapc_mail *)_mail; + + if (mail->body != NULL) + buffer_free(&mail->body); + index_mail_free(_mail); +} + static int imapc_mail_get_received_date(struct mail *_mail, time_t *date_r) { struct index_mail *mail = (struct index_mail *)_mail; @@ -126,7 +135,7 @@ struct mail_vfuncs imapc_mail_vfuncs = { index_mail_close, - index_mail_free, + imapc_mail_free, index_mail_set_seq, index_mail_set_uid, index_mail_set_uid_cache_updates, diff -r 1e88287fc721 -r 3dde816d945d src/lib-storage/index/imapc/imapc-mail.h --- a/src/lib-storage/index/imapc/imapc-mail.h Mon Jan 31 04:00:44 2011 +0200 +++ b/src/lib-storage/index/imapc/imapc-mail.h Mon Jan 31 04:02:04 2011 +0200 @@ -5,6 +5,8 @@ struct imapc_mail { struct index_mail imail; + + buffer_t *body; unsigned int searching:1; unsigned int fetch_one:1; }; diff -r 1e88287fc721 -r 3dde816d945d src/lib-storage/index/imapc/imapc-mailbox.c --- a/src/lib-storage/index/imapc/imapc-mailbox.c Mon Jan 31 04:00:44 2011 +0200 +++ b/src/lib-storage/index/imapc/imapc-mailbox.c Mon Jan 31 04:02:04 2011 +0200 @@ -131,7 +131,7 @@ if (mbox->cur_fetch_mail != NULL && mbox->cur_fetch_mail->seq == seq) { i_assert(uid == 0 || mbox->cur_fetch_mail->uid == uid); - imapc_fetch_mail_update(mbox->cur_fetch_mail, list); + imapc_fetch_mail_update(mbox->cur_fetch_mail, reply, list); } imapc_mailbox_init_delayed_trans(mbox); diff -r 1e88287fc721 -r 3dde816d945d src/lib-storage/index/imapc/imapc-save.c --- a/src/lib-storage/index/imapc/imapc-save.c Mon Jan 31 04:00:44 2011 +0200 +++ b/src/lib-storage/index/imapc/imapc-save.c Mon Jan 31 04:02:04 2011 +0200 @@ -64,7 +64,7 @@ i_assert(ctx->fd == -1); - ctx->fd = imapc_create_temp_fd(storage->user, &path); + ctx->fd = imapc_client_create_temp_fd(ctx->mbox->storage->client, &path); if (ctx->fd == -1) { mail_storage_set_critical(storage, "Couldn't create temp file %s", path); diff -r 1e88287fc721 -r 3dde816d945d src/lib-storage/index/imapc/imapc-search.c --- a/src/lib-storage/index/imapc/imapc-search.c Mon Jan 31 04:00:44 2011 +0200 +++ b/src/lib-storage/index/imapc/imapc-search.c Mon Jan 31 04:02:04 2011 +0200 @@ -230,29 +230,67 @@ return TRUE; } +static bool imapc_find_lfile_arg(const struct imapc_untagged_reply *reply, + const struct imap_arg *arg, int *fd_r) +{ + const struct imap_arg *list; + unsigned int i, count; + + for (i = 0; i < reply->file_args_count; i++) { + const struct imapc_arg_file *farg = &reply->file_args[i]; + + if (farg->parent_arg == arg->parent && + imap_arg_get_list_full(arg->parent, &list, &count) && + farg->list_idx < count && &list[farg->list_idx] == arg) { + *fd_r = farg->fd; + return TRUE; + } + } + return FALSE; +} + static void -imapc_fetch_stream(struct index_mail *imail, const char *value, bool body) +imapc_fetch_stream(struct imapc_mail *mail, + const struct imapc_untagged_reply *reply, + const struct imap_arg *arg, bool body) { + struct index_mail *imail = &mail->imail; struct mail *_mail = &imail->mail.mail; struct istream *input; - size_t value_len = strlen(value); uoff_t size; - const char *path; + const char *value; int fd, ret; if (imail->data.stream != NULL) return; - fd = imapc_create_temp_fd(_mail->box->storage->user, &path); - if (fd == -1) - return; - if (write_full(fd, value, value_len) < 0) { - (void)close(fd); - return; + if (arg->type == IMAP_ARG_LITERAL_SIZE) { + if (!imapc_find_lfile_arg(reply, arg, &fd)) + return; + if ((fd = dup(fd)) == -1) { + i_error("dup() failed: %m"); + return; + } + imail->data.stream = i_stream_create_fd(fd, 0, TRUE); + } else { + if (!imap_arg_get_nstring(arg, &value)) + return; + if (value == NULL) { + mail_set_expunged(_mail); + return; + } + if (mail->body == NULL) { + mail->body = buffer_create_dynamic(default_pool, + arg->str_len + 1); + } + buffer_set_used_size(mail->body, 0); + buffer_append(mail->body, value, arg->str_len); + imail->data.stream = i_stream_create_from_data(mail->body->data, + mail->body->used); } - imail->data.stream = i_stream_create_fd(fd, 0, TRUE); - i_stream_set_name(imail->data.stream, path); + i_stream_set_name(imail->data.stream, + t_strdup_printf("imapc mail uid=%u", _mail->uid)); index_mail_set_read_buffer_size(_mail, imail->data.stream); if (imail->mail.v.istream_opened != NULL) { @@ -278,11 +316,13 @@ i_stream_unref(&imail->data.stream); } -void imapc_fetch_mail_update(struct mail *mail, const struct imap_arg *args) +void imapc_fetch_mail_update(struct mail *mail, + const struct imapc_untagged_reply *reply, + const struct imap_arg *args) { struct imapc_mail *imapmail = (struct imapc_mail *)mail; struct imapc_mailbox *mbox = (struct imapc_mailbox *)mail->box; - struct index_mail *imail = (struct index_mail *)mail; + struct imapc_mail *imail = (struct imapc_mail *)mail; const char *key, *value; unsigned int i; time_t t; @@ -291,23 +331,16 @@ for (i = 0; args[i].type != IMAP_ARG_EOL; i += 2) { if (!imap_arg_get_atom(&args[i], &key) || args[i+1].type == IMAP_ARG_EOL) - return; + break; - if (strcasecmp(key, "BODY[]") == 0) { - if (!imap_arg_get_nstring(&args[i+1], &value)) - return; - if (value != NULL) - imapc_fetch_stream(imail, value, TRUE); - } else if (strcasecmp(key, "BODY[HEADER]") == 0) { - if (!imap_arg_get_nstring(&args[i+1], &value)) - return; - if (value != NULL) - imapc_fetch_stream(imail, value, FALSE); - } else if (strcasecmp(key, "INTERNALDATE") == 0) { - if (!imap_arg_get_astring(&args[i+1], &value) || - !imap_parse_datetime(value, &t, &tz)) - return; - imail->data.received_date = t; + if (strcasecmp(key, "BODY[]") == 0) + imapc_fetch_stream(imail, reply, &args[i+1], TRUE); + else if (strcasecmp(key, "BODY[HEADER]") == 0) + imapc_fetch_stream(imail, reply, &args[i+1], FALSE); + else if (strcasecmp(key, "INTERNALDATE") == 0) { + if (imap_arg_get_astring(&args[i+1], &value) && + imap_parse_datetime(value, &t, &tz)) + imail->imail.data.received_date = t; } } if (!imapmail->fetch_one) diff -r 1e88287fc721 -r 3dde816d945d src/lib-storage/index/imapc/imapc-storage.c --- a/src/lib-storage/index/imapc/imapc-storage.c Mon Jan 31 04:00:44 2011 +0200 +++ b/src/lib-storage/index/imapc/imapc-storage.c Mon Jan 31 04:02:04 2011 +0200 @@ -3,7 +3,6 @@ #include "lib.h" #include "ioloop.h" #include "str.h" -#include "safe-mkstemp.h" #include "imap-arg.h" #include "imap-resp-code.h" #include "imapc-mail.h" @@ -165,6 +164,7 @@ struct imapc_storage *storage = (struct imapc_storage *)_storage; struct imapc_client_settings set; const char *port; + string_t *str; memset(&set, 0, sizeof(set)); set.host = ns->list->set.root_dir; @@ -192,6 +192,10 @@ set.dns_client_socket_path = t_strconcat(_storage->user->set->base_dir, "/", DNS_CLIENT_SOCKET_NAME, NULL); + str = t_str_new(128); + mail_user_set_get_temp_prefix(str, _storage->user->set); + set.temp_path_prefix = str_c(str); + storage->list = (struct imapc_mailbox_list *)ns->list; storage->list->storage = storage; storage->client = imapc_client_init(&set); @@ -477,30 +481,6 @@ /* we're doing IDLE all the time anyway - nothing to do here */ } -int imapc_create_temp_fd(struct mail_user *user, const char **path_r) -{ - string_t *path; - int fd; - - path = t_str_new(128); - mail_user_set_get_temp_prefix(path, user->set); - fd = safe_mkstemp(path, 0600, (uid_t)-1, (gid_t)-1); - if (fd == -1) { - i_error("safe_mkstemp(%s) failed: %m", str_c(path)); - return -1; - } - - /* we just want the fd, unlink it */ - if (unlink(str_c(path)) < 0) { - /* shouldn't happen.. */ - i_error("unlink(%s) failed: %m", str_c(path)); - (void)close(fd); - return -1; - } - *path_r = str_c(path); - return fd; -} - struct mail_storage imapc_storage = { .name = IMAPC_STORAGE_NAME, .class_flags = 0, diff -r 1e88287fc721 -r 3dde816d945d src/lib-storage/index/imapc/imapc-storage.h --- a/src/lib-storage/index/imapc/imapc-storage.h Mon Jan 31 04:00:44 2011 +0200 +++ b/src/lib-storage/index/imapc/imapc-storage.h Mon Jan 31 04:02:04 2011 +0200 @@ -81,7 +81,9 @@ bool imapc_search_next_nonblock(struct mail_search_context *_ctx, struct mail *mail, bool *tryagain_r); bool imapc_search_next_update_seq(struct mail_search_context *_ctx); -void imapc_fetch_mail_update(struct mail *mail, const struct imap_arg *args); +void imapc_fetch_mail_update(struct mail *mail, + const struct imapc_untagged_reply *reply, + const struct imap_arg *args); void imapc_copy_error_from_reply(struct imapc_storage *storage, enum mail_error default_error, @@ -102,6 +104,5 @@ imapc_mailbox_callback_t *callback); void imapc_mailbox_register_callbacks(struct imapc_mailbox *mbox); -int imapc_create_temp_fd(struct mail_user *user, const char **path_r); #endif