Mercurial > dovecot > core-2.2
changeset 12312:28eaaa23f2c6
lib-storage: Added support for saving mail attachments separately via filesystem API.
Currently this works only with sdbox and mdbox backends.
line wrap: on
line diff
--- a/src/config/settings-get.pl Tue Oct 19 18:30:51 2010 +0100 +++ b/src/config/settings-get.pl Tue Oct 19 18:47:17 2010 +0100 @@ -6,6 +6,7 @@ print '#include "var-expand.h"'."\n"; print '#include "file-lock.h"'."\n"; print '#include "fsync-mode.h"'."\n"; +print '#include "hash-format.h"'."\n"; print '#include "settings-parser.h"'."\n"; print '#include "all-settings.h"'."\n"; print '#include <stddef.h>'."\n";
--- a/src/lib-storage/index/Makefile.am Tue Oct 19 18:30:51 2010 +0100 +++ b/src/lib-storage/index/Makefile.am Tue Oct 19 18:47:17 2010 +0100 @@ -5,13 +5,16 @@ AM_CPPFLAGS = \ -I$(top_srcdir)/src/lib \ -I$(top_srcdir)/src/lib-test \ + -I$(top_srcdir)/src/lib-fs \ -I$(top_srcdir)/src/lib-mail \ -I$(top_srcdir)/src/lib-imap \ -I$(top_srcdir)/src/lib-index \ -I$(top_srcdir)/src/lib-storage libstorage_index_la_SOURCES = \ + istream-attachment.c \ istream-mail-stats.c \ + index-attachment.c \ index-fetch.c \ index-mail.c \ index-mail-headers.c \ @@ -34,7 +37,9 @@ libstorage_index_la_DEPENDENCIES = @LINKED_STORAGE_LIBS@ headers = \ + istream-attachment.h \ istream-mail-stats.h \ + index-attachment.h \ index-mail.h \ index-search-result.h \ index-sort.h \
--- a/src/lib-storage/index/cydir/cydir-storage.c Tue Oct 19 18:30:51 2010 +0100 +++ b/src/lib-storage/index/cydir/cydir-storage.c Tue Oct 19 18:47:17 2010 +0100 @@ -179,6 +179,7 @@ cydir_save_finish, cydir_save_cancel, mail_storage_copy, + NULL, index_storage_is_inconsistent } };
--- a/src/lib-storage/index/dbox-common/Makefile.am Tue Oct 19 18:30:51 2010 +0100 +++ b/src/lib-storage/index/dbox-common/Makefile.am Tue Oct 19 18:47:17 2010 +0100 @@ -3,6 +3,7 @@ AM_CPPFLAGS = \ -I$(top_srcdir)/src/lib \ -I$(top_srcdir)/src/lib-settings \ + -I$(top_srcdir)/src/lib-fs \ -I$(top_srcdir)/src/lib-mail \ -I$(top_srcdir)/src/lib-imap \ -I$(top_srcdir)/src/lib-index \ @@ -10,6 +11,7 @@ -I$(top_srcdir)/src/lib-storage/index libstorage_dbox_common_la_SOURCES = \ + dbox-attachment.c \ dbox-file.c \ dbox-file-fix.c \ dbox-mail.c \ @@ -18,6 +20,7 @@ dbox-sync-rebuild.c headers = \ + dbox-attachment.h \ dbox-file.h \ dbox-mail.h \ dbox-save.h \
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-storage/index/dbox-common/dbox-attachment.c Tue Oct 19 18:47:17 2010 +0100 @@ -0,0 +1,235 @@ +/* Copyright (c) 2010 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "istream.h" +#include "istream-concat.h" +#include "str.h" +#include "istream-attachment.h" +#include "istream-base64-encoder.h" +#include "dbox-file.h" +#include "dbox-save.h" +#include "dbox-attachment.h" + +enum dbox_attachment_decode_option { + DBOX_ATTACHMENT_DECODE_OPTION_NONE = '-', + DBOX_ATTACHMENT_DECODE_OPTION_BASE64 = 'B', + DBOX_ATTACHMENT_DECODE_OPTION_CRLF = 'C' +}; + +void dbox_attachment_save_write_metadata(struct mail_save_context *ctx, + string_t *str) +{ + const ARRAY_TYPE(mail_attachment_extref) *extrefs; + const struct mail_attachment_extref *extref; + bool add_space = FALSE; + unsigned int startpos; + + extrefs = index_attachment_save_get_extrefs(ctx); + if (extrefs == NULL || array_count(extrefs) == 0) + return; + + str_append_c(str, DBOX_METADATA_EXT_REF); + array_foreach(extrefs, extref) { + if (!add_space) + add_space = TRUE; + else + str_append_c(str, ' '); + str_printfa(str, "%"PRIuUOFF_T" %"PRIuUOFF_T" ", + extref->start_offset, extref->size); + + startpos = str_len(str); + if (extref->base64_have_crlf) + str_append_c(str, DBOX_ATTACHMENT_DECODE_OPTION_CRLF); + if (extref->base64_blocks_per_line > 0) { + str_printfa(str, "%c%u", + DBOX_ATTACHMENT_DECODE_OPTION_BASE64, + extref->base64_blocks_per_line * 4); + } + if (startpos == str_len(str)) { + /* make it clear there are no options */ + str_append_c(str, DBOX_ATTACHMENT_DECODE_OPTION_NONE); + } + str_append_c(str, ' '); + str_append(str, extref->path); + } + str_append_c(str, '\n'); +} + +static bool +parse_extref_decode_options(const char *str, + struct mail_attachment_extref *extref) +{ + unsigned int num; + + if (*str == DBOX_ATTACHMENT_DECODE_OPTION_NONE) + return str[1] == '\0'; + + while (*str != '\0') { + switch (*str) { + case DBOX_ATTACHMENT_DECODE_OPTION_BASE64: + str++; num = 0; + while (*str >= '0' && *str <= '9') { + num = num*10 + (*str-'0'); + str++; + } + if (num == 0 || num % 4 != 0) + return FALSE; + + extref->base64_blocks_per_line = num/4; + break; + case DBOX_ATTACHMENT_DECODE_OPTION_CRLF: + extref->base64_have_crlf = TRUE; + str++; + break; + default: + return FALSE; + } + } + return TRUE; +} + +static bool +dbox_attachment_parse_extref_real(const char *line, pool_t pool, + ARRAY_TYPE(mail_attachment_extref) *extrefs) +{ + struct mail_attachment_extref extref; + const char *const *args; + unsigned int i, len; + uoff_t last_voffset; + + args = t_strsplit(line, " "); + len = str_array_length(args); + if ((len % 4) != 0) + return FALSE; + + last_voffset = 0; + for (i = 0; args[i] != NULL; i += 4) { + const char *start_offset_str = args[i+0]; + const char *size_str = args[i+1]; + const char *decode_options = args[i+2]; + const char *path = args[i+3]; + + memset(&extref, 0, sizeof(extref)); + if (str_to_uoff(start_offset_str, &extref.start_offset) < 0 || + str_to_uoff(size_str, &extref.size) < 0 || + extref.start_offset < last_voffset || + !parse_extref_decode_options(decode_options, &extref)) + return FALSE; + + last_voffset += extref.size + + (extref.start_offset - last_voffset); + + extref.path = p_strdup(pool, path); + array_append(extrefs, &extref, 1); + } + return TRUE; +} + +bool dbox_attachment_parse_extref(const char *line, pool_t pool, + ARRAY_TYPE(mail_attachment_extref) *extrefs) +{ + bool ret; + + T_BEGIN { + ret = dbox_attachment_parse_extref_real(line, pool, extrefs); + } T_END; + return ret; +} + +static int +dbox_attachment_file_get_stream_from(struct dbox_file *file, + const char *ext_refs, + struct istream **stream_r) +{ + ARRAY_TYPE(mail_attachment_extref) extrefs_arr; + ARRAY_DEFINE(streams, struct istream *); + const struct mail_attachment_extref *extref; + struct istream **inputs, *input, *input2; + const char *path, *path_suffix; + uoff_t root_offset, last_voffset = 0; + unsigned int i; + + t_array_init(&extrefs_arr, 16); + if (!dbox_attachment_parse_extref_real(ext_refs, pool_datastack_create(), + &extrefs_arr)) + return 0; + + root_offset = file->input->v_offset; + t_array_init(&streams, 8); + array_foreach(&extrefs_arr, extref) { + path_suffix = file->storage->v.get_attachment_path_suffix(file); + path = t_strdup_printf("%s/%s%s", file->storage->attachment_dir, + extref->path, path_suffix); + + if (extref->start_offset != last_voffset) { + uoff_t part_size = extref->start_offset - last_voffset; + + input = i_stream_create_limit(file->input, part_size); + array_append(&streams, &input, 1); + i_stream_seek(file->input, + file->input->v_offset + part_size); + last_voffset += part_size; + } + + last_voffset += extref->size; + input2 = i_stream_create_file(path, IO_BLOCK_SIZE); + + if (extref->base64_blocks_per_line > 0) { + input = i_stream_create_base64_encoder(input2, + extref->base64_blocks_per_line*4, + extref->base64_have_crlf); + i_stream_unref(&input2); + input2 = input; + } + + input = i_stream_create_attachment(input2, extref->size); + i_stream_unref(&input2); + array_append(&streams, &input, 1); + } + + if (file->cur_physical_size != file->input->v_offset-root_offset) { + uoff_t trailer_size = file->cur_physical_size - + (file->input->v_offset - root_offset); + + input = i_stream_create_limit(file->input, trailer_size); + array_append(&streams, &input, 1); + (void)array_append_space(&streams); + } + + inputs = array_idx_modifiable(&streams, 0); + *stream_r = i_stream_create_concat(inputs); + for (i = 0; inputs[i] != NULL; i++) + i_stream_unref(&inputs[i]); + return 1; +} + +int dbox_attachment_file_get_stream(struct dbox_file *file, + struct istream **stream_r) +{ + const char *ext_refs; + int ret; + + /* need to read metadata in case there are external references */ + if ((ret = dbox_file_metadata_read(file)) <= 0) + return ret; + + i_stream_seek(file->input, file->cur_offset + file->msg_header_size); + + ext_refs = dbox_file_metadata_get(file, DBOX_METADATA_EXT_REF); + if (ext_refs == NULL) { + *stream_r = i_stream_create_limit(file->input, + file->cur_physical_size); + return 1; + } + + /* we have external references. */ + T_BEGIN { + ret = dbox_attachment_file_get_stream_from(file, ext_refs, + stream_r); + } T_END; + if (ret == 0) { + dbox_file_set_corrupted(file, "Ext refs metadata corrupted: %s", + ext_refs); + } + return ret; +}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-storage/index/dbox-common/dbox-attachment.h Tue Oct 19 18:47:17 2010 +0100 @@ -0,0 +1,20 @@ +#ifndef DBOX_ATTACHMENT_H +#define DBOX_ATTACHMENT_H + +#include "index-attachment.h" + +struct dbox_file; + +void dbox_attachment_save_write_metadata(struct mail_save_context *ctx, + string_t *str); + +/* Parse DBOX_METADATA_EXT_REF line to given array. Names are allocated + from the given pool. */ +bool dbox_attachment_parse_extref(const char *line, pool_t pool, + ARRAY_TYPE(mail_attachment_extref) *extrefs); +/* Build a single message body stream out of the current message and all of its + attachments. */ +int dbox_attachment_file_get_stream(struct dbox_file *file, + struct istream **stream_r); + +#endif
--- a/src/lib-storage/index/dbox-common/dbox-file.c Tue Oct 19 18:30:51 2010 +0100 +++ b/src/lib-storage/index/dbox-common/dbox-file.c Tue Oct 19 18:47:17 2010 +0100 @@ -14,6 +14,7 @@ #include "eacces-error.h" #include "str.h" #include "dbox-storage.h" +#include "dbox-attachment.h" #include "dbox-file.h" #include <stdio.h> @@ -397,6 +398,9 @@ if ((ret = dbox_file_seek(file, offset)) <= 0) return ret; + if (file->storage->attachment_dir != NULL) + return dbox_attachment_file_get_stream(file, stream_r); + *stream_r = i_stream_create_limit(file->input, file->cur_physical_size); return 1; }
--- a/src/lib-storage/index/dbox-common/dbox-save.c Tue Oct 19 18:30:51 2010 +0100 +++ b/src/lib-storage/index/dbox-common/dbox-save.c Tue Oct 19 18:47:17 2010 +0100 @@ -7,6 +7,7 @@ #include "str.h" #include "hex-binary.h" #include "index-mail.h" +#include "dbox-attachment.h" #include "dbox-file.h" #include "dbox-save.h" @@ -31,6 +32,8 @@ void dbox_save_begin(struct dbox_save_context *ctx, struct istream *input) { struct mail_save_context *_ctx = &ctx->ctx; + struct mail_storage *_storage = _ctx->transaction->box->storage; + struct dbox_storage *storage = (struct dbox_storage *)_storage; struct dbox_message_header dbox_msg_hdr; struct istream *crlf_input; @@ -53,7 +56,7 @@ o_stream_cork(ctx->dbox_output); if (o_stream_send(ctx->dbox_output, &dbox_msg_hdr, sizeof(dbox_msg_hdr)) < 0) { - mail_storage_set_critical(_ctx->transaction->box->storage, + mail_storage_set_critical(_storage, "o_stream_send(%s) failed: %m", ctx->cur_file->cur_path); ctx->failed = TRUE; @@ -62,6 +65,7 @@ if (_ctx->received_date == (time_t)-1) _ctx->received_date = ioloop_time; + index_attachment_save_begin(_ctx, storage->attachment_fs, ctx->input); } int dbox_save_continue(struct mail_save_context *_ctx) @@ -72,6 +76,9 @@ if (ctx->failed) return -1; + if (_ctx->attach != NULL) + return index_attachment_save_continue(_ctx); + do { if (o_stream_send_istream(_ctx->output, ctx->input) < 0) { if (!mail_storage_set_error_from_errno(storage)) { @@ -95,6 +102,10 @@ { struct ostream *dbox_output = ctx->dbox_output; + if (ctx->ctx.attach != NULL) { + if (index_attachment_save_finish(&ctx->ctx) < 0) + ctx->failed = TRUE; + } if (ctx->ctx.output == dbox_output) return; @@ -159,6 +170,8 @@ orig_mailbox_name); } + dbox_attachment_save_write_metadata(_ctx, str); + str_append_c(str, '\n'); o_stream_send(output, str_data(str), str_len(str)); }
--- a/src/lib-storage/index/dbox-common/dbox-storage.c Tue Oct 19 18:30:51 2010 +0100 +++ b/src/lib-storage/index/dbox-common/dbox-storage.c Tue Oct 19 18:47:17 2010 +0100 @@ -2,6 +2,7 @@ #include "lib.h" #include "ioloop.h" +#include "fs-api.h" #include "mkdir-parents.h" #include "unlink-old-files.h" #include "mailbox-uidvalidity.h" @@ -26,6 +27,43 @@ set->mailbox_dir_name = DBOX_MAILBOX_DIR_NAME; } +int dbox_storage_create(struct mail_storage *_storage, + struct mail_namespace *ns, + const char **error_r ATTR_UNUSED) +{ + struct dbox_storage *storage = (struct dbox_storage *)_storage; + const struct mail_storage_settings *set = _storage->set; + struct fs_settings fs_set; + + memset(&fs_set, 0, sizeof(fs_set)); + fs_set.temp_file_prefix = mailbox_list_get_global_temp_prefix(ns->list); + + if (*set->mail_attachment_fs != '\0') T_BEGIN { + const char *name, *args, *dir; + + args = strchr(set->mail_attachment_fs, ' '); + if (args == NULL) { + name = set->mail_attachment_fs; + args = ""; + } else { + name = t_strdup_until(set->mail_attachment_fs, args++); + } + dir = mail_user_home_expand(_storage->user, + set->mail_attachment_dir); + storage->attachment_dir = p_strdup(_storage->pool, dir); + storage->attachment_fs = fs_init(name, args, &fs_set); + } T_END; + return 0; +} + +void dbox_storage_destroy(struct mail_storage *_storage) +{ + struct dbox_storage *storage = (struct dbox_storage *)_storage; + + if (storage->attachment_fs != NULL) + fs_deinit(&storage->attachment_fs); +} + uint32_t dbox_get_uidvalidity_next(struct mailbox_list *list) { const char *path;
--- a/src/lib-storage/index/dbox-common/dbox-storage.h Tue Oct 19 18:30:51 2010 +0100 +++ b/src/lib-storage/index/dbox-common/dbox-storage.h Tue Oct 19 18:47:17 2010 +0100 @@ -6,6 +6,7 @@ struct dbox_file; struct dbox_mail; struct dbox_storage; +struct dbox_save_context; #define DBOX_SUBSCRIPTION_FILE_NAME "subscriptions" #define DBOX_UIDVALIDITY_FILE_NAME "dovecot-uidvalidity" @@ -39,6 +40,9 @@ int (*mailbox_create_indexes)(struct mailbox *box, const struct mailbox_update *update, struct mail_index_transaction *trans); + /* returns attachment path suffix. mdbox returns "", sdbox returns + "-<mailbox_guid>-<uid>" */ + const char *(*get_attachment_path_suffix)(struct dbox_file *file); /* mark the mailbox corrupted */ void (*set_mailbox_corrupted)(struct mailbox *box); /* mark the file corrupted */ @@ -48,10 +52,17 @@ struct dbox_storage { struct mail_storage storage; struct dbox_storage_vfuncs v; + + struct fs *attachment_fs; + const char *attachment_dir; }; void dbox_storage_get_list_settings(const struct mail_namespace *ns, struct mailbox_list_settings *set); +int dbox_storage_create(struct mail_storage *storage, + struct mail_namespace *ns, + const char **error_r); +void dbox_storage_destroy(struct mail_storage *storage); uint32_t dbox_get_uidvalidity_next(struct mailbox_list *list); void dbox_notify_changes(struct mailbox *box); int dbox_mailbox_open(struct mailbox *box);
--- a/src/lib-storage/index/dbox-multi/mdbox-purge.c Tue Oct 19 18:30:51 2010 +0100 +++ b/src/lib-storage/index/dbox-multi/mdbox-purge.c Tue Oct 19 18:47:17 2010 +0100 @@ -6,6 +6,7 @@ #include "ostream.h" #include "str.h" #include "hash.h" +#include "dbox-attachment.h" #include "mdbox-storage.h" #include "mdbox-storage-rebuild.h" #include "mdbox-file.h" @@ -63,16 +64,15 @@ } static int -mdbox_file_copy_metadata(struct dbox_file *file, struct ostream *output) +mdbox_file_read_metadata_hdr(struct dbox_file *file, + struct dbox_metadata_header *meta_hdr_r) { - struct dbox_metadata_header meta_hdr; - const char *line; const unsigned char *data; - size_t size, buf_size; + size_t size; int ret; ret = i_stream_read_data(file->input, &data, &size, - sizeof(meta_hdr)); + sizeof(*meta_hdr_r)); if (ret <= 0) { i_assert(ret == -1); if (file->input->stream_errno == 0) { @@ -84,26 +84,74 @@ return -1; } - memcpy(&meta_hdr, data, sizeof(meta_hdr)); - if (memcmp(meta_hdr.magic_post, DBOX_MAGIC_POST, - sizeof(meta_hdr.magic_post)) != 0) { + memcpy(meta_hdr_r, data, sizeof(*meta_hdr_r)); + if (memcmp(meta_hdr_r->magic_post, DBOX_MAGIC_POST, + sizeof(meta_hdr_r->magic_post)) != 0) { dbox_file_set_corrupted(file, "invalid metadata magic"); return 0; } - i_stream_skip(file->input, sizeof(meta_hdr)); - if (output != NULL) - o_stream_send(output, &meta_hdr, sizeof(meta_hdr)); + i_stream_skip(file->input, sizeof(*meta_hdr_r)); + return 1; +} + +static int +mdbox_file_metadata_copy(struct dbox_file *file, struct ostream *output) +{ + struct dbox_metadata_header meta_hdr; + const char *line; + size_t buf_size; + int ret; + + if ((ret = mdbox_file_read_metadata_hdr(file, &meta_hdr)) <= 0) + return ret; + + o_stream_send(output, &meta_hdr, sizeof(meta_hdr)); + buf_size = i_stream_get_max_buffer_size(file->input); + i_stream_set_max_buffer_size(file->input, 0); + while ((line = i_stream_read_next_line(file->input)) != NULL) { + if (*line == '\0') { + /* end of metadata */ + break; + } + o_stream_send_str(output, line); + o_stream_send(output, "\n", 1); + } + i_stream_set_max_buffer_size(file->input, buf_size); + + if (line == NULL) { + dbox_file_set_corrupted(file, "missing end-of-metadata line"); + return 0; + } + o_stream_send(output, "\n", 1); + return 1; +} + +static int +mdbox_metadata_get_extrefs(struct dbox_file *file, pool_t ext_refs_pool, + ARRAY_TYPE(mail_attachment_extref) *extrefs) +{ + struct dbox_metadata_header meta_hdr; + const char *line; + size_t buf_size; + int ret; + + /* skip and ignore the header */ + if ((ret = mdbox_file_read_metadata_hdr(file, &meta_hdr)) <= 0) + return ret; buf_size = i_stream_get_max_buffer_size(file->input); i_stream_set_max_buffer_size(file->input, 0); while ((line = i_stream_read_next_line(file->input)) != NULL) { - if (*line == DBOX_METADATA_OLDV1_SPACE || *line == '\0') { + if (*line == '\0') { /* end of metadata */ break; } - if (output != NULL) { - o_stream_send_str(output, line); - o_stream_send(output, "\n", 1); + if (*line == DBOX_METADATA_EXT_REF) { + if (dbox_attachment_parse_extref(line+1, ext_refs_pool, + extrefs) < 0) { + i_warning("%s: Ignoring corrupted extref: %s", + file->cur_path, line); + } } } i_stream_set_max_buffer_size(file->input, buf_size); @@ -112,8 +160,6 @@ dbox_file_set_corrupted(file, "missing end-of-metadata line"); return 0; } - if (output != NULL) - o_stream_send(output, "\n", 1); return 1; } @@ -179,7 +225,7 @@ i_assert(ret == (off_t)msg_size); /* copy metadata */ - if ((ret = mdbox_file_copy_metadata(file, output)) <= 0) + if ((ret = mdbox_file_metadata_copy(file, output)) <= 0) return ret; mdbox_map_append_finish(ctx->append_ctx); @@ -222,6 +268,23 @@ } static int +mdbox_purge_attachments(struct mdbox_purge_context *ctx, + const ARRAY_TYPE(mail_attachment_extref) *extrefs_arr) +{ + struct dbox_storage *storage = &ctx->storage->storage; + const struct mail_attachment_extref *extref; + int ret = 0; + + array_foreach(extrefs_arr, extref) { + if (index_attachment_delete(&storage->storage, + storage->attachment_fs, + extref->path) < 0) + ret = -1; + } + return ret; +} + +static int mdbox_file_purge(struct mdbox_purge_context *ctx, struct dbox_file *file) { struct mdbox_storage *dstorage = (struct mdbox_storage *)file->storage; @@ -230,6 +293,8 @@ const struct mdbox_map_file_msg *msgs; ARRAY_TYPE(seq_range) expunged_map_uids; ARRAY_TYPE(uint32_t) copied_map_uids; + ARRAY_TYPE(mail_attachment_extref) ext_refs; + pool_t ext_refs_pool; unsigned int i, count; uoff_t offset; int ret; @@ -265,8 +330,10 @@ /* sort messages by their offset */ array_sort(&msgs_arr, mdbox_map_file_msg_offset_cmp); + ext_refs_pool = pool_alloconly_create("mdbox purge ext refs", 1024); ctx->atomic = mdbox_map_atomic_begin(ctx->storage->map); msgs = array_get(&msgs_arr, &count); + i_array_init(&ext_refs, 32); i_array_init(&copied_map_uids, I_MIN(count, 1)); i_array_init(&expunged_map_uids, I_MIN(count, 1)); offset = file->file_header_size; @@ -290,7 +357,9 @@ file->msg_header_size + file->cur_physical_size); /* skip metadata */ - if ((ret = mdbox_file_copy_metadata(file, NULL)) <= 0) + ret = mdbox_metadata_get_extrefs(file, ext_refs_pool, + &ext_refs); + if (ret <= 0) break; seq_range_array_add(&expunged_map_uids, 0, msgs[i].map_uid); @@ -352,6 +421,10 @@ dbox_file_unlock(file); array_free(&copied_map_uids); array_free(&expunged_map_uids); + + (void)mdbox_purge_attachments(ctx, &ext_refs); + array_free(&ext_refs); + pool_unref(&ext_refs_pool); return ret; }
--- a/src/lib-storage/index/dbox-multi/mdbox-storage.c Tue Oct 19 18:30:51 2010 +0100 +++ b/src/lib-storage/index/dbox-multi/mdbox-storage.c Tue Oct 19 18:47:17 2010 +0100 @@ -60,7 +60,7 @@ i_array_init(&storage->open_files, 64); storage->map = mdbox_map_init(storage, ns->list); - return 0; + return dbox_storage_create(_storage, ns, error_r); } static void mdbox_storage_destroy(struct mail_storage *_storage) @@ -77,6 +77,7 @@ if (array_is_created(&storage->move_to_alt_map_uids)) array_free(&storage->move_to_alt_map_uids); array_free(&storage->open_files); + dbox_storage_destroy(_storage); } struct mailbox * @@ -271,6 +272,12 @@ } } +static const char * +mdbox_get_attachment_path_suffix(struct dbox_file *file ATTR_UNUSED) +{ + return ""; +} + static void mdbox_set_mailbox_corrupted(struct mailbox *box) { struct mdbox_storage *mstorage = (struct mdbox_storage *)box->storage; @@ -425,6 +432,7 @@ mdbox_save_finish, mdbox_save_cancel, mdbox_copy, + NULL, index_storage_is_inconsistent } }; @@ -434,6 +442,7 @@ mdbox_file_create_fd, mdbox_mail_open, mdbox_mailbox_create_indexes, + mdbox_get_attachment_path_suffix, mdbox_set_mailbox_corrupted, mdbox_set_file_corrupted };
--- a/src/lib-storage/index/dbox-single/Makefile.am Tue Oct 19 18:30:51 2010 +0100 +++ b/src/lib-storage/index/dbox-single/Makefile.am Tue Oct 19 18:47:17 2010 +0100 @@ -4,6 +4,7 @@ -I$(top_srcdir)/src/lib \ -I$(top_srcdir)/src/lib-settings \ -I$(top_srcdir)/src/lib-master \ + -I$(top_srcdir)/src/lib-fs \ -I$(top_srcdir)/src/lib-mail \ -I$(top_srcdir)/src/lib-imap \ -I$(top_srcdir)/src/lib-index \
--- a/src/lib-storage/index/dbox-single/sdbox-copy.c Tue Oct 19 18:30:51 2010 +0100 +++ b/src/lib-storage/index/dbox-single/sdbox-copy.c Tue Oct 19 18:47:17 2010 +0100 @@ -2,12 +2,79 @@ #include "lib.h" #include "nfs-workarounds.h" +#include "fs-api.h" #include "dbox-save.h" +#include "dbox-attachment.h" #include "sdbox-storage.h" #include "sdbox-file.h" #include "mail-copy.h" static int +sdbox_file_copy_attachments(struct sdbox_file *src_file, + struct sdbox_file *dest_file) +{ + struct dbox_storage *src_storage = src_file->file.storage; + struct dbox_storage *dest_storage = dest_file->file.storage; + ARRAY_TYPE(mail_attachment_extref) extrefs; + const struct mail_attachment_extref *extref; + const char *extrefs_line, *src, *dest, *dest_relpath; + pool_t pool; + int ret; + + if (src_storage->attachment_dir == NULL) { + /* no attachments in source storage */ + return 1; + } + if (dest_storage->attachment_dir == NULL || + strcmp(src_storage->attachment_dir, + dest_storage->attachment_dir) != 0) { + /* different attachment dirs between storages. + have to copy the slow way. */ + return 0; + } + + if ((ret = sdbox_file_get_attachments(&src_file->file, + &extrefs_line)) <= 0) + return ret < 0 ? -1 : 1; + + pool = pool_alloconly_create("sdbox attachments copy", 1024); + p_array_init(&extrefs, pool, 16); + if (!dbox_attachment_parse_extref(extrefs_line, pool, &extrefs)) { + mail_storage_set_critical(&dest_storage->storage, + "Can't copy %s with corrupted extref metadata: %s", + src_file->file.cur_path, extrefs_line); + pool_unref(&pool); + return -1; + } + + dest_file->attachment_pool = + pool_alloconly_create("sdbox attachment copy paths", 512); + p_array_init(&dest_file->attachment_paths, dest_file->attachment_pool, + array_count(&extrefs)); + + ret = 1; + array_foreach(&extrefs, extref) T_BEGIN { + src = t_strdup_printf("%s/%s", dest_storage->attachment_dir, + sdbox_file_attachment_relpath(src_file, extref->path)); + dest_relpath = p_strconcat(dest_file->attachment_pool, + extref->path, "-", + mail_generate_guid_string(), NULL); + dest = t_strdup_printf("%s/%s", dest_storage->attachment_dir, + dest_relpath); + if (fs_link(dest_storage->attachment_fs, src, dest) < 0) { + mail_storage_set_critical(&dest_storage->storage, "%s", + fs_last_error(dest_storage->attachment_fs)); + ret = -1; + } else { + array_append(&dest_file->attachment_paths, + &dest_relpath, 1); + } + } T_END; + pool_unref(&pool); + return ret; +} + +static int sdbox_copy_hardlink(struct mail_save_context *_ctx, struct mail *mail) { struct dbox_save_context *ctx = (struct dbox_save_context *)_ctx; @@ -50,6 +117,15 @@ return ret; } + ret = sdbox_file_copy_attachments((struct sdbox_file *)src_file, + (struct sdbox_file *)dest_file); + if (ret <= 0) { + sdbox_file_unlink_aborted_save((struct sdbox_file *)dest_file); + dbox_file_unref(&src_file); + dbox_file_unref(&dest_file); + return ret; + } + dbox_save_add_to_index(ctx); sdbox_save_add_file(_ctx, dest_file); if (_ctx->dest_mail != NULL) {
--- a/src/lib-storage/index/dbox-single/sdbox-file.c Tue Oct 19 18:30:51 2010 +0100 +++ b/src/lib-storage/index/dbox-single/sdbox-file.c Tue Oct 19 18:47:17 2010 +0100 @@ -6,6 +6,9 @@ #include "mkdir-parents.h" #include "istream.h" #include "ostream.h" +#include "str.h" +#include "fs-api.h" +#include "dbox-attachment.h" #include "sdbox-storage.h" #include "sdbox-file.h" @@ -61,6 +64,84 @@ return file; } +void sdbox_file_free(struct dbox_file *file) +{ + struct sdbox_file *sfile = (struct sdbox_file *)file; + + if (sfile->attachment_pool != NULL) + pool_unref(&sfile->attachment_pool); + dbox_file_free(file); +} + +int sdbox_file_get_attachments(struct dbox_file *file, const char **extrefs_r) +{ + const char *line; + bool deleted; + int ret; + + *extrefs_r = NULL; + + /* read the metadata */ + ret = dbox_file_open(file, &deleted); + if (ret > 0) { + if (deleted) + return 0; + if ((ret = dbox_file_seek(file, 0)) > 0) + ret = dbox_file_metadata_read(file); + } + if (ret <= 0) { + if (ret < 0) + return -1; + /* corrupted file. we're deleting it anyway. */ + line = NULL; + } else { + line = dbox_file_metadata_get(file, DBOX_METADATA_EXT_REF); + } + if (line == NULL) { + /* no attachments */ + return 0; + } + *extrefs_r = line; + return 1; +} + +const char * +sdbox_file_attachment_relpath(struct sdbox_file *file, const char *srcpath) +{ + const char *p; + + p = strchr(srcpath, '-'); + if (p == NULL) { + mail_storage_set_critical(file->mbox->box.storage, + "sdbox attachment path in invalid format: %s", srcpath); + } else { + p = strchr(p+1, '-'); + } + return t_strdup_printf("%s-%s-%u", + p == NULL ? srcpath : t_strdup_until(srcpath, p), + mail_guid_128_to_string(file->mbox->mailbox_guid), + file->uid); +} + +static int sdbox_file_rename_attachments(struct sdbox_file *file) +{ + struct dbox_storage *storage = file->file.storage; + const char *const *pathp, *src, *dest; + int ret = 0; + + array_foreach(&file->attachment_paths, pathp) T_BEGIN { + src = t_strdup_printf("%s/%s", storage->attachment_dir, *pathp); + dest = t_strdup_printf("%s/%s", storage->attachment_dir, + sdbox_file_attachment_relpath(file, *pathp)); + if (fs_rename(storage->attachment_fs, src, dest) < 0) { + mail_storage_set_critical(&storage->storage, "%s", + fs_last_error(storage->attachment_fs)); + ret = -1; + } + } T_END; + return ret; +} + int sdbox_file_assign_uid(struct sdbox_file *file, uint32_t uid) { const char *old_path, *new_fname, *new_path; @@ -80,9 +161,62 @@ } sdbox_file_init_paths(file, new_fname); file->uid = uid; + + if (array_is_created(&file->attachment_paths)) { + if (sdbox_file_rename_attachments(file) < 0) + return -1; + } return 0; } +static int sdbox_file_unlink_aborted_save_attachments(struct sdbox_file *file) +{ + struct dbox_storage *storage = file->file.storage; + struct fs *fs = storage->attachment_fs; + const char *const *pathp, *path; + int ret = 0; + + array_foreach(&file->attachment_paths, pathp) T_BEGIN { + /* we don't know if we aborted before renaming this attachment, + so try deleting both source and dest path. the source paths + point to temporary files (not to source messages' + attachment paths), so it's safe to delete them. */ + path = t_strdup_printf("%s/%s", storage->attachment_dir, + *pathp); + if (fs_unlink(fs, path) < 0 && + errno != ENOENT) { + mail_storage_set_critical(&storage->storage, "%s", + fs_last_error(fs)); + ret = -1; + } + path = t_strdup_printf("%s/%s", storage->attachment_dir, + sdbox_file_attachment_relpath(file, *pathp)); + if (fs_unlink(fs, path) < 0 && + errno != ENOENT) { + mail_storage_set_critical(&storage->storage, "%s", + fs_last_error(fs)); + ret = -1; + } + } T_END; + return ret; +} + +int sdbox_file_unlink_aborted_save(struct sdbox_file *file) +{ + int ret = 0; + + if (unlink(file->file.cur_path) < 0) { + mail_storage_set_critical(file->mbox->box.storage, + "unlink(%s) failed: %m", file->file.cur_path); + ret = -1; + } + if (array_is_created(&file->attachment_paths)) { + if (sdbox_file_unlink_aborted_save_attachments(file) < 0) + ret = -1; + } + return ret; +} + int sdbox_file_create_fd(struct dbox_file *file, const char *path, bool parents) { struct sdbox_file *sfile = (struct sdbox_file *)file; @@ -238,3 +372,52 @@ } return 0; } + +static int +sdbox_unlink_attachments(struct sdbox_file *sfile, + const ARRAY_TYPE(mail_attachment_extref) *extrefs) +{ + struct dbox_storage *storage = sfile->file.storage; + const struct mail_attachment_extref *extref; + const char *path; + int ret = 0; + + array_foreach(extrefs, extref) T_BEGIN { + path = sdbox_file_attachment_relpath(sfile, extref->path); + if (index_attachment_delete(&storage->storage, + storage->attachment_fs, path) < 0) + ret = -1; + } T_END; + return ret; +} + +int sdbox_file_unlink_with_attachments(struct sdbox_file *sfile) +{ + ARRAY_TYPE(mail_attachment_extref) extrefs; + const char *extrefs_line; + pool_t pool; + int ret; + + ret = sdbox_file_get_attachments(&sfile->file, &extrefs_line); + if (ret < 0) + return -1; + if (ret == 0) { + /* no attachments */ + return dbox_file_unlink(&sfile->file); + } + + pool = pool_alloconly_create("sdbox attachments unlink", 1024); + p_array_init(&extrefs, pool, 16); + if (!dbox_attachment_parse_extref(extrefs_line, pool, &extrefs)) { + i_warning("%s: Ignoring corrupted extref: %s", + sfile->file.cur_path, extrefs_line); + array_clear(&extrefs); + } + + /* try to delete the file first, so if it fails we don't have + missing attachments */ + if ((ret = dbox_file_unlink(&sfile->file)) >= 0) + (void)sdbox_unlink_attachments(sfile, &extrefs); + pool_unref(&pool); + return ret; +}
--- a/src/lib-storage/index/dbox-single/sdbox-file.h Tue Oct 19 18:30:51 2010 +0100 +++ b/src/lib-storage/index/dbox-single/sdbox-file.h Tue Oct 19 18:47:17 2010 +0100 @@ -9,10 +9,23 @@ /* 0 while file is being created */ uint32_t uid; + + /* list of attachment paths while saving/copying message */ + pool_t attachment_pool; + ARRAY_TYPE(const_string) attachment_paths; }; struct dbox_file *sdbox_file_init(struct sdbox_mailbox *mbox, uint32_t uid); struct dbox_file *sdbox_file_create(struct sdbox_mailbox *mbox); +void sdbox_file_free(struct dbox_file *file); + +/* Get file's extrefs metadata. */ +int sdbox_file_get_attachments(struct dbox_file *file, const char **extrefs_r); +/* Returns attachment path for this file, given the source path. The result is + always <hash>-<guid>-<mailbox_guid>-<uid>. The source path is expected to + contain <hash>-<guid>[-*]. */ +const char * +sdbox_file_attachment_relpath(struct sdbox_file *file, const char *srcpath); /* Assign UID for a newly created file (by renaming it) */ int sdbox_file_assign_uid(struct sdbox_file *file, uint32_t uid); @@ -21,5 +34,9 @@ bool parents); /* Move the file to alt path or back. */ int sdbox_file_move(struct dbox_file *file, bool alt_path); +/* Unlink file and all of its referenced attachments. */ +int sdbox_file_unlink_with_attachments(struct sdbox_file *sfile); +/* Unlink file and its attachments when rollbacking a saved message. */ +int sdbox_file_unlink_aborted_save(struct sdbox_file *file); #endif
--- a/src/lib-storage/index/dbox-single/sdbox-save.c Tue Oct 19 18:30:51 2010 +0100 +++ b/src/lib-storage/index/dbox-single/sdbox-save.c Tue Oct 19 18:47:17 2010 +0100 @@ -12,6 +12,7 @@ #include "write-full.h" #include "index-mail.h" #include "mail-copy.h" +#include "dbox-attachment.h" #include "dbox-save.h" #include "sdbox-storage.h" #include "sdbox-file.h" @@ -110,9 +111,13 @@ static int dbox_save_mail_write_metadata(struct dbox_save_context *ctx, struct dbox_file *file) { + struct sdbox_file *sfile = (struct sdbox_file *)file; + const ARRAY_TYPE(mail_attachment_extref) *extrefs_arr; + const struct mail_attachment_extref *extrefs; struct dbox_message_header dbox_msg_hdr; uoff_t message_size; uint8_t guid_128[MAIL_GUID_128_SIZE]; + unsigned int i, count; i_assert(file->msg_header_size == sizeof(dbox_msg_hdr)); @@ -128,6 +133,26 @@ dbox_file_set_syscall_error(file, "pwrite()"); return -1; } + + /* remember the attachment paths until commit time */ + extrefs_arr = index_attachment_save_get_extrefs(&ctx->ctx); + if (extrefs_arr != NULL) + extrefs = array_get(extrefs_arr, &count); + else { + extrefs = NULL; + count = 0; + } + if (count > 0) { + sfile->attachment_pool = + pool_alloconly_create("sdbox attachment paths", 512); + p_array_init(&sfile->attachment_paths, + sfile->attachment_pool, count); + for (i = 0; i < count; i++) { + const char *path = p_strdup(sfile->attachment_pool, + extrefs[i].path); + array_append(&sfile->attachment_paths, &path, 1); + } + } return 0; } @@ -213,18 +238,16 @@ static void dbox_save_unref_files(struct sdbox_save_context *ctx) { - struct mail_storage *storage = &ctx->mbox->storage->storage.storage; struct dbox_file **files; unsigned int i, count; files = array_get_modifiable(&ctx->files, &count); for (i = 0; i < count; i++) { if (ctx->ctx.failed) { - if (unlink(files[i]->cur_path) < 0) { - mail_storage_set_critical(storage, - "unlink(%s) failed: %m", - files[i]->cur_path); - } + struct sdbox_file *sfile = + (struct sdbox_file *)files[i]; + + (void)sdbox_file_unlink_aborted_save(sfile); } dbox_file_unref(&files[i]); }
--- a/src/lib-storage/index/dbox-single/sdbox-storage.c Tue Oct 19 18:30:51 2010 +0100 +++ b/src/lib-storage/index/dbox-single/sdbox-storage.c Tue Oct 19 18:47:17 2010 +0100 @@ -3,6 +3,7 @@ #include "lib.h" #include "master-service.h" #include "mail-index-modseq.h" +#include "mail-search-build.h" #include "mailbox-list-private.h" #include "dbox-mail.h" #include "dbox-save.h" @@ -27,14 +28,6 @@ return &storage->storage.storage; } -static int -sdbox_storage_create(struct mail_storage *storage ATTR_UNUSED, - struct mail_namespace *ns ATTR_UNUSED, - const char **error_r ATTR_UNUSED) -{ - return 0; -} - static struct mailbox * sdbox_mailbox_alloc(struct mail_storage *storage, struct mailbox_list *list, const char *name, enum mailbox_flags flags) @@ -198,6 +191,16 @@ return ret; } +static const char * +sdbox_get_attachment_path_suffix(struct dbox_file *_file) +{ + struct sdbox_file *file = (struct sdbox_file *)_file; + + return t_strdup_printf("-%s-%u", + mail_guid_128_to_string(file->mbox->mailbox_guid), + file->uid); +} + static void sdbox_set_mailbox_corrupted(struct mailbox *box) { struct sdbox_mailbox *mbox = (struct sdbox_mailbox *)box; @@ -216,6 +219,29 @@ sdbox_set_mailbox_corrupted(&file->mbox->box); } +static int sdbox_mailbox_open(struct mailbox *box) +{ + struct sdbox_mailbox *mbox = (struct sdbox_mailbox *)box; + struct sdbox_index_header hdr; + + if (dbox_mailbox_open(box) < 0) + return -1; + + /* get/generate mailbox guid */ + if (sdbox_read_header(mbox, &hdr, TRUE) < 0) + memset(&hdr, 0, sizeof(hdr)); + + if (mail_guid_128_is_empty(hdr.mailbox_guid)) { + /* regenerate it */ + if (sdbox_write_index_header(box, NULL, NULL) < 0 || + sdbox_read_header(mbox, &hdr, TRUE) < 0) + return -1; + } + memcpy(mbox->mailbox_guid, hdr.mailbox_guid, + sizeof(mbox->mailbox_guid)); + return 0; +} + static void sdbox_mailbox_close(struct mailbox *box) { struct sdbox_mailbox *mbox = (struct sdbox_mailbox *)box; @@ -225,22 +251,54 @@ index_storage_mailbox_close(box); } +static int sdbox_mailbox_delete(struct mailbox *box) +{ + struct sdbox_mailbox *mbox = (struct sdbox_mailbox *)box; + struct mail_search_context *ctx; + struct mailbox_transaction_context *t; + struct mail *mail; + struct mail_search_args *search_args; + struct dbox_file *file; + struct sdbox_file *sfile; + + if (!box->opened || mbox->storage->storage.attachment_dir == NULL) + return index_storage_mailbox_delete(box); + + /* mark the mailbox deleted to avoid race conditions */ + if (mailbox_mark_index_deleted(box, TRUE) < 0) + return -1; + + /* ulink all dbox mails and their attachements in the mailbox. */ + t = mailbox_transaction_begin(box, 0); + + search_args = mail_search_build_init(); + mail_search_build_add_all(search_args); + ctx = mailbox_search_init(t, search_args, NULL); + mail_search_args_unref(&search_args); + + mail = mail_alloc(t, 0, NULL); + while (mailbox_search_next(ctx, mail)) { + file = sdbox_file_init(mbox, mail->uid); + sfile = (struct sdbox_file *)file; + (void)sdbox_file_unlink_with_attachments(sfile); + dbox_file_unref(&file); + } + mail_free(&mail); + + if (mailbox_search_deinit(&ctx) < 0) { + /* maybe we missed some mails. oh well, can't help it. */ + } + mailbox_transaction_rollback(&t); + + return index_storage_mailbox_delete(box); +} + static int sdbox_mailbox_get_guid(struct mailbox *box, uint8_t guid[MAIL_GUID_128_SIZE]) { struct sdbox_mailbox *mbox = (struct sdbox_mailbox *)box; - struct sdbox_index_header hdr; - if (sdbox_read_header(mbox, &hdr, TRUE) < 0) - memset(&hdr, 0, sizeof(hdr)); - - if (mail_guid_128_is_empty(hdr.mailbox_guid)) { - /* regenerate it */ - if (sdbox_write_index_header(box, NULL, NULL) < 0 || - sdbox_read_header(mbox, &hdr, TRUE) < 0) - return -1; - } - memcpy(guid, hdr.mailbox_guid, MAIL_GUID_128_SIZE); + memcpy(guid, mbox->mailbox_guid, MAIL_GUID_128_SIZE); return 0; } @@ -263,8 +321,8 @@ .v = { NULL, sdbox_storage_alloc, - sdbox_storage_create, - NULL, + dbox_storage_create, + dbox_storage_destroy, NULL, dbox_storage_get_list_settings, NULL, @@ -280,8 +338,8 @@ .v = { NULL, sdbox_storage_alloc, - sdbox_storage_create, - NULL, + dbox_storage_create, + dbox_storage_destroy, NULL, dbox_storage_get_list_settings, NULL, @@ -295,12 +353,12 @@ index_storage_is_readonly, index_storage_allow_new_keywords, index_storage_mailbox_enable, - dbox_mailbox_open, + sdbox_mailbox_open, sdbox_mailbox_close, index_storage_mailbox_free, dbox_mailbox_create, dbox_mailbox_update, - index_storage_mailbox_delete, + sdbox_mailbox_delete, index_storage_mailbox_rename, index_storage_get_status, sdbox_mailbox_get_guid, @@ -339,15 +397,17 @@ sdbox_save_finish, sdbox_save_cancel, sdbox_copy, + NULL, index_storage_is_inconsistent } }; struct dbox_storage_vfuncs sdbox_dbox_storage_vfuncs = { - dbox_file_free, + sdbox_file_free, sdbox_file_create_fd, sdbox_mail_open, sdbox_mailbox_create_indexes, + sdbox_get_attachment_path_suffix, sdbox_set_mailbox_corrupted, sdbox_set_file_corrupted };
--- a/src/lib-storage/index/dbox-single/sdbox-storage.h Tue Oct 19 18:30:51 2010 +0100 +++ b/src/lib-storage/index/dbox-single/sdbox-storage.h Tue Oct 19 18:47:17 2010 +0100 @@ -28,6 +28,7 @@ has changed from this value) */ uint32_t corrupted_rebuild_count; + uint8_t mailbox_guid[MAIL_GUID_128_SIZE]; unsigned int creating:1; };
--- a/src/lib-storage/index/dbox-single/sdbox-sync.c Tue Oct 19 18:30:51 2010 +0100 +++ b/src/lib-storage/index/dbox-single/sdbox-sync.c Tue Oct 19 18:47:17 2010 +0100 @@ -1,6 +1,7 @@ /* Copyright (c) 2007-2010 Dovecot authors, see the included COPYING file */ #include "lib.h" +#include "dbox-attachment.h" #include "sdbox-storage.h" #include "sdbox-file.h" #include "sdbox-sync.h" @@ -27,12 +28,18 @@ { struct sdbox_file *sfile = (struct sdbox_file *)file; struct mailbox *box = &ctx->mbox->box; + int ret; if (mail_index_transaction_is_expunged(ctx->trans, seq)) { /* already expunged within this transaction */ return; } - if (dbox_file_unlink(file) < 0) { + + if (file->storage->attachment_dir != NULL) + ret = sdbox_file_unlink_with_attachments(sfile); + else + ret = dbox_file_unlink(file); + if (ret < 0) { /* some non-ENOENT error trying to unlink the file */ return; }
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-storage/index/index-attachment.c Tue Oct 19 18:47:17 2010 +0100 @@ -0,0 +1,788 @@ +/* Copyright (c) 2010 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "safe-mkstemp.h" +#include "fs-api.h" +#include "istream.h" +#include "ostream.h" +#include "base64.h" +#include "hash-format.h" +#include "str.h" +#include "message-parser.h" +#include "rfc822-parser.h" +#include "mail-user.h" +#include "index-mail.h" +#include "index-attachment.h" + +#define BASE64_ATTACHMENT_MAX_EXTRA_BYTES 1024 + +enum mail_attachment_state { + MAIL_ATTACHMENT_STATE_NO, + MAIL_ATTACHMENT_STATE_MAYBE, + MAIL_ATTACHMENT_STATE_YES +}; + +enum base64_state { + BASE64_STATE_0 = 0, + BASE64_STATE_1, + BASE64_STATE_2, + BASE64_STATE_3, + BASE64_STATE_CR, + BASE64_STATE_EOB, + BASE64_STATE_EOM +}; + +struct mail_save_attachment_part { + char *content_type, *content_disposition; + enum mail_attachment_state state; + /* start offset of the message part in the original input stream */ + uoff_t start_offset; + + /* for saving attachments base64-decoded: */ + enum base64_state base64_state, base64_prev_state; + unsigned int base64_line_blocks, cur_base64_blocks; + unsigned int base64_last_newline_size; + uoff_t base64_bytes; + bool base64_have_crlf; /* CRLF linefeeds */ + bool base64_failed; + + int temp_fd; + struct ostream *output; + struct hash_format *part_hash; + buffer_t *part_buf; +}; + +struct mail_save_attachment { + pool_t pool; + struct message_parser_ctx *parser; + struct fs *fs; + struct istream *input; + + /* per-MIME part data */ + struct mail_save_attachment_part part; + struct message_part *prev_part; + + ARRAY_TYPE(mail_attachment_extref) extrefs; +}; + +static const char *index_attachment_dir_get(struct mail_storage *storage) +{ + return mail_user_home_expand(storage->user, + storage->set->mail_attachment_dir); +} + +void index_attachment_save_begin(struct mail_save_context *ctx, + struct fs *fs, struct istream *input) +{ + struct mail_storage *storage = ctx->transaction->box->storage; + pool_t pool; + + i_assert(ctx->attach == NULL); + + if (*storage->set->mail_attachment_dir == '\0') + return; + + pool = pool_alloconly_create("save attachment", 1024*4); + ctx->attach = p_new(pool, struct mail_save_attachment, 1); + ctx->attach->pool = pool; + ctx->attach->fs = fs; + ctx->attach->input = input; + ctx->attach->parser = + message_parser_init(ctx->attach->pool, input, 0, 0); + p_array_init(&ctx->attach->extrefs, ctx->attach->pool, 8); +} + +static void parse_content_type(struct mail_save_context *ctx, + const struct message_header_line *hdr) +{ + struct rfc822_parser_context parser; + string_t *content_type; + + rfc822_parser_init(&parser, hdr->full_value, hdr->full_value_len, NULL); + (void)rfc822_skip_lwsp(&parser); + + T_BEGIN { + content_type = t_str_new(64); + if (rfc822_parse_content_type(&parser, content_type) >= 0) { + i_free(ctx->attach->part.content_type); + ctx->attach->part.content_type = + i_strdup(str_c(content_type)); + } + } T_END; +} + +static void +parse_content_disposition(struct mail_save_context *ctx, + const struct message_header_line *hdr) +{ + /* just pass it as-is to backend. */ + i_free(ctx->attach->part.content_disposition); + ctx->attach->part.content_disposition = + i_strndup(hdr->full_value, hdr->full_value_len); +} + +static void index_attachment_save_mail_header(struct mail_save_context *ctx, + struct message_header_line *hdr) +{ + if (hdr->continues) { + hdr->use_full_value = TRUE; + return; + } + + if (strcasecmp(hdr->name, "Content-Type") == 0) + parse_content_type(ctx, hdr); + else if (strcasecmp(hdr->name, "Content-Disposition") == 0) + parse_content_disposition(ctx, hdr); + + o_stream_send(ctx->output, hdr->name, hdr->name_len); + o_stream_send(ctx->output, hdr->middle, hdr->middle_len); + o_stream_send(ctx->output, hdr->full_value, hdr->full_value_len); + if (!hdr->no_newline) { + if (hdr->crlf_newline) + o_stream_send(ctx->output, "\r\n", 2); + else + o_stream_send(ctx->output, "\n", 1); + } +} + +static bool save_is_attachment(struct mail_save_context *ctx, + struct message_part *part) +{ + struct mailbox *box = ctx->transaction->box; + struct mail_attachment_part apart; + + if ((part->flags & MESSAGE_PART_FLAG_MULTIPART) != 0) { + /* multiparts may contain attachments as children, + but they're never themselves */ + return FALSE; + } + if (box->v.save_is_attachment == NULL) + return TRUE; + + memset(&apart, 0, sizeof(apart)); + apart.part = part; + apart.content_type = ctx->attach->part.content_type; + apart.content_disposition = ctx->attach->part.content_disposition; + return box->v.save_is_attachment(ctx, &apart); +} + +static int index_attachment_save_temp_open_fd(struct mail_storage *storage) +{ + string_t *temp_path; + int fd; + + temp_path = t_str_new(256); + mail_user_set_get_temp_prefix(temp_path, storage->user->set); + fd = safe_mkstemp_hostpid(temp_path, 0600, (uid_t)-1, (gid_t)-1); + if (fd == -1) { + mail_storage_set_critical(storage, + "safe_mkstemp(%s) failed: %m", str_c(temp_path)); + return -1; + } + if (unlink(str_c(temp_path)) < 0) { + mail_storage_set_critical(storage, + "unlink(%s) failed: %m", str_c(temp_path)); + (void)close(fd); + return -1; + } + return fd; +} + +static struct hash_format * +index_attachment_hash_format_init(struct mail_save_context *ctx) +{ + struct mail_storage *storage = ctx->transaction->box->storage; + struct hash_format *format; + const char *error; + + if (hash_format_init(storage->set->mail_attachment_hash, + &format, &error) < 0) { + /* we already checked this when verifying settings */ + i_panic("mail_attachment_hash=%s unexpectedly failed: %s", + storage->set->mail_attachment_hash, error); + } + return format; +} + +static int index_attachment_save_temp_open(struct mail_save_context *ctx) +{ + int fd; + + fd = index_attachment_save_temp_open_fd(ctx->transaction->box->storage); + if (fd == -1) + return -1; + + ctx->attach->part.temp_fd = fd; + ctx->attach->part.output = o_stream_create_fd(fd, 0, FALSE); + o_stream_cork(ctx->attach->part.output); + + ctx->attach->part.part_hash = index_attachment_hash_format_init(ctx); + return 0; +} + +static int save_check_write_error(struct mail_storage *storage, + struct ostream *output) +{ + if (output->last_failed_errno == 0) + return 0; + + errno = output->last_failed_errno; + if (!mail_storage_set_error_from_errno(storage)) { + mail_storage_set_critical(storage, "write(%s) failed: %m", + o_stream_get_name(output)); + } + return -1; +} + +static int index_attachment_base64_decode(struct mail_save_context *ctx) +{ + struct mail_save_attachment_part *part = &ctx->attach->part; + struct mail_storage *storage = ctx->transaction->box->storage; + buffer_t *extra_buf = NULL; + struct istream *input, *base64_input; + struct ostream *output; + struct hash_format *hash; + const unsigned char *data; + size_t size; + ssize_t ret; + buffer_t *buf; + int outfd; + bool failed = FALSE; + + if (part->base64_bytes < storage->set->mail_attachment_min_size || + part->output->offset > part->base64_bytes + + BASE64_ATTACHMENT_MAX_EXTRA_BYTES) { + /* only a small part of the MIME part is base64-encoded. */ + return -1; + } + + if (part->base64_line_blocks == 0) { + /* only one line of base64 */ + part->base64_line_blocks = part->cur_base64_blocks; + i_assert(part->base64_line_blocks > 0); + } + + /* decode base64 data and write it to another temp file */ + outfd = index_attachment_save_temp_open_fd(storage); + if (outfd == -1) + return -1; + + hash = index_attachment_hash_format_init(ctx); + buf = buffer_create_dynamic(default_pool, 1024); + input = i_stream_create_fd(part->temp_fd, IO_BLOCK_SIZE, FALSE); + base64_input = i_stream_create_limit(input, part->base64_bytes); + output = o_stream_create_fd_file(outfd, 0, FALSE); + o_stream_cork(output); + + while ((ret = i_stream_read(base64_input)) > 0) { + data = i_stream_get_data(base64_input, &size); + buffer_set_used_size(buf, 0); + if (base64_decode(data, size, &size, buf) < 0) { + mail_storage_set_critical(storage, + "Attachment base64 data unexpectedly broke"); + failed = TRUE; + break; + } + i_stream_skip(base64_input, size); + o_stream_send(output, buf->data, buf->used); + hash_format_loop(hash, buf->data, buf->used); + } + if (ret != -1) { + i_assert(failed); + } else if (base64_input->stream_errno != 0) { + mail_storage_set_critical(storage, + "read(attachment-temp) failed: %m"); + + failed = TRUE; + } + (void)o_stream_flush(output); + if (save_check_write_error(storage, output) < 0) + failed = TRUE; + + buffer_free(&buf); + i_stream_unref(&base64_input); + o_stream_unref(&output); + + if (input->v_offset != part->output->offset && !failed) { + /* write the rest of the data to the message stream */ + extra_buf = buffer_create_dynamic(default_pool, 1024); + while ((ret = i_stream_read_data(input, &data, &size, 0)) > 0) { + buffer_append(extra_buf, data, size); + i_stream_skip(input, size); + } + i_assert(ret == -1); + if (input->stream_errno != 0) { + mail_storage_set_critical(storage, + "read(attachment-temp) failed: %m"); + failed = TRUE; + } + } + i_stream_unref(&input); + + if (failed) { + hash_format_deinit_free(&hash); + if (close(outfd) < 0) { + mail_storage_set_critical(storage, + "close(attachment-temp) failed: %m"); + } + return -1; + } + + /* successfully wrote it. switch to using it. */ + o_stream_destroy(&part->output); + if (close(part->temp_fd) < 0) { + mail_storage_set_critical(storage, + "close(attachment-decoded-temp) failed: %m"); + } + part->temp_fd = outfd; + + if (extra_buf != NULL) { + o_stream_send(ctx->output, extra_buf->data, extra_buf->used); + buffer_free(&extra_buf); + } + hash_format_deinit_free(&part->part_hash); + part->part_hash = hash; + return 0; +} + +static int index_attachment_save_finish_part(struct mail_save_context *ctx) +{ + struct mail_save_attachment_part *part = &ctx->attach->part; + struct mail_storage *storage = ctx->transaction->box->storage; + struct fs_file *file; + struct istream *input; + struct ostream *output; + uint8_t guid_128[MAIL_GUID_128_SIZE]; + const char *attachment_dir, *path, *digest; + string_t *digest_str; + const unsigned char *data; + size_t size; + uoff_t attachment_size; + enum fs_open_flags flags = FS_OPEN_FLAG_MKDIR; + int ret = 0; + + if (o_stream_flush(part->output) < 0) { + save_check_write_error(storage, part->output); + return -1; + } + + if (part->base64_state == BASE64_STATE_0) { + part->base64_bytes = part->output->offset - + part->base64_last_newline_size; + part->base64_state = BASE64_STATE_EOM; + } + + if (!part->base64_failed && + part->base64_state == BASE64_STATE_EOM) { + /* base64 data looks ok. STATE_0 happens when + there is no trailing LF or '=' characters. */ + if (index_attachment_base64_decode(ctx) < 0) + part->base64_failed = TRUE; + } + + /* open the attachment destination file */ + if (storage->set->parsed_fsync_mode != FSYNC_MODE_NEVER) + flags |= FS_OPEN_FLAG_FDATASYNC; + + digest_str = t_str_new(128); + hash_format_deinit(&part->part_hash, digest_str); + digest = str_c(digest_str); + if (strlen(digest) < 4) { + /* make sure we can access first 4 bytes without accessing + out of bounds memory */ + digest = t_strconcat(digest, "\0\0\0\0", NULL); + } + + mail_generate_guid_128(guid_128); + attachment_dir = index_attachment_dir_get(storage); + path = t_strdup_printf("%s/%c%c/%c%c/%s-%s", attachment_dir, + digest[0], digest[1], + digest[2], digest[3], digest, + mail_guid_128_to_string(guid_128)); + if (fs_open(ctx->attach->fs, path, + FS_OPEN_MODE_CREATE | flags, &file) < 0) { + mail_storage_set_critical(storage, "%s", + fs_last_error(ctx->attach->fs)); + return -1; + } + + /* copy data to it from temp file */ + input = i_stream_create_fd(part->temp_fd, IO_BLOCK_SIZE, FALSE); + output = fs_write_stream(file); + while (i_stream_read_data(input, &data, &size, 0) > 0) { + o_stream_send(output, data, size); + i_stream_skip(input, size); + } + + if (input->stream_errno != 0) { + mail_storage_set_critical(storage, + "read(%s) failed: %m", i_stream_get_name(input)); + ret = -1; + } + attachment_size = !part->base64_failed ? + part->base64_bytes : input->v_offset; + i_stream_destroy(&input); + + if (ret < 0) + fs_write_stream_abort(file, &output); + else if (fs_write_stream_finish(file, &output) < 0) { + mail_storage_set_critical(storage, "%s", + fs_file_last_error(file)); + ret = -1; + } + fs_close(&file); + + if (ret == 0) { + struct mail_attachment_extref *extref; + + extref = array_append_space(&ctx->attach->extrefs); + extref->start_offset = part->start_offset; + extref->size = attachment_size; + extref->path = p_strdup(ctx->attach->pool, + path + strlen(attachment_dir) + 1); + extref->base64_blocks_per_line = + part->base64_failed ? 0 : part->base64_line_blocks; + extref->base64_have_crlf = part->base64_have_crlf; + } + return ret; +} + +static int +index_attachment_try_base64_decode_char(struct mail_save_attachment_part *part, + size_t pos, char chr) +{ + enum base64_state prev_state = part->base64_prev_state; + + part->base64_last_newline_size = 0; + part->base64_prev_state = part->base64_state; + switch (part->base64_state) { + case BASE64_STATE_0: + if (base64_is_valid_char(chr)) + part->base64_state++; + else if (chr == '\r') + part->base64_state = BASE64_STATE_CR; + else if (chr == '\n') { + part->base64_state = BASE64_STATE_0; + part->base64_last_newline_size = + prev_state == BASE64_STATE_CR ? 2 : 1; + if (part->cur_base64_blocks < + part->base64_line_blocks) { + /* last line */ + part->base64_bytes = + part->output->offset + pos; + if (prev_state == BASE64_STATE_CR) + part->base64_bytes--; + part->base64_state = BASE64_STATE_EOM; + return 0; + } else if (part->base64_line_blocks == 0) { + /* first line */ + if (part->cur_base64_blocks == 0) + return -1; + part->base64_line_blocks = + part->cur_base64_blocks; + } else if (part->cur_base64_blocks == + part->base64_line_blocks) { + /* line is ok */ + } else { + return -1; + } + part->cur_base64_blocks = 0; + } else { + return -1; + } + break; + case BASE64_STATE_1: + if (!base64_is_valid_char(chr)) + return -1; + part->base64_state++; + break; + case BASE64_STATE_2: + if (base64_is_valid_char(chr)) + part->base64_state++; + else if (chr == '=') + part->base64_state = BASE64_STATE_EOB; + else + return -1; + break; + case BASE64_STATE_3: + if (base64_is_valid_char(chr)) { + part->base64_state = BASE64_STATE_0; + part->cur_base64_blocks++; + } else if (chr == '=') { + part->base64_bytes = part->output->offset + pos + 1; + part->base64_state = BASE64_STATE_EOM; + part->cur_base64_blocks++; + return 0; + } else { + return -1; + } + break; + case BASE64_STATE_CR: + if (chr != '\n') + return -1; + part->base64_have_crlf = TRUE; + break; + case BASE64_STATE_EOB: + if (chr != '=') + return -1; + + part->base64_bytes = part->output->offset + pos + 1; + part->base64_state = BASE64_STATE_EOM; + part->cur_base64_blocks++; + return 0; + case BASE64_STATE_EOM: + i_unreached(); + } + return 1; +} + +static void +index_attachment_try_base64_decode(struct mail_save_attachment_part *part, + const unsigned char *data, size_t size) +{ + size_t i; + int ret; + + if (part->base64_failed || part->base64_state == BASE64_STATE_EOM) + return; + + for (i = 0; i < size; i++) { + ret = index_attachment_try_base64_decode_char(part, i, + (char)data[i]); + if (ret <= 0) { + if (ret < 0) + part->base64_failed = TRUE; + break; + } + } +} + +static void index_attachment_save_body(struct mail_save_context *ctx, + const struct message_block *block) +{ + struct mail_save_attachment_part *part = &ctx->attach->part; + struct mail_storage *storage = ctx->transaction->box->storage; + buffer_t *part_buf; + size_t new_size; + + switch (part->state) { + case MAIL_ATTACHMENT_STATE_NO: + o_stream_send(ctx->output, block->data, block->size); + break; + case MAIL_ATTACHMENT_STATE_MAYBE: + if (part->part_buf == NULL) { + part->part_buf = + buffer_create_dynamic(default_pool, + storage->set->mail_attachment_min_size); + } + part_buf = part->part_buf; + new_size = part_buf->used + block->size; + if (new_size < storage->set->mail_attachment_min_size) { + buffer_append(part_buf, block->data, block->size); + break; + } + /* attachment is large enough. we'll first write it to + temp file. */ + if (index_attachment_save_temp_open(ctx) < 0) { + /* failed, fallback to just saving it inline */ + part->state = MAIL_ATTACHMENT_STATE_NO; + o_stream_send(ctx->output, part_buf->data, + part_buf->used); + o_stream_send(ctx->output, block->data, block->size); + break; + } + part->state = MAIL_ATTACHMENT_STATE_YES; + index_attachment_try_base64_decode(part, part_buf->data, + part_buf->used); + hash_format_loop(part->part_hash, + part_buf->data, part_buf->used); + o_stream_send(part->output, part_buf->data, part_buf->used); + buffer_set_used_size(part_buf, 0); + /* fall through */ + case MAIL_ATTACHMENT_STATE_YES: + index_attachment_try_base64_decode(part, block->data, + block->size); + hash_format_loop(part->part_hash, block->data, block->size); + o_stream_send(part->output, block->data, block->size); + break; + } +} + +static void index_attachment_save_close(struct mail_save_context *ctx) +{ + struct mail_save_attachment_part *part = &ctx->attach->part; + struct mail_storage *storage = ctx->transaction->box->storage; + + if (part->output != NULL) + o_stream_destroy(&part->output); + if (close(part->temp_fd) < 0) { + mail_storage_set_critical(storage, + "close(attachment-temp) failed: %m"); + } + part->temp_fd = -1; +} + +static int +index_attachment_save_body_part_changed(struct mail_save_context *ctx) +{ + struct mail_save_attachment_part *part = &ctx->attach->part; + int ret = 0; + + /* body part changed. we're now parsing the end of a + boundary, possibly followed by message epilogue */ + switch (part->state) { + case MAIL_ATTACHMENT_STATE_NO: + break; + case MAIL_ATTACHMENT_STATE_MAYBE: + /* body part wasn't large enough. write to main file. */ + if (part->part_buf != NULL) { + o_stream_send(ctx->output, part->part_buf->data, + part->part_buf->used); + } + break; + case MAIL_ATTACHMENT_STATE_YES: + if (index_attachment_save_finish_part(ctx) < 0) + ret = -1; + index_attachment_save_close(ctx); + break; + } + part->state = MAIL_ATTACHMENT_STATE_NO; + + i_free_and_null(part->content_type); + i_free_and_null(part->content_disposition); + if (part->part_buf != NULL) + buffer_free(&part->part_buf); + memset(part, 0, sizeof(*part)); + return ret; +} + +int index_attachment_save_continue(struct mail_save_context *ctx) +{ + struct mail_storage *storage = ctx->transaction->box->storage; + struct message_parser_ctx *parser = ctx->attach->parser; + struct message_block block; + struct ostream *output; + int ret; + + while ((ret = message_parser_parse_next_block(parser, &block)) > 0) { + if (block.part != ctx->attach->prev_part) { + if (index_attachment_save_body_part_changed(ctx) < 0) + return -1; + ctx->attach->prev_part = block.part; + } + + if (block.hdr != NULL) + index_attachment_save_mail_header(ctx, block.hdr); + else if (block.size == 0) { + /* end of headers */ + if (save_is_attachment(ctx, block.part)) { + ctx->attach->part.state = + MAIL_ATTACHMENT_STATE_MAYBE; + ctx->attach->part.start_offset = + ctx->attach->input->v_offset; + } + } else { + /* body */ + index_attachment_save_body(ctx, &block); + } + + output = ctx->attach->part.output != NULL ? + ctx->attach->part.output : ctx->output; + if (output->last_failed_errno != 0) + break; + index_mail_cache_parse_continue(ctx->dest_mail); + } + if (ret == 0) + return 0; + + if (ctx->attach->input->stream_errno != 0) { + errno = ctx->attach->input->stream_errno; + mail_storage_set_critical(storage, "read(%s) failed: %m", + i_stream_get_name(ctx->attach->input)); + return -1; + } + if (ctx->attach->part.output != NULL) { + if (save_check_write_error(storage, + ctx->attach->part.output) < 0) + return -1; + } + if (ctx->output != NULL) { + if (save_check_write_error(storage, ctx->output) < 0) + return -1; + } + return 0; +} + +int index_attachment_save_finish(struct mail_save_context *ctx) +{ + struct message_part *parts; + int ret = 0, ret2; + + if (ctx->attach->parser != NULL) { + if (index_attachment_save_body_part_changed(ctx) < 0) + ret = -1; + ret2 = message_parser_deinit(&ctx->attach->parser, &parts); + i_assert(ret2 == 0); + } + i_assert(ctx->attach->part.output == NULL); + return ret; +} + +void index_attachment_save_free(struct mail_save_context *ctx) +{ + if (ctx->attach != NULL) { + pool_unref(&ctx->attach->pool); + ctx->attach = NULL; + } +} + +const ARRAY_TYPE(mail_attachment_extref) * +index_attachment_save_get_extrefs(struct mail_save_context *ctx) +{ + return ctx->attach == NULL ? NULL : + &ctx->attach->extrefs; +} + +static int +index_attachment_delete_real(struct mail_storage *storage, + struct fs *fs, const char *name) +{ + const char *path, *p, *attachment_dir; + int ret; + + path = t_strdup_printf("%s/%s", index_attachment_dir_get(storage), name); + if ((ret = fs_unlink(fs, path)) < 0) + mail_storage_set_critical(storage, "%s", fs_last_error(fs)); + + /* if the directory is now empty, rmdir it and its parents + until it fails */ + attachment_dir = index_attachment_dir_get(storage); + while ((p = strrchr(path, '/')) != NULL) { + path = t_strdup_until(path, p); + if (strcmp(path, attachment_dir) == 0) + break; + + if (fs_rmdir(fs, path) == 0) { + /* success, continue to parent */ + } else if (errno == ENOTEMPTY || errno == EEXIST) { + /* there are other entries in this directory */ + break; + } else { + mail_storage_set_critical(storage, "%s", + fs_last_error(fs)); + break; + } + } + return ret; +} + +int index_attachment_delete(struct mail_storage *storage, + struct fs *fs, const char *name) +{ + int ret; + + T_BEGIN { + ret = index_attachment_delete_real(storage, fs, name); + } T_END; + return ret; +}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-storage/index/index-attachment.h Tue Oct 19 18:47:17 2010 +0100 @@ -0,0 +1,40 @@ +#ifndef INDEX_ATTACHMENT_H +#define INDEX_ATTACHMENT_H + +#include "sha1.h" + +struct fs; +struct mail_save_context; +struct mail_storage; + +struct mail_attachment_extref { + /* path without attachment_dir/ prefix */ + const char *path; + /* offset in input stream where part begins */ + uoff_t start_offset; + uoff_t size; + + /* If non-zero, this attachment was saved as base64-decoded and it + need to be encoded back before presenting it to client. Each line + (except last one) consists of this many base64 blocks (4 chars of + base64 encoded data). */ + unsigned int base64_blocks_per_line; + /* Line feeds are CRLF instead of LF */ + bool base64_have_crlf; +}; +ARRAY_DEFINE_TYPE(mail_attachment_extref, struct mail_attachment_extref); + +void index_attachment_save_begin(struct mail_save_context *ctx, + struct fs *fs, struct istream *input); +int index_attachment_save_continue(struct mail_save_context *ctx); +int index_attachment_save_finish(struct mail_save_context *ctx); +void index_attachment_save_free(struct mail_save_context *ctx); +const ARRAY_TYPE(mail_attachment_extref) * +index_attachment_save_get_extrefs(struct mail_save_context *ctx); + +/* Delete a given attachment name from storage + (name is same as mail_attachment_extref.name). */ +int index_attachment_delete(struct mail_storage *storage, + struct fs *fs, const char *name); + +#endif
--- a/src/lib-storage/index/index-storage.c Tue Oct 19 18:30:51 2010 +0100 +++ b/src/lib-storage/index/index-storage.c Tue Oct 19 18:47:17 2010 +0100 @@ -14,6 +14,7 @@ #include "mailbox-list-private.h" #include "index-storage.h" #include "index-mail.h" +#include "index-attachment.h" #include "index-thread-private.h" #include <stdlib.h> @@ -597,4 +598,5 @@ i_free_and_null(ctx->from_envelope); i_free_and_null(ctx->guid); i_free_and_null(ctx->pop3_uidl); + index_attachment_save_free(ctx); }
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-storage/index/istream-attachment.c Tue Oct 19 18:47:17 2010 +0100 @@ -0,0 +1,123 @@ +/* Copyright (c) 2003-2010 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "istream-internal.h" +#include "istream-attachment.h" + +struct attachment_istream { + struct istream_private istream; + + uoff_t size; +}; + +static ssize_t i_stream_attachment_read(struct istream_private *stream) +{ + struct attachment_istream *astream = + (struct attachment_istream *)stream; + uoff_t left; + ssize_t ret; + size_t pos; + + if (stream->istream.v_offset + + (stream->pos - stream->skip) >= astream->size) { + stream->istream.eof = TRUE; + return -1; + } + + i_stream_seek(stream->parent, astream->istream.parent_start_offset + + stream->istream.v_offset); + + stream->pos -= stream->skip; + stream->skip = 0; + + stream->buffer = i_stream_get_data(stream->parent, &pos); + if (pos > stream->pos) + ret = 0; + else do { + if ((ret = i_stream_read(stream->parent)) == -2) + return -2; + + stream->istream.stream_errno = stream->parent->stream_errno; + stream->istream.eof = stream->parent->eof; + stream->buffer = i_stream_get_data(stream->parent, &pos); + } while (pos <= stream->pos && ret > 0); + + left = astream->size - stream->istream.v_offset; + if (pos == left) + stream->istream.eof = TRUE; + else if (pos > left) { + i_error("Attachment file %s larger than expected " + "(%"PRIuUOFF_T")", i_stream_get_name(stream->parent), + astream->size); + pos = left; + stream->istream.eof = TRUE; + } else if (!stream->istream.eof) { + /* still more to read */ + } else { + i_error("Attachment file %s smaller than expected " + "(%"PRIuUOFF_T")", i_stream_get_name(stream->parent), + astream->size); + } + + ret = pos > stream->pos ? (ssize_t)(pos - stream->pos) : + (ret == 0 ? 0 : -1); + stream->pos = pos; + i_assert(ret != -1 || stream->istream.eof || + stream->istream.stream_errno != 0); + return ret; +} + +static void +i_stream_attachment_seek(struct istream_private *stream, + uoff_t v_offset, bool mark ATTR_UNUSED) +{ + struct attachment_istream *astream = + (struct attachment_istream *)stream; + + i_assert(v_offset <= astream->size); + + stream->istream.v_offset = v_offset; + stream->skip = stream->pos = 0; +} + +static const struct stat * +i_stream_attachment_stat(struct istream_private *stream, bool exact) +{ + struct attachment_istream *astream = + (struct attachment_istream *)stream; + const struct stat *st; + + st = i_stream_stat(stream->parent, exact); + if (st == NULL) + return NULL; + + stream->statbuf = *st; + stream->statbuf.st_size = astream->size; + if (st->st_size != 0 && (uoff_t)st->st_size != astream->size) { + i_error("Attachment file %s size mismatch: " + "%"PRIuUOFF_T" != %"PRIuUOFF_T, + i_stream_get_name(stream->parent), + st->st_size, astream->size); + } + return &stream->statbuf; +} + +struct istream *i_stream_create_attachment(struct istream *input, uoff_t size) +{ + struct attachment_istream *astream; + + astream = i_new(struct attachment_istream, 1); + astream->size = size; + astream->istream.max_buffer_size = input->real_stream->max_buffer_size; + + astream->istream.parent = input; + astream->istream.read = i_stream_attachment_read; + astream->istream.seek = i_stream_attachment_seek; + astream->istream.stat = i_stream_attachment_stat; + + astream->istream.istream.readable_fd = input->readable_fd; + astream->istream.istream.blocking = input->blocking; + astream->istream.istream.seekable = input->seekable; + return i_stream_create(&astream->istream, input, + i_stream_get_fd(input)); +}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-storage/index/istream-attachment.h Tue Oct 19 18:47:17 2010 +0100 @@ -0,0 +1,6 @@ +#ifndef ISTREAM_ATTACHMENT_H +#define ISTREAM_ATTACHMENT_H + +struct istream *i_stream_create_attachment(struct istream *input, uoff_t size); + +#endif
--- a/src/lib-storage/index/maildir/maildir-storage.c Tue Oct 19 18:30:51 2010 +0100 +++ b/src/lib-storage/index/maildir/maildir-storage.c Tue Oct 19 18:47:17 2010 +0100 @@ -636,6 +636,7 @@ maildir_save_finish, maildir_save_cancel, maildir_copy, + NULL, index_storage_is_inconsistent } };
--- a/src/lib-storage/index/mbox/mbox-storage.c Tue Oct 19 18:30:51 2010 +0100 +++ b/src/lib-storage/index/mbox/mbox-storage.c Tue Oct 19 18:47:17 2010 +0100 @@ -754,6 +754,7 @@ mbox_save_finish, mbox_save_cancel, mail_storage_copy, + NULL, index_storage_is_inconsistent } };
--- a/src/lib-storage/index/raw/raw-storage.c Tue Oct 19 18:30:51 2010 +0100 +++ b/src/lib-storage/index/raw/raw-storage.c Tue Oct 19 18:47:17 2010 +0100 @@ -179,6 +179,7 @@ NULL, NULL, mail_storage_copy, + NULL, index_storage_is_inconsistent } };
--- a/src/lib-storage/mail-storage-private.h Tue Oct 19 18:30:51 2010 +0100 +++ b/src/lib-storage/mail-storage-private.h Tue Oct 19 18:47:17 2010 +0100 @@ -91,6 +91,11 @@ ARRAY_DEFINE(module_contexts, union mail_storage_module_context *); }; +struct mail_attachment_part { + struct message_part *part; + const char *content_type, *content_disposition; +}; + struct mailbox_vfuncs { bool (*is_readonly)(struct mailbox *box); bool (*allow_new_keywords)(struct mailbox *box); @@ -205,6 +210,9 @@ int (*save_finish)(struct mail_save_context *ctx); void (*save_cancel)(struct mail_save_context *ctx); int (*copy)(struct mail_save_context *ctx, struct mail *mail); + /* returns TRUE if message part is an attachment. */ + bool (*save_is_attachment)(struct mail_save_context *ctx, + const struct mail_attachment_part *part); bool (*is_inconsistent)(struct mailbox *box); }; @@ -421,6 +429,8 @@ char *guid, *pop3_uidl, *from_envelope; struct ostream *output; + struct mail_save_attachment *attach; + /* we came here from mailbox_copy() */ unsigned int copying:1; };
--- a/src/lib-storage/mail-storage-settings.c Tue Oct 19 18:30:51 2010 +0100 +++ b/src/lib-storage/mail-storage-settings.c Tue Oct 19 18:47:17 2010 +0100 @@ -2,6 +2,7 @@ #include "lib.h" #include "array.h" +#include "hash-format.h" #include "var-expand.h" #include "settings-parser.h" #include "mail-index.h" @@ -23,6 +24,10 @@ static const struct setting_define mail_storage_setting_defines[] = { DEF(SET_STR_VARS, mail_location), { SET_ALIAS, "mail", 0, NULL }, + DEF(SET_STR_VARS, mail_attachment_fs), + DEF(SET_STR_VARS, mail_attachment_dir), + DEF(SET_STR, mail_attachment_hash), + DEF(SET_SIZE, mail_attachment_min_size), DEF(SET_STR, mail_cache_fields), DEF(SET_STR, mail_never_cache_fields), DEF(SET_UINT, mail_cache_min_mail_count), @@ -47,6 +52,10 @@ const struct mail_storage_settings mail_storage_default_settings = { .mail_location = "", + .mail_attachment_fs = "sis posix", + .mail_attachment_dir = "", + .mail_attachment_hash = "%{sha1}", + .mail_attachment_min_size = 1024*128, .mail_cache_fields = "flags", .mail_never_cache_fields = "imap.envelope", .mail_cache_min_mail_count = 0, @@ -287,7 +296,8 @@ const char **error_r) { struct mail_storage_settings *set = _set; - const char *p; + struct hash_format *format; + const char *p, *error; bool uidl_format_ok; char c; @@ -347,6 +357,23 @@ "%% variables."; return FALSE; } + + if (strchr(set->mail_attachment_hash, '/') != NULL) { + *error_r = "mail_attachment_hash setting " + "must not contain '/' characters"; + return FALSE; + } + if (hash_format_init(set->mail_attachment_hash, &format, &error) < 0) { + *error_r = t_strconcat("Invalid mail_attachment_hash setting: ", + error, NULL); + return FALSE; + } + if (strchr(set->mail_attachment_hash, '-') != NULL) { + *error_r = "mail_attachment_hash setting " + "must not contain '-' characters"; + return FALSE; + } + hash_format_deinit_free(&format); return TRUE; }
--- a/src/lib-storage/mail-storage-settings.h Tue Oct 19 18:30:51 2010 +0100 +++ b/src/lib-storage/mail-storage-settings.h Tue Oct 19 18:47:17 2010 +0100 @@ -11,6 +11,10 @@ struct mail_storage_settings { const char *mail_location; + const char *mail_attachment_fs; + const char *mail_attachment_dir; + const char *mail_attachment_hash; + uoff_t mail_attachment_min_size; const char *mail_cache_fields; const char *mail_never_cache_fields; unsigned int mail_cache_min_mail_count;