Mercurial > dovecot > core-2.2
changeset 10765:19df4309e389 HEAD
lib-storage: Added support for plugins to specify message's physical size when saving.
Changed dbox to save this value to metadata, and use when present.
author | Timo Sirainen <tss@iki.fi> |
---|---|
date | Fri, 19 Feb 2010 07:15:46 +0200 |
parents | 90f4663211ba |
children | 300ac0c2cac3 |
files | src/lib-storage/index/dbox-common/dbox-file.c src/lib-storage/index/dbox-common/dbox-file.h src/lib-storage/index/dbox-common/dbox-mail.c src/lib-storage/index/dbox-common/dbox-save.c src/lib-storage/index/dbox-multi/mdbox-file-purge.c src/lib-storage/index/dbox-multi/mdbox-storage-rebuild.c src/lib-storage/index/dbox-single/sdbox-sync-rebuild.c src/lib-storage/mail-storage-private.h |
diffstat | 8 files changed, 71 insertions(+), 42 deletions(-) [+] |
line wrap: on
line diff
--- a/src/lib-storage/index/dbox-common/dbox-file.c Fri Feb 19 07:14:41 2010 +0200 +++ b/src/lib-storage/index/dbox-common/dbox-file.c Fri Feb 19 07:15:46 2010 +0200 @@ -310,7 +310,6 @@ } int dbox_file_get_mail_stream(struct dbox_file *file, uoff_t offset, - uoff_t *physical_size_r, struct istream **stream_r) { uoff_t size; @@ -334,7 +333,6 @@ *stream_r = i_stream_create_limit(file->input, file->cur_physical_size); } - *physical_size_r = file->cur_physical_size; return 1; } @@ -366,7 +364,7 @@ int dbox_file_seek_next(struct dbox_file *file, uoff_t *offset_r, bool *last_r) { - uoff_t offset, size; + uoff_t offset; int ret; i_assert(file->input != NULL); @@ -391,7 +389,7 @@ } *last_r = FALSE; - ret = dbox_file_get_mail_stream(file, offset, &size, NULL); + ret = dbox_file_get_mail_stream(file, offset, NULL); if (*offset_r == 0) *offset_r = file->file_header_size; return ret;
--- a/src/lib-storage/index/dbox-common/dbox-file.h Fri Feb 19 07:14:41 2010 +0200 +++ b/src/lib-storage/index/dbox-common/dbox-file.h Fri Feb 19 07:15:46 2010 +0200 @@ -41,6 +41,10 @@ DBOX_METADATA_RECEIVED_TIME = 'R', /* Saved UNIX timestamp in hex */ DBOX_METADATA_SAVE_TIME = 'S', + /* Physical message size in hex. Necessary only if it differs from + the dbox_message_header.message_size_hex, for example because the + message is compressed. */ + DBOX_METADATA_PHYSICAL_SIZE = 'Z', /* Virtual message size in hex (line feeds counted as CRLF) */ DBOX_METADATA_VIRTUAL_SIZE = 'V', /* Pointer to external message data. Format is: @@ -135,11 +139,9 @@ int dbox_file_try_lock(struct dbox_file *file); void dbox_file_unlock(struct dbox_file *file); -/* Seek to given offset in file and return the message's input stream - and physical size. Returns 1 if ok/expunged, 0 if file/offset is corrupted, - -1 if I/O error. */ +/* Seek to given offset in file and return the message's input stream. + Returns 1 if ok/expunged, 0 if file/offset is corrupted, -1 if I/O error. */ int dbox_file_get_mail_stream(struct dbox_file *file, uoff_t offset, - uoff_t *physical_size_r, struct istream **input_r); /* Start seeking at the beginning of the file. */ void dbox_file_seek_rewind(struct dbox_file *file);
--- a/src/lib-storage/index/dbox-common/dbox-mail.c Fri Feb 19 07:14:41 2010 +0200 +++ b/src/lib-storage/index/dbox-common/dbox-mail.c Fri Feb 19 07:15:46 2010 +0200 @@ -40,31 +40,59 @@ { struct dbox_storage *storage = (struct dbox_storage *)mail->imail.mail.mail.box->storage; - uoff_t offset, size; + uoff_t offset; if (storage->v.mail_open(mail, &offset, file_r) < 0) return -1; - if (dbox_file_get_mail_stream(*file_r, offset, &size, NULL) <= 0) + if (dbox_file_get_mail_stream(*file_r, offset, NULL) <= 0) return -1; if (dbox_file_metadata_read(*file_r) <= 0) return -1; + + if (mail->imail.data.stream != NULL) { + /* we just messed up mail's input stream by reading metadata */ + i_stream_seek((*file_r)->input, offset); + i_stream_sync(mail->imail.data.stream); + } + return 0; +} + +static int +dbox_mail_metadata_get(struct dbox_mail *mail, enum dbox_metadata_key key, + const char **value_r) +{ + struct dbox_file *file; + + if (dbox_mail_metadata_read(mail, &file) < 0) + return -1; + + *value_r = dbox_file_metadata_get(file, key); return 0; } int dbox_mail_get_physical_size(struct mail *_mail, uoff_t *size_r) { - struct index_mail *mail = (struct index_mail *)_mail; - struct index_mail_data *data = &mail->data; + struct dbox_mail *mail = (struct dbox_mail *)_mail; + struct index_mail_data *data = &mail->imail.data; struct istream *input; + const char *value; if (index_mail_get_physical_size(_mail, size_r) == 0) return 0; - if (mail_get_stream(_mail, NULL, NULL, &input) < 0) + /* see if we have it in metadata */ + if (dbox_mail_metadata_get(mail, DBOX_METADATA_PHYSICAL_SIZE, + &value) < 0) return -1; - i_assert(data->physical_size != (uoff_t)-1); + if (value != NULL) + data->physical_size = strtoul(value, NULL, 16); + else { + if (mail_get_stream(_mail, NULL, NULL, &input) < 0) + return -1; + i_assert(data->physical_size != (uoff_t)-1); + } *size_r = data->physical_size; return 0; } @@ -73,16 +101,14 @@ { struct dbox_mail *mail = (struct dbox_mail *)_mail; struct index_mail_data *data = &mail->imail.data; - struct dbox_file *file; const char *value; if (index_mail_get_cached_virtual_size(&mail->imail, size_r)) return 0; - if (dbox_mail_metadata_read(mail, &file) < 0) + if (dbox_mail_metadata_get(mail, DBOX_METADATA_VIRTUAL_SIZE, + &value) < 0) return -1; - - value = dbox_file_metadata_get(file, DBOX_METADATA_VIRTUAL_SIZE); if (value == NULL) return index_mail_get_virtual_size(_mail, size_r); @@ -95,16 +121,15 @@ { struct dbox_mail *mail = (struct dbox_mail *)_mail; struct index_mail_data *data = &mail->imail.data; - struct dbox_file *file; const char *value; if (index_mail_get_received_date(_mail, date_r) == 0) return 0; - if (dbox_mail_metadata_read(mail, &file) < 0) + if (dbox_mail_metadata_get(mail, DBOX_METADATA_RECEIVED_TIME, + &value) < 0) return -1; - value = dbox_file_metadata_get(file, DBOX_METADATA_RECEIVED_TIME); data->received_date = value == NULL ? 0 : strtoul(value, NULL, 16); *date_r = data->received_date; return 0; @@ -150,7 +175,6 @@ struct index_mail *imail = &mail->imail; struct index_mailbox_context *ibox = INDEX_STORAGE_CONTEXT(imail->mail.mail.box); - struct dbox_file *file; const char *value; string_t *str; @@ -162,10 +186,9 @@ return 0; } - if (dbox_mail_metadata_read(mail, &file) < 0) + if (dbox_mail_metadata_get(mail, key, &value) < 0) return -1; - value = dbox_file_metadata_get(file, key); if (value == NULL) value = ""; index_mail_cache_add_idx(imail, ibox->cache_fields[cache_field].idx, @@ -205,7 +228,7 @@ struct dbox_mail *mail = (struct dbox_mail *)_mail; struct index_mail_data *data = &mail->imail.data; struct istream *input; - uoff_t offset, size; + uoff_t offset; int ret; if (data->stream == NULL) { @@ -213,7 +236,7 @@ return -1; ret = dbox_file_get_mail_stream(mail->open_file, offset, - &size, &input); + &input); if (ret <= 0) { if (ret < 0) return -1; @@ -222,7 +245,6 @@ "%"PRIuUOFF_T, _mail->uid, offset); return -1; } - data->physical_size = size; data->stream = input; }
--- a/src/lib-storage/index/dbox-common/dbox-save.c Fri Feb 19 07:14:41 2010 +0200 +++ b/src/lib-storage/index/dbox-common/dbox-save.c Fri Feb 19 07:15:46 2010 +0200 @@ -94,12 +94,13 @@ { struct ostream *dbox_output = ctx->dbox_output; - if (ctx->ctx.output != dbox_output) { - /* e.g. zlib plugin had changed this */ - o_stream_ref(dbox_output); - o_stream_destroy(&ctx->ctx.output); - ctx->ctx.output = dbox_output; - } + if (ctx->ctx.output == dbox_output) + return; + + /* e.g. zlib plugin had changed this */ + o_stream_ref(dbox_output); + o_stream_destroy(&ctx->ctx.output); + ctx->ctx.output = dbox_output; } void dbox_save_write_metadata(struct mail_save_context *ctx, @@ -118,6 +119,10 @@ o_stream_send(output, &metadata_hdr, sizeof(metadata_hdr)); str = t_str_new(256); + if (ctx->saved_physical_size != 0) { + str_printfa(str, "%c%llx\n", DBOX_METADATA_PHYSICAL_SIZE, + (unsigned long long)ctx->saved_physical_size); + } str_printfa(str, "%c%lx\n", DBOX_METADATA_RECEIVED_TIME, (unsigned long)ctx->received_date); str_printfa(str, "%c%lx\n", DBOX_METADATA_SAVE_TIME,
--- a/src/lib-storage/index/dbox-multi/mdbox-file-purge.c Fri Feb 19 07:14:41 2010 +0200 +++ b/src/lib-storage/index/dbox-multi/mdbox-file-purge.c Fri Feb 19 07:15:46 2010 +0200 @@ -135,10 +135,9 @@ i_array_init(&expunged_map_uids, I_MIN(count, 1)); offset = file->file_header_size; for (i = 0; i < count; i++) { - if ((ret = dbox_file_get_mail_stream(file, offset, - &physical_size, - NULL)) <= 0) + if ((ret = dbox_file_get_mail_stream(file, offset, NULL)) <= 0) break; + physical_size = file->cur_physical_size; msg_size = file->msg_header_size + physical_size; if (msgs[i].offset != offset) {
--- a/src/lib-storage/index/dbox-multi/mdbox-storage-rebuild.c Fri Feb 19 07:14:41 2010 +0200 +++ b/src/lib-storage/index/dbox-multi/mdbox-storage-rebuild.c Fri Feb 19 07:15:46 2010 +0200 @@ -133,7 +133,7 @@ { const char *guid; struct mdbox_rebuild_msg *rec; - uoff_t offset, prev_offset, size; + uoff_t offset, prev_offset; bool last, first, fixed = FALSE; int ret; @@ -162,7 +162,7 @@ if (!first) { /* seek to the offset where we last left off */ ret = dbox_file_get_mail_stream(file, - prev_offset, &size, NULL); + prev_offset, NULL); if (ret <= 0) break; } @@ -508,7 +508,6 @@ struct mdbox_mailbox *mbox; enum mail_error error; bool deleted, created; - uoff_t size; int ret; uint32_t seq; @@ -517,7 +516,7 @@ file = mdbox_file_init(ctx->storage, msg->file_id); ret = dbox_file_open(file, &deleted); if (ret > 0) - ret = dbox_file_get_mail_stream(file, msg->offset, &size, NULL); + ret = dbox_file_get_mail_stream(file, msg->offset, NULL); if (ret > 0 && !deleted && dbox_file_metadata_read(file) > 0) { mailbox = dbox_file_metadata_get(file, DBOX_METADATA_ORIG_MAILBOX);
--- a/src/lib-storage/index/dbox-single/sdbox-sync-rebuild.c Fri Feb 19 07:14:41 2010 +0200 +++ b/src/lib-storage/index/dbox-single/sdbox-sync-rebuild.c Fri Feb 19 07:15:46 2010 +0200 @@ -28,14 +28,13 @@ struct dbox_file *file, uint32_t uid) { uint32_t seq; - uoff_t size; bool deleted; int ret; if ((ret = dbox_file_open(file, &deleted)) > 0) { if (deleted) return 0; - ret = dbox_file_get_mail_stream(file, 0, &size, NULL); + ret = dbox_file_get_mail_stream(file, 0, NULL); } if (ret <= 0) {
--- a/src/lib-storage/mail-storage-private.h Fri Feb 19 07:14:41 2010 +0200 +++ b/src/lib-storage/mail-storage-private.h Fri Feb 19 07:15:46 2010 +0200 @@ -410,6 +410,11 @@ char *guid, *pop3_uidl, *from_envelope; struct ostream *output; + /* if non-zero, overrides the physical size that should be saved. + for example when using zlib plugin, this would contain the mail's + uncompressed size. */ + uoff_t saved_physical_size; + /* we came here from mailbox_copy() */ unsigned int copying:1; };