Mercurial > dovecot > original-hg > dovecot-1.2
changeset 106:5fe3e04ca8d9 HEAD
Added support for caching of MessagePart data. This is useful for fetching
body[part]s. Also BODY and BODYSTRUCTURE can be generated fast using it.
Also fixed index corruption in some situations when more cached data was
added to index.
author | Timo Sirainen <tss@iki.fi> |
---|---|
date | Mon, 02 Sep 2002 05:31:18 +0300 |
parents | 31034993473c |
children | e2fe8921e8ad |
files | src/lib-imap/imap-bodystructure.c src/lib-imap/imap-message-cache.c src/lib-imap/imap-message-cache.h src/lib-index/mail-index-update.c src/lib-index/mail-index.h src/lib-mail/Makefile.am src/lib-mail/message-parser.c src/lib-mail/message-parser.h src/lib-storage/index/Makefile.am src/lib-storage/index/index-fetch-section.c src/lib-storage/index/index-fetch.c src/lib-storage/index/index-storage.c src/lib-storage/index/index-storage.h |
diffstat | 13 files changed, 282 insertions(+), 354 deletions(-) [+] |
line wrap: on
line diff
--- a/src/lib-imap/imap-bodystructure.c Sun Sep 01 02:57:37 2002 +0300 +++ b/src/lib-imap/imap-bodystructure.c Mon Sep 02 05:31:18 2002 +0300 @@ -113,7 +113,8 @@ MessagePartBodyData *part_data; int parent_rfc822; - parent_rfc822 = part->parent != NULL && part->parent->message_rfc822; + parent_rfc822 = part->parent != NULL && + (part->parent->flags & MESSAGE_PART_FLAG_MESSAGE_RFC822); if (!parent_rfc822 && (name_len <= 8 || strncasecmp(name, "Content-", 8) != 0)) return; @@ -271,10 +272,10 @@ NVL(data->content_transfer_encoding, "\"8bit\""), part->body_size.virtual_size); - if (part->text) { + if (part->flags & MESSAGE_PART_FLAG_TEXT) { /* text/.. contains line count */ t_string_printfa(str, " %u", part->body_size.lines); - } else if (part->message_rfc822) { + } else if (part->flags & MESSAGE_PART_FLAG_MESSAGE_RFC822) { /* message/rfc822 contains envelope + body + line count */ MessagePartBodyData *child_data; @@ -336,7 +337,7 @@ { while (part != NULL) { t_string_append_c(str, '('); - if (part->multipart) + if (part->flags & MESSAGE_PART_FLAG_MULTIPART) part_write_body_multipart(part, str, extended); else part_write_body(part, str, extended);
--- a/src/lib-imap/imap-message-cache.c Sun Sep 01 02:57:37 2002 +0300 +++ b/src/lib-imap/imap-message-cache.c Mon Sep 02 05:31:18 2002 +0300 @@ -5,6 +5,7 @@ #include "temp-string.h" #include "mmap-util.h" #include "message-parser.h" +#include "message-part-serialize.h" #include "message-size.h" #include "imap-bodystructure.h" #include "imap-envelope.h" @@ -40,6 +41,8 @@ }; struct _ImapMessageCache { + ImapMessageCacheIface *iface; + CachedMessage *messages; int messages_count; @@ -47,13 +50,16 @@ IOBuffer *open_inbuf; uoff_t open_virtual_size; - IOBuffer *(*inbuf_rewind)(IOBuffer *inbuf, void *context); void *context; }; -ImapMessageCache *imap_msgcache_alloc(void) +ImapMessageCache *imap_msgcache_alloc(ImapMessageCacheIface *iface) { - return i_new(ImapMessageCache, 1); + ImapMessageCache *cache; + + cache = i_new(ImapMessageCache, 1); + cache->iface = iface; + return cache; } static void cached_message_free(CachedMessage *msg) @@ -151,132 +157,128 @@ } } -static CachedMessage *cache_find(ImapMessageCache *cache, unsigned int uid) +static void imap_msgcache_get_inbuf(ImapMessageCache *cache, uoff_t offset) { - CachedMessage *msg; - - for (msg = cache->messages; msg != NULL; msg = msg->next) { - if (msg->uid == uid) - return msg; + if (cache->open_inbuf == NULL) + cache->open_inbuf = cache->iface->open_mail(cache->context); + else if (offset < cache->open_inbuf->offset) { + /* need to rewind */ + cache->open_inbuf = + cache->iface->inbuf_rewind(cache->open_inbuf, + cache->context); } - return NULL; -} - -static void imap_msgcache_get_inbuf(ImapMessageCache *cache, uoff_t offset) -{ - if (offset < cache->open_inbuf->offset) { - /* need to rewind */ - cache->open_inbuf = cache->inbuf_rewind(cache->open_inbuf, - cache->context); - if (cache->open_inbuf == NULL) - i_fatal("Can't rewind message buffer"); - } + if (cache->open_inbuf == NULL) + i_fatal("Can't get input buffer for message"); io_buffer_skip(cache->open_inbuf, offset - cache->open_inbuf->offset); } -int imap_msgcache_is_cached(ImapMessageCache *cache, unsigned int uid, - ImapCacheField fields) +static void msg_get_part(ImapMessageCache *cache) { - CachedMessage *msg; - - if (cache->open_msg != NULL && cache->open_msg->uid == uid) - return TRUE; - - /* not open, see if the wanted fields are cached */ - msg = cache_find(cache, uid); - if (msg == NULL) - return FALSE; - - if ((fields & IMAP_CACHE_BODY) && msg->cached_body == NULL) - return FALSE; - if ((fields & IMAP_CACHE_BODYSTRUCTURE) && - msg->cached_bodystructure == NULL) - return FALSE; - if ((fields & IMAP_CACHE_ENVELOPE) && msg->cached_envelope == NULL) - return FALSE; - - if ((fields & IMAP_CACHE_MESSAGE_OPEN) && msg != cache->open_msg) - return FALSE; - if ((fields & IMAP_CACHE_MESSAGE_PART) && msg->part == NULL) - return FALSE; - if ((fields & IMAP_CACHE_MESSAGE_HDR_SIZE) && msg->hdr_size == NULL) - return FALSE; - if ((fields & IMAP_CACHE_MESSAGE_BODY_SIZE) && msg->body_size == NULL) - return FALSE; - - return TRUE; + if (cache->open_msg->part == NULL) { + cache->open_msg->part = + cache->iface->get_cached_parts(cache->open_msg->pool, + cache->context); + } } /* Caches the fields for given message if possible */ -static void cache_fields(ImapMessageCache *cache, CachedMessage *msg, - ImapCacheField fields) +static void cache_fields(ImapMessageCache *cache, ImapCacheField fields) { + CachedMessage *msg; const char *value; + msg = cache->open_msg; + t_push(); - if ((fields & IMAP_CACHE_BODY) && msg->cached_body == NULL && - msg == cache->open_msg) { - imap_msgcache_get_inbuf(cache, 0); - value = imap_part_get_bodystructure(msg->pool, &msg->part, - cache->open_inbuf, FALSE); + if ((fields & IMAP_CACHE_BODY) && msg->cached_body == NULL) { + value = cache->iface->get_cached_field(IMAP_CACHE_BODY, + cache->context); + if (value == NULL) { + msg_get_part(cache); + + imap_msgcache_get_inbuf(cache, 0); + value = imap_part_get_bodystructure(msg->pool, + &msg->part, + cache->open_inbuf, + FALSE); + } msg->cached_body = p_strdup(msg->pool, value); } if ((fields & IMAP_CACHE_BODYSTRUCTURE) && - msg->cached_bodystructure == NULL && msg == cache->open_msg) { - imap_msgcache_get_inbuf(cache, 0); - value = imap_part_get_bodystructure(msg->pool, &msg->part, - cache->open_inbuf, TRUE); + msg->cached_bodystructure == NULL) { + value = cache->iface->get_cached_field(IMAP_CACHE_BODYSTRUCTURE, + cache->context); + if (value == NULL) { + msg_get_part(cache); + + imap_msgcache_get_inbuf(cache, 0); + value = imap_part_get_bodystructure(msg->pool, + &msg->part, + cache->open_inbuf, + TRUE); + } msg->cached_bodystructure = p_strdup(msg->pool, value); } if ((fields & IMAP_CACHE_ENVELOPE) && msg->cached_envelope == NULL) { - if (msg->envelope == NULL && msg == cache->open_msg) { - /* envelope isn't parsed yet, do it. header size - is calculated anyway so save it */ - if (msg->hdr_size == NULL) { - msg->hdr_size = p_new(msg->pool, - MessageSize, 1); + value = cache->iface->get_cached_field(IMAP_CACHE_ENVELOPE, + cache->context); + if (value == NULL) { + if (msg->envelope == NULL) { + /* envelope isn't parsed yet, do it. header + size is calculated anyway so save it */ + if (msg->hdr_size == NULL) { + msg->hdr_size = p_new(msg->pool, + MessageSize, 1); + } + + imap_msgcache_get_inbuf(cache, 0); + message_parse_header(NULL, cache->open_inbuf, + msg->hdr_size, + parse_envelope_header, + msg); + } + + if (msg->envelope != NULL) { + value = imap_envelope_get_part_data( + msg->envelope); + } + } + + if (value != NULL) + msg->cached_envelope = p_strdup(msg->pool, value); + } + + if (fields & IMAP_CACHE_MESSAGE_PART) { + msg_get_part(cache); + + if (msg->part == NULL) { + /* we need to parse the message */ + MessageHeaderFunc func; + + if ((fields & IMAP_CACHE_ENVELOPE) && + msg->cached_envelope == NULL) { + /* we need envelope too, fill the info + while parsing headers */ + func = parse_envelope_header; + } else { + func = NULL; } imap_msgcache_get_inbuf(cache, 0); - message_parse_header(NULL, cache->open_inbuf, - msg->hdr_size, - parse_envelope_header, msg); - } - - if (msg->envelope != NULL) { - value = imap_envelope_get_part_data(msg->envelope); - msg->cached_envelope = p_strdup(msg->pool, value); + msg->part = message_parse(msg->pool, cache->open_inbuf, + func, msg); } } - if ((fields & IMAP_CACHE_MESSAGE_PART) && msg->part == NULL && - msg == cache->open_msg) { - /* we need to parse the message */ - MessageHeaderFunc func; + if ((fields & IMAP_CACHE_MESSAGE_BODY_SIZE) && msg->body_size == NULL) { + msg_get_part(cache); - if ((fields & IMAP_CACHE_ENVELOPE) && - msg->cached_envelope == NULL) { - /* we need envelope too, fill the info - while parsing headers */ - func = parse_envelope_header; - } else { - func = NULL; - } - - imap_msgcache_get_inbuf(cache, 0); - msg->part = message_parse(msg->pool, cache->open_inbuf, - func, msg); - } - - if ((fields & IMAP_CACHE_MESSAGE_BODY_SIZE) && - msg->body_size == NULL && - (msg == cache->open_msg || msg->part != NULL)) { - /* fill the body size, and while at it fill the header size - as well */ + /* fill the body size, and while at it fill the header + size as well */ if (msg->hdr_size == NULL) msg->hdr_size = p_new(msg->pool, MessageSize, 1); msg->body_size = p_new(msg->pool, MessageSize, 1); @@ -286,14 +288,14 @@ *msg->hdr_size = msg->part->header_size; *msg->body_size = msg->part->body_size; } else { - /* first get the header's size, then calculate the - body size from it and the total virtual size */ + /* first get the header's size, then calculate the body + size from it and the total virtual size */ imap_msgcache_get_inbuf(cache, 0); message_get_header_size(cache->open_inbuf, msg->hdr_size); - /* FIXME: this may actually happen if file size is - shrinked.. */ + /* FIXME: this may actually happen if file size + is shrinked.. */ i_assert(msg->hdr_size->physical_size <= cache->open_inbuf->size); i_assert(msg->hdr_size->virtual_size <= @@ -309,10 +311,10 @@ } } - if ((fields & IMAP_CACHE_MESSAGE_HDR_SIZE) && msg->hdr_size == NULL && - (msg == cache->open_msg || msg->part != NULL)) { + if ((fields & IMAP_CACHE_MESSAGE_HDR_SIZE) && msg->hdr_size == NULL) { + msg_get_part(cache); + msg->hdr_size = p_new(msg->pool, MessageSize, 1); - if (msg->part != NULL) { /* easy, get it from root part */ *msg->hdr_size = msg->part->header_size; @@ -327,13 +329,10 @@ t_pop(); } -void imap_msgcache_message(ImapMessageCache *cache, unsigned int uid, - ImapCacheField fields, uoff_t virtual_size, - uoff_t pv_headers_size, uoff_t pv_body_size, - IOBuffer *inbuf, - IOBuffer *(*inbuf_rewind)(IOBuffer *inbuf, - void *context), - void *context) +void imap_msgcache_open(ImapMessageCache *cache, unsigned int uid, + ImapCacheField fields, uoff_t virtual_size, + uoff_t pv_headers_size, uoff_t pv_body_size, + void *context) { CachedMessage *msg; @@ -342,10 +341,7 @@ imap_msgcache_close(cache); cache->open_msg = msg; - cache->open_inbuf = inbuf; cache->open_virtual_size = virtual_size; - - cache->inbuf_rewind = inbuf_rewind; cache->context = context; } @@ -363,7 +359,7 @@ pv_body_size; } - cache_fields(cache, msg, fields); + cache_fields(cache, fields); } void imap_msgcache_close(ImapMessageCache *cache) @@ -376,53 +372,28 @@ cache->open_msg = NULL; cache->open_virtual_size = 0; + cache->context = NULL; } -void imap_msgcache_set(ImapMessageCache *cache, unsigned int uid, - ImapCacheField field, const char *value) +const char *imap_msgcache_get(ImapMessageCache *cache, ImapCacheField field) { CachedMessage *msg; - msg = cache_find(cache, uid); - if (msg == NULL) - msg = cache_new(cache, uid); + i_assert(cache->open_msg != NULL); - switch (field) { - case IMAP_CACHE_BODY: - msg->cached_body = p_strdup(msg->pool, value); - break; - case IMAP_CACHE_BODYSTRUCTURE: - msg->cached_bodystructure = p_strdup(msg->pool, value); - break; - case IMAP_CACHE_ENVELOPE: - msg->cached_envelope = p_strdup(msg->pool, value); - break; - default: - i_assert(0); - } -} - -const char *imap_msgcache_get(ImapMessageCache *cache, unsigned int uid, - ImapCacheField field) -{ - CachedMessage *msg; - - msg = cache_find(cache, uid); - if (msg == NULL) - return NULL; - + msg = cache->open_msg; switch (field) { case IMAP_CACHE_BODY: if (msg->cached_body == NULL) - cache_fields(cache, msg, field); + cache_fields(cache, field); return msg->cached_body; case IMAP_CACHE_BODYSTRUCTURE: if (msg->cached_bodystructure == NULL) - cache_fields(cache, msg, field); + cache_fields(cache, field); return msg->cached_bodystructure; case IMAP_CACHE_ENVELOPE: if (msg->cached_envelope == NULL) - cache_fields(cache, msg, field); + cache_fields(cache, field); return msg->cached_envelope; default: i_assert(0); @@ -431,45 +402,34 @@ return NULL; } -MessagePart *imap_msgcache_get_parts(ImapMessageCache *cache, unsigned int uid) +MessagePart *imap_msgcache_get_parts(ImapMessageCache *cache) { - CachedMessage *msg; + i_assert(cache->open_msg != NULL); - msg = cache_find(cache, uid); - if (msg == NULL) - return NULL; - - if (msg->part == NULL) - cache_fields(cache, msg, IMAP_CACHE_MESSAGE_PART); - return msg->part; + if (cache->open_msg->part == NULL) + cache_fields(cache, IMAP_CACHE_MESSAGE_PART); + return cache->open_msg->part; } -int imap_msgcache_get_rfc822(ImapMessageCache *cache, unsigned int uid, - MessageSize *hdr_size, MessageSize *body_size, - IOBuffer **inbuf) +int imap_msgcache_get_rfc822(ImapMessageCache *cache, IOBuffer **inbuf, + MessageSize *hdr_size, MessageSize *body_size) { CachedMessage *msg; uoff_t offset; + i_assert(cache->open_msg != NULL); + + msg = cache->open_msg; if (inbuf != NULL) { - if (cache->open_msg == NULL || cache->open_msg->uid != uid) - return FALSE; - - msg = cache->open_msg; - offset = hdr_size != NULL ? 0 : msg->hdr_size->physical_size; imap_msgcache_get_inbuf(cache, offset); *inbuf = cache->open_inbuf; - } else { - msg = cache_find(cache, uid); - if (msg == NULL) - return FALSE; } if (body_size != NULL) { if (msg->body_size == NULL) - cache_fields(cache, msg, IMAP_CACHE_MESSAGE_BODY_SIZE); + cache_fields(cache, IMAP_CACHE_MESSAGE_BODY_SIZE); if (msg->body_size == NULL) return FALSE; *body_size = *msg->body_size; @@ -477,7 +437,7 @@ if (hdr_size != NULL) { if (msg->hdr_size == NULL) - cache_fields(cache, msg, IMAP_CACHE_MESSAGE_HDR_SIZE); + cache_fields(cache, IMAP_CACHE_MESSAGE_HDR_SIZE); if (msg->hdr_size == NULL) return FALSE; *hdr_size = *msg->hdr_size; @@ -518,7 +478,7 @@ message_get_body_size(inbuf, dest, max_virtual_size); } -int imap_msgcache_get_rfc822_partial(ImapMessageCache *cache, unsigned int uid, +int imap_msgcache_get_rfc822_partial(ImapMessageCache *cache, uoff_t virtual_skip, uoff_t max_virtual_size, int get_header, MessageSize *size, @@ -528,10 +488,9 @@ uoff_t physical_skip; int size_got; + i_assert(cache->open_msg != NULL); + msg = cache->open_msg; - if (msg == NULL || msg->uid != uid) - return FALSE; - if (msg->hdr_size == NULL) { msg->hdr_size = p_new(msg->pool, MessageSize, 1); imap_msgcache_get_inbuf(cache, 0); @@ -580,11 +539,9 @@ return TRUE; } -int imap_msgcache_get_data(ImapMessageCache *cache, unsigned int uid, - IOBuffer **inbuf) +int imap_msgcache_get_data(ImapMessageCache *cache, IOBuffer **inbuf) { - if (cache->open_msg == NULL || cache->open_msg->uid != uid) - return FALSE; + i_assert(cache->open_msg != NULL); imap_msgcache_get_inbuf(cache, 0); *inbuf = cache->open_inbuf;
--- a/src/lib-imap/imap-message-cache.h Sun Sep 01 02:57:37 2002 +0300 +++ b/src/lib-imap/imap-message-cache.h Mon Sep 02 05:31:18 2002 +0300 @@ -22,60 +22,57 @@ IMAP_CACHE_MESSAGE_BODY_SIZE = 0x40 } ImapCacheField; +typedef struct { + /* Open mail for reading. */ + IOBuffer *(*open_mail)(void *context); + /* Rewind input buffer to beginning, possibly closing the old buffer + if it can't directly be rewinded. */ + IOBuffer *(*inbuf_rewind)(IOBuffer *inbuf, void *context); + + /* Returns field if it's already cached, or NULL. */ + const char *(*get_cached_field)(ImapCacheField field, void *context); + /* Returns MessagePart if it's already cached, or NULL. */ + MessagePart *(*get_cached_parts)(Pool pool, void *context); +} ImapMessageCacheIface; + typedef struct _ImapMessageCache ImapMessageCache; -ImapMessageCache *imap_msgcache_alloc(void); +ImapMessageCache *imap_msgcache_alloc(ImapMessageCacheIface *iface); void imap_msgcache_clear(ImapMessageCache *cache); void imap_msgcache_free(ImapMessageCache *cache); -/* Returns TRUE if all given fields are fully cached, or at least the - message is open (ie. you don't need imap_msgcache_message()). */ -int imap_msgcache_is_cached(ImapMessageCache *cache, unsigned int uid, - ImapCacheField fields); - -/* Parse and cache the message. If pv_headers_size and pv_body_size is +/* Open the specified message. If pv_headers_size and pv_body_size is non-zero, they're set to saved to message's both physical and virtual sizes (ie. doesn't need to be calculated). */ -void imap_msgcache_message(ImapMessageCache *cache, unsigned int uid, - ImapCacheField fields, uoff_t virtual_size, - uoff_t pv_headers_size, uoff_t pv_body_size, - IOBuffer *inbuf, - IOBuffer *(*inbuf_rewind)(IOBuffer *inbuf, - void *context), - void *context); +void imap_msgcache_open(ImapMessageCache *cache, unsigned int uid, + ImapCacheField fields, uoff_t virtual_size, + uoff_t pv_headers_size, uoff_t pv_body_size, + void *context); -/* Close the IOBuffer for cached message. */ +/* Close the IOBuffer for opened message. */ void imap_msgcache_close(ImapMessageCache *cache); -/* Store a value for field in cache */ -void imap_msgcache_set(ImapMessageCache *cache, unsigned int uid, - ImapCacheField field, const char *value); +/* Returns the field from cache, or NULL if it's not cached. */ +const char *imap_msgcache_get(ImapMessageCache *cache, ImapCacheField field); -/* Returns the field from cache, or NULL if it's not cached. */ -const char *imap_msgcache_get(ImapMessageCache *cache, unsigned int uid, - ImapCacheField field); - -/* Returns the root MessagePart for message, or NULL if it's not cached. */ -MessagePart *imap_msgcache_get_parts(ImapMessageCache *cache, unsigned int uid); +/* Returns the root MessagePart for message, or NULL if failed. */ +MessagePart *imap_msgcache_get_parts(ImapMessageCache *cache); -/* Returns FALSE if message isn't in cache. If inbuf is not NULL, it's set - to point to beginning of message, or to beginning of message body if - hdr_size is NULL. */ -int imap_msgcache_get_rfc822(ImapMessageCache *cache, unsigned int uid, - MessageSize *hdr_size, MessageSize *body_size, - IOBuffer **inbuf); +/* Returns TRUE if successful. If inbuf is not NULL, it's set to point to + beginning of message, or to beginning of message body if hdr_size is NULL. */ +int imap_msgcache_get_rfc822(ImapMessageCache *cache, IOBuffer **inbuf, + MessageSize *hdr_size, MessageSize *body_size); -/* Returns FALSE if message isn't in cache. *inbuf is set to point to the first - non-skipped character. size is set to specify the full size of message. */ -int imap_msgcache_get_rfc822_partial(ImapMessageCache *cache, unsigned int uid, +/* Returns TRUE if successful. *inbuf is set to point to the first non-skipped + character. size is set to specify the full size of message. */ +int imap_msgcache_get_rfc822_partial(ImapMessageCache *cache, uoff_t virtual_skip, uoff_t max_virtual_size, int get_header, MessageSize *size, IOBuffer **inbuf); -/* Returns FALSE if message isn't in cache. *inbuf is set to point to - beginning of message. */ -int imap_msgcache_get_data(ImapMessageCache *cache, unsigned int uid, - IOBuffer **inbuf); +/* Returns TRUE if successful. *inbuf is set to point to beginning of + message. */ +int imap_msgcache_get_data(ImapMessageCache *cache, IOBuffer **inbuf); #endif
--- a/src/lib-index/mail-index-update.c Sun Sep 01 02:57:37 2002 +0300 +++ b/src/lib-index/mail-index-update.c Mon Sep 02 05:31:18 2002 +0300 @@ -6,6 +6,7 @@ #include "rfc822-date.h" #include "rfc822-tokenize.h" #include "message-parser.h" +#include "message-part-serialize.h" #include "message-size.h" #include "imap-envelope.h" #include "imap-bodystructure.h" @@ -155,7 +156,7 @@ update->field_extra_sizes[i]; src = update->fields[i]; src_size = update->field_sizes[i]; - } else if (rec != NULL) { + } else if (rec != NULL && rec->field == field) { /* use the old value */ destrec->full_field_size = rec->full_field_size; src = rec->data; @@ -171,7 +172,9 @@ index_set_error(update->index, "Error in index file %s: " "full_field_size points outside " - "data_size", update->index->filepath); + "data_size (field %d?)", + update->index->filepath, + rec == NULL ? -1 : rec->field); update->index->header->flags |= MAIL_INDEX_FLAG_REBUILD; return FALSE; } @@ -387,6 +390,7 @@ MessageSize hdr_size, body_size; Pool pool; const char *value; + unsigned int size; ctx.update = update; ctx.envelope_pool = NULL; @@ -397,7 +401,7 @@ if (cache_fields == 0) cache_fields = update->index->header->cache_fields; - if ((cache_fields & (FIELD_TYPE_BODY|FIELD_TYPE_BODYSTRUCTURE)) != 0) { + if (IS_BODYSTRUCTURE_FIELD(cache_fields)) { /* for body / bodystructure, we need need to fully parse the message */ pool = pool_create("index message parser", 2048, FALSE); @@ -431,6 +435,15 @@ t_pop(); } + if (cache_fields & FIELD_TYPE_MESSAGEPART) { + t_push(); + value = message_part_serialize(part, &size); + update->index->update_field_raw(update, + FIELD_TYPE_MESSAGEPART, + value, size); + t_pop(); + } + pool_unref(pool); } else { message_parse_header(NULL, inbuf, &hdr_size,
--- a/src/lib-index/mail-index.h Sun Sep 01 02:57:37 2002 +0300 +++ b/src/lib-index/mail-index.h Mon Sep 02 05:31:18 2002 +0300 @@ -43,15 +43,20 @@ FIELD_TYPE_BCC = 0x0080, FIELD_TYPE_SUBJECT = 0x0100, FIELD_TYPE_MD5 = 0x0200, + FIELD_TYPE_MESSAGEPART = 0x0400, - FIELD_TYPE_LAST = 0x0400, - FIELD_TYPE_MAX_BITS = 10 + FIELD_TYPE_LAST = 0x0800, + FIELD_TYPE_MAX_BITS = 11 } MailField; #define IS_HEADER_FIELD(field) \ (((field) & (FIELD_TYPE_FROM | FIELD_TYPE_TO | FIELD_TYPE_CC | \ FIELD_TYPE_BCC | FIELD_TYPE_SUBJECT)) != 0) +#define IS_BODYSTRUCTURE_FIELD(field) \ + (((field) & (FIELD_TYPE_BODY|FIELD_TYPE_BODYSTRUCTURE| \ + FIELD_TYPE_MESSAGEPART)) != 0) + typedef enum { MAIL_LOCK_UNLOCK = 0, MAIL_LOCK_SHARED,
--- a/src/lib-mail/Makefile.am Sun Sep 01 02:57:37 2002 +0300 +++ b/src/lib-mail/Makefile.am Mon Sep 02 05:31:18 2002 +0300 @@ -4,8 +4,9 @@ -I$(top_srcdir)/src/lib libmail_a_SOURCES = \ + message-content-parser.c \ message-parser.c \ - message-content-parser.c \ + message-part-serialize.c \ message-send.c \ message-size.c \ rfc822-address.c \ @@ -13,8 +14,9 @@ rfc822-tokenize.c noinst_HEADERS = \ + message-content-parser.h \ message-parser.h \ - message-content-parser.h \ + message-part-serialize.h \ message-send.h \ message-size.h \ rfc822-address.h \
--- a/src/lib-mail/message-parser.c Sun Sep 01 02:57:37 2002 +0300 +++ b/src/lib-mail/message-parser.c Mon Sep 02 05:31:18 2002 +0300 @@ -78,14 +78,16 @@ parse_ctx->last_content_type = p_strdup(parse_ctx->pool, str); if (strcasecmp(str, "message/rfc822") == 0) - parse_ctx->part->message_rfc822 = TRUE; + parse_ctx->part->flags |= MESSAGE_PART_FLAG_MESSAGE_RFC822; else if (strncasecmp(str, "text/", 5) == 0) - parse_ctx->part->text = TRUE; + parse_ctx->part->flags |= MESSAGE_PART_FLAG_TEXT; else if (strncasecmp(str, "multipart/", 10) == 0) { - parse_ctx->part->multipart = TRUE; + parse_ctx->part->flags |= MESSAGE_PART_FLAG_MULTIPART; - if (strcasecmp(str+10, "digest") == 0) - parse_ctx->part->multipart_digest = TRUE; + if (strcasecmp(str+10, "digest") == 0) { + parse_ctx->part->flags |= + MESSAGE_PART_FLAG_MULTIPART_DIGEST; + } } } @@ -96,8 +98,8 @@ MessageParseContext *parse_ctx = context; const char *str; - if (!parse_ctx->part->multipart || name->len != 8 || - strncasecmp(name->ptr, "boundary", 8) != 0) + if ((parse_ctx->part->flags & MESSAGE_PART_FLAG_MULTIPART) == 0 || + name->len != 8 || strncasecmp(name->ptr, "boundary", 8) != 0) return; if (parse_ctx->last_boundary == NULL) { @@ -201,21 +203,23 @@ if (parse_ctx->last_content_type == NULL) { if (parse_ctx->part->parent != NULL && - parse_ctx->part->parent->multipart_digest) { + (parse_ctx->part->parent->flags & + MESSAGE_PART_FLAG_MULTIPART_DIGEST)) { /* when there's no content-type specified and we're below multipart/digest, the assume message/rfc822 content-type */ - parse_ctx->part->message_rfc822 = TRUE; + parse_ctx->part->flags |= + MESSAGE_PART_FLAG_MESSAGE_RFC822; } else { /* otherwise we default to text/plain */ - parse_ctx->part->text = TRUE; + parse_ctx->part->flags |= MESSAGE_PART_FLAG_TEXT; } } parse_ctx->last_boundary = NULL; parse_ctx->last_content_type = NULL; - if (parse_ctx->part->message_rfc822) { + if (parse_ctx->part->flags & MESSAGE_PART_FLAG_MESSAGE_RFC822) { /* message/rfc822 part - the message body begins with headers again, this works pretty much the same as a single multipart/mixed item */
--- a/src/lib-mail/message-parser.h Sun Sep 01 02:57:37 2002 +0300 +++ b/src/lib-mail/message-parser.h Mon Sep 02 05:31:18 2002 +0300 @@ -5,6 +5,18 @@ typedef struct _MessagePosition MessagePosition; typedef struct _MessageSize MessageSize; +typedef enum { + MESSAGE_PART_FLAG_MULTIPART = 0x01, + MESSAGE_PART_FLAG_MULTIPART_DIGEST = 0x02, + MESSAGE_PART_FLAG_MESSAGE_RFC822 = 0x04, + + /* content-type: text/... */ + MESSAGE_PART_FLAG_TEXT = 0x08, + + /* content-transfer-encoding: binary */ + MESSAGE_PART_FLAG_BINARY = 0x10 +} MessagePartFlags; + struct _MessageSize { uoff_t physical_size; uoff_t virtual_size; @@ -20,12 +32,7 @@ MessageSize header_size; MessageSize body_size; - unsigned int multipart:1; - unsigned int multipart_digest:1; - unsigned int message_rfc822:1; - unsigned int text:1; /* content-type: text/.. */ - unsigned int binary:1; /* content-transfer-encoding: binary */ - + MessagePartFlags flags; void *context; };
--- a/src/lib-storage/index/Makefile.am Sun Sep 01 02:57:37 2002 +0300 +++ b/src/lib-storage/index/Makefile.am Mon Sep 02 05:31:18 2002 +0300 @@ -14,6 +14,7 @@ index-expunge.c \ index-fetch.c \ index-fetch-section.c \ + index-msgcache.c \ index-save.c \ index-search.c \ index-status.c \
--- a/src/lib-storage/index/index-fetch-section.c Sun Sep 01 02:57:37 2002 +0300 +++ b/src/lib-storage/index/index-fetch-section.c Mon Sep 02 05:31:18 2002 +0300 @@ -38,9 +38,9 @@ IOBuffer *inbuf; const char *str; - if (!imap_msgcache_get_rfc822_partial(ctx->cache, rec->uid, - sect->skip, sect->max_size, - fetch_header, &size, &inbuf)) { + if (!imap_msgcache_get_rfc822_partial(ctx->cache, sect->skip, + sect->max_size, fetch_header, + &size, &inbuf)) { i_error("Couldn't get BODY[] for UID %u (index %s)", rec->uid, ctx->index->filepath); return FALSE; @@ -252,14 +252,12 @@ } /* fetch BODY[HEADER...] */ -static int fetch_header(MailIndexRecord *rec, MailFetchBodyData *sect, - FetchContext *ctx) +static int fetch_header(MailFetchBodyData *sect, FetchContext *ctx) { MessageSize hdr_size; IOBuffer *inbuf; - if (!imap_msgcache_get_rfc822(ctx->cache, rec->uid, - &hdr_size, NULL, &inbuf)) + if (!imap_msgcache_get_rfc822(ctx->cache, &inbuf, &hdr_size, NULL)) return FALSE; fetch_header_from(inbuf, &hdr_size, sect->section, sect, ctx); @@ -267,14 +265,14 @@ } /* Find MessagePart for section (eg. 1.3.4) */ -static MessagePart *part_find(MailIndexRecord *rec, MailFetchBodyData *sect, - FetchContext *ctx, const char **section) +static MessagePart *part_find(MailFetchBodyData *sect, FetchContext *ctx, + const char **section) { MessagePart *part; const char *path; unsigned int num; - part = imap_msgcache_get_parts(ctx->cache, rec->uid); + part = imap_msgcache_get_parts(ctx->cache); path = sect->section; while (*path >= '0' && *path <= '9' && part != NULL) { @@ -290,7 +288,7 @@ if (*path == '.') path++; - if (part->multipart) { + if (part->flags & MESSAGE_PART_FLAG_MULTIPART) { /* find the part */ part = part->children; for (; num > 1 && part != NULL; num--) @@ -307,14 +305,14 @@ } /* fetch BODY[1.2] or BODY[1.2.TEXT] */ -static int fetch_part_body(MessagePart *part, unsigned int uid, - MailFetchBodyData *sect, FetchContext *ctx) +static int fetch_part_body(MessagePart *part, MailFetchBodyData *sect, + FetchContext *ctx) { IOBuffer *inbuf; const char *str; uoff_t skip_pos; - if (!imap_msgcache_get_data(ctx->cache, uid, &inbuf)) + if (!imap_msgcache_get_data(ctx->cache, &inbuf)) return FALSE; /* jump to beginning of wanted data */ @@ -333,13 +331,12 @@ } /* fetch BODY[1.2.MIME|HEADER...] */ -static int fetch_part_header(MessagePart *part, unsigned int uid, - const char *section, MailFetchBodyData *sect, - FetchContext *ctx) +static int fetch_part_header(MessagePart *part, const char *section, + MailFetchBodyData *sect, FetchContext *ctx) { IOBuffer *inbuf; - if (!imap_msgcache_get_data(ctx->cache, uid, &inbuf)) + if (!imap_msgcache_get_data(ctx->cache, &inbuf)) return FALSE; io_buffer_skip(inbuf, part->physical_pos); @@ -347,23 +344,22 @@ return TRUE; } -static int fetch_part(MailIndexRecord *rec, MailFetchBodyData *sect, - FetchContext *ctx) +static int fetch_part(MailFetchBodyData *sect, FetchContext *ctx) { MessagePart *part; const char *section; - part = part_find(rec, sect, ctx, §ion); + part = part_find(sect, ctx, §ion); if (part == NULL) return FALSE; if (*section == '\0' || strcasecmp(section, "TEXT") == 0) - return fetch_part_body(part, rec->uid, sect, ctx); + return fetch_part_body(part, sect, ctx); if (strncasecmp(section, "HEADER", 6) == 0) - return fetch_part_header(part, rec->uid, section, sect, ctx); + return fetch_part_header(part, section, sect, ctx); if (strcasecmp(section, "MIME") == 0) - return fetch_part_header(part, rec->uid, section, sect, ctx); + return fetch_part_header(part, section, sect, ctx); return FALSE; } @@ -386,9 +382,9 @@ } else if (strcasecmp(sect->section, "TEXT") == 0) { fetch_ok = fetch_body(rec, sect, ctx, FALSE); } else if (strncasecmp(sect->section, "HEADER", 6) == 0) { - fetch_ok = fetch_header(rec, sect, ctx); + fetch_ok = fetch_header(sect, ctx); } else if (*sect->section >= '0' && *sect->section <= '9') { - fetch_ok = fetch_part(rec, sect, ctx); + fetch_ok = fetch_part(sect, ctx); } else { fetch_ok = FALSE; }
--- a/src/lib-storage/index/index-fetch.c Sun Sep 01 02:57:37 2002 +0300 +++ b/src/lib-storage/index/index-fetch.c Mon Sep 02 05:31:18 2002 +0300 @@ -17,7 +17,7 @@ { const char *body; - body = imap_msgcache_get(ctx->cache, rec->uid, IMAP_CACHE_BODY); + body = imap_msgcache_get(ctx->cache, IMAP_CACHE_BODY); if (body != NULL) t_string_printfa(ctx->str, " BODY %s", body); else { @@ -30,8 +30,7 @@ { const char *bodystructure; - bodystructure = imap_msgcache_get(ctx->cache, rec->uid, - IMAP_CACHE_BODYSTRUCTURE); + bodystructure = imap_msgcache_get(ctx->cache, IMAP_CACHE_BODYSTRUCTURE); if (bodystructure != NULL) { t_string_printfa(ctx->str, " BODYSTRUCTURE %s", bodystructure); @@ -45,8 +44,7 @@ { const char *envelope; - envelope = imap_msgcache_get(ctx->cache, rec->uid, - IMAP_CACHE_ENVELOPE); + envelope = imap_msgcache_get(ctx->cache, IMAP_CACHE_ENVELOPE); if (envelope != NULL) t_string_printfa(ctx->str, " ENVELOPE (%s)", envelope); else { @@ -92,8 +90,8 @@ IOBuffer *inbuf; const char *str; - if (!imap_msgcache_get_rfc822(ctx->cache, rec->uid, - &hdr_size, &body_size, &inbuf)) { + if (!imap_msgcache_get_rfc822(ctx->cache, &inbuf, + &hdr_size, &body_size)) { i_error("Couldn't get RFC822 for UID %u (index %s)", rec->uid, ctx->index->filepath); return; @@ -115,8 +113,7 @@ IOBuffer *inbuf; const char *str; - if (!imap_msgcache_get_rfc822(ctx->cache, rec->uid, - &hdr_size, NULL, &inbuf)) { + if (!imap_msgcache_get_rfc822(ctx->cache, &inbuf, &hdr_size, NULL)) { i_error("Couldn't get RFC822.HEADER for UID %u (index %s)", rec->uid, ctx->index->filepath); return; @@ -134,8 +131,7 @@ IOBuffer *inbuf; const char *str; - if (!imap_msgcache_get_rfc822(ctx->cache, rec->uid, - NULL, &body_size, &inbuf)) { + if (!imap_msgcache_get_rfc822(ctx->cache, &inbuf, NULL, &body_size)) { i_error("Couldn't get RFC822.TEXT for UID %u (index %s)", rec->uid, ctx->index->filepath); return; @@ -180,78 +176,27 @@ return field; } -static IOBuffer *inbuf_rewind(IOBuffer *inbuf, void *context __attr_unused__) +static void index_msgcache_open(FetchContext *ctx, MailIndexRecord *rec) { - if (!io_buffer_seek(inbuf, 0)) { - i_error("inbuf_rewind: lseek() failed: %m"); - - (void)close(inbuf->fd); - io_buffer_destroy(inbuf); - return NULL; - } + ImapCacheField fields; + void *mail_cache_context; - return inbuf; -} + fields = index_get_cache(ctx->fetch_data); + if (fields == 0) + return; -static int index_cache_message(MailIndexRecord *rec, FetchContext *ctx, - ImapCacheField field) -{ - IOBuffer *inbuf; - - inbuf = ctx->index->open_mail(ctx->index, rec); - if (inbuf == NULL) - return FALSE; + mail_cache_context = index_msgcache_get_context(ctx->index, rec); if (MSG_HAS_VALID_CRLF_DATA(rec)) { - imap_msgcache_message(ctx->cache, rec->uid, - field, rec->full_virtual_size, - rec->header_size, rec->body_size, - inbuf, inbuf_rewind, NULL); + imap_msgcache_open(ctx->cache, rec->uid, fields, + rec->full_virtual_size, + rec->header_size, rec->body_size, + mail_cache_context); } else { - imap_msgcache_message(ctx->cache, rec->uid, - field, rec->full_virtual_size, - 0, 0, inbuf, inbuf_rewind, NULL); + imap_msgcache_open(ctx->cache, rec->uid, fields, + rec->full_virtual_size, 0, 0, + mail_cache_context); } - - return TRUE; -} - -static void index_cache_mail(FetchContext *ctx, MailIndexRecord *rec) -{ - ImapCacheField fields; - const char *value; - - fields = index_get_cache(ctx->fetch_data); - if (imap_msgcache_is_cached(ctx->cache, rec->uid, fields)) - return; - - /* see if we can get some of the values from our index */ - if (fields & IMAP_CACHE_BODY) { - value = ctx->index->lookup_field(ctx->index, rec, - FIELD_TYPE_BODY); - imap_msgcache_set(ctx->cache, rec->uid, - IMAP_CACHE_BODY, value); - } - - if (fields & IMAP_CACHE_BODYSTRUCTURE) { - value = ctx->index->lookup_field(ctx->index, rec, - FIELD_TYPE_BODYSTRUCTURE); - imap_msgcache_set(ctx->cache, rec->uid, - IMAP_CACHE_BODYSTRUCTURE, value); - } - - if (fields & IMAP_CACHE_ENVELOPE) { - value = ctx->index->lookup_field(ctx->index, rec, - FIELD_TYPE_ENVELOPE); - imap_msgcache_set(ctx->cache, rec->uid, - IMAP_CACHE_ENVELOPE, value); - } - - /* if we still don't have everything, open the message and - cache the needed fields */ - if (fields != 0 && - !imap_msgcache_is_cached(ctx->cache, rec->uid, fields)) - (void)index_cache_message(rec, ctx, fields); } static int index_fetch_mail(MailIndex *index __attr_unused__, @@ -270,7 +215,7 @@ /* first see what we need to do. this way we don't first do some light parsing and later notice that we need to do heavier parsing anyway */ - index_cache_mail(ctx, rec); + index_msgcache_open(ctx, rec); if (ctx->fetch_data->uid) index_fetch_uid(rec, ctx); @@ -308,6 +253,7 @@ (void)io_buffer_send(ctx->outbuf, ")\r\n", 3); + imap_msgcache_close(ctx->cache); return TRUE; } @@ -345,11 +291,6 @@ fetch_data->uidset, index_fetch_mail, &ctx); - /* close open message files in cache, they're reopened at next fetch - anyway, and especially with mbox the old data may not be valid - then */ - imap_msgcache_close(ibox->cache); - flags_file_list_unref(ibox->flagsfile); if (!ibox->index->set_lock(ibox->index, MAIL_LOCK_UNLOCK))
--- a/src/lib-storage/index/index-storage.c Sun Sep 01 02:57:37 2002 +0300 +++ b/src/lib-storage/index/index-storage.c Mon Sep 02 05:31:18 2002 +0300 @@ -38,7 +38,7 @@ ibox->index = index; ibox->flagsfile = flagsfile; - ibox->cache = imap_msgcache_alloc(); + ibox->cache = imap_msgcache_alloc(&index_msgcache_iface); return ibox; }
--- a/src/lib-storage/index/index-storage.h Sun Sep 01 02:57:37 2002 +0300 +++ b/src/lib-storage/index/index-storage.h Mon Sep 02 05:31:18 2002 +0300 @@ -23,6 +23,8 @@ unsigned int synced_messages_count; }; +extern ImapMessageCacheIface index_msgcache_iface; + IndexMailbox *index_storage_init(MailStorage *storage, Mailbox *box, MailIndex *index, const char *name, int readonly); @@ -43,6 +45,8 @@ int index_storage_save_into_fd(MailStorage *storage, int fd, const char *path, IOBuffer *buf, size_t data_size); +void *index_msgcache_get_context(MailIndex *index, MailIndexRecord *rec); + /* Mailbox methods: */ int index_storage_copy(Mailbox *box, Mailbox *destbox, const char *messageset, int uidset);