Mercurial > dovecot > original-hg > dovecot-1.2
changeset 5:1b34ec11fff8 HEAD
Message data is parsed in blocks (no longer entirely mmap()ed). Several
IOBuffer changes. All mixed signed/unsigned comparisions were fixed so code
can now be compiled with gcc's -W flag. mbox support is broken currently,
and there's most likely several other problems too.
line wrap: on
line diff
--- a/configure.in Tue Aug 13 18:30:02 2002 +0300 +++ b/configure.in Thu Aug 22 01:10:20 2002 +0300 @@ -95,7 +95,7 @@ dnl * gcc specific options if test "x$ac_cv_prog_gcc" = "xyes"; then # -W -Wchar-subscripts -Wpointer-arith -Wcast-align -Wconversion -Wstrict-prototypes -Wmissing-prototypes -Wmissing-declarations - CFLAGS="$CFLAGS -Wall" + CFLAGS="$CFLAGS -Wall -W" fi dnl * OS specific options
--- a/src/auth/auth-digest-md5.c Tue Aug 13 18:30:02 2002 +0300 +++ b/src/auth/auth-digest-md5.c Thu Aug 22 01:10:20 2002 +0300 @@ -48,7 +48,6 @@ unsigned char response[32]; unsigned long maxbuf; unsigned int nonce_found:1; - unsigned int utf8:1; /* final reply: */ char *rspauth; @@ -114,7 +113,7 @@ /* get the MD5 password */ if (!userinfo->lookup_digest_md5(auth->username, auth->realm != NULL ? - auth->realm : "", auth->utf8, digest, + auth->realm : "", digest, &auth->cookie_reply)) return FALSE; @@ -414,7 +413,6 @@ return FALSE; } - auth->utf8 = TRUE; return TRUE; }
--- a/src/auth/login-connection.c Tue Aug 13 18:30:02 2002 +0300 +++ b/src/auth/login-connection.c Thu Aug 22 01:10:20 2002 +0300 @@ -80,7 +80,7 @@ return; memcpy(&request, data, sizeof(request)); - conn->inbuf->skip += sizeof(request); + io_buffer_skip(conn->inbuf, sizeof(request)); /* we have a full init request */ auth_init_request(&request, request_callback, conn); @@ -95,7 +95,8 @@ if (size < sizeof(request) + request.data_size) return; - conn->inbuf->skip += sizeof(request) + request.data_size; + io_buffer_skip(conn->inbuf, + sizeof(request) + request.data_size); /* we have a full continued request */ auth_continue_request(&request, data + sizeof(request),
--- a/src/auth/userinfo-passwd-file.c Tue Aug 13 18:30:02 2002 +0300 +++ b/src/auth/userinfo-passwd-file.c Thu Aug 22 01:10:20 2002 +0300 @@ -148,7 +148,7 @@ } static int passwd_file_lookup_digest_md5(const char *user, const char *realm, - int utf8, unsigned char digest[16], + unsigned char digest[16], AuthCookieReplyData *reply) { const char *id;
--- a/src/auth/userinfo.h Tue Aug 13 18:30:02 2002 +0300 +++ b/src/auth/userinfo.h Thu Aug 22 01:10:20 2002 +0300 @@ -18,7 +18,7 @@ "user:realm:password". If utf8 is TRUE, the user and realm are in UTF-8, otherwise ISO-8859-1. Returns TRUE if user was found. */ int (*lookup_digest_md5)(const char *user, const char *realm, - int utf8, unsigned char digest[16], + unsigned char digest[16], AuthCookieReplyData *reply); } UserInfoModule;
--- a/src/imap/client.c Tue Aug 13 18:30:02 2002 +0300 +++ b/src/imap/client.c Thu Aug 22 01:10:20 2002 +0300 @@ -238,7 +238,7 @@ for (i = 0; i < data_size; i++) { if (data[i] == '\n') { client->inbuf_skip_line = FALSE; - client->inbuf->skip += i+1; + io_buffer_skip(client->inbuf, i+1); break; } }
--- a/src/imap/cmd-fetch.c Tue Aug 13 18:30:02 2002 +0300 +++ b/src/imap/cmd-fetch.c Thu Aug 22 01:10:20 2002 +0300 @@ -49,7 +49,8 @@ *p++ = '\0'; /* <start.end> */ - body->skip = body->max_size = 0; + body->skip = 0; + body->max_size = -1; if (*p != '<' && *p != '\0') { client_send_tagline(client, t_strconcat( "BAD Unexpected character after ']' with ",
--- a/src/imap/cmd-list.c Tue Aug 13 18:30:02 2002 +0300 +++ b/src/imap/cmd-list.c Thu Aug 22 01:10:20 2002 +0300 @@ -49,7 +49,7 @@ /* escaping is done here to make sure we don't try to escape the separator char */ - name = imap_escape(t_strndup(name, (unsigned int) (path-name))); + name = imap_escape(t_strdup_until(name, path)); /* find the node */ while (*node != NULL) {
--- a/src/lib-imap/imap-bodystructure.c Tue Aug 13 18:30:02 2002 +0300 +++ b/src/lib-imap/imap-bodystructure.c Thu Aug 22 01:10:20 2002 +0300 @@ -1,6 +1,7 @@ /* Copyright (C) 2002 Timo Sirainen */ #include "lib.h" +#include "iobuffer.h" #include "temp-string.h" #include "rfc822-tokenize.h" #include "message-parser.h" @@ -169,18 +170,18 @@ t_pop(); } -static void part_parse_headers(MessagePart *part, const char *msg, - size_t size, Pool pool) +static void part_parse_headers(MessagePart *part, IOBuffer *inbuf, Pool pool) { while (part != NULL) { /* note that we want to parse the header of all the message parts, multiparts too. */ - message_parse_header(part, msg + part->pos.physical_pos, - part->header_size.physical_size, - NULL, parse_header, pool); + i_assert(part->pos.physical_pos >= inbuf->offset); + io_buffer_skip(inbuf, part->pos.physical_pos - inbuf->offset); + + message_parse_header(part, inbuf, NULL, parse_header, pool); if (part->children != NULL) - part_parse_headers(part->children, msg, size, pool); + part_parse_headers(part->children, inbuf, pool); part = part->next; } @@ -355,13 +356,12 @@ } const char *imap_part_get_bodystructure(Pool pool, MessagePart **part, - const char *msg, size_t size, - int extended) + IOBuffer *inbuf, int extended) { if (*part == NULL) - *part = message_parse(pool, msg, size, parse_header, pool); + *part = message_parse(pool, inbuf, parse_header, pool); else - part_parse_headers(*part, msg, size, pool); + part_parse_headers(*part, inbuf, pool); return part_get_bodystructure(*part, extended); }
--- a/src/lib-imap/imap-bodystructure.h Tue Aug 13 18:30:02 2002 +0300 +++ b/src/lib-imap/imap-bodystructure.h Thu Aug 22 01:10:20 2002 +0300 @@ -4,7 +4,6 @@ /* If *part is non-NULL, it's used as base for building the body structure. Otherwise it's set to the root MessagePart and parsed. */ const char *imap_part_get_bodystructure(Pool pool, MessagePart **part, - const char *msg, size_t size, - int extended); + IOBuffer *inbuf, int extended); #endif
--- a/src/lib-imap/imap-message-cache.c Tue Aug 13 18:30:02 2002 +0300 +++ b/src/lib-imap/imap-message-cache.c Thu Aug 22 01:10:20 2002 +0300 @@ -1,6 +1,7 @@ /* Copyright (C) 2002 Timo Sirainen */ #include "lib.h" +#include "iobuffer.h" #include "temp-string.h" #include "mmap-util.h" #include "message-parser.h" @@ -43,20 +44,16 @@ int messages_count; CachedMessage *open_msg; - int open_fd; - void *open_mmap_base; - const char *open_msg_data; - off_t open_start_pos; - size_t open_mmap_size, open_size, open_virtual_size; + IOBuffer *open_inbuf; + size_t open_size, open_virtual_size; + + IOBuffer *(*inbuf_rewind)(IOBuffer *inbuf, void *user_data); + void *user_data; }; ImapMessageCache *imap_msgcache_alloc(void) { - ImapMessageCache *cache; - - cache = i_new(ImapMessageCache, 1); - cache->open_fd = -1; - return cache; + return i_new(ImapMessageCache, 1); } static void cached_message_free(CachedMessage *msg) @@ -64,29 +61,11 @@ pool_unref(msg->pool); } -static void cache_close_msg(ImapMessageCache *cache) -{ - if (cache->open_mmap_base != NULL) { - (void)munmap(cache->open_mmap_base, cache->open_mmap_size); - cache->open_mmap_base = NULL; - cache->open_msg_data = NULL; - } - - if (cache->open_fd != -1) { - (void)close(cache->open_fd); - cache->open_fd = -1; - } - - cache->open_msg = NULL; - cache->open_size = 0; - cache->open_start_pos = 0; -} - void imap_msgcache_clear(ImapMessageCache *cache) { CachedMessage *next; - cache_close_msg(cache); + imap_msgcache_close(cache); while (cache->messages != NULL) { next = cache->messages->next; @@ -129,6 +108,34 @@ return msg; } +static CachedMessage *cache_open_or_create(ImapMessageCache *cache, + unsigned int uid) +{ + CachedMessage **pos, *msg; + + pos = &cache->messages; + for (; *pos != NULL; pos = &(*pos)->next) { + if ((*pos)->uid == uid) + break; + } + + if (*pos == NULL) { + /* not found, add it */ + msg = cache_new(cache, uid); + } else if (*pos != cache->messages) { + /* move it to first in list */ + msg = *pos; + *pos = msg->next; + + msg->next = cache->messages; + cache->messages = msg; + } else { + msg = *pos; + } + + return msg; +} + static void parse_envelope_header(MessagePart *part, const char *name, unsigned int name_len, const char *value, unsigned int value_len, @@ -156,6 +163,19 @@ return NULL; } +static void imap_msgcache_get_inbuf(ImapMessageCache *cache, off_t offset) +{ + if (offset < cache->open_inbuf->offset) { + /* need to rewind */ + cache->open_inbuf = cache->inbuf_rewind(cache->open_inbuf, + cache->user_data); + if (cache->open_inbuf == NULL) + i_fatal("Can't rewind message buffer"); + } + + io_buffer_skip(cache->open_inbuf, offset - cache->open_inbuf->offset); +} + int imap_msgcache_is_cached(ImapMessageCache *cache, unsigned int uid, ImapCacheField fields) { @@ -198,20 +218,41 @@ t_push(); if ((fields & IMAP_CACHE_BODY) && msg->cached_body == NULL && msg == cache->open_msg) { + imap_msgcache_get_inbuf(cache, 0); value = imap_part_get_bodystructure(msg->pool, &msg->part, - cache->open_msg_data, - cache->open_size, FALSE); + cache->open_inbuf, FALSE); msg->cached_body = p_strdup(msg->pool, value); } if ((fields & IMAP_CACHE_BODYSTRUCTURE) && msg->cached_bodystructure == NULL && msg == cache->open_msg) { + imap_msgcache_get_inbuf(cache, 0); value = imap_part_get_bodystructure(msg->pool, &msg->part, - cache->open_msg_data, - cache->open_size, TRUE); + cache->open_inbuf, TRUE); msg->cached_bodystructure = p_strdup(msg->pool, value); } + if ((fields & IMAP_CACHE_ENVELOPE) && msg->cached_envelope == NULL) { + if (msg->envelope == NULL && msg == cache->open_msg) { + /* envelope isn't parsed yet, do it. header size + is calculated anyway so save it */ + if (msg->hdr_size == NULL) { + msg->hdr_size = p_new(msg->pool, + MessageSize, 1); + } + + imap_msgcache_get_inbuf(cache, 0); + message_parse_header(NULL, cache->open_inbuf, + msg->hdr_size, + parse_envelope_header, msg); + } + + if (msg->envelope != NULL) { + value = imap_envelope_get_part_data(msg->envelope); + msg->cached_envelope = p_strdup(msg->pool, value); + } + } + if ((fields & IMAP_CACHE_MESSAGE_PART) && msg->part == NULL && msg == cache->open_msg) { /* we need to parse the message */ @@ -226,27 +267,9 @@ func = NULL; } - msg->part = message_parse(msg->pool, cache->open_msg_data, - cache->open_size, func, msg); - } - - if ((fields & IMAP_CACHE_ENVELOPE) && msg->cached_envelope == NULL) { - if (msg->envelope == NULL && msg == cache->open_msg) { - /* envelope isn't parsed yet, do it. header size - is calculated anyway so save it */ - if (msg->hdr_size == NULL) { - msg->hdr_size = p_new(msg->pool, - MessageSize, 1); - } - message_parse_header(NULL, cache->open_msg_data, - cache->open_size, msg->hdr_size, - parse_envelope_header, msg); - } - - if (msg->envelope != NULL) { - value = imap_envelope_get_part_data(msg->envelope); - msg->cached_envelope = p_strdup(msg->pool, value); - } + imap_msgcache_get_inbuf(cache, 0); + msg->part = message_parse(msg->pool, cache->open_inbuf, + func, msg); } if ((fields & IMAP_CACHE_MESSAGE_BODY_SIZE) && @@ -265,8 +288,8 @@ } else { /* first get the header's size, then calculate the body size from it and the total virtual size */ - message_get_header_size(cache->open_msg_data, - cache->open_size, + imap_msgcache_get_inbuf(cache, 0); + message_get_header_size(cache->open_inbuf, msg->hdr_size); msg->body_size->lines = 0; @@ -287,8 +310,8 @@ *msg->hdr_size = msg->part->header_size; } else { /* need to do some light parsing */ - message_get_header_size(cache->open_msg_data, - cache->open_size, + imap_msgcache_get_inbuf(cache, 0); + message_get_header_size(cache->open_inbuf, msg->hdr_size); } } @@ -296,69 +319,30 @@ t_pop(); } -static int cache_mmap(ImapMessageCache *cache) +void imap_msgcache_message(ImapMessageCache *cache, unsigned int uid, + ImapCacheField fields, size_t virtual_size, + size_t pv_headers_size, size_t pv_body_size, + IOBuffer *inbuf, + IOBuffer *(*inbuf_rewind)(IOBuffer *inbuf, + void *user_data), + void *user_data) { - if (cache->open_mmap_base == NULL) { - cache->open_mmap_base = - mmap_aligned(cache->open_fd, PROT_READ, - cache->open_start_pos, cache->open_size, - (void **) &cache->open_msg_data, - &cache->open_mmap_size); - if (cache->open_mmap_base == MAP_FAILED) { - i_error("mmap() failed for msg %u: %m", - cache->open_msg->uid); - return FALSE; - } - } - return TRUE; -} - -void imap_msgcache_message(ImapMessageCache *cache, unsigned int uid, - int fd, off_t offset, size_t size, - size_t virtual_size, size_t pv_headers_size, - size_t pv_body_size, ImapCacheField fields) -{ - CachedMessage **pos, *msg; + CachedMessage *msg; - i_assert(fd != -1); - - if (cache->open_msg == NULL || cache->open_msg->uid != uid) { - cache_close_msg(cache); - - pos = &cache->messages; - for (; *pos != NULL; pos = &(*pos)->next) { - if ((*pos)->uid == uid) - break; - } - - if (*pos == NULL) { - /* not found, add it */ - msg = cache_new(cache, uid); - } else if (*pos != cache->messages) { - /* move it to first in list */ - msg = *pos; - *pos = msg->next; - - msg->next = cache->messages; - cache->messages = msg; - } else { - msg = *pos; - } + msg = cache_open_or_create(cache, uid); + if (cache->open_msg != msg) { + imap_msgcache_close(cache); cache->open_msg = msg; - cache->open_fd = fd; - cache->open_size = size; + cache->open_inbuf = inbuf; + cache->open_size = cache->open_inbuf->stop_offset - + cache->open_inbuf->offset; cache->open_virtual_size = virtual_size; - cache->open_start_pos = offset; - if (!cache_mmap(cache)) { - cache_close_msg(cache); - return; - } + cache->inbuf_rewind = inbuf_rewind; + cache->user_data = user_data; } - msg = cache->open_msg; - if (pv_headers_size != 0 && msg->hdr_size == NULL) { /* physical size == virtual size */ msg->hdr_size = p_new(msg->pool, MessageSize, 1); @@ -376,6 +360,18 @@ cache_fields(cache, msg, fields); } +void imap_msgcache_close(ImapMessageCache *cache) +{ + if (cache->open_inbuf != NULL) { + (void)close(cache->open_inbuf->fd); + io_buffer_destroy(cache->open_inbuf); + cache->open_inbuf = NULL; + } + + cache->open_msg = NULL; + cache->open_size = cache->open_virtual_size = 0; +} + void imap_msgcache_set(ImapMessageCache *cache, unsigned int uid, ImapCacheField field, const char *value) { @@ -444,23 +440,21 @@ int imap_msgcache_get_rfc822(ImapMessageCache *cache, unsigned int uid, MessageSize *hdr_size, MessageSize *body_size, - const char **data, int *fd) + IOBuffer **inbuf) { CachedMessage *msg; + off_t offset; - if (data != NULL || fd != NULL) { + if (inbuf != NULL) { if (cache->open_msg == NULL || cache->open_msg->uid != uid) return FALSE; msg = cache->open_msg; - if (data != NULL) - *data = cache->open_msg_data; - if (fd != NULL) { - *fd = cache->open_fd; - if (*fd != -1 && lseek(*fd, cache->open_start_pos, - SEEK_SET) == (off_t)-1) - *fd = -1; - } + + offset = hdr_size != NULL ? 0 : + msg->hdr_size->physical_size; + imap_msgcache_get_inbuf(cache, offset); + *inbuf = cache->open_inbuf; } else { msg = cache_find(cache, uid); if (msg == NULL) @@ -481,98 +475,49 @@ if (msg->hdr_size == NULL) return FALSE; *hdr_size = *msg->hdr_size; - } else { - /* header isn't wanted, skip it */ - if (data != NULL) - *data += msg->hdr_size->physical_size; - if (fd != NULL) { - if (lseek(*fd, (off_t) msg->hdr_size->physical_size, - SEEK_CUR) == (off_t)-1) - return FALSE; - } } return TRUE; } -static void get_partial_size(const char *msg, size_t max_physical_size, - off_t virtual_skip, size_t max_virtual_size, +static void get_partial_size(IOBuffer *inbuf, + off_t virtual_skip, off_t max_virtual_size, MessageSize *partial, MessageSize *dest) { - const char *msg_start, *msg_end, *cr; - - msg_end = msg + max_physical_size; + unsigned char *msg; + unsigned int size; + int cr_skipped; /* see if we can use the existing partial */ - if (partial->virtual_size > (size_t) virtual_skip) + if ((off_t)partial->virtual_size > virtual_skip) memset(partial, 0, sizeof(MessageSize)); - - /* first do the virtual skip - FIXME: <..\r><\n..> skipping! */ - if (virtual_skip > 0) { - msg = msg_start = msg + partial->physical_size; - - cr = NULL; - while (msg != msg_end && - partial->virtual_size < (size_t) virtual_skip) { - if (*msg == '\r') - cr = msg; - else if (*msg == '\n') { - partial->lines++; - - if (cr != msg-1) { - if (++partial->virtual_size == - (size_t) virtual_skip) { - /* FIXME: CR thingy */ - } - } - } - - msg++; - partial->virtual_size++; - } - - partial->physical_size += (int) (msg-msg_start); - max_physical_size -= partial->physical_size; + else { + io_buffer_skip(inbuf, partial->physical_size); + virtual_skip -= partial->virtual_size; } - if (max_virtual_size == 0) { - /* read the rest of the message */ - message_get_body_size(msg, max_physical_size, dest); - return; + message_skip_virtual(inbuf, virtual_skip, partial, &cr_skipped); + + if (!cr_skipped) { + /* see if we need to add virtual CR */ + while (io_buffer_read_data(inbuf, &msg, &size, 0) >= 0) { + if (size > 0) { + if (msg[0] == '\n') + dest->virtual_size++; + break; + } + } } - /* now read until the message is either finished or we've read - max_virtual_size */ - msg_start = msg; - memset(dest, 0, sizeof(MessageSize)); - - cr = NULL; - while (msg != msg_end && dest->virtual_size < (size_t) virtual_skip) { - if (*msg == '\r') - cr = msg; - else if (*msg == '\n') { - dest->lines++; - - if (cr != msg-1) - dest->virtual_size++; - } - - msg++; - dest->virtual_size++; - } - - dest->physical_size = (int) (msg-msg_start); + message_get_body_size(inbuf, dest, max_virtual_size); } int imap_msgcache_get_rfc822_partial(ImapMessageCache *cache, unsigned int uid, - off_t virtual_skip, - size_t max_virtual_size, + off_t virtual_skip, off_t max_virtual_size, int get_header, MessageSize *size, - const char **data, int *fd) + IOBuffer **inbuf) { CachedMessage *msg; - const char *body; - size_t body_size; off_t physical_skip; int size_got; @@ -582,21 +527,16 @@ if (msg->hdr_size == NULL) { msg->hdr_size = p_new(msg->pool, MessageSize, 1); - message_get_header_size(cache->open_msg_data, - cache->open_size, - msg->hdr_size); + imap_msgcache_get_inbuf(cache, 0); + message_get_header_size(cache->open_inbuf, msg->hdr_size); } - body = cache->open_msg_data + msg->hdr_size->physical_size; - body_size = cache->open_size - msg->hdr_size->physical_size; - - if (fd != NULL) *fd = cache->open_fd; physical_skip = get_header ? 0 : msg->hdr_size->physical_size; /* see if we can do this easily */ size_got = FALSE; if (virtual_skip == 0) { - if (max_virtual_size == 0 && msg->body_size == NULL) { + if (max_virtual_size < 0 && msg->body_size == NULL) { msg->body_size = p_new(msg->pool, MessageSize, 1); msg->body_size->physical_size = cache->open_size - msg->hdr_size->physical_size; @@ -606,8 +546,8 @@ } if (msg->body_size != NULL && - (max_virtual_size == 0 || - max_virtual_size >= msg->body_size->virtual_size)) { + (max_virtual_size < 0 || + max_virtual_size >= (off_t)msg->body_size->virtual_size)) { *size = *msg->body_size; size_got = TRUE; } @@ -617,7 +557,8 @@ if (msg->partial_size == NULL) msg->partial_size = p_new(msg->pool, MessageSize, 1); - get_partial_size(body, body_size, virtual_skip, + imap_msgcache_get_inbuf(cache, msg->hdr_size->physical_size); + get_partial_size(cache->open_inbuf, virtual_skip, max_virtual_size, msg->partial_size, size); physical_skip += msg->partial_size->physical_size; @@ -627,30 +568,18 @@ message_size_add(size, msg->hdr_size); /* seek to wanted position */ - *data = cache->open_msg_data + physical_skip; - if (fd != NULL && *fd != -1) { - if (lseek(*fd, cache->open_start_pos + physical_skip, - SEEK_SET) == (off_t)-1) - *fd = -1; - } + imap_msgcache_get_inbuf(cache, physical_skip); + *inbuf = cache->open_inbuf; return TRUE; } int imap_msgcache_get_data(ImapMessageCache *cache, unsigned int uid, - const char **data, int *fd, size_t *size) + IOBuffer **inbuf) { if (cache->open_msg == NULL || cache->open_msg->uid != uid) return FALSE; - *data = cache->open_msg_data; - if (fd != NULL) { - *fd = cache->open_fd; - if (lseek(*fd, cache->open_start_pos, SEEK_SET) == (off_t)-1) - return FALSE; - } - - if (size != NULL) - *size = cache->open_size; - + imap_msgcache_get_inbuf(cache, 0); + *inbuf = cache->open_inbuf; return TRUE; }
--- a/src/lib-imap/imap-message-cache.h Tue Aug 13 18:30:02 2002 +0300 +++ b/src/lib-imap/imap-message-cache.h Thu Aug 22 01:10:20 2002 +0300 @@ -1,6 +1,8 @@ #ifndef __IMAP_MESSAGE_CACHE_H #define __IMAP_MESSAGE_CACHE_H +// FIXME: update comments + /* IMAP message cache. Caches are mailbox-specific and must be cleared if UID validity changes. @@ -32,14 +34,19 @@ int imap_msgcache_is_cached(ImapMessageCache *cache, unsigned int uid, ImapCacheField fields); -/* Parse and cache the message. The given file handle is automatically - closed by cache and must not be closed elsewhere. If pv_headers_size - and pv_body_size is non-zero, they're set to saved to message's both - physical and virtual sizes (ie. doesn't need to be calculated). */ +/* Parse and cache the message. If pv_headers_size and pv_body_size is + non-zero, they're set to saved to message's both physical and virtual + sizes (ie. doesn't need to be calculated). */ void imap_msgcache_message(ImapMessageCache *cache, unsigned int uid, - int fd, off_t offset, size_t size, - size_t virtual_size, size_t pv_headers_size, - size_t pv_body_size, ImapCacheField fields); + ImapCacheField fields, size_t virtual_size, + size_t pv_headers_size, size_t pv_body_size, + IOBuffer *inbuf, + IOBuffer *(*inbuf_rewind)(IOBuffer *inbuf, + void *user_data), + void *user_data); + +/* Close the IOBuffer for cached message. */ +void imap_msgcache_close(ImapMessageCache *cache); /* Store a value for field in cache */ void imap_msgcache_set(ImapMessageCache *cache, unsigned int uid, @@ -58,19 +65,17 @@ is NULL, *data contains only the message body. */ int imap_msgcache_get_rfc822(ImapMessageCache *cache, unsigned int uid, MessageSize *hdr_size, MessageSize *body_size, - const char **data, int *fd); + IOBuffer **inbuf); -/* Returns FALSE if message isn't in cache. *data is set to point to the first - non-skipped character. size is set to specify the real size for message. */ +/* Returns FALSE if message isn't in cache. *inbuf is set to point to the first + non-skipped character. size is set to specify the full size for message. */ int imap_msgcache_get_rfc822_partial(ImapMessageCache *cache, unsigned int uid, - off_t virtual_skip, - size_t max_virtual_size, + off_t virtual_skip, off_t max_virtual_size, int get_header, MessageSize *size, - const char **data, int *fd); + IOBuffer **inbuf); -/* Like imap_msgcache_get_rfc822() without calculating virtual sizes. - size and fd may be NULL. */ +/* Like imap_msgcache_get_rfc822() without calculating virtual sizes. */ int imap_msgcache_get_data(ImapMessageCache *cache, unsigned int uid, - const char **data, int *fd, size_t *size); + IOBuffer **inbuf); #endif
--- a/src/lib-imap/imap-message-send.c Tue Aug 13 18:30:02 2002 +0300 +++ b/src/lib-imap/imap-message-send.c Thu Aug 22 01:10:20 2002 +0300 @@ -3,88 +3,71 @@ #include "lib.h" #include "iobuffer.h" #include "imap-message-send.h" +#include "message-size.h" -int imap_message_send(IOBuffer *outbuf, const char *msg, int msg_fd, - MessageSize *size, off_t virtual_skip, - size_t max_virtual_size) +int imap_message_send(IOBuffer *outbuf, IOBuffer *inbuf, + MessageSize *msg_size, off_t virtual_skip, + off_t max_virtual_size) { - const char *msg_start, *msg_end, *cr; - unsigned int len; + unsigned char *msg; + unsigned int i, size; + int cr_skipped, add_cr; - if (size->physical_size == 0) + if (max_virtual_size == -1 || + max_virtual_size > (off_t)msg_size->virtual_size - virtual_skip) + max_virtual_size = msg_size->virtual_size - virtual_skip; + + if (msg_size->physical_size == 0 || virtual_skip >= max_virtual_size) return TRUE; - if (size->physical_size == size->virtual_size) { + if (msg_size->physical_size == msg_size->virtual_size) { /* no need to kludge with CRs, we can use sendfile() */ - size_t send_size; - - send_size = size->physical_size - virtual_skip; - if (msg_fd == -1) { - return io_buffer_send(outbuf, msg + virtual_skip, - send_size) > 0; - } else { - return io_buffer_send_file(outbuf, msg_fd, virtual_skip, - msg + virtual_skip, - send_size) > 0; - } + io_buffer_skip(inbuf, virtual_skip); + return io_buffer_send_buf(outbuf, inbuf, max_virtual_size) > 0; } - msg_start = msg; - msg_end = msg + size->physical_size; + message_skip_virtual(inbuf, virtual_skip, NULL, &cr_skipped); - /* first do the virtual skip - FIXME: <..\r><\n..> skipping! */ - if (virtual_skip > 0) { - cr = NULL; - while (msg != msg_end && virtual_skip > 0) { - if (*msg == '\r') - cr = msg; - else if (*msg == '\n') { - if (cr != msg-1) { - if (--virtual_skip == 0) { - /* FIXME: cr thingy */ - } + /* go through the message data and insert CRs where needed. */ + while (io_buffer_read_data(inbuf, &msg, &size, 0) >= 0) { + add_cr = FALSE; + for (i = 0; i < size; i++) { + if (msg[i] == '\n') { + if ((i == 0 && !cr_skipped) || + (i > 0 && msg[i-1] != '\r')) { + /* missing CR */ + if (max_virtual_size > 0) + max_virtual_size--; + add_cr = TRUE; + break; } + } - msg++; - virtual_skip--; + if (max_virtual_size > 0) { + if (--max_virtual_size == 0) { + i++; + break; + } + } } - msg_start = msg; + /* send the data read so far */ + if (io_buffer_send(outbuf, msg, i) <= 0) + return FALSE; + + if (add_cr) { + if (io_buffer_send(outbuf, "\r", 1) <= 0) + return FALSE; + } + + /* see if we've reached the limit */ + if (max_virtual_size == 0) + break; + + cr_skipped = TRUE; + io_buffer_skip(inbuf, i); } - /* go through the message data and insert CRs where needed. */ - cr = NULL; - while (msg != msg_end) { - if (*msg == '\r') - cr = msg; - else if (*msg == '\n' && cr != msg-1) { - len = (unsigned int) (msg - msg_start); - if (max_virtual_size != 0 && max_virtual_size <= len) { - /* reached max. size limit */ - return io_buffer_send(outbuf, msg_start, - max_virtual_size) > 0; - } - - if (io_buffer_send(outbuf, msg_start, len) <= 0) - return FALSE; - - if (io_buffer_send(outbuf, "\r", 1) <= 0) - return FALSE; - - /* update max. size */ - if (max_virtual_size == len+1) - return TRUE; - max_virtual_size -= len+1; - - msg_start = msg; - } - msg++; - } - - /* send the rest */ - len = (unsigned int) (msg - msg_start); - if (max_virtual_size != 0 && max_virtual_size < len) - len = max_virtual_size; - return io_buffer_send(outbuf, msg_start, len) > 0; + return TRUE; }
--- a/src/lib-imap/imap-message-send.h Tue Aug 13 18:30:02 2002 +0300 +++ b/src/lib-imap/imap-message-send.h Thu Aug 22 01:10:20 2002 +0300 @@ -4,10 +4,10 @@ #include "message-parser.h" /* Send message to client inserting CRs if needed. If max_virtual_size is - non-zero, only that much of the message is sent. If msg_fd is -1, only + not negative, only that much of the message is sent. If msg_fd is -1, only msg is used. Returns TRUE if successful. */ -int imap_message_send(IOBuffer *outbuf, const char *msg, int msg_fd, - MessageSize *size, off_t virtual_skip, - size_t max_virtual_size); +int imap_message_send(IOBuffer *outbuf, IOBuffer *inbuf, + MessageSize *msg_size, off_t virtual_skip, + off_t max_virtual_size); #endif
--- a/src/lib-imap/imap-parser.c Tue Aug 13 18:30:02 2002 +0300 +++ b/src/lib-imap/imap-parser.c Thu Aug 22 01:10:20 2002 +0300 @@ -84,7 +84,7 @@ break; } - parser->inbuf->skip += i; + io_buffer_skip(parser->inbuf, i); parser->cur_pos = 0; *data += i; @@ -294,7 +294,7 @@ parser->cur_type = ARG_PARSE_LITERAL_DATA; parser->literal_skip_crlf = TRUE; - parser->inbuf->skip += i+1; + io_buffer_skip(parser->inbuf, i+1); parser->cur_pos = 0; break; } @@ -325,14 +325,14 @@ return TRUE; data++; data_size--; - parser->inbuf->skip++; + io_buffer_skip(parser->inbuf, 1); } if (*data != '\n') return FALSE; data++; data_size--; - parser->inbuf->skip++; + io_buffer_skip(parser->inbuf, 1); parser->literal_skip_crlf = FALSE; i_assert(parser->cur_pos == 0); @@ -497,7 +497,7 @@ } if (i < data_size) { - parser->inbuf->skip += i + (data[i] == ' ' ? 1 : 0); + io_buffer_skip(parser->inbuf, i + (data[i] == ' ' ? 1 : 0)); return p_strndup(parser->pool, data, i); } else { return NULL; @@ -518,7 +518,7 @@ } if (i < data_size) { - parser->inbuf->skip += i; + io_buffer_skip(parser->inbuf, i); return p_strndup(parser->pool, data, i); } else { return NULL;
--- a/src/lib-index/Makefile.am Tue Aug 13 18:30:02 2002 +0300 +++ b/src/lib-index/Makefile.am Thu Aug 22 01:10:20 2002 +0300 @@ -1,4 +1,4 @@ -SUBDIRS = maildir mbox +SUBDIRS = maildir noinst_LIBRARIES = libstorage_index.a
--- a/src/lib-index/mail-hash.c Tue Aug 13 18:30:02 2002 +0300 +++ b/src/lib-index/mail-hash.c Thu Aug 22 01:10:20 2002 +0300 @@ -163,7 +163,7 @@ /* skip the existing data in file */ pos = lseek(fd, 0, SEEK_END); - if (pos == (off_t)-1) + if (pos == -1) return FALSE; size -= pos;
--- a/src/lib-index/mail-index-data.c Tue Aug 13 18:30:02 2002 +0300 +++ b/src/lib-index/mail-index-data.c Thu Aug 22 01:10:20 2002 +0300 @@ -183,7 +183,7 @@ hdr.indexid = data->index->indexid; hdr.deleted_space = 0; - if (lseek(data->fd, 0, SEEK_SET) == (off_t)-1) { + if (lseek(data->fd, 0, SEEK_SET) == -1) { index_set_error(data->index, "lseek() failed for data file " "%s: %m", data->filepath); return FALSE; @@ -210,16 +210,16 @@ i_assert((size & (MEM_ALIGN_SIZE-1)) == 0); pos = lseek(data->fd, 0, SEEK_END); - if (pos == (off_t)-1) { + if (pos == -1) { index_set_error(data->index, "lseek() failed with file %s: %m", data->filepath); - return (off_t)-1; + return -1; } if ((size_t) write(data->fd, buffer, size) != size) { index_set_error(data->index, "Error appending to file %s: %m", data->filepath); - return (off_t)-1; + return -1; } mail_index_data_new_data_notify(data); @@ -243,7 +243,7 @@ /* see if we've reached the max. deleted space in file */ if (data->mmap_length >= COMPRESS_MIN_SIZE) { max_del_space = data->mmap_length / 100 * COMPRESS_PERCENTAGE; - if ((size_t) hdr->deleted_space >= max_del_space) + if (hdr->deleted_space >= (off_t)max_del_space) data->index->set_flags |= MAIL_INDEX_FLAG_COMPRESS_DATA; } return TRUE; @@ -283,7 +283,7 @@ if (!mmap_update(data, index_rec->data_position, index_rec->data_size)) return NULL; - max_pos = (size_t) index_rec->data_position + index_rec->data_size; + max_pos = index_rec->data_position + (off_t)index_rec->data_size; if (max_pos > data->mmap_length) { INDEX_MARK_CORRUPTED(data->index); index_set_error(data->index, "Error in data file %s: " @@ -335,7 +335,7 @@ return NULL; /* get position to next record */ - pos = (size_t) DATA_FILE_POSITION(data, rec) + DATA_RECORD_SIZE(rec); + pos = DATA_FILE_POSITION(data, rec) + (off_t)DATA_RECORD_SIZE(rec); max_pos = index_rec->data_position + index_rec->data_size; /* make sure it's within range */
--- a/src/lib-index/mail-index-fsck.c Tue Aug 13 18:30:02 2002 +0300 +++ b/src/lib-index/mail-index-fsck.c Thu Aug 22 01:10:20 2002 +0300 @@ -42,9 +42,9 @@ if (hdr->first_hole_position == 0) { hdr->first_hole_position = pos; hdr->first_hole_records = 1; - } else if (hdr->first_hole_position + - (hdr->first_hole_records * - sizeof(MailIndexRecord)) == (size_t) pos) { + } else if ((off_t) (hdr->first_hole_position + + (hdr->first_hole_records * + sizeof(MailIndexRecord))) == pos) { /* hole continues */ hdr->first_hole_records++; }
--- a/src/lib-index/mail-index-update.c Tue Aug 13 18:30:02 2002 +0300 +++ b/src/lib-index/mail-index-update.c Thu Aug 22 01:10:20 2002 +0300 @@ -1,6 +1,7 @@ /* Copyright (C) 2002 Timo Sirainen */ #include "lib.h" +#include "iobuffer.h" #include "ioloop.h" #include "rfc822-date.h" #include "rfc822-tokenize.h" @@ -151,7 +152,7 @@ /* append the data at the end of the data file */ fpos = mail_index_data_append(update->index->data, mem, pos); - if (fpos == (off_t)-1) + if (fpos == -1) return FALSE; /* update index file position - it's mmap()ed so it'll be writte @@ -327,8 +328,7 @@ } } -void mail_index_update_headers(MailIndexUpdate *update, - const char *msg, size_t size, +void mail_index_update_headers(MailIndexUpdate *update, IOBuffer *inbuf, MessageHeaderFunc header_func, void *user_data) { HeaderUpdateData data; @@ -349,8 +349,7 @@ /* for body / bodystructure, we need need to fully parse the message */ pool = pool_create("index message parser", 2048, FALSE); - part = message_parse(pool, msg, size, - update_header_func, &data); + part = message_parse(pool, inbuf, update_header_func, &data); /* update our sizes */ update->rec->header_size = part->header_size.physical_size; @@ -362,7 +361,7 @@ if (cache_fields & FIELD_TYPE_BODY) { t_push(); value = imap_part_get_bodystructure(pool, &part, - msg, size, FALSE); + inbuf, FALSE); update->index->update_field(update, FIELD_TYPE_BODY, value, 0); t_pop(); @@ -371,7 +370,7 @@ if (cache_fields & FIELD_TYPE_BODYSTRUCTURE) { t_push(); value = imap_part_get_bodystructure(pool, &part, - msg, size, TRUE); + inbuf, TRUE); update->index->update_field(update, FIELD_TYPE_BODYSTRUCTURE, value, 0); @@ -380,18 +379,18 @@ pool_unref(pool); } else { - message_parse_header(NULL, msg, size, &hdr_size, + message_parse_header(NULL, inbuf, &hdr_size, update_header_func, &data); update->rec->header_size = hdr_size.physical_size; - update->rec->body_size = size - hdr_size.physical_size; + update->rec->body_size = (inbuf->stop_offset - inbuf->offset) - + hdr_size.physical_size; if (update->rec->full_virtual_size == 0) { /* we need to calculate virtual size of the - body as well */ - message_get_body_size(msg + hdr_size.physical_size, - size - hdr_size.physical_size, - &body_size); + body as well. message_parse_header() left the + inbuf point to beginning of the body. */ + message_get_body_size(inbuf, &body_size, -1); update->rec->full_virtual_size = hdr_size.virtual_size + body_size.virtual_size;
--- a/src/lib-index/mail-index.c Tue Aug 13 18:30:02 2002 +0300 +++ b/src/lib-index/mail-index.c Thu Aug 22 01:10:20 2002 +0300 @@ -740,7 +740,7 @@ seekpos = sizeof(MailIndexHeader) + (off_t)(lookup_seq-1) * sizeof(MailIndexRecord); - if ((size_t) seekpos > index->mmap_length - sizeof(MailIndexRecord)) { + if (seekpos > (off_t) (index->mmap_length - sizeof(MailIndexRecord))) { /* out of range */ return NULL; } @@ -1042,14 +1042,14 @@ /* first deleted message in index */ hdr->first_hole_position = pos; hdr->first_hole_records = 1; - } else if (hdr->first_hole_position - - sizeof(MailIndexRecord) == (size_t) pos) { + } else if ((off_t) (hdr->first_hole_position - + sizeof(MailIndexRecord)) == pos) { /* deleted the previous record before hole */ hdr->first_hole_position -= sizeof(MailIndexRecord); hdr->first_hole_records++; - } else if (hdr->first_hole_position + - (hdr->first_hole_records * - sizeof(MailIndexRecord)) == (size_t) pos) { + } else if ((off_t) (hdr->first_hole_position + + (hdr->first_hole_records * + sizeof(MailIndexRecord))) == pos) { /* deleted the next record after hole */ hdr->first_hole_records++; update_first_hole_records(index); @@ -1101,7 +1101,7 @@ (*rec)->uid = index->header->next_uid++; pos = lseek(index->fd, 0, SEEK_END); - if (pos == (off_t)-1) { + if (pos == -1) { index_set_error(index, "lseek() failed with file %s: %m", index->filepath); return FALSE;
--- a/src/lib-index/mail-index.h Tue Aug 13 18:30:02 2002 +0300 +++ b/src/lib-index/mail-index.h Thu Aug 22 01:10:20 2002 +0300 @@ -202,10 +202,9 @@ /* Returns sequence for given message. */ unsigned int (*get_sequence)(MailIndex *index, MailIndexRecord *rec); - /* Open mail file lseek()ed to beginning position and return the - file handle. */ - int (*open_mail)(MailIndex *index, MailIndexRecord *rec, - off_t *offset, size_t *size); + /* Open mail file and return it as mmap()ed IOBuffer, or + NULL if failed. */ + IOBuffer *(*open_mail)(MailIndex *index, MailIndexRecord *rec); /* Expunge a mail from index. Hash and modifylog is also updated. The index must be exclusively locked before calling this function. @@ -303,6 +302,13 @@ unsigned int dirty_mmap:1; }; +/* needed to remove annoying warnings about not initializing all struct + members.. */ +#define MAIL_INDEX_PRIVATE_FILL \ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, \ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, \ + 0, 0, 0, 0, 0, 0 + /* defaults - same as above but prefixed with mail_index_. */ int mail_index_open(MailIndex *index, int update_recent); int mail_index_open_or_create(MailIndex *index, int update_recent); @@ -337,10 +343,12 @@ void mail_index_close(MailIndex *index); int mail_index_rebuild_all(MailIndex *index); int mail_index_sync_file(MailIndex *index); -void mail_index_update_headers(MailIndexUpdate *update, - const char *msg, size_t size, +void mail_index_update_headers(MailIndexUpdate *update, IOBuffer *inbuf, MessageHeaderFunc header_func, void *user_data); +/* Max. mmap()ed size for a message */ +#define MAIL_MMAP_BLOCK_SIZE (1024*256) + /* off_t to index file for given record */ #define INDEX_FILE_POSITION(index, ptr) \ ((off_t) ((char *) (ptr) - (char *) ((index)->mmap_base)))
--- a/src/lib-index/mail-modifylog.c Tue Aug 13 18:30:02 2002 +0300 +++ b/src/lib-index/mail-modifylog.c Thu Aug 22 01:10:20 2002 +0300 @@ -391,7 +391,7 @@ } } - if (lseek(log->fd, 0, SEEK_END) == (off_t)-1) { + if (lseek(log->fd, 0, SEEK_END) == -1) { index_set_error(log->index, "lseek() failed with file %s: %m", log->filepath); return FALSE;
--- a/src/lib-index/maildir/maildir-build.c Tue Aug 13 18:30:02 2002 +0300 +++ b/src/lib-index/maildir/maildir-build.c Thu Aug 22 01:10:20 2002 +0300 @@ -67,7 +67,7 @@ MAILDIR_LOCATION_EXTRA_SPACE); /* parse the header and update record's fields */ - failed = !maildir_record_update(index, update, fd, path); + failed = !maildir_record_update(update, fd, path); if (!index->update_end(update) || failed) { /* failed - delete the record */
--- a/src/lib-index/maildir/maildir-index.c Tue Aug 13 18:30:02 2002 +0300 +++ b/src/lib-index/maildir/maildir-index.c Thu Aug 22 01:10:20 2002 +0300 @@ -21,7 +21,7 @@ oldflags = ""; if (info != NULL) { - fname = t_strndup(fname, (unsigned int) (info-fname)); + fname = t_strdup_until(fname, info); if (info[1] == '2' && info[2] == ',') oldflags = info+3; } @@ -178,5 +178,7 @@ mail_index_update_end, mail_index_update_field, mail_index_get_last_error, - mail_index_is_inconsistency_error + mail_index_is_inconsistency_error, + + MAIL_INDEX_PRIVATE_FILL };
--- a/src/lib-index/maildir/maildir-index.h Tue Aug 13 18:30:02 2002 +0300 +++ b/src/lib-index/maildir/maildir-index.h Thu Aug 22 01:10:20 2002 +0300 @@ -18,10 +18,8 @@ int maildir_index_build_dir(MailIndex *index, const char *source_dir, const char *dest_dir); -int maildir_open_mail(MailIndex *index, MailIndexRecord *rec, - off_t *offset, size_t *size); +IOBuffer *maildir_open_mail(MailIndex *index, MailIndexRecord *rec); -int maildir_record_update(MailIndex *index, MailIndexUpdate *update, - int fd, const char *path); +int maildir_record_update(MailIndexUpdate *update, int fd, const char *path); #endif
--- a/src/lib-index/maildir/maildir-open.c Tue Aug 13 18:30:02 2002 +0300 +++ b/src/lib-index/maildir/maildir-open.c Thu Aug 22 01:10:20 2002 +0300 @@ -1,16 +1,15 @@ /* Copyright (C) 2002 Timo Sirainen */ #include "lib.h" +#include "iobuffer.h" #include "maildir-index.h" #include "mail-index-util.h" #include <unistd.h> #include <fcntl.h> -int maildir_open_mail(MailIndex *index, MailIndexRecord *rec, - off_t *offset, size_t *size) +IOBuffer *maildir_open_mail(MailIndex *index, MailIndexRecord *rec) { - off_t pos; const char *fname, *path; int fd; @@ -20,7 +19,7 @@ index_set_error(index, "Corrupted index file %s: " "Missing location field for record %u", index->filepath, rec->uid); - return -1; + return NULL; } path = t_strconcat(index->dir, "/cur/", fname, NULL); @@ -28,18 +27,8 @@ fd = open(path, O_RDONLY); if (fd == -1) { index_set_error(index, "Error opening mail file %s: %m", path); - return -1; + return NULL; } - pos = lseek(fd, 0, SEEK_END); - if (pos == (off_t)-1 || lseek(fd, 0, SEEK_SET) == (off_t)-1) { - index_set_error(index, "lseek() failed with mail file %s: %m", - path); - (void)close(fd); - return -1; - } - - *offset = 0; - *size = (size_t) pos; - return fd; + return io_buffer_create_mmap(fd, default_pool, MAIL_MMAP_BLOCK_SIZE, 0); }
--- a/src/lib-index/maildir/maildir-sync.c Tue Aug 13 18:30:02 2002 +0300 +++ b/src/lib-index/maildir/maildir-sync.c Thu Aug 22 01:10:20 2002 +0300 @@ -79,7 +79,7 @@ path); failed = TRUE; } else { - if (!maildir_record_update(index, update, fd, path)) + if (!maildir_record_update(update, fd, path)) failed = TRUE; (void)close(fd); } @@ -151,8 +151,8 @@ return FALSE; } - file_changed = rec->body_size + rec->header_size != - (size_t) st.st_size; + file_changed = st.st_size != (off_t) (rec->body_size + + rec->header_size); } /* changed - update */ @@ -252,9 +252,7 @@ continue; value = p_strdup(pool, d->d_name); - key = p == NULL ? value : - p_strndup(pool, d->d_name, - (unsigned int) (p - d->d_name)); + key = p == NULL ? value : p_strdup_until(pool, d->d_name, p); hash_insert(files, key, value); } (void)closedir(dirp);
--- a/src/lib-index/maildir/maildir-update.c Tue Aug 13 18:30:02 2002 +0300 +++ b/src/lib-index/maildir/maildir-update.c Thu Aug 22 01:10:20 2002 +0300 @@ -1,34 +1,17 @@ /* Copyright (C) 2002 Timo Sirainen */ #include "lib.h" -#include "mmap-util.h" +#include "iobuffer.h" #include "maildir-index.h" -#include "mail-index-util.h" -int maildir_record_update(MailIndex *index, MailIndexUpdate *update, - int fd, const char *path) +int maildir_record_update(MailIndexUpdate *update, int fd, const char *path) { - void *mmap_base; - size_t mmap_length; + IOBuffer *inbuf; i_assert(path != NULL); - /* we need only the header which probably fits into one page, - so don't use MADV_SEQUENTIAL which would just read more than - is needed. */ - mmap_base = mmap_ro_file(fd, &mmap_length); - if (mmap_base == MAP_FAILED) { - index_set_error(index, "update: mmap() failed with file %s: %m", - path); - return FALSE; - } - - if (mmap_base == NULL) { - /* empty file */ - return TRUE; - } - - mail_index_update_headers(update, mmap_base, mmap_length, NULL, NULL); - (void)munmap(mmap_base, mmap_length); + inbuf = io_buffer_create_mmap(fd, default_pool, + MAIL_MMAP_BLOCK_SIZE, 0); + mail_index_update_headers(update, inbuf, NULL, NULL); return TRUE; }
--- a/src/lib-index/mbox/mbox-append.c Tue Aug 13 18:30:02 2002 +0300 +++ b/src/lib-index/mbox/mbox-append.c Thu Aug 22 01:10:20 2002 +0300 @@ -160,7 +160,7 @@ i_snprintf(location, sizeof(location), "%lu", (unsigned long) offset); index->update_field(update, FIELD_TYPE_LOCATION, location, 0); - /* parse the header and add cache wanted fields */ + /* parse the header and cache wanted fields */ mail_index_update_headers(update, msg, physical_size, header_func, rec); if (!index->update_end(update)) { @@ -238,7 +238,7 @@ /* get the size of the file */ end_pos = lseek(fd, 0, SEEK_END); - if (pos == (off_t)-1 || end_pos == (off_t)-1) { + if (pos == -1 || end_pos == -1) { index_set_error(index, "lseek() failed with mbox file %s: %m", path); return FALSE;
--- a/src/lib-index/mbox/mbox-fsck.c Tue Aug 13 18:30:02 2002 +0300 +++ b/src/lib-index/mbox/mbox-fsck.c Thu Aug 22 01:10:20 2002 +0300 @@ -32,6 +32,7 @@ match_next_record(MailIndex *index, MailIndexRecord *rec, unsigned int *seq, const char **data, const char *data_end) { +#if 0 // FIXME MessageSize hdr_size; HeaderData hdr_data; const char *rec_msgid, *data_next; @@ -86,7 +87,7 @@ (void)index->expunge(index, rec, *seq, TRUE); rec = index->next(index, rec); } while (rec != NULL); - +#endif return NULL; }
--- a/src/lib-index/mbox/mbox-index.c Tue Aug 13 18:30:02 2002 +0300 +++ b/src/lib-index/mbox/mbox-index.c Thu Aug 22 01:10:20 2002 +0300 @@ -87,5 +87,7 @@ mail_index_update_end, mail_index_update_field, mail_index_get_last_error, - mail_index_is_inconsistency_error + mail_index_is_inconsistency_error, + + MAIL_INDEX_PRIVATE_FILL };
--- a/src/lib-index/mbox/mbox-index.h Tue Aug 13 18:30:02 2002 +0300 +++ b/src/lib-index/mbox/mbox-index.h Thu Aug 22 01:10:20 2002 +0300 @@ -11,8 +11,7 @@ int mbox_index_rebuild(MailIndex *index); int mbox_index_sync(MailIndex *index); int mbox_index_fsck(MailIndex *index); -int mbox_open_mail(MailIndex *index, MailIndexRecord *rec, - off_t *offset, size_t *size); +IOBuffer *mbox_open_mail(MailIndex *index, MailIndexRecord *rec); int mbox_index_append(MailIndex *index, int fd, const char *path); int mbox_index_append_mmaped(MailIndex *index, const char *data,
--- a/src/lib-index/mbox/mbox-open.c Tue Aug 13 18:30:02 2002 +0300 +++ b/src/lib-index/mbox/mbox-open.c Thu Aug 22 01:10:20 2002 +0300 @@ -1,6 +1,7 @@ /* Copyright (C) 2002 Timo Sirainen */ #include "lib.h" +#include "iobuffer.h" #include "mbox-index.h" #include "mail-index-util.h" @@ -8,11 +9,10 @@ #include <unistd.h> #include <fcntl.h> -int mbox_open_mail(MailIndex *index, MailIndexRecord *rec, - off_t *offset, size_t *size) +IOBuffer *mbox_open_mail(MailIndex *index, MailIndexRecord *rec) { const char *location; - off_t pos; + off_t pos, offset, stop_offset; char buf[5]; int fd, ret, ok; @@ -24,33 +24,32 @@ index_set_error(index, "Corrupted index file %s: " "Missing location field for record %u", index->filepath, rec->uid); - return -1; + return NULL; } /* location = offset */ - *offset = (off_t)strtoul(location, NULL, 10); - *size = rec->header_size + rec->body_size; + offset = (off_t)strtoul(location, NULL, 10); + stop_offset = offset + rec->header_size + rec->body_size; fd = open(index->mbox_path, O_RDONLY); if (fd == -1) { index_set_error(index, "Can't open mbox file %s: %m", index->mbox_path); - return -1; + return NULL; } - pos = lseek(fd, *offset, SEEK_SET); - if (pos == (off_t)-1) { + pos = lseek(fd, offset, SEEK_SET); + if (pos == -1) { index_set_error(index, "lseek() failed with mbox file %s: %m", index->mbox_path); (void)close(fd); - return -1; + return NULL; } ok = FALSE; - if (pos == *offset) { + if (pos == offset) { /* make sure message size is valid */ - pos = *offset + *size; - if (lseek(fd, pos, SEEK_SET) == pos) { + if (lseek(fd, stop_offset, SEEK_SET) == stop_offset) { /* and check that we end with either EOF or to beginning of next message */ ret = read(fd, buf, 5); @@ -62,8 +61,13 @@ } if (ok) { - if (lseek(fd, *offset, SEEK_SET) == *offset) - return fd; + if (lseek(fd, offset, SEEK_SET) == offset) { + /* everything ok */ + return io_buffer_create_mmap(fd, default_pool, + MAIL_MMAP_BLOCK_SIZE, + stop_offset); + } + index_set_error(index, "lseek() failed with mbox file %s: %m", index->mbox_path); @@ -73,5 +77,5 @@ } (void)close(fd); - return -1; + return NULL; }
--- a/src/lib-index/mbox/mbox-sync.c Tue Aug 13 18:30:02 2002 +0300 +++ b/src/lib-index/mbox/mbox-sync.c Thu Aug 22 01:10:20 2002 +0300 @@ -64,7 +64,7 @@ } pos = lseek(fd, index->mbox_size, SEEK_SET); - if (pos == (off_t)-1) { + if (pos == -1) { index_set_error(index, "lseek() failed with mbox file %s: %m", index->mbox_path); (void)close(fd);
--- a/src/lib-mail/message-parser.c Tue Aug 13 18:30:02 2002 +0300 +++ b/src/lib-mail/message-parser.c Thu Aug 22 01:10:20 2002 +0300 @@ -1,6 +1,7 @@ /* Copyright (C) 2002 Timo Sirainen */ #include "lib.h" +#include "iobuffer.h" #include "rfc822-tokenize.h" #include "message-content-parser.h" #include "message-parser.h" @@ -25,12 +26,12 @@ void *user_data; } MessageParseData; -static MessagePart *message_parse_part(const char *msg, size_t size, +static MessagePart *message_parse_part(IOBuffer *inbuf, MessageParseData *parse_data); -static MessagePart *message_parse_body(const char *msg, size_t size, +static MessagePart *message_parse_body(IOBuffer *inbuf, MessageBoundary *boundaries, MessageSize *body_size); -static MessagePart *message_skip_boundary(const char *msg, size_t size, +static MessagePart *message_skip_boundary(IOBuffer *inbuf, MessageBoundary *boundaries, MessageSize *boundary_size); @@ -126,29 +127,28 @@ } } -static MessagePart *message_parse_multipart(const char *msg, size_t size, +static MessagePart *message_parse_multipart(IOBuffer *inbuf, MessageParseData *parse_data) { MessagePart *parent_part, *next_part, *part; MessageBoundary *b; - off_t offset; /* multipart message. add new boundary */ b = t_new(MessageBoundary, 1); b->part = parse_data->part; - b->boundary = t_strdup(parse_data->last_boundary); + b->boundary = parse_data->last_boundary; b->len = strlen(b->boundary); b->next = parse_data->boundaries; parse_data->boundaries = b; /* reset fields */ - p_free_and_null(parse_data->pool, parse_data->last_boundary); - p_free_and_null(parse_data->pool, parse_data->last_content_type); + parse_data->last_boundary = NULL; + parse_data->last_content_type = NULL; /* skip the data before the first boundary */ parent_part = parse_data->part; - next_part = message_skip_boundary(msg, size, parse_data->boundaries, + next_part = message_skip_boundary(inbuf, parse_data->boundaries, &parent_part->body_size); /* now, parse the parts */ @@ -163,10 +163,8 @@ part->pos.virtual_pos += parent_part->body_size.virtual_size + parent_part->header_size.virtual_size; - offset = parent_part->body_size.physical_size; parse_data->part = part; - next_part = message_parse_part(msg + offset, size - offset, - parse_data); + next_part = message_parse_part(inbuf, parse_data); /* update our size */ message_size_add_part(&parent_part->body_size, part); @@ -175,9 +173,7 @@ break; /* skip the boundary */ - offset = parent_part->body_size.physical_size; - next_part = message_skip_boundary(msg + offset, size - offset, - parse_data->boundaries, + next_part = message_skip_boundary(inbuf, parse_data->boundaries, &parent_part->body_size); } @@ -187,22 +183,21 @@ return next_part; } -static MessagePart *message_parse_part(const char *msg, size_t size, +static MessagePart *message_parse_part(IOBuffer *inbuf, MessageParseData *parse_data) { MessagePart *next_part, *part; size_t hdr_size; - message_parse_header(parse_data->part, msg, size, + message_parse_header(parse_data->part, inbuf, &parse_data->part->header_size, parse_header_field, parse_data); /* update message position/size */ hdr_size = parse_data->part->header_size.physical_size; - msg += hdr_size; size -= hdr_size; if (parse_data->last_boundary != NULL) - return message_parse_multipart(msg, size, parse_data); + return message_parse_multipart(inbuf, parse_data); if (parse_data->last_content_type == NULL) { if (parse_data->part->parent != NULL && @@ -227,7 +222,7 @@ part = message_part_append(parse_data->pool, parse_data->part); parse_data->part = part; - next_part = message_parse_part(msg, size, parse_data); + next_part = message_parse_part(inbuf, parse_data); parse_data->part = part->parent; /* our body size is the size of header+body in message/rfc822 */ @@ -235,15 +230,14 @@ } else { /* normal message, read until the next boundary */ part = parse_data->part; - next_part = message_parse_body(msg, size, - parse_data->boundaries, + next_part = message_parse_body(inbuf, parse_data->boundaries, &part->body_size); } return next_part; } -MessagePart *message_parse(Pool pool, const char *msg, size_t size, +MessagePart *message_parse(Pool pool, IOBuffer *inbuf, MessageHeaderFunc func, void *user_data) { MessagePart *part; @@ -256,107 +250,182 @@ parse_data.part = part = p_new(pool, MessagePart, 1); t_push(); - message_parse_part(msg, size, &parse_data); + message_parse_part(inbuf, &parse_data); t_pop(); return part; } -void message_parse_header(MessagePart *part, const char *msg, size_t size, +/* skip over to next line increasing message size */ +static void message_skip_line(IOBuffer *inbuf, MessageSize *msg_size) +{ + unsigned char *msg; + unsigned int i, size, startpos; + + startpos = 0; + + while (io_buffer_read_data(inbuf, &msg, &size, startpos) >= 0) { + for (i = startpos; i < size; i++) { + if (msg[i] == '\n') { + if (i > 0 && msg[i-1] != '\r') + msg_size->virtual_size++; + if (msg_size != NULL) + msg_size->lines++; + break; + } + } + + if (i < size) { + startpos = i; + break; + } + + if (i > 0) { + /* leave the last character, it may be \r */ + io_buffer_skip(inbuf, i - 1); + startpos = 1; + + if (msg_size != NULL) { + msg_size->physical_size += i - 1; + msg_size->virtual_size += i - 1; + } + } + } + + io_buffer_skip(inbuf, startpos); + + if (msg_size != NULL) { + msg_size->physical_size += startpos; + msg_size->virtual_size += startpos; + } +} + +void message_parse_header(MessagePart *part, IOBuffer *inbuf, MessageSize *hdr_size, MessageHeaderFunc func, void *user_data) { - const char *msg_start, *msg_end, *cr, *last_lf; - const char *name, *value, *name_end, *value_end; - int missing_cr_count, stop; - - msg_start = msg; - msg_end = msg + size; - - missing_cr_count = 0; cr = NULL; - name = msg; name_end = value = last_lf = NULL; + unsigned char *msg; + unsigned int i, size, startpos, missing_cr_count; + unsigned int line_start, colon_pos, end_pos, name_len, value_len; + int ret; if (hdr_size != NULL) - hdr_size->lines = 0; + memset(hdr_size, 0, sizeof(MessageSize)); + + missing_cr_count = startpos = line_start = 0; + colon_pos = UINT_MAX; + while ((ret = io_buffer_read_data(inbuf, &msg, + &size, startpos+1)) != -1) { + if (size == 0) { + /* no, we never want empty buffer */ + continue; + } + + if (ret == -2) { + /* overflow, line is too long. just skip it. */ + i_assert(size > 2); - stop = FALSE; - while (msg != msg_end && !stop) { - switch (*msg) { - case '\n': + message_skip_line(inbuf, hdr_size); + startpos = line_start = 0; + colon_pos = UINT_MAX; + continue; + } + + /* don't parse the last character, so we can always have + one character read-ahead. we never care about the last + character anyway, it's either the first character in + message body, or if there's no body for any reason, it's + the \n ending the header. */ + size--; + for (i = startpos; i < size; i++) { + if (msg[i] == ':' && colon_pos != UINT_MAX) { + colon_pos = i; + continue; + } + + if (msg[i] != '\n') + continue; + if (hdr_size != NULL) hdr_size->lines++; - if (msg == msg_start || - (cr == msg_start && cr == msg-1)) { - /* no headers at all */ - if (cr != msg-1) - missing_cr_count++; - stop = TRUE; - break; - } else if (cr == msg-1) { - /* CR+LF */ - value_end = cr; - - if (last_lf == cr-1) { - /* LF+CR+LF -> end of headers */ - stop = TRUE; - break; - } - } else { + if (i == 0 || msg[i-1] != '\r') { /* missing CR */ missing_cr_count++; - value_end = msg; + } - if (last_lf == msg-1) { - /* LF+LF -> end of headers */ - stop = TRUE; - break; - } + if (i == 0 || (i == 1 && msg[i-1] == '\r')) { + /* no headers at all */ + i++; + break; } - last_lf = msg; - if (msg+1 != msg_end && IS_LWSP(msg[1])) { - /* long header continuing in next line */ + if ((i > 0 && msg[i-1] == '\n') || + (i > 1 && msg[i-2] == '\n' && msg[i-1] == '\r')) { + /* \n\n or \n\r\n - end of headers */ + i++; break; } - /* Ignore header lines missing ':' (value == NULL) */ - if (func != NULL && value != NULL) { - func(part, name, (unsigned int) (name_end-name), - value, (unsigned int) (value_end-value), - user_data); - } + /* make sure the header doesn't continue to next line */ + if (!IS_LWSP(msg[i+1])) { + if (colon_pos != UINT_MAX && + colon_pos != line_start && func != NULL && + !IS_LWSP(msg[line_start])) { + /* we have a valid header line */ + + /* get length of name-field */ + end_pos = colon_pos-1; + while (end_pos > line_start && + IS_LWSP(msg[end_pos])) + end_pos--; + name_len = end_pos - line_start + 1; - /* reset the data */ - name = msg+1; - name_end = NULL; - value = NULL; - break; - case '\r': - cr = msg; - break; - case ':': - if (value != NULL) - break; - name_end = msg; + /* get length of value field */ + colon_pos++; + while (colon_pos < i && + IS_LWSP(msg[colon_pos])) + colon_pos++; + value_len = i - colon_pos; + if (msg[i-1] == '\r') value_len--; - /* skip the ending whitespace for field */ - while (name_end != name && IS_LWSP(name_end[-1])) - name_end--; + /* and finally call the function */ + func(part, msg + line_start, name_len, + msg + colon_pos, value_len, + user_data); + } - /* get beginning of field value */ - value = msg+1; - if (msg+1 != msg_end && IS_LWSP(msg[1])) - value++; + colon_pos = UINT_MAX; + line_start = i+1; + } + } + + if (i < size) { + /* end of header */ + startpos = i; break; } - msg++; + if (i > 0) { + /* leave the last line to buffer */ + startpos = line_start; + line_start = 0; + + if (colon_pos != UINT_MAX) + colon_pos -= startpos; + + io_buffer_skip(inbuf, i - startpos); + + if (hdr_size != NULL) + hdr_size->physical_size += i - startpos; + } } + io_buffer_skip(inbuf, startpos); if (hdr_size != NULL) { - hdr_size->physical_size = (int) (msg - msg_start); - hdr_size->virtual_size = + hdr_size->physical_size += startpos; + hdr_size->virtual_size += hdr_size->physical_size + missing_cr_count; + i_assert(hdr_size->virtual_size >= hdr_size->physical_size); } } @@ -374,128 +443,157 @@ return NULL; } -static MessagePart *message_parse_body(const char *msg, size_t size, +/* read until next boundary is found. if skip_over = FALSE, stop at the + [\r]\n before the boundary, otherwise leave it right after the known + boundary so the ending "--" can be checked. */ +static MessageBoundary * +message_find_boundary(IOBuffer *inbuf, MessageBoundary *boundaries, + MessageSize *msg_size, int skip_over) +{ + // FIXME: probably lots of bugs in this function + MessageBoundary *boundary; + unsigned char *msg; + unsigned int i, size, startpos, line_start, missing_cr_count; + + memset(msg_size, 0, sizeof(MessageSize)); + + boundary = NULL; + missing_cr_count = startpos = line_start = 0; + + while (io_buffer_read_data(inbuf, &msg, &size, startpos) >= 0) { + for (i = startpos; i < size; i++) { + if (msg[i] != '\n') + continue; + + if (i > line_start+2 && msg[line_start] == '-' && + msg[line_start+1] == '-') { + /* possible boundary */ + boundary = boundary_find(boundaries, + msg + line_start + 2, + i - line_start - 2); + if (boundary != NULL) + break; + } + + if (i > 0 && msg[i-1] != '\r') { + /* missing CR */ + missing_cr_count++; + } + + msg_size->lines++; + line_start = i+1; + } + + if (boundary != NULL) { + /* boundary found */ + break; + } + + if (i > 0) { + if (i - line_start > 128 && + msg[line_start] == '-' && msg[line_start+1] == '-') { + /* long partial line, see if it's a boundary. + RFC-2046 says that the boundaries must be + 70 chars without "--" or less. We allow + a bit larger.. */ + boundary = boundary_find(boundaries, + msg + line_start + 2, + i - line_start - 2); + if (boundary != NULL) + break; + + /* nope, we can skip over the line, just + leave the last char since it may be \r */ + i--; + } else { + /* leave the last line to buffer, it may be + boundary */ + i = line_start; + if (i > 2) i -= 2; /* leave the \r\n too */ + line_start -= i; + } + + io_buffer_skip(inbuf, i); + msg_size->physical_size += i; + + startpos = size - i; + } + } + + if (boundary != NULL) { + msg_size->physical_size += line_start - startpos; + + if (skip_over) { + /* leave the pointer right after the boundary */ + line_start += 2 + boundary->len; + } else if (line_start > 0 && msg[line_start-1] == '\n') { + /* leave the \r\n before the boundary */ + line_start--; + msg_size->physical_size--; + msg_size->lines--; + + if (line_start > 0 && msg[line_start-1] == '\r') { + line_start--; + msg_size->physical_size--; + } else { + missing_cr_count--; + } + } + startpos = line_start; + } + + io_buffer_skip(inbuf, startpos); + msg_size->physical_size += startpos; + + msg_size->virtual_size += + msg_size->physical_size + missing_cr_count; + i_assert(msg_size->virtual_size >= msg_size->physical_size); + + return boundary; +} + +static MessagePart *message_parse_body(IOBuffer *inbuf, MessageBoundary *boundaries, MessageSize *body_size) { MessageBoundary *boundary; - const char *msg_start, *msg_end, *cr; - unsigned int missing_cr_count, len; - msg_start = msg; - msg_end = msg + size; - - missing_cr_count = 0; cr = NULL; - - boundary = NULL; - while (msg != msg_end) { - if (*msg == '\r') - cr = msg; - else if (*msg == '\n') { - if (cr != msg-1) - missing_cr_count++; - body_size->lines++; - } else if (*msg == '-' && msg+2 < msg_end && msg[1] == '-' && - (msg == msg_start || msg[-1] == '\n')) { - /* "\n--", could be boundary */ - len = (unsigned int) (msg_end - (msg+2)); - - boundary = boundary_find(boundaries, msg+2, len); - if (boundary != NULL) { - /* boundary found, move the pointer - before the [CR]LF */ - if (msg != msg_start) { - msg--; - if (cr == msg-1) - msg--; - else - missing_cr_count--; - body_size->lines--; - } - break; - } - } - - msg++; - } - - len = (unsigned int) (msg - msg_start); - body_size->physical_size += len; - body_size->virtual_size += len + missing_cr_count; - + boundary = message_find_boundary(inbuf, boundaries, body_size, FALSE); return boundary == NULL ? NULL : boundary->part; } -static MessagePart *message_skip_boundary(const char *msg, size_t size, +/* skip data until next boundary is found. if it's end boundary, + skip the footer as well. */ +static MessagePart *message_skip_boundary(IOBuffer *inbuf, MessageBoundary *boundaries, MessageSize *boundary_size) { MessageBoundary *boundary; - const char *msg_start, *msg_end, *cr; - unsigned int len, missing_cr_count; + unsigned char *msg; + unsigned int size; int end_boundary; - /* first find and skip the boundary */ - msg_start = msg; - msg_end = msg + size; - - cr = NULL; missing_cr_count = 0; - - boundary = NULL; - while (msg != msg_end) { - if (*msg == '-' && msg+2 < msg_end && msg[1] == '-' && - (msg == msg_start || msg[-1] == '\n')) { - /* possible boundary */ - len = (unsigned int) (msg_end - (msg+2)); - boundary = boundary_find(boundaries, msg+2, len); - if (boundary != NULL) { - /* skip the boundary */ - msg += 2 + boundary->len; - break; - } - } else if (*msg == '\r') - cr = msg; - else if (*msg == '\n') { - if (cr != msg-1) - missing_cr_count++; - boundary_size->lines++; - } - msg++; - } - - len = (unsigned int) (msg - msg_start); - boundary_size->physical_size += len; - boundary_size->virtual_size += len + missing_cr_count; - + boundary = message_find_boundary(inbuf, boundaries, + boundary_size, TRUE); if (boundary == NULL) return NULL; - /* now read the boundary until we reach the end of line */ - msg_start = msg; - end_boundary = msg+2 <= msg_end && msg[0] == '-' && msg[1] == '-'; - while (msg != msg_end) { - if (*msg == '\r') - cr = msg; - else if (*msg == '\n') { - if (cr != msg-1) - boundary_size->virtual_size++; - boundary_size->lines++; - msg++; + /* now, see if it's end boundary */ + end_boundary = FALSE; + while (io_buffer_read_data(inbuf, &msg, &size, 1) >= 0) { + if (size >= 2) { + end_boundary = msg[0] == '-' && msg[1] == '-'; break; } - - msg++; } - len = (unsigned int) (msg - msg_start); - boundary_size->physical_size += len; - boundary_size->virtual_size += len; + /* skip the rest of the line */ + message_skip_line(inbuf, boundary_size); if (end_boundary) { /* skip the footer */ - return message_parse_body(msg, (unsigned int) (msg_end-msg), - boundaries, boundary_size); + return message_parse_body(inbuf, boundaries, boundary_size); } - return boundary->part; + return boundary == NULL ? NULL : boundary->part; }
--- a/src/lib-mail/message-parser.h Tue Aug 13 18:30:02 2002 +0300 +++ b/src/lib-mail/message-parser.h Thu Aug 22 01:10:20 2002 +0300 @@ -41,14 +41,15 @@ void *user_data); /* func is called for each field in message header. */ -MessagePart *message_parse(Pool pool, const char *msg, size_t size, +MessagePart *message_parse(Pool pool, IOBuffer *inbuf, MessageHeaderFunc func, void *user_data); /* Call func for each field in message header. Fills the hdr_size. part can be NULL, just make sure your header function works with it. This function doesn't use temp. mempool so your header function may save - return values to it. */ -void message_parse_header(MessagePart *part, const char *msg, size_t size, + return values to it. When finished, inbuf will point to beginning of + message body. */ +void message_parse_header(MessagePart *part, IOBuffer *inbuf, MessageSize *hdr_size, MessageHeaderFunc func, void *user_data);
--- a/src/lib-mail/message-size.c Tue Aug 13 18:30:02 2002 +0300 +++ b/src/lib-mail/message-size.c Thu Aug 22 01:10:20 2002 +0300 @@ -1,90 +1,170 @@ /* Copyright (C) 2002 Timo Sirainen */ #include "lib.h" +#include "iobuffer.h" #include "message-parser.h" #include "message-size.h" -void message_get_header_size(const char *msg, size_t size, MessageSize *hdr) +void message_get_header_size(IOBuffer *inbuf, MessageSize *hdr) { - const char *msg_start, *msg_end, *cr, *last_lf; - int missing_cr_count; + unsigned char *msg; + unsigned int i, size, startpos, missing_cr_count; - hdr->lines = 0; - - msg_start = msg; - msg_end = msg + size; - - /* get header size */ + memset(hdr, 0, sizeof(MessageSize)); - cr = last_lf = NULL; missing_cr_count = 0; - while (msg != msg_end) { - if (*msg == '\r') - cr = msg; - else if (*msg == '\n') { - hdr->lines++; + missing_cr_count = 0; startpos = 0; + while (io_buffer_read_data(inbuf, &msg, &size, startpos) >= 0) { + for (i = startpos; i < size; i++) { + if (msg[i] != '\n') + continue; - if (msg == msg_start || - (cr == msg_start && cr == msg-1)) { + hdr->lines++; + if (i == 0 || msg[i-1] != '\r') { + /* missing CR */ + missing_cr_count++; + } + + if (i == 0 || (i == 1 && msg[i-1] == '\r')) { /* no headers at all */ - if (cr != msg-1) - missing_cr_count++; - msg++; break; } - if (cr == msg-1) { - /* CR+LF */ - if (last_lf == cr-1) { - /* LF+CR+LF -> end of headers */ - msg++; - break; - } - } else { + if ((i > 0 && msg[i-1] == '\n') || + (i > 1 && msg[i-2] == '\n' && msg[i-1] == '\r')) { + /* \n\n or \n\r\n - end of headers */ + break; + } + } + + if (i < size) { + /* end of header */ + startpos = i + 1; + break; + } + + if (i > 0) { + /* leave the last two characters, they may be \r\n */ + startpos = size == 1 ? 1 : 2; + io_buffer_skip(inbuf, i - startpos); + + hdr->physical_size += i - startpos; + } + } + io_buffer_skip(inbuf, startpos); + hdr->physical_size += startpos; + + hdr->virtual_size = hdr->physical_size + missing_cr_count; + i_assert(hdr->virtual_size >= hdr->physical_size); +} + +void message_get_body_size(IOBuffer *inbuf, MessageSize *body, + off_t max_virtual_size) +{ + unsigned char *msg; + unsigned int i, size, startpos, missing_cr_count; + + memset(body, 0, sizeof(MessageSize)); + + missing_cr_count = 0; startpos = 0; + while (max_virtual_size != 0 && + io_buffer_read_data(inbuf, &msg, &size, startpos) >= 0) { + for (i = startpos; i < size && max_virtual_size != 0; i++) { + if (max_virtual_size > 0) + max_virtual_size--; + + if (msg[i] != '\n') + continue; + + if (i == 0 || msg[i-1] != '\r') { /* missing CR */ missing_cr_count++; - if (last_lf == msg-1) { - /* LF+LF -> end of headers */ - msg++; - break; + if (max_virtual_size > 0) { + if (max_virtual_size == 0) + break; + + max_virtual_size--; } } - last_lf = msg; + + /* increase after making sure we didn't break + at virtual \r */ + body->lines++; } - msg++; - } + if (i > 0) { + /* leave the last character, it may be \r */ + io_buffer_skip(inbuf, i - 1); + startpos = 1; - hdr->physical_size = (int) (msg-msg_start); - hdr->virtual_size = hdr->physical_size + missing_cr_count; + body->physical_size += i - 1; + } + } + io_buffer_skip(inbuf, startpos); + body->physical_size += startpos; + + body->virtual_size = body->physical_size + missing_cr_count; + i_assert(body->virtual_size >= body->physical_size); } -void message_get_body_size(const char *msg, size_t size, MessageSize *body) +void message_skip_virtual(IOBuffer *inbuf, off_t virtual_skip, + MessageSize *msg_size, int *cr_skipped) { - const char *msg_start, *msg_end, *cr; - int missing_cr_count; + unsigned char *msg; + unsigned int i, size, startpos; - msg_start = msg; - msg_end = msg + size; + *cr_skipped = FALSE; + if (virtual_skip == 0) + return; - body->lines = 0; + startpos = 0; + while (io_buffer_read_data(inbuf, &msg, &size, startpos) >= 0) { + for (i = startpos; i < size && virtual_skip > 0; i++) { + virtual_skip--; - cr = NULL; missing_cr_count = 0; - while (msg != msg_end) { - if (*msg == '\r') - cr = msg; - else if (*msg == '\n') { - body->lines++; + if (msg[i] == '\r') { + /* CR */ + if (virtual_skip == 0) + *cr_skipped = TRUE; + } else if (msg[i] == '\n') { + /* LF */ + if (i == 0 || msg[i-1] != '\r') { + /* missing CR */ + if (msg_size != NULL) + msg_size->virtual_size++; - if (cr != msg-1) - missing_cr_count++; + if (virtual_skip == 0) { + /* CR/LF boundary */ + *cr_skipped = TRUE; + break; + } + + virtual_skip--; + } + + /* increase after making sure we didn't break + at virtual \r */ + if (msg_size != NULL) + msg_size->lines++; + } } - msg++; - } + if (msg_size != NULL) { + msg_size->physical_size += i; + msg_size->virtual_size += i; + } - body->physical_size = (int) (msg-msg_start); - body->virtual_size = (int) (msg-msg_start) + missing_cr_count; + if (i < size) { + io_buffer_skip(inbuf, i); + break; + } + + if (i > 0) { + /* leave the last character, it may be \r */ + io_buffer_skip(inbuf, i - 1); + startpos = 1; + } + } } void message_size_add(MessageSize *dest, MessageSize *src)
--- a/src/lib-mail/message-size.h Tue Aug 13 18:30:02 2002 +0300 +++ b/src/lib-mail/message-size.h Thu Aug 22 01:10:20 2002 +0300 @@ -3,8 +3,21 @@ #include "message-parser.h" -void message_get_header_size(const char *msg, size_t size, MessageSize *hdr); -void message_get_body_size(const char *msg, size_t size, MessageSize *body); +/* Calculate size of message header. Leave the inbuf point to first + character in body. */ +void message_get_header_size(IOBuffer *inbuf, MessageSize *hdr); +/* Calculate size of message body. Read only max_virtual_size virtual bytes + if it's >= 0. */ +void message_get_body_size(IOBuffer *inbuf, MessageSize *body, + off_t max_virtual_size); + +/* Skip number of virtual bytes from buffer. If first character is \n, and + cr_skipped is FALSE, \r must be sent before it. msg_size is updated if + it's not NULL. */ +void message_skip_virtual(IOBuffer *inbuf, off_t virtual_skip, + MessageSize *msg_size, int *cr_skipped); + +/* Sum contents of src into dest. */ void message_size_add(MessageSize *dest, MessageSize *src); #endif
--- a/src/lib-storage/flags-file/flags-file.c Tue Aug 13 18:30:02 2002 +0300 +++ b/src/lib-storage/flags-file/flags-file.c Thu Aug 22 01:10:20 2002 +0300 @@ -54,10 +54,10 @@ /* make sure it's still empty after locking */ pos = lseek(ff->fd, 0, SEEK_END); - if (pos != (off_t)-1 && pos < HEADER_SIZE) + if (pos != -1 && pos < HEADER_SIZE) pos = lseek(ff->fd, 0, SEEK_SET); - if (pos == (off_t)-1) { + if (pos == -1) { mail_storage_set_critical(ff->storage, "lseek() failed for " "flags file %s: %m", ff->path); return FALSE; @@ -123,8 +123,7 @@ while (data != data_end && *data != '\n') data++; - ff->custom_flags[num] = - i_strndup(line, (unsigned int) (data - line)); + ff->custom_flags[num] = i_strdup_until(line, data); } } } @@ -256,7 +255,7 @@ { int i; - if (lseek(ff->fd, 0, SEEK_SET) == (off_t)-1) { + if (lseek(ff->fd, 0, SEEK_SET) == -1) { mail_storage_set_critical(ff->storage, "lseek() failed for " "flags file %s: %m", ff->path); return FALSE; @@ -300,13 +299,13 @@ /* add the flag */ pos = lseek(ff->fd, 0, SEEK_END); - if (pos == (off_t)-1) { + if (pos == -1) { mail_storage_set_critical(ff->storage, "lseek() failed for " "flags file %s: %m", ff->path); return FALSE; } - if ((size_t) pos != ff->mmap_length) { + if (pos != (off_t)ff->mmap_length) { mail_storage_set_critical(ff->storage, "flags file %s was " "changed by someone while we were" "trying to modify it", ff->path);
--- a/src/lib-storage/index/index-copy.c Tue Aug 13 18:30:02 2002 +0300 +++ b/src/lib-storage/index/index-copy.c Thu Aug 22 01:10:20 2002 +0300 @@ -16,22 +16,19 @@ unsigned int seq __attr_unused__, void *user_data) { CopyData *cd = user_data; - IOBuffer *buf; - off_t offset; - size_t size; - int fd, failed; + IOBuffer *inbuf; + int failed; - fd = index->open_mail(index, rec, &offset, &size); - if (fd == -1) + inbuf = index->open_mail(index, rec); + if (inbuf == NULL) return FALSE; /* save it in destination mailbox */ - buf = io_buffer_create_file(fd, default_pool, 4096); failed = !cd->dest->save(cd->dest, rec->msg_flags, cd->custom_flags, rec->internal_date, - buf, size); - - (void)close(fd); + inbuf, inbuf->stop_offset - inbuf->offset); + (void)close(inbuf->fd); + io_buffer_destroy(inbuf); return !failed; }
--- a/src/lib-storage/index/index-fetch-section.c Tue Aug 13 18:30:02 2002 +0300 +++ b/src/lib-storage/index/index-fetch-section.c Thu Aug 22 01:10:20 2002 +0300 @@ -35,12 +35,12 @@ FetchData *data, int fetch_header) { MessageSize size; - const char *msg, *str; - int fd; + IOBuffer *inbuf; + const char *str; if (!imap_msgcache_get_rfc822_partial(data->cache, rec->uid, sect->skip, sect->max_size, - fetch_header, &size, &msg, &fd)) { + fetch_header, &size, &inbuf)) { i_error("Couldn't get BODY[] for UID %u (index %s)", rec->uid, data->index->filepath); return FALSE; @@ -49,8 +49,7 @@ str = t_strdup_printf("{%lu}\r\n", (unsigned long) size.virtual_size); (void)io_buffer_send(data->outbuf, str, strlen(str)); - (void)imap_message_send(data->outbuf, msg, fd, &size, - 0, sect->max_size); + (void)imap_message_send(data->outbuf, inbuf, &size, 0, sect->max_size); return TRUE; } @@ -74,33 +73,30 @@ return field_list; } -static int header_match(char *const *fields, const char *data, size_t size) +static int header_match(char *const *fields, const char *name, + unsigned int size) { - const char *field, *data_start, *data_end; + const char *field, *name_start, *name_end; i_assert(size > 0); - data_start = data; - data_end = data + size; + name_start = name; + name_end = name + size; for (; *fields != NULL; fields++) { field = *fields; if (*field == '\0') continue; - for (data = data_start; data != data_end; data++) { + for (name = name_start; name != name_end; name++) { /* field has been uppercased long time ago while parsing FETCH command */ - if (i_toupper(*data) != *field) + if (i_toupper(*name) != *field) break; field++; if (*field == '\0') { - /* "field : value" is valid */ - while (data+1 != data_end && IS_LWSP(data[1])) - data++; - - if (data+1 != data_end && data[1] == ':') + if (name+1 == name_end) return TRUE; break; } @@ -110,84 +106,92 @@ return FALSE; } -static int header_match_not(char *const *fields, const char *data, size_t size) +static int header_match_not(char *const *fields, const char *name, + unsigned int size) { - return !header_match(fields, data, size); + return !header_match(fields, name, size); } static int header_match_mime(char *const *fields __attr_unused__, - const char *data, size_t size) + const char *name, unsigned int size) { - if (size > 8 && strncasecmp(data, "Content-", 8) == 0) + if (size > 8 && strncasecmp(name, "Content-", 8) == 0) return TRUE; - if (size >= 13 && strncasecmp(data, "Mime-Version:", 13) == 0) + if (size == 12 && strncasecmp(name, "Mime-Version", 13) == 0) return TRUE; return FALSE; } -/* Store headers into dest, returns number of bytes written. */ -static unsigned int -fetch_header_fields(const char *msg, size_t size, - char *dest, char *const *fields, - int (*match_func) (char *const *, const char *, size_t)) +typedef struct { + char *dest; + char *const *fields; + int (*match_func) (char *const *, const char *, unsigned int); +} FetchHeaderFieldData; + +static void fetch_header_field(MessagePart *part __attr_unused__, + const char *name, unsigned int name_len, + const char *value __attr_unused__, + unsigned int value_len __attr_unused__, + void *user_data) { - const char *msg_start, *msg_end, *cr; - char *dest_start; - unsigned int i; - int matched; + FetchHeaderFieldData *data = user_data; + const char *name_start, *name_end, *cr; + unsigned int len; - dest_start = dest; - - /* parse fields uppercased into array - no error checking */ - msg_start = msg; - msg_end = msg + size; + /* see if we want this field */ + if (!data->match_func(data->fields, name, name_len)) + return; - cr = NULL; matched = FALSE; - for (; msg != msg_end; msg++) { - if (*msg == '\r') - cr = msg; - else if (*msg == '\n') { - if (!matched && msg != msg_start && - !IS_LWSP(*msg_start)) { - matched = match_func(fields, msg_start, - (size_t) (msg-msg_start)); - } + /* add the field, inserting CRs when needed. FIXME: is this too + kludgy? we assume name continues with ": value\n".. */ + name_start = name; + name_end = name + name_len; - if (matched) { - if (cr == msg-1) { - /* contains CR+LF, copy them */ - i = (unsigned int) (msg-msg_start)+1; - memcpy(dest, msg_start, i); - dest += i; - } else { - /* copy line without LF, appending - CR+LF afterwards */ - i = (unsigned int) (msg-msg_start); - memcpy(dest, msg_start, i); - dest += i; + cr = NULL; + for (name_start = name; name != name_end; name++) { + if (*name == '\r') + cr = name; + else if (*name == '\n' && cr != name-1) { + /* missing CR */ + len = (unsigned int) (name-name_start); + memcpy(data->dest, name_start, len); - *dest++ = '\r'; - *dest++ = '\n'; - } + data->dest[len++] = '\r'; + data->dest[len++] = '\n'; + data->dest += len; - /* see if it continues in next line */ - matched = msg+1 != msg_end && IS_LWSP(msg[1]); - } - - msg_start = msg+1; + name_start = name+1; } } - /* headers should always end with \n\n, so we don't need to - check the last line here */ + if (name_start != name_end) { + /* last linebreak was \r\n */ + len = (unsigned int) (name_end-name_start); + memcpy(data->dest, name_start, len); + data->dest += len; + } +} - return (unsigned int) (dest - dest_start); +/* Store headers into dest, returns number of bytes written. */ +static unsigned int +fetch_header_fields(IOBuffer *inbuf, char *dest, char *const *fields, + int (*match_func) (char *const *, const char *, + unsigned int)) +{ + FetchHeaderFieldData data; + + data.dest = dest; + data.fields = fields; + data.match_func = match_func; + + message_parse_header(NULL, inbuf, NULL, fetch_header_field, &data); + return (unsigned int) (data.dest - dest); } /* fetch wanted headers from given data */ -static void fetch_header_from(const char *msg, int fd, MessageSize *size, +static void fetch_header_from(IOBuffer *inbuf, MessageSize *size, const char *section, MailFetchBodyData *sect, FetchData *data) { @@ -202,8 +206,8 @@ str = t_strdup_printf("{%lu}\r\n", (unsigned long) size->virtual_size); (void)io_buffer_send(data->outbuf, str, strlen(str)); - (void)imap_message_send(data->outbuf, msg, fd, - size, sect->skip, sect->max_size); + (void)imap_message_send(data->outbuf, inbuf, size, + sect->skip, sect->max_size); return; } @@ -213,17 +217,16 @@ dest = t_malloc(size->virtual_size); if (strncasecmp(section, "HEADER.FIELDS ", 14) == 0) { - len = fetch_header_fields(msg, size->physical_size, dest, + len = fetch_header_fields(inbuf, dest, get_fields_array(section + 14), header_match); } else if (strncasecmp(section, "HEADER.FIELDS.NOT ", 18) == 0) { - len = fetch_header_fields(msg, size->physical_size, dest, + len = fetch_header_fields(inbuf, dest, get_fields_array(section + 18), header_match_not); } else if (strcasecmp(section, "MIME") == 0) { /* Mime-Version + Content-* fields */ - len = fetch_header_fields(msg, size->physical_size, dest, - NULL, header_match_mime); + len = fetch_header_fields(inbuf, dest, NULL, header_match_mime); } else { /* error */ len = 0; @@ -231,13 +234,13 @@ i_assert(len <= size->virtual_size); - if ((off_t) len <= sect->skip) + if ((off_t)len <= sect->skip) len = 0; else { dest += sect->skip; len -= sect->skip; - if (sect->max_size > 0 && len > sect->max_size) + if (sect->max_size >= 0 && (off_t)len > sect->max_size) len = sect->max_size; } @@ -253,14 +256,13 @@ FetchData *data) { MessageSize hdr_size; - const char *msg; - int fd; + IOBuffer *inbuf; if (!imap_msgcache_get_rfc822(data->cache, rec->uid, - &hdr_size, NULL, &msg, &fd)) + &hdr_size, NULL, &inbuf)) return FALSE; - fetch_header_from(msg, fd, &hdr_size, sect->section, sect, data); + fetch_header_from(inbuf, &hdr_size, sect->section, sect, data); return TRUE; } @@ -308,19 +310,17 @@ static int fetch_part_body(MessagePart *part, unsigned int uid, MailFetchBodyData *sect, FetchData *data) { - const char *msg, *str; + IOBuffer *inbuf; + const char *str; off_t skip_pos; - int fd; - if (!imap_msgcache_get_data(data->cache, uid, &msg, &fd, NULL)) + if (!imap_msgcache_get_data(data->cache, uid, &inbuf)) return FALSE; /* jump to beginning of wanted data */ skip_pos = (off_t) (part->pos.physical_pos + part->header_size.physical_size); - msg += skip_pos; - if (fd != -1 && lseek(fd, skip_pos, SEEK_CUR) == (off_t)-1) - fd = -1; + io_buffer_skip(inbuf, skip_pos); str = t_strdup_printf("{%lu}\r\n", (unsigned long) part->body_size.virtual_size); @@ -328,7 +328,7 @@ /* FIXME: potential performance problem with big messages: FETCH BODY[1]<100000..1024>, hopefully no clients do this */ - (void)imap_message_send(data->outbuf, msg, -1, &part->body_size, + (void)imap_message_send(data->outbuf, inbuf, &part->body_size, sect->skip, sect->max_size); return TRUE; } @@ -338,13 +338,13 @@ const char *section, MailFetchBodyData *sect, FetchData *data) { - const char *msg; + IOBuffer *inbuf; - if (!imap_msgcache_get_data(data->cache, uid, &msg, NULL, NULL)) + if (!imap_msgcache_get_data(data->cache, uid, &inbuf)) return FALSE; - fetch_header_from(msg + part->pos.physical_pos, -1, - &part->header_size, section, sect, data); + io_buffer_skip(inbuf, part->pos.physical_pos); + fetch_header_from(inbuf, &part->header_size, section, sect, data); return TRUE; }
--- a/src/lib-storage/index/index-fetch.c Tue Aug 13 18:30:02 2002 +0300 +++ b/src/lib-storage/index/index-fetch.c Thu Aug 22 01:10:20 2002 +0300 @@ -89,11 +89,11 @@ static void index_fetch_rfc822(MailIndexRecord *rec, FetchData *data) { MessageSize hdr_size, body_size; - const char *msg, *str; - int fd; + IOBuffer *inbuf; + const char *str; if (!imap_msgcache_get_rfc822(data->cache, rec->uid, - &hdr_size, &body_size, &msg, &fd)) { + &hdr_size, &body_size, &inbuf)) { i_error("Couldn't get RFC822 for UID %u (index %s)", rec->uid, data->index->filepath); return; @@ -106,17 +106,17 @@ body_size.physical_size += hdr_size.physical_size; body_size.virtual_size += hdr_size.virtual_size; - (void)imap_message_send(data->outbuf, msg, fd, &body_size, 0, 0); + (void)imap_message_send(data->outbuf, inbuf, &body_size, 0, -1); } static void index_fetch_rfc822_header(MailIndexRecord *rec, FetchData *data) { MessageSize hdr_size; - const char *msg, *str; - int fd; + IOBuffer *inbuf; + const char *str; if (!imap_msgcache_get_rfc822(data->cache, rec->uid, - &hdr_size, NULL, &msg, &fd)) { + &hdr_size, NULL, &inbuf)) { i_error("Couldn't get RFC822.HEADER for UID %u (index %s)", rec->uid, data->index->filepath); return; @@ -125,17 +125,17 @@ str = t_strdup_printf(" RFC822.HEADER {%lu}\r\n", (unsigned long) hdr_size.virtual_size); (void)io_buffer_send(data->outbuf, str, strlen(str)); - (void)imap_message_send(data->outbuf, msg, fd, &hdr_size, 0, 0); + (void)imap_message_send(data->outbuf, inbuf, &hdr_size, 0, -1); } static void index_fetch_rfc822_text(MailIndexRecord *rec, FetchData *data) { MessageSize body_size; - const char *msg, *str; - int fd; + IOBuffer *inbuf; + const char *str; if (!imap_msgcache_get_rfc822(data->cache, rec->uid, - NULL, &body_size, &msg, &fd)) { + NULL, &body_size, &inbuf)) { i_error("Couldn't get RFC822.TEXT for UID %u (index %s)", rec->uid, data->index->filepath); return; @@ -144,7 +144,7 @@ str = t_strdup_printf(" RFC822.TEXT {%lu}\r\n", (unsigned long) body_size.virtual_size); (void)io_buffer_send(data->outbuf, str, strlen(str)); - (void)imap_message_send(data->outbuf, msg, fd, &body_size, 0, 0); + (void)imap_message_send(data->outbuf, inbuf, &body_size, 0, -1); } static ImapCacheField index_get_cache(MailFetchData *fetch_data) @@ -180,28 +180,42 @@ return field; } +static IOBuffer *inbuf_rewind(IOBuffer *inbuf, void *user_data __attr_unused__) +{ + if (!io_buffer_seek(inbuf, 0)) { + i_error("inbuf_rewind: lseek() failed: %m"); + + (void)close(inbuf->fd); + io_buffer_destroy(inbuf); + return NULL; + } + + return inbuf; +} + static int index_cache_message(MailIndexRecord *rec, FetchData *data, ImapCacheField field) { - off_t offset; - size_t size; - int fd; + IOBuffer *inbuf; - fd = data->index->open_mail(data->index, rec, &offset, &size); - if (fd == -1) { + inbuf = data->index->open_mail(data->index, rec); + if (inbuf == NULL) { i_error("Couldn't open message UID %u (index %s)", rec->uid, data->index->filepath); return FALSE; } if (MSG_HAS_VALID_CRLF_DATA(rec)) { - imap_msgcache_message(data->cache, rec->uid, fd, offset, size, - rec->full_virtual_size, - rec->header_size, rec->body_size, field); + imap_msgcache_message(data->cache, rec->uid, + field, rec->full_virtual_size, + rec->header_size, rec->body_size, + inbuf, inbuf_rewind, NULL); } else { - imap_msgcache_message(data->cache, rec->uid, fd, offset, size, - rec->full_virtual_size, 0, 0, field); + imap_msgcache_message(data->cache, rec->uid, + field, rec->full_virtual_size, + 0, 0, inbuf, inbuf_rewind, NULL); } + return TRUE; } @@ -240,7 +254,7 @@ cache the needed fields */ if (fields != 0 && !imap_msgcache_is_cached(data->cache, rec->uid, fields)) - index_cache_message(rec, data, fields); + (void)index_cache_message(rec, data, fields); } static int index_fetch_mail(MailIndex *index __attr_unused__,
--- a/src/lib-storage/index/index-save.c Tue Aug 13 18:30:02 2002 +0300 +++ b/src/lib-storage/index/index-save.c Thu Aug 22 01:10:20 2002 +0300 @@ -60,7 +60,6 @@ if (size > data_size) size = data_size; - buf->pos += size; data_size -= size; if (write_with_crlf(fd, data, size, &last_cr)) { @@ -68,6 +67,9 @@ "for file %s: %m", path); return FALSE; } + + io_buffer_skip(buf, size); + } return TRUE;
--- a/src/lib-storage/index/index-search.c Tue Aug 13 18:30:02 2002 +0300 +++ b/src/lib-storage/index/index-search.c Thu Aug 22 01:10:20 2002 +0300 @@ -34,7 +34,8 @@ MailSearchArg *args; const char *msg; size_t size; - int last_block; + + unsigned int max_searchword_len; } SearchTextData; /* truncate timestamp to day */ @@ -379,18 +380,20 @@ return; len = strlen(arg->value.str); - max = data->size-len; - for (i = 0, p = data->msg; i <= max; i++, p++) { - if (i_toupper(*p) == arg->value.str[0] && - strncasecmp(p, arg->value.str, len) == 0) { - /* match */ - ARG_SET_RESULT(arg, 1); - return; + if (len > data->max_searchword_len) + data->max_searchword_len = len; + + if (data->size >= len) { + max = data->size-len; + for (i = 0, p = data->msg; i <= max; i++, p++) { + if (i_toupper(*p) == arg->value.str[0] && + strncasecmp(p, arg->value.str, len) == 0) { + /* match */ + ARG_SET_RESULT(arg, 1); + return; + } } } - - if (data->last_block) - ARG_SET_RESULT(arg, -1); } static void search_text_header(MailSearchArg *arg, void *user_data) @@ -409,14 +412,46 @@ search_text(arg, data); } +static void search_text_set_unmatched(MailSearchArg *arg, + void *user_data __attr_unused__) +{ + if (arg->type == SEARCH_TEXT || arg->type == SEARCH_BODY) + ARG_SET_RESULT(arg, -1); +} + +static void search_arg_match_data(IOBuffer *inbuf, unsigned int max_size, + MailSearchArg *args, + MailSearchForeachFunc search_func) +{ + SearchTextData data; + unsigned int size; + int ret; + + memset(&data, 0, sizeof(data)); + data.args = args; + + /* do this in blocks: read data, compare it for all search words, skip + for block size - (strlen(largest_searchword)-1) and continue. */ + while (max_size > 0 && + (ret = io_buffer_read_max(inbuf, max_size)) > 0) { + data.msg = io_buffer_get_data(inbuf, &size); + if (size > 0) { + data.size = max_size < size ? max_size : size; + max_size -= data.size; + + mail_search_args_foreach(args, search_func, &data); + + if (data.max_searchword_len < size) + size -= data.max_searchword_len-1; + io_buffer_skip(inbuf, size); + } + } +} + static int search_arg_match_text(IndexMailbox *ibox, MailIndexRecord *rec, MailSearchArg *args) { - const char *msg; - void *mmap_base; - off_t offset; - size_t size, mmap_length; - int fd, failed; + IOBuffer *inbuf; int have_headers, have_body, have_text; /* first check what we need to use */ @@ -424,63 +459,50 @@ if (!have_headers && !have_body && !have_text) return TRUE; - fd = ibox->index->open_mail(ibox->index, rec, &offset, &size); - if (fd == -1) + inbuf = ibox->index->open_mail(ibox->index, rec); + if (inbuf == NULL) return FALSE; - mmap_base = mmap_aligned(fd, PROT_READ, offset, size, - (void **) &msg, &mmap_length); - if (mmap_base == MAP_FAILED) { - failed = TRUE; - mail_storage_set_critical(ibox->box.storage, "mmap() failed " - "for msg %u: %m", rec->uid); - } else { - failed = FALSE; - (void)madvise(mmap_base, mmap_length, MADV_SEQUENTIAL); + if (have_headers) { + SearchHeaderData data; + + memset(&data, 0, sizeof(data)); - if (have_headers) { - SearchHeaderData data; - - memset(&data, 0, sizeof(data)); + /* header checks */ + data.custom_header = TRUE; + data.args = args; + message_parse_header(NULL, inbuf, NULL, search_header, &data); + } - /* header checks */ - data.custom_header = TRUE; - data.args = args; - message_parse_header(NULL, msg, size, NULL, - search_header, &data); + if (have_text) { + if (inbuf->offset != 0) { + /* need to rewind back to beginning of headers */ + if (!io_buffer_seek(inbuf, 0)) { + i_error("io_buffer_seek() failed: %m"); + return FALSE; + } } - if (have_text) { - /* first search text from header*/ - SearchTextData data; + search_arg_match_data(inbuf, rec->header_size, + args, search_text_header); + } - data.args = args; - data.msg = msg; - data.size = rec->header_size; - data.last_block = FALSE; - - mail_search_args_foreach(args, search_text_header, - &data); + if (have_text || have_body) { + if (inbuf->offset != (off_t)rec->header_size) { + /* skip over headers */ + i_assert(inbuf->offset == 0); + io_buffer_skip(inbuf, rec->header_size); } - if (have_text || have_body) { - /* search text from body */ - SearchTextData data; + search_arg_match_data(inbuf, UINT_MAX, args, search_text_body); - /* FIXME: we should check this in blocks, so the whole - message doesn't need to be in memory */ - data.args = args; - data.msg = msg + rec->header_size; - data.size = size - rec->header_size; - data.last_block = TRUE; - - mail_search_args_foreach(args, search_text_body, &data); - } + /* set the rest as unmatched */ + mail_search_args_foreach(args, search_text_set_unmatched, NULL); } - (void)munmap(mmap_base, mmap_length); - (void)close(fd); - return !failed; + (void)close(inbuf->fd); + io_buffer_destroy(inbuf); + return TRUE; } static void seq_update(const char *set, unsigned int *first_seq,
--- a/src/lib-storage/index/maildir/maildir-storage.c Tue Aug 13 18:30:02 2002 +0300 +++ b/src/lib-storage/index/maildir/maildir-storage.c Thu Aug 22 01:10:20 2002 +0300 @@ -353,7 +353,10 @@ subsfile_set_subscribed, maildir_find_subscribed, maildir_get_mailbox_name_status, - mail_storage_get_last_error + mail_storage_get_last_error, + + NULL, + NULL }; static Mailbox maildir_mailbox = { @@ -369,5 +372,8 @@ index_storage_fetch, index_storage_search, maildir_storage_save, - mail_storage_is_inconsistency_error + mail_storage_is_inconsistency_error, + + FALSE, + FALSE };
--- a/src/lib-storage/index/mbox/mbox-save.c Tue Aug 13 18:30:02 2002 +0300 +++ b/src/lib-storage/index/mbox/mbox-save.c Thu Aug 22 01:10:20 2002 +0300 @@ -42,7 +42,7 @@ failed = FALSE; pos = lseek(fd, 0, SEEK_END); - if (pos == (off_t)-1) { + if (pos == -1) { mail_storage_set_error(box->storage, "lseek() failed for mbox " "file %s: %m", ibox->index->mbox_path); failed = TRUE;
--- a/src/lib-storage/index/mbox/mbox-storage.c Tue Aug 13 18:30:02 2002 +0300 +++ b/src/lib-storage/index/mbox/mbox-storage.c Thu Aug 22 01:10:20 2002 +0300 @@ -95,7 +95,7 @@ if (p == NULL) return t_strconcat(".imap/", mbox_path); else { - rootpath = t_strndup(mbox_path, (unsigned int) (p-mbox_path)); + rootpath = t_strdup_until(mbox_path, p); return t_strconcat(rootpath, "/.imap/", p+1, NULL); } } @@ -105,9 +105,7 @@ const char *index_dir, *imap_dir; index_dir = mbox_get_index_dir(mbox_path); - imap_dir = strstr(index_dir, ".imap/"); - imap_dir = t_strndup(index_dir, - (unsigned int) (imap_dir - index_dir) + 5); + imap_dir = t_strdup_until(index_dir, strstr(index_dir, ".imap/") + 5); if (mkdir(imap_dir, CREATE_MODE) == -1 && errno != EEXIST) return FALSE; @@ -345,7 +343,10 @@ subsfile_set_subscribed, mbox_find_subscribed, mbox_get_mailbox_name_status, - mail_storage_get_last_error + mail_storage_get_last_error, + + NULL, + NULL }; static Mailbox mbox_mailbox = { @@ -361,5 +362,8 @@ index_storage_fetch, index_storage_search, mbox_storage_save, - mail_storage_is_inconsistency_error + mail_storage_is_inconsistency_error, + + FALSE, + FALSE };
--- a/src/lib-storage/mail-storage.c Tue Aug 13 18:30:02 2002 +0300 +++ b/src/lib-storage/mail-storage.c Thu Aug 22 01:10:20 2002 +0300 @@ -98,7 +98,7 @@ while (i_isalnum(*p)) p++; if (*p == ':') { - name = t_strndup(data, (unsigned int) (p-data)); + name = t_strdup_until(data, p); storage = mail_storage_create(name, p+1); } else { storage = mail_storage_autodetect(data);
--- a/src/lib-storage/mail-storage.h Tue Aug 13 18:30:02 2002 +0300 +++ b/src/lib-storage/mail-storage.h Thu Aug 22 01:10:20 2002 +0300 @@ -207,8 +207,7 @@ MailFetchBodyData *next; const char *section; /* NOTE: always uppercased */ - off_t skip; - size_t max_size; + off_t skip, max_size; /* max_size is ignored if it's < 0 */ unsigned int skip_set:1; unsigned int peek:1; };
--- a/src/lib-storage/subscription-file/subscription-file.c Tue Aug 13 18:30:02 2002 +0300 +++ b/src/lib-storage/subscription-file/subscription-file.c Thu Aug 22 01:10:20 2002 +0300 @@ -77,7 +77,7 @@ { char *buf; - if (lseek(fd, 0, SEEK_END) == (off_t)-1) { + if (lseek(fd, 0, SEEK_END) == -1) { mail_storage_set_critical(storage, "lseek() failed for " "subscription file %s: %m", path); return FALSE; @@ -203,7 +203,7 @@ break; } - line = t_strndup(start, (unsigned int) (p-start)); + line = t_strdup_until(start, p); if (line != NULL && *line != '\0' && imap_match(glob, line, 0, NULL) >= 0) ret = func(storage, line, user_data);
--- a/src/lib/compat.c Tue Aug 13 18:30:02 2002 +0300 +++ b/src/lib/compat.c Thu Aug 22 01:10:20 2002 +0300 @@ -23,11 +23,12 @@ SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ +#include "lib.h" + #include <ctype.h> +#include <unistd.h> #include <syslog.h> -#include "lib.h" - #ifndef INADDR_NONE # define INADDR_NONE INADDR_BROADCAST #endif @@ -120,3 +121,15 @@ syslog(priority, "%s", str); } #endif + +#ifndef HAVE_GETPAGESIZE +int my_getpagesize(void) +{ +#ifdef _SC_PAGESIZE + return sysconf(_SC_PAGESIZE); +#else +# warning Guessing page size to be 4096 + return 4096; +#endif +} +#endif
--- a/src/lib/compat.h Tue Aug 13 18:30:02 2002 +0300 +++ b/src/lib/compat.h Thu Aug 22 01:10:20 2002 +0300 @@ -33,6 +33,11 @@ void my_vsyslog(int priority, const char *format, va_list args); #endif +#ifndef HAVE_GETPAGESIZE +# define getpagesize my_getpagesize +int my_getpagesize(void); +#endif + /* ctype.h isn't safe with signed chars, use our own instead if really needed */ #define i_toupper(x) toupper((int) (unsigned char) (x))
--- a/src/lib/imem.c Tue Aug 13 18:30:02 2002 +0300 +++ b/src/lib/imem.c Thu Aug 22 01:10:20 2002 +0300 @@ -52,6 +52,11 @@ return p_strdup_empty(default_pool, str); } +char *i_strdup_until(const char *str, const char *end) +{ + return p_strdup_until(default_pool, str, end); +} + char *i_strndup(const char *str, unsigned int max_chars) { return p_strndup(default_pool, str, max_chars);
--- a/src/lib/imem.h Tue Aug 13 18:30:02 2002 +0300 +++ b/src/lib/imem.h Thu Aug 22 01:10:20 2002 +0300 @@ -13,6 +13,7 @@ /* string functions */ char *i_strdup(const char *str); char *i_strdup_empty(const char *str); /* like i_strdup(), but if str == "", return NULL */ +char *i_strdup_until(const char *str, const char *end); /* *end isn't included */ char *i_strndup(const char *str, unsigned int max_chars); char *i_strdup_printf(const char *format, ...) __attr_format__(1, 2); char *i_strdup_vprintf(const char *format, va_list args);
--- a/src/lib/iobuffer.c Tue Aug 13 18:30:02 2002 +0300 +++ b/src/lib/iobuffer.c Thu Aug 22 01:10:20 2002 +0300 @@ -26,6 +26,7 @@ #include "lib.h" #include "ioloop.h" #include "iobuffer.h" +#include "mmap-util.h" #include "network.h" #include <unistd.h> @@ -34,8 +35,11 @@ # include <sys/sendfile.h> #endif +static unsigned int mmap_pagesize = 0; +static unsigned int mmap_pagemask = 0; + IOBuffer *io_buffer_create(int fd, Pool pool, int priority, - unsigned int max_size) + unsigned int max_buffer_size) { IOBuffer *buf; @@ -46,17 +50,58 @@ buf->fd = fd; buf->pool = pool; buf->priority = priority; - buf->max_size = max_size; + buf->max_size = max_buffer_size; return buf; } -IOBuffer *io_buffer_create_file(int fd, Pool pool, unsigned int max_size) +IOBuffer *io_buffer_create_file(int fd, Pool pool, + unsigned int max_buffer_size) +{ + IOBuffer *buf; + + buf = io_buffer_create(fd, pool, IO_PRIORITY_DEFAULT, max_buffer_size); + buf->file = TRUE; + return buf; +} + +IOBuffer *io_buffer_create_mmap(int fd, Pool pool, unsigned int block_size, + off_t stop_offset) { IOBuffer *buf; - buf = io_buffer_create(fd, pool, IO_PRIORITY_DEFAULT, max_size); - buf->file = TRUE; - return buf; + /* block size must be page aligned, and at least two pages long */ + if (mmap_pagesize == 0) { + mmap_pagesize = getpagesize(); + mmap_pagemask = mmap_pagesize-1; + } + + if (block_size < mmap_pagesize*2) + block_size = mmap_pagesize*2; + else if ((block_size & mmap_pagemask) != 0) { + block_size &= ~mmap_pagemask; + block_size += mmap_pagesize; + } + + buf = io_buffer_create_file(fd, pool, block_size); + buf->stop_offset = stop_offset; + buf->mmaped = TRUE; + buf->receive = TRUE; + + /* set offsets */ + buf->start_offset = lseek(fd, 0, SEEK_CUR); + buf->stop_offset = stop_offset > 0 ? stop_offset : + lseek(fd, 0, SEEK_END); + + if (buf->start_offset == -1 || buf->stop_offset == -1) { + i_error("io_buffer_create_mmap(): lseek() failed: %m"); + buf->start_offset = buf->stop_offset = 0; + } + + /* fix offset alignment */ + buf->mmap_offset = buf->start_offset & ~mmap_pagemask; + buf->skip = buf->mmap_offset & mmap_pagemask; + + return buf; } void io_buffer_destroy(IOBuffer *buf) @@ -66,7 +111,12 @@ if (buf->io != NULL) io_remove(buf->io); - p_free(buf->pool, buf->buffer); + if (buf->buffer != NULL) { + if (!buf->mmaped) + p_free(buf->pool, buf->buffer); + else + (void)munmap(buf->buffer, buf->size); + } p_free(buf->pool, buf); } @@ -81,7 +131,15 @@ void io_buffer_reset(IOBuffer *buf) { buf->pos = buf->skip = buf->cr_lookup_pos = 0; - buf->last_cr = FALSE; + buf->last_cr = FALSE; + + if (buf->mmaped && buf->buffer != NULL) { + (void)munmap(buf->buffer, buf->size); + buf->buffer = NULL; + buf->size = 0; + } + + buf->mmap_offset = buf->offset = 0; } IOBuffer *io_buffer_set_pool(IOBuffer *buf, Pool pool) @@ -95,22 +153,28 @@ memcpy(newbuf, buf, sizeof(IOBuffer)); newbuf->pool = pool; - newbuf->buffer = p_malloc(pool, buf->size); - memcpy(newbuf->buffer, buf->buffer + buf->skip, - buf->size - buf->skip); + + if (!newbuf->mmaped) { + newbuf->buffer = p_malloc(pool, buf->size); + memcpy(newbuf->buffer, buf->buffer + buf->skip, + buf->size - buf->skip); - newbuf->cr_lookup_pos -= newbuf->skip; - newbuf->pos -= newbuf->skip; - newbuf->skip = 0; + newbuf->cr_lookup_pos -= newbuf->skip; + newbuf->pos -= newbuf->skip; + newbuf->skip = 0; - p_free(buf->pool, buf->buffer); + p_free(buf->pool, buf->buffer); + } + p_free(buf->pool, buf); return newbuf; } void io_buffer_set_max_size(IOBuffer *buf, unsigned int max_size) { - buf->max_size = max_size; + i_assert(!buf->mmaped); + + buf->max_size = max_size; } void io_buffer_set_send_blocking(IOBuffer *buf, unsigned int max_size, @@ -158,7 +222,7 @@ if (ret < 0) { buf->closed = TRUE; } else { - buf->transfd += ret; + buf->offset += ret; buf->skip += ret; if (buf->skip == buf->pos) { /* everything sent */ @@ -194,13 +258,11 @@ typedef struct { IOLoop ioloop; - IOBuffer *buf; + IOBuffer *outbuf; const char *data; unsigned int size; - - int in_fd; - off_t offset; + IOBuffer *inbuf; int timeout; } IOBufferBlockData; @@ -209,23 +271,24 @@ { int ret; - if (bd->buf->skip != bd->buf->pos) { - buf_send_real(bd->buf); + if (bd->outbuf->skip != bd->outbuf->pos) { + buf_send_real(bd->outbuf); } else { /* send the data */ - ret = !bd->buf->file ? - net_transmit(bd->buf->fd, bd->data, bd->size) : - my_write(bd->buf->fd, bd->data, bd->size); + ret = !bd->outbuf->file ? + net_transmit(bd->outbuf->fd, bd->data, bd->size) : + my_write(bd->outbuf->fd, bd->data, bd->size); if (ret < 0) { - bd->buf->closed = TRUE; + bd->outbuf->closed = TRUE; } else { + bd->outbuf->offset += ret; bd->data += ret; bd->size -= ret; } } - if (bd->buf->closed || bd->size == 0) + if (bd->outbuf->closed || bd->size == 0) io_loop_stop(bd->ioloop); } @@ -241,6 +304,7 @@ void (*send_func)(IOBufferBlockData *bd)) { Timeout to; + int save_errno; /* close old IO */ if (buf->io != NULL) @@ -248,13 +312,14 @@ /* create a new I/O loop */ bd->ioloop = io_loop_create(); - bd->buf = buf; + bd->outbuf = buf; buf->io = io_add(buf->fd, IO_WRITE, (IOFunc) send_func, bd); to = buf->timeout_msecs <= 0 ? NULL : timeout_add(buf->timeout_msecs, block_loop_timeout, bd); io_loop_run(bd->ioloop); + save_errno = errno; if (buf->corked) { /* remove cork */ @@ -276,6 +341,8 @@ } io_loop_destroy(bd->ioloop); + + errno = save_errno; return bd->size > 0 ? -1 : 1; } @@ -304,6 +371,8 @@ static void buffer_alloc_more(IOBuffer *buf, unsigned int size) { + i_assert(!buf->mmaped); + buf->size = buf->pos+size; buf->size = buf->size <= IO_BUFFER_MIN_SIZE ? IO_BUFFER_MIN_SIZE : nearest_power(buf->size); @@ -318,7 +387,7 @@ } } -static inline void io_buffer_compress(IOBuffer *buf) +static void io_buffer_compress(IOBuffer *buf) { memmove(buf->buffer, buf->buffer + buf->skip, buf->pos - buf->skip); @@ -354,7 +423,7 @@ return -1; } - buf->transfd += ret; + buf->offset += ret; data = (const char *) data + ret; size -= ret; } @@ -386,59 +455,120 @@ { int ret; - ret = sendfile(bd->buf->fd, bd->in_fd, &bd->offset, bd->size); + ret = sendfile(bd->outbuf->fd, bd->inbuf->fd, + &bd->inbuf->mmap_offset, bd->size); if (ret < 0) { if (errno != EINTR && errno != EAGAIN) - bd->buf->closed = TRUE; + bd->outbuf->closed = TRUE; ret = 0; } bd->size -= ret; - if (bd->buf->closed || bd->size == 0) + if (bd->outbuf->closed || bd->size == 0) io_loop_stop(bd->ioloop); } -#endif -int io_buffer_send_file(IOBuffer *buf, int fd, off_t offset, - const void *data, unsigned int size) +static int io_buffer_sendfile(IOBuffer *outbuf, IOBuffer *inbuf, + unsigned int size) { -#ifdef HAVE_SYS_SENDFILE_H IOBufferBlockData bd; int ret; -#endif - i_assert(fd >= 0); - i_assert(data != NULL); - i_assert(size < INT_MAX); - -#ifdef HAVE_SYS_SENDFILE_H - io_buffer_send_flush(buf); + io_buffer_send_flush(outbuf); /* first try if we can do it with a single sendfile() call */ - ret = sendfile(buf->fd, fd, &offset, size); + ret = sendfile(outbuf->fd, inbuf->fd, &inbuf->offset, size); if (ret < 0) { if (errno != EINTR && errno != EAGAIN) return -1; ret = 0; } - if ((unsigned int) ret == size) + if ((unsigned int) ret == size) { + /* yes, all sent */ return 1; + } + + memset(&bd, 0, sizeof(IOBufferBlockData)); + + bd.inbuf = inbuf; + bd.size = size - ret; + + ret = io_buffer_ioloop(outbuf, &bd, block_loop_sendfile); + if (ret < 0 && errno == EINVAL) { + /* this shouldn't happen, must be a bug. It would also + mess up later if we let this pass. */ + i_fatal("io_buffer_sendfile() failed: %m"); + } + return ret; +} +#endif - if (buf->blocking) { - memset(&bd, 0, sizeof(IOBufferBlockData)); +static void block_loop_copy(IOBufferBlockData *bd) +{ + unsigned char *in_data; + unsigned int size; + int ret; + + while ((ret = io_buffer_read_data(bd->inbuf, &in_data, + &size, 0)) <= 0) { + if (ret == -1) { + /* disconnected */ + bd->outbuf->closed = TRUE; + break; + } + } + + /* send the data */ + bd->data = (const char *) in_data; + bd->size = size; + block_loop_send(bd); + + io_buffer_skip(bd->inbuf, size - bd->size); +} - bd.in_fd = fd; - bd.offset = offset + ret; - bd.size = size - ret; +int io_buffer_send_buf(IOBuffer *outbuf, IOBuffer *inbuf, unsigned int size) +{ + IOBufferBlockData bd; + unsigned char *in_data; + unsigned int in_size; + int ret; + + i_assert(size < INT_MAX); + +#ifdef HAVE_SYS_SENDFILE_H + ret = io_buffer_sendfile(outbuf, inbuf, size); + if (ret >= 0 || errno != EINVAL) + return ret; - return io_buffer_ioloop(buf, &bd, block_loop_sendfile); - } else { - data = (char *) data + ret; - size -= ret; + /* sendfile() not supported with fd, fallback to + regular sending */ +#endif + /* see if we can do it at one go */ + ret = io_buffer_read_data(inbuf, &in_data, &in_size, 0); + if (ret == -1) + return -1; + + ret = !outbuf->file ? + net_transmit(outbuf->fd, in_data, in_size) : + my_write(outbuf->fd, in_data, in_size); + if (ret < 0) + return -1; + + outbuf->offset += ret; + io_buffer_skip(inbuf, ret); + if ((unsigned int) ret == size) { + /* all sent */ + return 1; } -#endif - return io_buffer_send(buf, data, size); + + /* create blocking send loop */ + memset(&bd, 0, sizeof(IOBufferBlockData)); + + bd.inbuf = inbuf; + bd.size = size; + + return io_buffer_ioloop(outbuf, &bd, block_loop_copy); } void io_buffer_send_flush(IOBuffer *buf) @@ -466,6 +596,48 @@ buf->flush_user_data = user_data; } +int io_buffer_read_mmaped(IOBuffer *buf, unsigned int size) +{ + unsigned int aligned_skip; + + if (buf->stop_offset - buf->mmap_offset <= (off_t)buf->size) { + /* end of file is already mapped */ + return -1; + } + + if (buf->pos != 0) { + aligned_skip = buf->skip & ~mmap_pagemask; + if (buf->buffer != NULL) { + /* didn't skip enough bytes */ + return -2; + } + + buf->skip -= aligned_skip; + buf->mmap_offset += aligned_skip; + } + + if (buf->buffer != NULL) + (void)munmap(buf->buffer, buf->size); + + buf->size = buf->stop_offset - buf->mmap_offset < (off_t)buf->max_size ? + buf->stop_offset - buf->mmap_offset : buf->max_size; + + if (buf->size > size) + buf->size = size; + + buf->buffer = mmap(NULL, buf->size, PROT_READ, MAP_PRIVATE, + buf->fd, buf->mmap_offset); + if (buf->buffer == MAP_FAILED) { + i_error("io_buffer_read_mmaped(): mmap() failed: %m"); + return -1; + } + + (void)madvise(buf->buffer, buf->size, MADV_SEQUENTIAL); + + buf->pos = buf->size; + return buf->size; +} + int io_buffer_read_max(IOBuffer *buf, unsigned int size) { int ret; @@ -475,7 +647,10 @@ buf->receive = TRUE; if (buf->closed) - return -1; + return -1; + + if (buf->mmaped) + return io_buffer_read_mmaped(buf, size); if (buf->pos == buf->size) { if (buf->skip > 0) { @@ -491,7 +666,7 @@ } /* fill the buffer */ - if (size == UINT_MAX || buf->size-buf->pos < size) + if (buf->size-buf->pos < size) size = buf->size - buf->pos; if (!buf->file) { @@ -511,7 +686,7 @@ return -1; } - buf->transfd += ret; + buf->offset += ret; buf->pos += ret; return ret; } @@ -521,8 +696,71 @@ return io_buffer_read_max(buf, UINT_MAX); } +void io_buffer_skip(IOBuffer *buf, unsigned int size) +{ + int ret; + + if (size <= buf->pos - buf->skip) { + buf->skip += size; + buf->offset += size; + return; + } + + if (buf->mmaped) { + if (buf->stop_offset - buf->mmap_offset <= (off_t)size) { + /* end of file */ + buf->mmap_offset = buf->stop_offset; + buf->size = 0; + return; + } + + /* these point outside mmap now, next io_buffer_read_mmaped() + will fix them */ + buf->skip += size; + buf->pos = buf->skip; + } else { + if (buf->size == 0) + buffer_alloc_more(buf, IO_BUFFER_MIN_SIZE); + + size -= buf->skip; + while (size > buf->size) { + ret = io_buffer_read_max(buf, buf->size); + if (ret <= 0) + break; + + size -= ret; + } + + (void)io_buffer_read_max(buf, size); + } +} + +int io_buffer_seek(IOBuffer *buf, off_t offset) +{ + off_t real_offset; + + i_assert(offset >= 0); + + if (buf->mmaped) { + /* first reset everything */ + io_buffer_reset(buf); + + /* then set the wanted position, next read will + pick up from there */ + buf->mmap_offset = buf->start_offset; + buf->pos = buf->skip = offset; + } else { + real_offset = buf->start_offset + offset; + if (lseek(buf->fd, real_offset, SEEK_SET) != real_offset) + return FALSE; + } + + buf->offset = offset; + return TRUE; +} + /* skip the first LF, if it exists */ -static inline void io_buffer_skip_lf(IOBuffer *buf) +static void io_buffer_skip_lf(IOBuffer *buf) { if (!buf->last_cr || buf->skip >= buf->pos) return; @@ -537,6 +775,7 @@ char *io_buffer_next_line(IOBuffer *buf) { + // FIXME: update buf->offset unsigned char *ret_buf; unsigned int i; @@ -577,6 +816,25 @@ return buf->buffer + buf->skip; } +int io_buffer_read_data(IOBuffer *buf, unsigned char **data, + unsigned int *size, unsigned int threshold) +{ + int ret; + + if (buf->pos - buf->skip <= threshold) { + /* we need more data */ + ret = io_buffer_read(buf); + if (ret <= 0) { + *size = 0; + *data = NULL; + return ret; + } + } + + *data = io_buffer_get_data(buf, size); + return 1; +} + unsigned char *io_buffer_get_space(IOBuffer *buf, unsigned int size) { i_assert(size <= INT_MAX); @@ -618,7 +876,7 @@ return -1; } - buf->transfd += ret; + buf->offset += ret; if ((unsigned int) ret == size) { /* all sent */ return 1; @@ -639,6 +897,8 @@ int io_buffer_set_data(IOBuffer *buf, const void *data, unsigned int size) { + i_assert(!buf->mmaped); + io_buffer_reset(buf); if (buf->size < size) { @@ -647,10 +907,13 @@ return -2; } + buf->offset += size; + buf->offset -= buf->pos - buf->skip; + memcpy(buf->buffer, data, size); buf->pos = size; - buf->transfd += size; - return 1; + buf->skip = 0; + return 1; } int io_buffer_is_empty(IOBuffer *buf)
--- a/src/lib/iobuffer.h Tue Aug 13 18:30:02 2002 +0300 +++ b/src/lib/iobuffer.h Thu Aug 22 01:10:20 2002 +0300 @@ -9,9 +9,15 @@ struct _IOBuffer { int fd; - IO io; + unsigned int pos, skip; + unsigned int size, max_size; + off_t start_offset, stop_offset; + off_t offset; /* virtual offset, 0 = start_offset */ + +/* private: */ Pool pool; + IO io; int priority; int timeout_msecs; @@ -22,13 +28,11 @@ void *flush_user_data; unsigned char *buffer; - unsigned int cr_lookup_pos; /* used only when reading a line */ - unsigned int pos, skip; - unsigned int size, max_size; - unsigned int transfd; + off_t mmap_offset; unsigned int file:1; /* reading/writing a file */ + unsigned int mmaped:1; /* reading a file with mmap() */ unsigned int closed:1; /* all further read/writes will return 0 */ unsigned int transmit:1; /* this is a transmit buffer */ unsigned int receive:1; /* this is a receive buffer */ @@ -39,15 +43,16 @@ }; /* Create an I/O buffer. It can be used for either sending or receiving data, - NEVER BOTH AT SAME TIME. If max_size is 0, there's no limit. */ + NEVER BOTH AT SAME TIME. */ IOBuffer *io_buffer_create(int fd, Pool pool, int priority, - unsigned int max_size); + unsigned int max_buffer_size); /* Same as io_buffer_create(), but specify that we're reading/writing file. */ -IOBuffer *io_buffer_create_file(int fd, Pool pool, unsigned int max_size); -/* Read the file by mmap()ing it in blocks. max_size specifies the maximum - amount of data to read from the file. */ +IOBuffer *io_buffer_create_file(int fd, Pool pool, + unsigned int max_buffer_size); +/* Read the file by mmap()ing it in blocks. stop_offset specifies where to + stop reading, or 0 to end of file. */ IOBuffer *io_buffer_create_mmap(int fd, Pool pool, unsigned int block_size, - unsigned int max_size); + off_t stop_offset); /* Destroy a buffer. */ void io_buffer_destroy(IOBuffer *buf); /* Mark the buffer closed. Any sends/reads after this will return -1. @@ -77,10 +82,10 @@ /* Returns 1 if all was ok, -1 if disconnected, -2 if buffer is full */ int io_buffer_send(IOBuffer *buf, const void *data, unsigned int size); -/* Send data using sendfile(), of if it's not available fallback to regular - sending from data-pointer. Returns 1 if all was ok, -1 if disconnected. */ -int io_buffer_send_file(IOBuffer *buf, int fd, off_t offset, - const void *data, unsigned int size); +/* Send data from input buffer to output buffer using the fastest + possible method. Returns 1 if all was ok, -1 if disconnected. + Note that this function may block. */ +int io_buffer_send_buf(IOBuffer *outbuf, IOBuffer *inbuf, unsigned int size); /* Flush the output buffer, blocks until all is sent. If io_buffer_set_send_blocking() is called, it's timeout settings are used. */ void io_buffer_send_flush(IOBuffer *buf); @@ -91,10 +96,15 @@ void *user_data); /* Returns number of bytes read if read was ok, - -1 if disconnected, -2 if the buffer is full */ + -1 if disconnected / EOF, -2 if the buffer is full */ int io_buffer_read(IOBuffer *buf); /* Like io_buffer_read(), but don't read more than specified size. */ int io_buffer_read_max(IOBuffer *buf, unsigned int size); +/* Skip forward a number of bytes */ +void io_buffer_skip(IOBuffer *buf, unsigned int size); +/* Seek to specified position from beginning of file. This works only for + files. Returns TRUE if successful. */ +int io_buffer_seek(IOBuffer *buf, off_t offset); /* Returns the next line from input buffer, or NULL if more data is needed to make a full line. NOTE: call to io_buffer_read() invalidates the returned data. */ @@ -102,6 +112,11 @@ /* Returns pointer to beginning of data in buffer, or NULL if there's no data. */ unsigned char *io_buffer_get_data(IOBuffer *buf, unsigned int *size); +/* Like io_buffer_get_data(), but read it when needed. There always must be + more than `threshold' bytes in buffer. Returns 1 if data was read, 0 if + read was interrupted or nonblocking, -1 if EOF / error */ +int io_buffer_read_data(IOBuffer *buf, unsigned char **data, + unsigned int *size, unsigned int threshold); /* Returns a pointer to buffer wanted amount of space, or NULL if size is too big. */
--- a/src/lib/mmap-util.c Tue Aug 13 18:30:02 2002 +0300 +++ b/src/lib/mmap-util.c Thu Aug 22 01:10:20 2002 +0300 @@ -29,7 +29,7 @@ static void *mmap_file(int fd, size_t *length, int access) { *length = lseek(fd, 0, SEEK_END); - if ((off_t)*length == (off_t)-1) + if ((off_t)*length == -1) return MAP_FAILED; if (*length == 0) @@ -54,12 +54,10 @@ void **data_start, size_t *mmap_length) { void *mmap_base; - -#ifdef HAVE_GETPAGESIZE static int pagemask = 0; if (pagemask == 0) { - pagemask = getpagesize(); + pagemask = getpagesize()-1; i_assert(pagemask > 0); pagemask--; } @@ -70,13 +68,6 @@ fd, offset & ~pagemask); *data_start = mmap_base == MAP_FAILED || mmap_base == NULL ? NULL : (char *) mmap_base + (offset & pagemask); -#else - *mmap_length = length + offset; - - mmap_base = mmap(NULL, *mmap_length, access, MAP_SHARED, fd, 0); - *data_start = mmap_base == MAP_FAILED || mmap_base == NULL ? NULL : - (char *) mmap_base + offset; -#endif return mmap_base; }
--- a/src/lib/strfuncs.c Tue Aug 13 18:30:02 2002 +0300 +++ b/src/lib/strfuncs.c Thu Aug 22 01:10:20 2002 +0300 @@ -544,6 +544,33 @@ STRDUP_EMPTY_CORE(t_malloc(len), str); } +char *p_strdup_until(Pool pool, const char *start, const char *end) +{ + unsigned int size; + char *mem; + + i_assert(start <= end); + + size = (unsigned int) (end-start); + mem = p_malloc(pool, size+1); + memcpy(mem, start, size); + return mem; +} + +const char *t_strdup_until(const char *start, const char *end) +{ + unsigned int size; + char *mem; + + i_assert(start <= end); + + size = (unsigned int) (end-start); + mem = t_malloc(size+1); + memcpy(mem, start, size); + mem[size] = '\0'; + return mem; +} + static inline char * strndup_core(const char *str, unsigned int max_chars, ALLOC_FUNC alloc, Pool pool) @@ -719,7 +746,7 @@ for (p = str; *p != '\0'; p++) { if (*p == cutchar) - return t_strndup(str, (unsigned int) (p-str)); + return t_strdup_until(str, p); } return str;
--- a/src/lib/strfuncs.h Tue Aug 13 18:30:02 2002 +0300 +++ b/src/lib/strfuncs.h Thu Aug 22 01:10:20 2002 +0300 @@ -18,6 +18,7 @@ char *p_strdup(Pool pool, const char *str); char *p_strdup_empty(Pool pool, const char *str); /* return NULL if str = "" */ +char *p_strdup_until(Pool pool, const char *start, const char *end); /* *end isn't included */ char *p_strndup(Pool pool, const char *str, unsigned int max_chars); char *p_strdup_printf(Pool pool, const char *format, ...) __attr_format__(2, 3); char *p_strdup_vprintf(Pool pool, const char *format, va_list args); @@ -29,6 +30,7 @@ /* same with temporary memory allocations: */ const char *t_strdup(const char *str); const char *t_strdup_empty(const char *str); /* return NULL if str = "" */ +const char *t_strdup_until(const char *start, const char *end); /* *end isn't included */ const char *t_strndup(const char *str, unsigned int max_chars); const char *t_strdup_printf(const char *format, ...) __attr_format__(1, 2); const char *t_strdup_vprintf(const char *format, va_list args);
--- a/src/login/auth-connection.c Tue Aug 13 18:30:02 2002 +0300 +++ b/src/login/auth-connection.c Thu Aug 22 01:10:20 2002 +0300 @@ -214,7 +214,7 @@ if (!conn->init_received) { if (size == sizeof(AuthInitData)) { memcpy(&init_data, data, sizeof(AuthInitData)); - conn->inbuf->skip += sizeof(AuthInitData); + io_buffer_skip(conn->inbuf, sizeof(AuthInitData)); auth_handle_init(conn, &init_data); } else if (size > sizeof(AuthInitData)) { @@ -235,7 +235,7 @@ memcpy(&conn->in_reply, data, sizeof(AuthReplyData)); data += sizeof(AuthReplyData); size -= sizeof(AuthReplyData); - conn->inbuf->skip += sizeof(AuthReplyData); + io_buffer_skip(conn->inbuf, sizeof(AuthReplyData)); conn->in_reply_received = TRUE; } @@ -243,11 +243,9 @@ return; /* we've got a full reply */ - size = conn->in_reply.data_size; - conn->inbuf->skip += size; conn->in_reply_received = FALSE; - auth_handle_reply(conn, &conn->in_reply, data); + io_buffer_skip(conn->inbuf, conn->in_reply.data_size); } int auth_init_request(AuthMethod method, AuthCallback callback,