# HG changeset patch # User Timo Sirainen # Date 1032769652 -10800 # Node ID 3dcc2275b4caa06bab682c1a7dff9404ed7768c0 # Parent c020e1168cc3e5eb19554e61ea12d15f39ad96db IOBuffer cleanup, hopefully fixes some mbox problems. diff -r c020e1168cc3 -r 3dcc2275b4ca src/lib-index/mbox/mbox-append.c --- a/src/lib-index/mbox/mbox-append.c Sun Sep 22 12:27:21 2002 +0300 +++ b/src/lib-index/mbox/mbox-append.c Mon Sep 23 11:27:32 2002 +0300 @@ -79,7 +79,7 @@ MailIndexUpdate *update; MboxHeaderContext ctx; time_t internal_date; - uoff_t abs_start_offset, stop_offset, old_size; + uoff_t abs_start_offset, stop_offset; unsigned char *data, md5_digest[16]; size_t size, pos; int failed; @@ -135,13 +135,12 @@ side effects?) */ mbox_header_init_context(&ctx, index); - old_size = inbuf->size; - inbuf->size = stop_offset; io_buffer_seek(inbuf, abs_start_offset - inbuf->start_offset); + io_buffer_set_read_limit(inbuf, stop_offset); mail_index_update_headers(update, inbuf, 0, mbox_header_func, &ctx); + io_buffer_set_read_limit(inbuf, 0); - inbuf->size = old_size; io_buffer_seek(inbuf, stop_offset); /* save MD5 */ diff -r c020e1168cc3 -r 3dcc2275b4ca src/lib-index/mbox/mbox-rewrite.c --- a/src/lib-index/mbox/mbox-rewrite.c Sun Sep 22 12:27:21 2002 +0300 +++ b/src/lib-index/mbox/mbox-rewrite.c Mon Sep 23 11:27:32 2002 +0300 @@ -183,7 +183,9 @@ ctx.msg_flags = rec->msg_flags; ctx.custom_flags = custom_flags; + io_buffer_set_read_limit(inbuf, inbuf->offset + rec->header_size); message_parse_header(NULL, inbuf, &hdr_size, header_func, &ctx); + io_buffer_set_read_limit(inbuf, 0); i_assert(hdr_size.physical_size == rec->header_size); diff -r c020e1168cc3 -r 3dcc2275b4ca src/lib-mail/message-parser.c --- a/src/lib-mail/message-parser.c Sun Sep 22 12:27:21 2002 +0300 +++ b/src/lib-mail/message-parser.c Mon Sep 23 11:27:32 2002 +0300 @@ -329,12 +329,10 @@ continue; } - if (ret < 0) { - /* EOF, but we may still have something in buffer. - this is needed only when there's no message body */ - msg = io_buffer_get_data(inbuf, &size); - if (size == startpos) - break; + if (ret < 0 || (ret == 0 && size == startpos)) { + /* EOF and nothing in buffer. the later check is + needed only when there's no message body */ + break; } for (i = startpos; i < size; i++) { @@ -573,10 +571,8 @@ /* now, see if it's end boundary */ end_boundary = FALSE; - if (io_buffer_read_data_blocking(inbuf, &msg, &size, 1) > 0) { - i_assert(size >= 2); + if (io_buffer_read_data_blocking(inbuf, &msg, &size, 1) > 0) end_boundary = msg[0] == '-' && msg[1] == '-'; - } /* skip the rest of the line */ message_skip_line(inbuf, boundary_size); diff -r c020e1168cc3 -r 3dcc2275b4ca src/lib-storage/index/index-save.c --- a/src/lib-storage/index/index-save.c Sun Sep 22 12:27:21 2002 +0300 +++ b/src/lib-storage/index/index-save.c Mon Sep 23 11:27:32 2002 +0300 @@ -49,7 +49,7 @@ failed = FALSE; while (data_size > 0) { - ret = io_buffer_read_blocking(buf, SSIZE_T_MAX); + ret = io_buffer_read_blocking(buf); if (ret < 0) { mail_storage_set_critical(storage, "Error reading mail: %m"); diff -r c020e1168cc3 -r 3dcc2275b4ca src/lib-storage/index/index-search.c --- a/src/lib-storage/index/index-search.c Sun Sep 22 12:27:21 2002 +0300 +++ b/src/lib-storage/index/index-search.c Mon Sep 23 11:27:32 2002 +0300 @@ -473,34 +473,37 @@ MailSearchForeachFunc search_func) { SearchTextContext ctx; - size_t size; - ssize_t ret; + unsigned char *data; + size_t size, max_searchword_len; memset(&ctx, 0, sizeof(ctx)); ctx.args = args; + /* first get the max. search keyword length */ + mail_search_args_foreach(args, search_func, &ctx); + max_searchword_len = ctx.max_searchword_len; + + io_buffer_set_read_limit(inbuf, inbuf->offset + max_size); + /* do this in blocks: read data, compare it for all search words, skip for block size - (strlen(largest_searchword)-1) and continue. */ - while (max_size > 0) { - size = max_size < SSIZE_T_MAX ? max_size : SSIZE_T_MAX; - if ((ret = io_buffer_read_max(inbuf, size)) < 0) - break; - - ctx.msg = io_buffer_get_data(inbuf, &size); - if (size > 0) { - if (size > max_size) - size = max_size; + while (io_buffer_read_data_blocking(inbuf, &data, &size, + max_searchword_len-1) > 0) { + ctx.msg = data; + ctx.size = size; + mail_search_args_foreach(args, search_func, &ctx); + io_buffer_skip(inbuf, size - (max_searchword_len-1)); + } - ctx.size = size; - mail_search_args_foreach(args, search_func, &ctx); + if (size > 0) { + /* last block */ + ctx.msg = data; + ctx.size = size; + mail_search_args_foreach(args, search_func, &ctx); + io_buffer_skip(inbuf, size); + } - if (ctx.max_searchword_len < size && size < max_size) - size -= ctx.max_searchword_len-1; - - max_size -= size; - io_buffer_skip(inbuf, size); - } - } + io_buffer_set_read_limit(inbuf, 0); } static int search_arg_match_text(IndexMailbox *ibox, MailIndexRecord *rec, diff -r c020e1168cc3 -r 3dcc2275b4ca src/lib/iobuffer.c --- a/src/lib/iobuffer.c Sun Sep 22 12:27:21 2002 +0300 +++ b/src/lib/iobuffer.c Mon Sep 23 11:27:32 2002 +0300 @@ -118,6 +118,7 @@ buf->start_offset = start_offset; buf->size = size > 0 ? size : stop_offset - start_offset; + buf->limit = buf->size; buf->skip = buf->pos = buf->start_offset; return buf; @@ -133,8 +134,12 @@ if (buf->buffer != NULL) { if (!buf->mmaped) p_free(buf->pool, buf->buffer); - else - (void)munmap(buf->buffer, buf->buffer_size); + else { + if (munmap(buf->buffer, buf->buffer_size) < 0) { + i_error("io_buffer_destroy(): " + "munmap() failed: %m"); + } + } } p_free(buf->pool, buf); } @@ -153,7 +158,8 @@ buf->last_cr = FALSE; if (buf->mmaped && buf->buffer != NULL) { - (void)munmap(buf->buffer, buf->buffer_size); + if (munmap(buf->buffer, buf->buffer_size) < 0) + i_error("io_buffer_reset(): munmap() failed: %m"); buf->buffer = NULL; buf->buffer_size = 0; } @@ -579,6 +585,7 @@ int ret; i_assert(size < OFF_T_MAX); + i_assert(inbuf->limit > 0 || size <= inbuf->limit - inbuf->offset); ret = io_buffer_sendfile(outbuf, inbuf, size); if (ret > 0 || errno != EINVAL) @@ -621,17 +628,29 @@ buf->flush_context = context; } -static ssize_t io_buffer_read_mmaped(IOBuffer *buf, size_t size) +static ssize_t io_buffer_set_mmaped_pos(IOBuffer *buf) { - uoff_t stop_offset; + buf->pos = buf->buffer_size; + if (buf->pos - buf->skip > buf->limit - buf->offset) + buf->pos = buf->limit - buf->offset + buf->skip; + return buf->pos - buf->skip; +} + +static ssize_t io_buffer_read_mmaped(IOBuffer *buf) +{ size_t aligned_skip; - stop_offset = buf->start_offset + buf->size; - if (stop_offset - buf->mmap_offset <= buf->buffer_size) { - /* end of file is already mapped */ + if (buf->start_offset + buf->limit <= + (uoff_t)buf->mmap_offset + buf->pos) { + /* end of file */ return -1; } + if (buf->pos < buf->buffer_size) { + /* more bytes available without needing to mmap() */ + return io_buffer_set_mmaped_pos(buf); + } + aligned_skip = buf->skip & ~mmap_pagemask; if (aligned_skip == 0 && buf->buffer != NULL) { /* didn't skip enough bytes */ @@ -641,14 +660,17 @@ buf->skip -= aligned_skip; buf->mmap_offset += aligned_skip; - if (buf->buffer != NULL) - (void)munmap(buf->buffer, buf->buffer_size); + if (buf->buffer != NULL) { + if (munmap(buf->buffer, buf->buffer_size) < 0) + i_error("io_buffer_read_mmaped(): munmap() failed: %m"); + } - buf->buffer_size = stop_offset - buf->mmap_offset; + buf->buffer_size = buf->start_offset + buf->size - buf->mmap_offset; if (buf->buffer_size > buf->max_buffer_size) buf->buffer_size = buf->max_buffer_size; - if (buf->buffer_size > size + buf->skip) - buf->buffer_size = size + buf->skip; + + i_assert((uoff_t)buf->mmap_offset + buf->buffer_size <= + buf->start_offset + buf->size); buf->buffer = mmap(NULL, buf->buffer_size, PROT_READ, MAP_PRIVATE, buf->fd, buf->mmap_offset); @@ -660,20 +682,29 @@ (void)madvise(buf->buffer, buf->buffer_size, MADV_SEQUENTIAL); - buf->pos = buf->buffer_size; - return buf->buffer_size - buf->skip; + return io_buffer_set_mmaped_pos(buf); +} + +void io_buffer_set_read_limit(IOBuffer *inbuf, uoff_t offset) +{ + i_assert(offset <= inbuf->size); + + if (offset == 0) + inbuf->limit = inbuf->size; + else { + i_assert(offset >= inbuf->offset); + + inbuf->limit = offset; + if (inbuf->offset + (inbuf->pos - inbuf->skip) > offset) + inbuf->pos = offset - inbuf->offset + inbuf->skip; + } } ssize_t io_buffer_read(IOBuffer *buf) { - return io_buffer_read_max(buf, SSIZE_T_MAX); -} - -ssize_t io_buffer_read_max(IOBuffer *buf, size_t size) -{ + size_t size; ssize_t ret; - i_assert(size <= SSIZE_T_MAX); i_assert(!buf->transmit); buf->receive = TRUE; @@ -681,7 +712,7 @@ return -1; if (buf->mmaped) - return io_buffer_read_mmaped(buf, size); + return io_buffer_read_mmaped(buf); if (buf->pos == buf->buffer_size) { if (buf->skip > 0) { @@ -697,14 +728,21 @@ return -2; /* buffer full */ } + size = buf->buffer_size - buf->pos; + if (buf->limit > 0) { + i_assert(buf->limit >= buf->offset); + if (size >= buf->limit - buf->offset) { + size = buf->limit - buf->offset; + if (size == 0) + return -1; + } + } + /* fill the buffer */ - if (buf->buffer_size-buf->pos < size) - size = buf->buffer_size - buf->pos; - if (!buf->file) { ret = net_receive(buf->fd, buf->buffer + buf->pos, size); } else { - ret = read(buf->fd, buf->buffer + buf->pos, size); + ret = read(buf->fd, buf->buffer + buf->pos, size); if (ret == 0) ret = -1; /* EOF */ else if (ret < 0 && (errno == EINTR || errno == EAGAIN)) @@ -726,20 +764,20 @@ { IOBufferBlockContext *ctx = context; - if (io_buffer_read_max(ctx->inbuf, (size_t)ctx->size) != 0) { + if (io_buffer_read(ctx->inbuf) != 0) { /* got data / error */ io_loop_stop(ctx->ioloop); } } -ssize_t io_buffer_read_blocking(IOBuffer *buf, size_t size) +ssize_t io_buffer_read_blocking(IOBuffer *buf) { IOBufferBlockContext ctx; Timeout to; ssize_t ret; /* first check if we can get some data */ - ret = io_buffer_read_max(buf, size); + ret = io_buffer_read(buf); if (ret != 0) return ret; @@ -749,7 +787,6 @@ memset(&ctx, 0, sizeof(ctx)); ctx.ioloop = io_loop_create(); ctx.inbuf = buf; - ctx.size = size; buf->io = io_add(buf->fd, IO_READ, io_read_data, &ctx); to = buf->timeout_msecs <= 0 ? NULL : @@ -776,36 +813,36 @@ (ssize_t) (buf->pos-buf->skip) : -1; } -void io_buffer_skip(IOBuffer *buf, uoff_t size) +void io_buffer_skip(IOBuffer *buf, uoff_t count) { + uoff_t old_limit; ssize_t ret; - buf->offset += size; + buf->offset += count; - if (size <= buf->pos - buf->skip) { - buf->skip += size; + if (count <= buf->pos - buf->skip) { + buf->skip += count; return; } if (buf->mmaped) { /* these point outside mmap now, next io_buffer_read_mmaped() will fix them */ - buf->skip += size; + buf->skip += count; buf->pos = buf->skip; } else { if (buf->buffer_size == 0) buffer_alloc_more(buf, IO_BUFFER_MIN_SIZE); - size -= buf->skip; - while (size > buf->buffer_size) { - ret = io_buffer_read_max(buf, buf->buffer_size); - if (ret <= 0) - break; + count -= buf->skip; + + old_limit = buf->limit; + io_buffer_set_read_limit(buf, buf->offset + count); - size -= ret; - } + while ((ret = io_buffer_read_blocking(buf)) > 0) + io_buffer_skip(buf, ret); - (void)io_buffer_read_max(buf, (size_t)size); + io_buffer_set_read_limit(buf, old_limit); } } @@ -900,19 +937,19 @@ { ssize_t ret; - if (buf->pos - buf->skip <= threshold) { + while (buf->pos - buf->skip <= threshold) { /* we need more data */ - ret = io_buffer_read_blocking(buf, SSIZE_T_MAX); + ret = io_buffer_read_blocking(buf); if (ret < 0) { - *size = 0; - *data = NULL; - return ret; + if (ret == -2) + return -2; + else + break; } } *data = io_buffer_get_data(buf, size); - i_assert(*size > 0); - return 1; + return *size > threshold ? 1 : *size > 0 ? 0 : -1; } unsigned char *io_buffer_get_space(IOBuffer *buf, size_t size) diff -r c020e1168cc3 -r 3dcc2275b4ca src/lib/iobuffer.h --- a/src/lib/iobuffer.h Sun Sep 22 12:27:21 2002 +0300 +++ b/src/lib/iobuffer.h Mon Sep 23 11:27:32 2002 +0300 @@ -11,7 +11,7 @@ int fd; uoff_t start_offset; - uoff_t offset, size; /* virtual offset, 0 = start_offset */ + uoff_t offset, size, limit; /* virtual offsets, 0 = start_offset */ int buf_errno; /* set when write() or read() failed. */ /* private: */ @@ -106,19 +106,20 @@ void io_buffer_send_flush_callback(IOBuffer *buf, IOBufferFlushFunc func, void *context); +/* IO buffer won't be read past specified offset. Giving 0 as offset removes + the limit. */ +void io_buffer_set_read_limit(IOBuffer *inbuf, uoff_t offset); + /* Returns number of bytes read if read was ok, -1 if disconnected / EOF, -2 if the buffer is full */ ssize_t io_buffer_read(IOBuffer *buf); -/* Like io_buffer_read(), but don't read more than specified size. */ -ssize_t io_buffer_read_max(IOBuffer *buf, size_t size); /* Blocking read, doesn't return until at least one byte is read, or until socket is disconnected or timeout has occured. Note that the fd must be - nonblocking, or the timeout doesn't work. If you don't want limit size, - set it to SSIZE_T_MAX. Returns number of bytes read (never 0), -1 if - error or -2 if buffer is full. */ -ssize_t io_buffer_read_blocking(IOBuffer *buf, size_t size); + nonblocking, or the timeout doesn't work. Returns number of bytes read + (never 0), -1 if error or -2 if buffer is full. */ +ssize_t io_buffer_read_blocking(IOBuffer *buf); /* Skip forward a number of bytes */ -void io_buffer_skip(IOBuffer *buf, uoff_t size); +void io_buffer_skip(IOBuffer *buf, uoff_t count); /* Seek to specified position from beginning of file. This works only for files. Returns TRUE if successful. */ int io_buffer_seek(IOBuffer *buf, uoff_t offset); @@ -129,9 +130,9 @@ /* Returns pointer to beginning of data in buffer, or NULL if there's no data. */ unsigned char *io_buffer_get_data(IOBuffer *buf, size_t *size); -/* Like io_buffer_get_data(), but read it when needed. There always must be - more than `threshold' bytes in buffer. Returns 1 if data was read, - -1 if EOF / error */ +/* Like io_buffer_get_data(), but read it when needed. Returns 1 if more + than threshold bytes were stored into buffer, 0 if less, -1 if error or + EOF with no bytes in buffer or -2 if buffer is full. */ int io_buffer_read_data_blocking(IOBuffer *buf, unsigned char **data, size_t *size, size_t threshold);