Mercurial > dovecot > core-2.2
changeset 1672:8920600a8cfc HEAD
Index cache file rewrite. It's not finished yet and mbox support is
completely broken. But it's getting difficult to maintain outside cvs :)
line wrap: on
line diff
--- a/src/imap/cmd-copy.c Wed Aug 06 23:13:57 2003 +0300 +++ b/src/imap/cmd-copy.c Wed Aug 06 23:15:30 2003 +0300 @@ -12,7 +12,8 @@ int failed = FALSE; fetch_ctx = box->fetch_init(box, MAIL_FETCH_STREAM_HEADER | - MAIL_FETCH_STREAM_BODY, messageset, uidset); + MAIL_FETCH_STREAM_BODY, NULL, + messageset, uidset); if (fetch_ctx == NULL) return FALSE;
--- a/src/imap/cmd-fetch.c Wed Aug 06 23:13:57 2003 +0300 +++ b/src/imap/cmd-fetch.c Wed Aug 06 23:15:30 2003 +0300 @@ -70,7 +70,10 @@ } if (strncmp(section, "HEADER", 6) == 0) { - *fetch_data |= MAIL_FETCH_STREAM_HEADER; + /* exact header matches could be cached */ + if (strncmp(section, "HEADER.FIELDS ", 14) != 0) + *fetch_data |= MAIL_FETCH_STREAM_HEADER; + if (check_header_section(section+6)) return TRUE; } else if (*section >= '0' && *section <= '9') {
--- a/src/imap/cmd-store.c Wed Aug 06 23:13:57 2003 +0300 +++ b/src/imap/cmd-store.c Wed Aug 06 23:15:30 2003 +0300 @@ -96,9 +96,16 @@ /* and update the flags */ box = client->mailbox; - failed = !box->lock(box, MAILBOX_LOCK_FLAGS | MAILBOX_LOCK_READ); + if (!box->is_readonly(box)) { + /* read-only, don't every try to get write locking */ + failed = FALSE; + } else { + failed = !box->lock(box, MAILBOX_LOCK_FLAGS | + MAILBOX_LOCK_READ); + } + fetch_ctx = failed ? NULL : - box->fetch_init(box, MAIL_FETCH_FLAGS, + box->fetch_init(box, MAIL_FETCH_FLAGS, NULL, messageset, client->cmd_uid); if (fetch_ctx == NULL) failed = TRUE;
--- a/src/imap/imap-fetch-body-section.c Wed Aug 06 23:13:57 2003 +0300 +++ b/src/imap/imap-fetch-body-section.c Wed Aug 06 23:15:30 2003 +0300 @@ -20,6 +20,9 @@ #define MAX_HEADER_BUFFER_SIZE (32*1024) struct fetch_header_field_context { + struct imap_fetch_context *fetch_ctx; + struct mail *mail; + buffer_t *dest; struct ostream *output; uoff_t dest_size; @@ -116,26 +119,6 @@ return ret >= 0; } -static const char **get_fields_array(const char *fields) -{ - const char **field_list, **field; - - while (*fields == ' ') - fields++; - if (*fields == '(') - fields++; - - field_list = t_strsplit(fields, " )"); - - /* array ends at ")" element */ - for (field = field_list; *field != NULL; field++) { - if (strcmp(*field, ")") == 0) - *field = NULL; - } - - return field_list; -} - static int header_match(const char *const *fields, const char *name, size_t size) { @@ -252,10 +235,16 @@ struct message_header_line *hdr; if (strncmp(section, "HEADER.FIELDS ", 14) == 0) { - ctx->fields = get_fields_array(section + 14); + ctx->fields = imap_fetch_get_body_fields(section + 14); ctx->match_func = header_match; + + if (ctx->fetch_ctx->body_fetch_from_cache) { + input = ctx->mail->get_headers(ctx->mail, ctx->fields); + if (input == NULL) + return FALSE; + } } else if (strncmp(section, "HEADER.FIELDS.NOT ", 18) == 0) { - ctx->fields = get_fields_array(section + 18); + ctx->fields = imap_fetch_get_body_fields(section + 18); ctx->match_func = header_match_not; } else if (strcmp(section, "MIME") == 0) { /* Mime-Version + Content-* fields */ @@ -329,18 +318,21 @@ the size first and then send the data directly to output stream. */ memset(&hdr_ctx, 0, sizeof(hdr_ctx)); + hdr_ctx.mail = mail; + hdr_ctx.fetch_ctx = ctx; hdr_ctx.skip = body->skip; hdr_ctx.max_size = body->max_size; hdr_ctx.fix_nuls = !mail->has_no_nuls; failed = FALSE; - start_offset = input->v_offset; + start_offset = input == NULL ? 0 : input->v_offset; t_push(); /* first pass, we need at least the size */ if (size->virtual_size > MAX_HEADER_BUFFER_SIZE && - body->max_size > MAX_HEADER_BUFFER_SIZE) { + body->max_size > MAX_HEADER_BUFFER_SIZE && + !ctx->body_fetch_from_cache) { if (!fetch_header_fields(input, header_section, &hdr_ctx)) failed = TRUE; @@ -392,9 +384,13 @@ struct istream *stream; struct message_size hdr_size; - stream = mail->get_stream(mail, &hdr_size, NULL); - if (stream == NULL) - return FALSE; + if (ctx->body_fetch_from_cache) + stream = NULL; + else { + stream = mail->get_stream(mail, &hdr_size, NULL); + if (stream == NULL) + return FALSE; + } return fetch_header_from(ctx, stream, &hdr_size, mail, body, body->section);
--- a/src/imap/imap-fetch.c Wed Aug 06 23:13:57 2003 +0300 +++ b/src/imap/imap-fetch.c Wed Aug 06 23:15:30 2003 +0300 @@ -1,6 +1,7 @@ /* Copyright (C) 2002 Timo Sirainen */ #include "common.h" +#include "buffer.h" #include "istream.h" #include "ostream.h" #include "str.h" @@ -12,6 +13,31 @@ #include <unistd.h> +const char *const *imap_fetch_get_body_fields(const char *fields) +{ + const char **field_list, **field, **dest; + + while (*fields == ' ') + fields++; + if (*fields == '(') + fields++; + + field_list = t_strsplit(fields, " )"); + + /* array ends at ")" element */ + for (field = dest = field_list; *field != NULL; field++) { + if (strcmp(*field, ")") == 0) + break; + if (**field != '\0') { + *dest = *field; + dest++; + } + } + *dest = NULL; + + return field_list; +} + static void fetch_uid(struct imap_fetch_context *ctx, struct mail *mail) { str_printfa(ctx->str, "UID %u ", mail->uid); @@ -265,6 +291,10 @@ struct mailbox *box = client->mailbox; struct imap_fetch_context ctx; struct mail *mail; + struct imap_fetch_body_data *body; + const char *null = NULL; + const char *const *wanted_headers, *const *arr; + buffer_t *buffer; int all_found; memset(&ctx, 0, sizeof(ctx)); @@ -277,9 +307,7 @@ if (!box->is_readonly(box)) { /* If we have any BODY[..] sections, \Seen flag is added for - all messages */ - struct imap_fetch_body_data *body; - + all messages. */ for (body = bodies; body != NULL; body = body->next) { if (!body->peek) { ctx.update_seen = TRUE; @@ -291,12 +319,34 @@ ctx.update_seen = TRUE; } + /* If we have only BODY[HEADER.FIELDS (...)] fetches, get them + separately rather than parsing the full header so mail storage + can try to cache them. */ + ctx.body_fetch_from_cache = TRUE; + buffer = buffer_create_dynamic(data_stack_pool, 64, (size_t)-1); + for (body = bodies; body != NULL; body = body->next) { + if (strncmp(body->section, "HEADER.FIELDS ", 14) != 0) { + ctx.body_fetch_from_cache = FALSE; + break; + } + + arr = imap_fetch_get_body_fields(body->section + 14); + while (*arr != NULL) { + buffer_append(buffer, arr, sizeof(*arr)); + arr++; + } + } + buffer_append(buffer, &null, sizeof(null)); + wanted_headers = !ctx.body_fetch_from_cache ? NULL : + buffer_get_data(buffer, NULL); + if (ctx.update_seen) { if (!box->lock(box, MAILBOX_LOCK_FLAGS | MAILBOX_LOCK_READ)) return -1; } - ctx.fetch_ctx = box->fetch_init(box, fetch_data, messageset, uidset); + ctx.fetch_ctx = box->fetch_init(box, fetch_data, wanted_headers, + messageset, uidset); if (ctx.fetch_ctx == NULL) ctx.failed = TRUE; else {
--- a/src/imap/imap-fetch.h Wed Aug 06 23:13:57 2003 +0300 +++ b/src/imap/imap-fetch.h Wed Aug 06 23:15:30 2003 +0300 @@ -33,7 +33,7 @@ int update_seen; struct mail_full_flags seen_flag; - int first, failed; + int first, failed, body_fetch_from_cache; }; int imap_fetch(struct client *client, @@ -46,4 +46,6 @@ const struct imap_fetch_body_data *body, struct mail *mail); +const char *const *imap_fetch_get_body_fields(const char *fields); + #endif
--- a/src/imap/imap-search.c Wed Aug 06 23:13:57 2003 +0300 +++ b/src/imap/imap-search.c Wed Aug 06 23:15:30 2003 +0300 @@ -21,20 +21,26 @@ return arg; } -#define ARG_NEW(type, value) \ - arg_new(data, args, next_sarg, type, value) +#define ARG_NEW(type) \ + arg_new(data, args, next_sarg, type, TRUE, NULL) + +#define ARG_NEW_FLAG(type) \ + arg_new(data, args, next_sarg, type, FALSE, NULL) + +#define ARG_NEW_HEADER(type, hdr_name) \ + arg_new(data, args, next_sarg, type, TRUE, hdr_name) static int arg_new(struct search_build_data *data, struct imap_arg **args, struct mail_search_arg **next_sarg, - enum mail_search_arg_type type, int value) + enum mail_search_arg_type type, int have_value, + const char *hdr_name) { struct mail_search_arg *sarg; *next_sarg = sarg = search_arg_new(data->pool, type); - if (value == 0) + if (!have_value) return TRUE; - /* first arg */ if ((*args)->type == IMAP_ARG_EOL) { data->error = "Missing parameter for argument"; return FALSE; @@ -49,23 +55,8 @@ sarg->value.str = str_ucase(IMAP_ARG_STR(*args)); *args += 1; - /* second arg */ - if (value == 2) { - if ((*args)->type == IMAP_ARG_EOL) { - data->error = "Missing parameter for argument"; - return FALSE; - } - - if ((*args)->type != IMAP_ARG_ATOM && - (*args)->type != IMAP_ARG_STRING) { - data->error = "Invalid parameter for argument"; - return FALSE; - } - - sarg->hdr_field_name = sarg->value.str; - sarg->value.str = str_ucase(IMAP_ARG_STR(*args)); - *args += 1; - } + if (hdr_name != NULL) + sarg->hdr_field_name = hdr_name; return TRUE; } @@ -122,40 +113,40 @@ switch (*str) { case 'A': if (strcmp(str, "ANSWERED") == 0) - return ARG_NEW(SEARCH_ANSWERED, 0); + return ARG_NEW_FLAG(SEARCH_ANSWERED); else if (strcmp(str, "ALL") == 0) - return ARG_NEW(SEARCH_ALL, 0); + return ARG_NEW_FLAG(SEARCH_ALL); break; case 'B': if (strcmp(str, "BODY") == 0) { /* <string> */ - return ARG_NEW(SEARCH_BODY, 1); + return ARG_NEW(SEARCH_BODY); } else if (strcmp(str, "BEFORE") == 0) { /* <date> */ - return ARG_NEW(SEARCH_BEFORE, 1); + return ARG_NEW(SEARCH_BEFORE); } else if (strcmp(str, "BCC") == 0) { /* <string> */ - return ARG_NEW(SEARCH_BCC, 1); + return ARG_NEW_HEADER(SEARCH_HEADER_ADDRESS, str); } break; case 'C': if (strcmp(str, "CC") == 0) { /* <string> */ - return ARG_NEW(SEARCH_CC, 1); + return ARG_NEW_HEADER(SEARCH_HEADER_ADDRESS, str); } break; case 'D': if (strcmp(str, "DELETED") == 0) - return ARG_NEW(SEARCH_DELETED, 0); + return ARG_NEW_FLAG(SEARCH_DELETED); else if (strcmp(str, "DRAFT") == 0) - return ARG_NEW(SEARCH_DRAFT, 0); + return ARG_NEW_FLAG(SEARCH_DRAFT); break; case 'F': if (strcmp(str, "FLAGGED") == 0) - return ARG_NEW(SEARCH_FLAGGED, 0); + return ARG_NEW_FLAG(SEARCH_FLAGGED); else if (strcmp(str, "FROM") == 0) { /* <string> */ - return ARG_NEW(SEARCH_FROM, 1); + return ARG_NEW_HEADER(SEARCH_HEADER_ADDRESS, str); } break; case 'H': @@ -174,43 +165,20 @@ } key = str_ucase(IMAP_ARG_STR(*args)); - - if (strcmp(key, "FROM") == 0) { - *args += 1; - return ARG_NEW(SEARCH_FROM, 1); - } else if (strcmp(key, "TO") == 0) { - *args += 1; - return ARG_NEW(SEARCH_TO, 1); - } else if (strcmp(key, "CC") == 0) { - *args += 1; - return ARG_NEW(SEARCH_CC, 1); - } else if (strcmp(key, "BCC") == 0) { - *args += 1; - return ARG_NEW(SEARCH_BCC, 1); - } else if (strcmp(key, "SUBJECT") == 0) { - *args += 1; - return ARG_NEW(SEARCH_SUBJECT, 1); - } else if (strcmp(key, "IN-REPLY-TO") == 0) { - *args += 1; - return ARG_NEW(SEARCH_IN_REPLY_TO, 1); - } else if (strcmp(key, "MESSAGE-ID") == 0) { - *args += 1; - return ARG_NEW(SEARCH_MESSAGE_ID, 1); - } else { - return ARG_NEW(SEARCH_HEADER, 2); - } + *args += 1; + return ARG_NEW_HEADER(SEARCH_HEADER, key); } break; case 'K': if (strcmp(str, "KEYWORD") == 0) { /* <flag> */ - return ARG_NEW(SEARCH_KEYWORD, 1); + return ARG_NEW(SEARCH_KEYWORD); } break; case 'L': if (strcmp(str, "LARGER") == 0) { /* <n> */ - return ARG_NEW(SEARCH_LARGER, 1); + return ARG_NEW(SEARCH_LARGER); } break; case 'N': @@ -260,10 +228,10 @@ return TRUE; } if (strcmp(str, "ON") == 0) { /* <date> */ - return ARG_NEW(SEARCH_ON, 1); + return ARG_NEW(SEARCH_ON); } if (strcmp(str, "OLD") == 0) { /* OLD == NOT RECENT */ - if (!ARG_NEW(SEARCH_RECENT, 0)) + if (!ARG_NEW_FLAG(SEARCH_RECENT)) return FALSE; (*next_sarg)->not = TRUE; @@ -272,71 +240,71 @@ break; case 'R': if (strcmp(str, "RECENT") == 0) - return ARG_NEW(SEARCH_RECENT, 0); + return ARG_NEW_FLAG(SEARCH_RECENT); break; case 'S': if (strcmp(str, "SEEN") == 0) - return ARG_NEW(SEARCH_SEEN, 0); + return ARG_NEW_FLAG(SEARCH_SEEN); else if (strcmp(str, "SUBJECT") == 0) { /* <string> */ - return ARG_NEW(SEARCH_SUBJECT, 1); + return ARG_NEW_HEADER(SEARCH_HEADER, str); } else if (strcmp(str, "SENTBEFORE") == 0) { /* <date> */ - return ARG_NEW(SEARCH_SENTBEFORE, 1); + return ARG_NEW(SEARCH_SENTBEFORE); } else if (strcmp(str, "SENTON") == 0) { /* <date> */ - return ARG_NEW(SEARCH_SENTON, 1); + return ARG_NEW(SEARCH_SENTON); } else if (strcmp(str, "SENTSINCE") == 0) { /* <date> */ - return ARG_NEW(SEARCH_SENTSINCE, 1); + return ARG_NEW(SEARCH_SENTSINCE); } else if (strcmp(str, "SINCE") == 0) { /* <date> */ - return ARG_NEW(SEARCH_SINCE, 1); + return ARG_NEW(SEARCH_SINCE); } else if (strcmp(str, "SMALLER") == 0) { /* <n> */ - return ARG_NEW(SEARCH_SMALLER, 1); + return ARG_NEW(SEARCH_SMALLER); } break; case 'T': if (strcmp(str, "TEXT") == 0) { /* <string> */ - return ARG_NEW(SEARCH_TEXT, 1); + return ARG_NEW(SEARCH_TEXT); } else if (strcmp(str, "TO") == 0) { /* <string> */ - return ARG_NEW(SEARCH_TO, 1); + return ARG_NEW_HEADER(SEARCH_HEADER_ADDRESS, str); } break; case 'U': if (strcmp(str, "UID") == 0) { /* <message set> */ - return ARG_NEW(SEARCH_UID, 1); + return ARG_NEW(SEARCH_UID); } else if (strcmp(str, "UNANSWERED") == 0) { - if (!ARG_NEW(SEARCH_ANSWERED, 0)) + if (!ARG_NEW_FLAG(SEARCH_ANSWERED)) return FALSE; (*next_sarg)->not = TRUE; return TRUE; } else if (strcmp(str, "UNDELETED") == 0) { - if (!ARG_NEW(SEARCH_DELETED, 0)) + if (!ARG_NEW_FLAG(SEARCH_DELETED)) return FALSE; (*next_sarg)->not = TRUE; return TRUE; } else if (strcmp(str, "UNDRAFT") == 0) { - if (!ARG_NEW(SEARCH_DRAFT, 0)) + if (!ARG_NEW_FLAG(SEARCH_DRAFT)) return FALSE; (*next_sarg)->not = TRUE; return TRUE; } else if (strcmp(str, "UNFLAGGED") == 0) { - if (!ARG_NEW(SEARCH_FLAGGED, 0)) + if (!ARG_NEW_FLAG(SEARCH_FLAGGED)) return FALSE; (*next_sarg)->not = TRUE; return TRUE; } else if (strcmp(str, "UNKEYWORD") == 0) { - if (!ARG_NEW(SEARCH_KEYWORD, 0)) + if (!ARG_NEW_FLAG(SEARCH_KEYWORD)) return FALSE; (*next_sarg)->not = TRUE; return TRUE; } else if (strcmp(str, "UNSEEN") == 0) { - if (!ARG_NEW(SEARCH_SEEN, 0)) + if (!ARG_NEW_FLAG(SEARCH_SEEN)) return FALSE; (*next_sarg)->not = TRUE; return TRUE; @@ -345,7 +313,7 @@ default: if (*str == '*' || (*str >= '0' && *str <= '9')) { /* <message-set> */ - if (!ARG_NEW(SEARCH_SET, 0)) + if (!ARG_NEW_FLAG(SEARCH_SET)) return FALSE; (*next_sarg)->value.str = str;
--- a/src/imap/imap-thread.c Wed Aug 06 23:13:57 2003 +0300 +++ b/src/imap/imap-thread.c Wed Aug 06 23:15:30 2003 +0300 @@ -103,7 +103,7 @@ struct mail_search_arg *args, enum mail_thread_type type) { static const char *wanted_headers[] = { - "message-id", "in-reply-to", "references", + "message-id", "in-reply-to", "references", "subject", NULL }; struct thread_context *ctx;
--- a/src/lib-imap/imap-envelope.c Wed Aug 06 23:13:57 2003 +0300 +++ b/src/lib-imap/imap-envelope.c Wed Aug 06 23:15:30 2003 +0300 @@ -19,6 +19,12 @@ char *in_reply_to, *message_id; }; +const char *imap_envelope_headers[] = { + "Date", "Subject", "From", "Sender", "Reply-To", + "To", "Cc", "Bcc", "In-Reply-To", "Message-ID", + NULL +}; + int imap_envelope_get_field(const char *name, enum imap_envelope_field *ret) { *ret = (enum imap_envelope_field)-1;
--- a/src/lib-imap/imap-envelope.h Wed Aug 06 23:13:57 2003 +0300 +++ b/src/lib-imap/imap-envelope.h Wed Aug 06 23:15:30 2003 +0300 @@ -26,6 +26,8 @@ struct message_part_envelope_data; +extern const char *imap_envelope_headers[]; + int imap_envelope_get_field(const char *name, enum imap_envelope_field *ret); /* Update envelope data based from given header field */
--- a/src/lib-index/Makefile.am Wed Aug 06 23:13:57 2003 +0300 +++ b/src/lib-index/Makefile.am Wed Aug 06 23:15:30 2003 +0300 @@ -8,21 +8,20 @@ -I$(top_srcdir)/src/lib-imap libindex_a_SOURCES = \ + mail-cache.c \ mail-custom-flags.c \ mail-index.c \ mail-index-compress.c \ mail-index-file.c \ mail-index-fsck.c \ - mail-index-data.c \ mail-index-open.c \ - mail-index-update.c \ - mail-index-update-cache.c \ + mail-index-rebuild.c \ mail-index-util.c \ mail-modifylog.c noinst_HEADERS = \ + mail-cache.h \ mail-custom-flags.h \ mail-index.h \ - mail-index-data.h \ mail-index-util.h \ mail-modifylog.h
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-index/mail-cache.c Wed Aug 06 23:15:30 2003 +0300 @@ -0,0 +1,1705 @@ +/* Copyright (C) 2003 Timo Sirainen */ + +#define _XOPEN_SOURCE 500 /* for pwrite() / Linux */ + +#include "lib.h" +#include "buffer.h" +#include "byteorder.h" +#include "file-lock.h" +#include "file-set-size.h" +#include "ioloop.h" +#include "mmap-util.h" +#include "write-full.h" +#include "mail-index.h" +#include "mail-index-util.h" +#include "mail-cache.h" + +#include <stddef.h> +#include <unistd.h> +#include <sys/stat.h> + +/* Never compress the file if it's smaller than this */ +#define COMPRESS_MIN_SIZE (1024*50) + +/* Compress the file when deleted space reaches n% of total size */ +#define COMPRESS_PERCENTAGE 20 + +/* Compress the file when n% of rows contain continued rows. + 200% means that there's 2 continued rows per record. */ +#define COMPRESS_CONTINUED_PERCENTAGE 200 + +/* Initial size for the file */ +#define MAIL_CACHE_INITIAL_SIZE (sizeof(struct mail_cache_header) + 10240) + +/* When more space is needed, grow the file n% larger than the previous size */ +#define MAIL_CACHE_GROW_PERCENTAGE 10 + +#define MAIL_CACHE_LOCK_TIMEOUT 120 +#define MAIL_CACHE_LOCK_STALE_TIMEOUT 60 + +#define CACHE_RECORD(cache, data_pos) \ + ((struct mail_cache_record *) \ + ((char *) (cache)->mmap_base + ((data_pos) & ~3))) + +struct mail_cache_header { + uint32_t indexid; + uint32_t continued_record_count; + + uint32_t used_file_size; + uint32_t deleted_space; + + uint32_t used_fields; /* enum mail_cache_field */ + + uint32_t field_usage_start; /* time_t */ + uint32_t field_usage_counts[32]; + + uint32_t header_offsets[MAIL_CACHE_HEADERS_COUNT]; +}; + +struct mail_cache_record { + uint32_t fields; /* enum mail_cache_field */ + uint32_t next_offset; + uint32_t size; /* full record size, including this header */ +}; + +struct mail_cache { + struct mail_index *index; + + char *filepath; + int fd; + + void *mmap_base; + size_t mmap_length; + uint32_t used_file_size; + + struct mail_cache_header *header; + + pool_t split_header_pool; + uint32_t split_offsets[MAIL_CACHE_HEADERS_COUNT]; + const char *const *split_headers[MAIL_CACHE_HEADERS_COUNT]; + + enum mail_cache_field default_cache_fields; + enum mail_cache_field never_cache_fields; + + struct mail_cache_transaction_ctx *trans_ctx; + unsigned int locks; + + unsigned int anon_mmap:1; + unsigned int mmap_refresh:1; +}; + +struct mail_cache_transaction_ctx { + struct mail_cache *cache; + + unsigned int next_unused_header_lowwater; + + unsigned int last_idx; + struct mail_cache_record cache_rec; + buffer_t *cache_data; + + unsigned int first_uid, last_uid; + buffer_t *index_marks, *cache_marks; +}; + +unsigned int mail_cache_field_sizes[32] = { + sizeof(enum mail_index_record_flag), + sizeof(uoff_t), + 16, + sizeof(struct mail_sent_date), + sizeof(time_t), + sizeof(uoff_t), + sizeof(uoff_t), + sizeof(uoff_t), + sizeof(uoff_t), + + 0, 0, 0, + + /* variable sized */ + (unsigned int)-1, (unsigned int)-1, (unsigned int)-1, (unsigned int)-1, + (unsigned int)-1, (unsigned int)-1, (unsigned int)-1, (unsigned int)-1, + (unsigned int)-1, (unsigned int)-1, (unsigned int)-1, (unsigned int)-1, + (unsigned int)-1, (unsigned int)-1, (unsigned int)-1, (unsigned int)-1, + (unsigned int)-1, (unsigned int)-1, (unsigned int)-1, (unsigned int)-1 +}; + +enum mail_cache_field mail_cache_header_fields[MAIL_CACHE_HEADERS_COUNT] = { + MAIL_CACHE_HEADERS1, + MAIL_CACHE_HEADERS2, + MAIL_CACHE_HEADERS3, + MAIL_CACHE_HEADERS4 +}; + +static const unsigned char *null4[] = { 0, 0, 0, 0 }; + +static const char * +mail_cache_get_header_fields_str(struct mail_cache *cache, unsigned int idx); +static int mail_cache_write(struct mail_cache_transaction_ctx *ctx); +static struct mail_cache_record * +mail_cache_lookup(struct mail_cache *cache, + const struct mail_index_record *rec); + +static int mail_cache_set_syscall_error(struct mail_cache *cache, + const char *function) +{ + i_assert(function != NULL); + + if (ENOSPACE(errno)) { + cache->index->nodiskspace = TRUE; + return FALSE; + } + + index_set_error(cache->index, "%s failed with index cache file %s: %m", + function, cache->filepath); + return FALSE; +} + +static int mail_cache_create_memory(struct mail_cache *cache, + struct mail_cache_header *hdr) +{ + cache->mmap_length = MAIL_CACHE_INITIAL_SIZE; + cache->mmap_base = mmap_anon(cache->mmap_length); + if (cache->mmap_base == MAP_FAILED) { + index_set_error(cache->index, "mmap_anon(%"PRIuSIZE_T")", + cache->mmap_length); + return FALSE; + } + + cache->header = cache->mmap_base; + *cache->header = *hdr; + + cache->anon_mmap = TRUE; + cache->filepath = i_strdup_printf("(in-memory index cache for %s)", + cache->index->mailbox_path); + return TRUE; +} + +static void mail_cache_file_close(struct mail_cache *cache) +{ + if (cache->anon_mmap) { + if (munmap_anon(cache->mmap_base, cache->mmap_length) < 0) + mail_cache_set_syscall_error(cache, "munmap_anon()"); + } else if (cache->mmap_base != NULL) { + if (munmap(cache->mmap_base, cache->mmap_length) < 0) + mail_cache_set_syscall_error(cache, "munmap()"); + } + + cache->mmap_base = NULL; + cache->header = NULL; + cache->mmap_length = 0; + + if (cache->fd != -1) { + if (close(cache->fd) < 0) + mail_cache_set_syscall_error(cache, "close()"); + cache->fd = -1; + } +} + +static int mail_cache_file_reopen(struct mail_cache *cache) +{ + int fd; + + i_assert(!cache->anon_mmap); + + fd = open(cache->filepath, O_RDWR); + if (fd == -1) + return mail_cache_set_syscall_error(cache, "open()"); + + mail_cache_file_close(cache); + + cache->fd = fd; + return TRUE; +} + +static int mmap_verify_header(struct mail_cache *cache) +{ + struct mail_cache_header *hdr = cache->header; + + if (cache->trans_ctx != NULL) { + /* we've updated used_file_size, do nothing */ + return TRUE; + } + + cache->used_file_size = nbo_to_uint32(hdr->used_file_size); + + /* only check the header if we're locked */ + if (cache->locks == 0) + return TRUE; + + if (cache->used_file_size < sizeof(struct mail_cache_header)) { + mail_cache_set_corrupted(cache, "used_file_size too small"); + return FALSE; + } + if ((cache->used_file_size % sizeof(uint32_t)) != 0) { + mail_cache_set_corrupted(cache, "used_file_size not aligned"); + return FALSE; + } + + if (cache->used_file_size > cache->mmap_length) { + /* maybe a crash truncated the file - just fix it */ + hdr->used_file_size = uint32_to_nbo(cache->mmap_length & ~3); + if (msync(cache->mmap_base, sizeof(*hdr), MS_SYNC) < 0) + return mail_cache_set_syscall_error(cache, "msync()"); + } + return TRUE; +} + +static int mmap_update(struct mail_cache *cache, size_t offset, size_t size) +{ + if (cache->header != NULL && + cache->header->indexid != cache->index->indexid) { + /* indexid changed, most likely it was rebuilt. + try reopening. */ + if (!mail_cache_file_reopen(cache)) + return FALSE; + + /* force mmap refresh */ + size = 0; + } + + if (size != 0 && offset < cache->mmap_length && + size <= cache->mmap_length - offset) { + /* already mapped */ + if (!cache->mmap_refresh) + return TRUE; + + cache->mmap_refresh = FALSE; + } + + if (cache->anon_mmap) + return TRUE; + + if (cache->mmap_base != NULL) { + if (cache->locks != 0) { + /* in the middle of transaction - write the changes */ + if (msync(cache->mmap_base, cache->mmap_length, + MS_SYNC) < 0) { + mail_cache_set_syscall_error(cache, "msync()"); + return FALSE; + } + } + + if (munmap(cache->mmap_base, cache->mmap_length) < 0) + mail_cache_set_syscall_error(cache, "munmap()"); + } + + i_assert(cache->fd != -1); + + /* map the whole file */ + cache->header = NULL; + cache->mmap_length = 0; + + cache->mmap_base = mmap_rw_file(cache->fd, &cache->mmap_length); + if (cache->mmap_base == MAP_FAILED) { + cache->mmap_base = NULL; + return mail_cache_set_syscall_error(cache, "mmap()"); + } + + /* check that the header is still ok */ + if (cache->mmap_length < sizeof(struct mail_cache_header)) + return mail_cache_set_corrupted(cache, "File too small"); + + cache->header = cache->mmap_base; + return mmap_verify_header(cache); +} + +static int mail_cache_open_and_verify(struct mail_cache *cache, int silent) +{ + struct stat st; + + mail_cache_file_close(cache); + + cache->fd = open(cache->filepath, O_RDWR); + if (cache->fd == -1) { + if (errno == ENOENT) + return 0; + + mail_cache_set_syscall_error(cache, "open()"); + return -1; + } + + if (fstat(cache->fd, &st) < 0) { + mail_cache_set_syscall_error(cache, "fstat()"); + return -1; + } + + if (st.st_size < sizeof(struct mail_cache_header)) + return 0; + + cache->mmap_refresh = TRUE; + if (!mmap_update(cache, 0, sizeof(struct mail_cache_header))) + return -1; + + /* verify that this really is the cache for wanted index */ + if (cache->header->indexid != cache->index->indexid) { + if (!silent) + mail_cache_set_corrupted(cache, "IndexID mismatch"); + return 0; + } + + return 1; +} + +static int mail_cache_open_or_create_file(struct mail_cache *cache, + struct mail_cache_header *hdr) +{ + int ret, fd; + + cache->filepath = i_strconcat(cache->index->filepath, + MAIL_CACHE_FILE_PREFIX, NULL); + + ret = mail_cache_open_and_verify(cache, FALSE); + if (ret != 0) + return ret > 0; + + /* maybe a rebuild.. */ + fd = file_dotlock_open(cache->filepath, NULL, MAIL_CACHE_LOCK_TIMEOUT, + MAIL_CACHE_LOCK_STALE_TIMEOUT, NULL, NULL); + if (fd == -1) { + mail_cache_set_syscall_error(cache, "file_dotlock_open()"); + return FALSE; + } + + /* see if someone else just created the cache file */ + ret = mail_cache_open_and_verify(cache, TRUE); + if (ret != 0) { + (void)file_dotlock_delete(cache->filepath, fd); + return ret > 0; + } + + /* rebuild then */ + if (write_full(fd, hdr, sizeof(*hdr)) < 0) { + mail_cache_set_syscall_error(cache, "write_full()"); + (void)file_dotlock_delete(cache->filepath, fd); + return FALSE; + } + if (file_set_size(fd, MAIL_CACHE_INITIAL_SIZE) < 0) { + mail_cache_set_syscall_error(cache, "file_set_size()"); + (void)file_dotlock_delete(cache->filepath, fd); + return FALSE; + } + + mail_cache_file_close(cache); + cache->fd = dup(fd); + + if (file_dotlock_replace(cache->filepath, fd, FALSE) < 0) { + mail_cache_set_syscall_error(cache, "file_dotlock_replace()"); + return FALSE; + } + + cache->mmap_refresh = TRUE; + if (!mmap_update(cache, 0, sizeof(struct mail_cache_header))) + return FALSE; + + return TRUE; +} + +int mail_cache_open_or_create(struct mail_index *index) +{ + struct mail_cache_header hdr; + struct mail_cache *cache; + + memset(&hdr, 0, sizeof(hdr)); + hdr.indexid = index->indexid; + hdr.used_file_size = uint32_to_nbo(sizeof(hdr)); + + cache = i_new(struct mail_cache, 1); + cache->index = index; + cache->fd = -1; + cache->split_header_pool = pool_alloconly_create("Headers", 512); + + index->cache = cache; + + /* we'll do anon-mmaping only if initially requested. if we fail + because of out of disk space, we'll just let the main index code + know it and fail. */ + if (INDEX_IS_IN_MEMORY(index)) { + if (!mail_cache_create_memory(cache, &hdr)) { + mail_cache_free(cache); + return FALSE; + } + } else { + if (!mail_cache_open_or_create_file(cache, &hdr)) { + mail_cache_free(cache); + return FALSE; + } + } + + return TRUE; +} + +void mail_cache_free(struct mail_cache *cache) +{ + i_assert(cache->trans_ctx == NULL); + + cache->index->cache = NULL; + + mail_cache_file_close(cache); + + pool_unref(cache->split_header_pool); + i_free(cache->filepath); + i_free(cache); +} + +void mail_cache_set_defaults(struct mail_cache *cache, + enum mail_cache_field default_cache_fields, + enum mail_cache_field never_cache_fields) +{ + cache->default_cache_fields = default_cache_fields; + cache->never_cache_fields = never_cache_fields; +} + +static const struct mail_cache_record * +mail_cache_compress_record(struct mail_cache *cache, + struct mail_index_record *rec, + enum mail_cache_field remove_fields, + uint32_t *size_r) +{ + enum mail_cache_field cached_fields, field; + struct mail_cache_record cache_rec; + buffer_t *buffer; + const void *data; + size_t size; + uint32_t nb_size; + int i; + + memset(&cache_rec, 0, sizeof(cache_rec)); + buffer = buffer_create_dynamic(data_stack_pool, 4096, (size_t)-1); + + cached_fields = mail_cache_get_fields(cache, rec) & ~remove_fields; + buffer_append(buffer, &cache_rec, sizeof(cache_rec)); + for (i = 0; i < 32; i++) { + field = nbo32_bitmasks[i]; + if ((cached_fields & field) == 0) + continue; + + if (!mail_cache_lookup_field(cache, rec, field, &data, &size)) { + cached_fields &= ~field; + continue; + } + + nb_size = uint32_to_nbo((uint32_t)size); + + if ((field & MAIL_CACHE_FIXED_MASK) == 0) + buffer_append(buffer, &nb_size, sizeof(nb_size)); + buffer_append(buffer, data, size); + if ((size & 3) != 0) + buffer_append(buffer, null4, 4 - (size & 3)); + } + + cache_rec.fields = cached_fields; + cache_rec.size = uint32_to_nbo(buffer_get_used_size(buffer)); + buffer_write(buffer, 0, &cache_rec, sizeof(cache_rec)); + + data = buffer_get_data(buffer, &size); + *size_r = size; + return data; +} + +static int mail_cache_copy(struct mail_cache *cache, int fd) +{ + struct mail_cache_header *hdr; + const struct mail_cache_record *cache_rec; + struct mail_index_record *rec; + enum mail_cache_field used_fields, remove_fields; + unsigned char *mmap_base; + const char *str; + uint32_t new_file_size, offset, size, nb_size; + int i; + + /* pick some reasonably good file size */ + new_file_size = cache->used_file_size - + nbo_to_uint32(cache->header->deleted_space); + new_file_size = (new_file_size + 1023) & ~1023; + if (new_file_size < MAIL_CACHE_INITIAL_SIZE) + new_file_size = MAIL_CACHE_INITIAL_SIZE; + + if (file_set_size(fd, new_file_size) < 0) + return mail_cache_set_syscall_error(cache, "file_set_size()"); + + mmap_base = mmap(NULL, new_file_size, PROT_READ | PROT_WRITE, + MAP_SHARED, fd, 0); + if (mmap_base == MAP_FAILED) + return mail_cache_set_syscall_error(cache, "mmap()"); + + /* skip file's header */ + hdr = (struct mail_cache_header *) mmap_base; + offset = sizeof(*hdr); + + /* get the newest message header list */ + remove_fields = 0; + for (i = MAIL_CACHE_HEADERS_COUNT-1; i >= 0; i--) { + str = mail_cache_get_header_fields_str(cache, i); + if (str != NULL) { + hdr->header_offsets[0] = uint32_to_nbo(offset | 2); + + size = strlen(str) + 1; + nb_size = uint32_to_nbo(size); + + memcpy(mmap_base + offset, &nb_size, sizeof(nb_size)); + offset += sizeof(nb_size); + memcpy(mmap_base + offset, str, size); + offset += (size + 3) & ~3; + break; + } + } + + /* remove other headers */ + for (i--; i >= 0; i--) + remove_fields |= mail_cache_header_fields[i]; + + used_fields = 0; + rec = cache->index->lookup(cache->index, 1); + while (rec != NULL) { + cache_rec = mail_cache_lookup(cache, rec); + if (cache_rec == NULL) + rec->data_offset = 0; + else if ((nbo_to_uint32(cache_rec->next_offset) & 2) == 0 && + remove_fields == 0) { + /* just one unmodified block, copy it */ + size = nbo_to_uint32(cache_rec->size); + i_assert(offset + size <= new_file_size); + + memcpy(mmap_base + offset, cache_rec, size); + rec->data_offset = uint32_to_nbo(offset | 2); + + size = (size + 3) & ~3; + offset += size; + } else { + /* multiple blocks, sort them into buffer */ + t_push(); + cache_rec = mail_cache_compress_record(cache, rec, + remove_fields, + &size); + i_assert(offset + size <= new_file_size); + memcpy(mmap_base + offset, cache_rec, size); + used_fields |= cache_rec->fields; + t_pop(); + + rec->data_offset = uint32_to_nbo(offset | 2); + offset += size; + } + + rec = cache->index->next(cache->index, rec); + } + + /* update header */ + hdr->indexid = cache->index->indexid; + hdr->used_file_size = uint32_to_nbo(offset); + hdr->used_fields = used_fields; + hdr->field_usage_start = uint32_to_nbo(ioloop_time); + + /* write everything to disk */ + if (msync(mmap_base, offset, MS_SYNC) < 0) + return mail_cache_set_syscall_error(cache, "msync()"); + + if (fdatasync(fd) < 0) + return mail_cache_set_syscall_error(cache, "fdatasync()"); + return TRUE; +} + +int mail_cache_compress(struct mail_cache *cache) +{ + int fd, ret = TRUE; + + i_assert(cache->trans_ctx == NULL); + + if (!cache->index->set_lock(cache->index, MAIL_LOCK_EXCLUSIVE)) + return FALSE; + + if (mail_cache_lock(cache, TRUE) <= 0) + return FALSE; + + fd = file_dotlock_open(cache->filepath, NULL, MAIL_CACHE_LOCK_TIMEOUT, + MAIL_CACHE_LOCK_STALE_TIMEOUT, NULL, NULL); + if (fd == -1) { + mail_cache_set_syscall_error(cache, "file_dotlock_open()"); + return FALSE; + } + + /* now we'll begin the actual moving. keep rebuild-flag on + while doing it. */ + cache->index->header->flags |= MAIL_INDEX_HDR_FLAG_REBUILD; + if (!mail_index_fmdatasync(cache->index, cache->index->header_size)) + return FALSE; + + if (!mail_cache_copy(cache, fd)) { + (void)file_dotlock_delete(cache->filepath, fd); + ret = FALSE; + } else { + mail_cache_file_close(cache); + cache->fd = dup(fd); + + if (file_dotlock_replace(cache->filepath, fd, FALSE) < 0) { + mail_cache_set_syscall_error(cache, + "file_dotlock_replace()"); + ret = FALSE; + } + + if (!mmap_update(cache, 0, 0)) + ret = FALSE; + } + cache->index->header->flags &= ~(MAIL_INDEX_HDR_FLAG_REBUILD | + MAIL_INDEX_HDR_FLAG_COMPRESS_CACHE); + + if (!mail_cache_unlock(cache)) + ret = FALSE; + + return ret; +} + +int mail_cache_truncate(struct mail_cache *cache) +{ + struct mail_cache_header hdr; + int ret, fd; + + memset(&hdr, 0, sizeof(hdr)); + hdr.indexid = cache->index->indexid; + hdr.used_file_size = uint32_to_nbo(sizeof(hdr)); + cache->used_file_size = sizeof(hdr); + + if (cache->anon_mmap) { + *cache->header = hdr; + return TRUE; + } + + ret = mail_cache_open_and_verify(cache, FALSE); + if (ret != 0) + return ret > 0; + + fd = file_dotlock_open(cache->filepath, NULL, MAIL_CACHE_LOCK_TIMEOUT, + MAIL_CACHE_LOCK_STALE_TIMEOUT, NULL, NULL); + if (fd == -1) { + mail_cache_set_syscall_error(cache, "file_dotlock_open()"); + return FALSE; + } + + if (write_full(fd, &hdr, sizeof(hdr)) < 0) { + mail_cache_set_syscall_error(cache, "write_full()"); + (void)file_dotlock_delete(cache->filepath, fd); + return FALSE; + } + if (file_set_size(fd, MAIL_CACHE_INITIAL_SIZE) < 0) { + mail_cache_set_syscall_error(cache, "file_set_size()"); + (void)file_dotlock_delete(cache->filepath, fd); + return FALSE; + } + + mail_cache_file_close(cache); + cache->fd = dup(fd); + + if (file_dotlock_replace(cache->filepath, fd, FALSE) < 0) { + mail_cache_set_syscall_error(cache, "file_dotlock_replace()"); + return FALSE; + } + + cache->mmap_refresh = TRUE; + if (!mmap_update(cache, 0, sizeof(struct mail_cache_header))) + return FALSE; + + return TRUE; +} + +int mail_cache_mark_file_deleted(struct mail_cache *cache) +{ + uint32_t indexid = 0; + + if (cache->anon_mmap) + cache->header->indexid = 0; + else { + if (pwrite(cache->fd, &indexid, sizeof(indexid), 0) < 0) + return mail_cache_set_syscall_error(cache, "pwrite()"); + } + return TRUE; +} + +int mail_cache_lock(struct mail_cache *cache, int nonblock) +{ + int ret; + + if (cache->locks++ != 0) + return TRUE; + + if (cache->anon_mmap) + return TRUE; + + if (nonblock) { + ret = file_try_lock(cache->fd, F_WRLCK); + if (ret < 0) + mail_cache_set_syscall_error(cache, "file_try_lock()"); + } else { + ret = file_wait_lock(cache->fd, F_WRLCK); + if (ret <= 0) + mail_cache_set_syscall_error(cache, "file_wait_lock()"); + } + + if (ret > 0) { + if (!mmap_verify_header(cache)) { + (void)mail_cache_unlock(cache); + ret = -1; + } + } + return ret; +} + +int mail_cache_unlock(struct mail_cache *cache) +{ + if (--cache->locks > 0) + return TRUE; + + if (cache->anon_mmap) + return TRUE; + + if (file_wait_lock(cache->fd, F_UNLCK) <= 0) { + mail_cache_set_syscall_error(cache, "file_wait_lock(F_UNLCK)"); + return FALSE; + } + + return TRUE; +} + +void mail_cache_unlock_later(struct mail_cache *cache) +{ + cache->index->cache_later_locks++; +} + +int mail_cache_is_locked(struct mail_cache *cache) +{ + return cache->locks > 0; +} + +int mail_cache_transaction_begin(struct mail_cache *cache, int nonblock, + struct mail_cache_transaction_ctx **ctx_r) +{ + int ret; + + i_assert(cache->trans_ctx == NULL); + + ret = mail_cache_lock(cache, nonblock); + if (ret <= 0) + return ret; + + *ctx_r = i_new(struct mail_cache_transaction_ctx, 1); + (*ctx_r)->cache = cache; + (*ctx_r)->cache_data = + buffer_create_dynamic(system_pool, 8192, (size_t)-1); + (*ctx_r)->last_idx = (unsigned int)-1; + + cache->trans_ctx = *ctx_r; + return 1; +} + +int mail_cache_transaction_end(struct mail_cache_transaction_ctx *ctx) +{ + int ret = TRUE; + + i_assert(ctx->cache->trans_ctx != NULL); + + (void)mail_cache_transaction_rollback(ctx); + + if (!mail_cache_unlock(ctx->cache)) + ret = FALSE; + + ctx->cache->trans_ctx = NULL; + + if (ctx->cache_marks != NULL) + buffer_free(ctx->cache_marks); + if (ctx->index_marks != NULL) + buffer_free(ctx->index_marks); + buffer_free(ctx->cache_data); + i_free(ctx); + return ret; +} + +static void mail_cache_transaction_flush(struct mail_cache_transaction_ctx *ctx) +{ + memset(&ctx->cache_rec, 0, sizeof(ctx->cache_rec)); + ctx->last_idx = (unsigned int)-1; + + ctx->next_unused_header_lowwater = 0; + ctx->first_uid = ctx->last_uid = 0; + + if (ctx->cache_marks != NULL) + buffer_set_used_size(ctx->cache_marks, 0); + if (ctx->index_marks != NULL) + buffer_set_used_size(ctx->index_marks, 0); + buffer_set_used_size(ctx->cache_data, 0); +} + +static void mark_update(buffer_t **buf, uint32_t offset, uint32_t data) +{ + unsigned char lower_data; + + if (*buf == NULL) + *buf = buffer_create_dynamic(system_pool, 1024, (size_t)-1); + + /* data is in big endian, we want to update only the lowest byte */ + offset += sizeof(uint32_t) - 1; + buffer_append(*buf, &offset, sizeof(offset)); + + lower_data = data & 0xff; + buffer_append(*buf, &lower_data, 1); +} + +static int write_mark_updates(struct mail_index *index, buffer_t *marks, + const char *path, int fd) +{ + const unsigned char *data, *end; + uint32_t offset; + size_t size; + + data = buffer_get_data(marks, &size); + end = data + size; + + while (data < end) { + memcpy(&offset, data, sizeof(offset)); + data += sizeof(offset); + + if (pwrite(fd, data, sizeof(*data), offset) < 0) { + index_file_set_syscall_error(index, path, "pwrite()"); + return FALSE; + } + data++; + } + return TRUE; +} + +static void write_mark_updates_in_memory(buffer_t *marks, void *mmap_base, + size_t mmap_length) +{ + const unsigned char *data, *end; + uint32_t offset; + size_t size; + + data = buffer_get_data(marks, &size); + end = data + size; + + while (data < end) { + memcpy(&offset, data, sizeof(offset)); + data += sizeof(offset); + + i_assert(offset < mmap_length); + ((char *) mmap_base)[offset] = *data; + data++; + } +} + +static void commit_all_changes_in_memory(struct mail_cache_transaction_ctx *ctx) +{ + struct mail_cache *cache = ctx->cache; + + if (ctx->cache_marks != NULL) { + write_mark_updates_in_memory(ctx->cache_marks, + cache->mmap_base, + cache->mmap_length); + } + if (ctx->index_marks != NULL) { + write_mark_updates_in_memory(ctx->index_marks, + cache->index->mmap_base, + cache->index->mmap_used_length); + } +} + +static int commit_all_changes(struct mail_cache_transaction_ctx *ctx) +{ + struct mail_cache *cache = ctx->cache; + uint32_t cont; + + if (ctx->cache->anon_mmap) { + commit_all_changes_in_memory(ctx); + return TRUE; + } + + /* write everything to disk */ + if (msync(cache->mmap_base, cache->mmap_length, MS_SYNC) < 0) + return mail_cache_set_syscall_error(cache, "msync()"); + + if (fdatasync(cache->fd) < 0) + return mail_cache_set_syscall_error(cache, "fdatasync()"); + + if (ctx->cache_marks != NULL && + buffer_get_used_size(ctx->cache_marks) != 0) { + /* now that we're sure it's there, set on all the used-bits */ + if (!write_mark_updates(cache->index, ctx->cache_marks, + cache->filepath, cache->fd)) + return FALSE; + + /* update continued records count */ + cont = nbo_to_uint32(cache->header->continued_record_count); + + cont += buffer_get_used_size(ctx->cache_marks) / + (sizeof(uint32_t)+1); + + if (cont * 100 / cache->index->header->messages_count >= + COMPRESS_CONTINUED_PERCENTAGE && + cache->used_file_size >= COMPRESS_MIN_SIZE) { + /* too many continued rows, compress */ + cache->index->set_flags |= + MAIL_INDEX_HDR_FLAG_COMPRESS_CACHE; + } + + cache->header->continued_record_count = uint32_to_nbo(cont); + } + + /* write index last */ + if (ctx->index_marks != NULL && + buffer_get_used_size(ctx->index_marks) != 0) { + if (!mail_index_fmdatasync(cache->index, + cache->index->mmap_used_length)) + return FALSE; + + if (!write_mark_updates(cache->index, ctx->index_marks, + cache->index->filepath, + cache->index->fd)) + return FALSE; + } + return TRUE; +} + +int mail_cache_transaction_commit(struct mail_cache_transaction_ctx *ctx) +{ + int ret = TRUE; + + if (ctx->last_idx != (unsigned int)-1) { + if (!mail_cache_write(ctx)) + return FALSE; + } + + ctx->cache->header->used_file_size = + uint32_to_nbo(ctx->cache->used_file_size); + + if (!commit_all_changes(ctx)) + ret = FALSE; + + mail_cache_transaction_flush(ctx); + return ret; +} + +int mail_cache_transaction_rollback(struct mail_cache_transaction_ctx *ctx) +{ + /* no need to actually modify the file - we just didn't update + used_file_size */ + ctx->cache->used_file_size = + nbo_to_uint32(ctx->cache->header->used_file_size); + + mail_cache_transaction_flush(ctx); + return TRUE; +} + +static int mail_cache_grow(struct mail_cache *cache, uint32_t size) +{ + struct stat st; + void *base; + uoff_t grow_size, new_fsize; + + new_fsize = cache->used_file_size + size; + grow_size = new_fsize / 100 * MAIL_CACHE_GROW_PERCENTAGE; + if (grow_size < 16384) + grow_size = 16384; + + new_fsize += grow_size; + new_fsize &= ~1023; + + if (cache->anon_mmap) { + i_assert(new_fsize < SSIZE_T_MAX); + + base = mremap_anon(cache->mmap_base, cache->mmap_length, + (size_t)new_fsize, MREMAP_MAYMOVE); + if (base == MAP_FAILED) { + mail_cache_set_syscall_error(cache, "mremap_anon()"); + return FALSE; + } + + cache->mmap_base = base; + cache->mmap_length = (size_t)new_fsize; + cache->header = cache->mmap_base; + return TRUE; + } + + if (fstat(cache->fd, &st) < 0) + return mail_cache_set_syscall_error(cache, "fstat()"); + + if (cache->used_file_size + size <= (uoff_t)st.st_size) { + /* no need to grow, just update mmap */ + if (!mmap_update(cache, 0, 0)) + return FALSE; + + i_assert(cache->mmap_length >= (uoff_t)st.st_size); + return TRUE; + } + + if (st.st_size < (off_t)sizeof(struct mail_cache_header)) + return mail_cache_set_corrupted(cache, "Header is missing"); + + if (file_set_size(cache->fd, (off_t)new_fsize) < 0) + return mail_cache_set_syscall_error(cache, "file_set_size()"); + + return mmap_update(cache, 0, 0); +} + +static uint32_t mail_cache_append_space(struct mail_cache_transaction_ctx *ctx, + uint32_t size) +{ + /* NOTE: must be done within transaction or rollback would break it */ + uint32_t offset; + + i_assert((size % sizeof(uint32_t)) == 0); + + offset = ctx->cache->used_file_size; + if (offset + size > ctx->cache->mmap_length) { + if (!mail_cache_grow(ctx->cache, size)) + return 0; + } + + ctx->cache->used_file_size += size; + return offset; +} + +static const char * +mail_cache_get_header_fields_str(struct mail_cache *cache, unsigned int idx) +{ + uint32_t offset, data_size; + unsigned char *buf; + + offset = nbo_to_uint32(cache->header->header_offsets[idx]); + + if ((offset & 2) == 0 && + (cache->trans_ctx == NULL || + cache->trans_ctx->next_unused_header_lowwater <= idx)) + return NULL; + offset &= ~2; + + if (!mmap_update(cache, offset, 1024)) + return NULL; + + if (offset + sizeof(data_size) > cache->mmap_length) { + mail_cache_set_corrupted(cache, "Header %u points outside file", + idx); + return NULL; + } + + buf = cache->mmap_base; + memcpy(&data_size, buf + offset, sizeof(data_size)); + data_size = nbo_to_uint32(data_size); + offset += sizeof(data_size); + + if (data_size == 0) { + mail_cache_set_corrupted(cache, + "Header %u points to empty string", idx); + return NULL; + } + + if (!mmap_update(cache, offset, data_size)) + return NULL; + + if (offset + data_size > cache->mmap_length) { + mail_cache_set_corrupted(cache, "Header %u points outside file", + idx); + return NULL; + } + + buf = cache->mmap_base; + if (buf[offset + data_size - 1] != '\0') { + mail_cache_set_corrupted(cache, + "Header %u points to invalid string", idx); + return NULL; + } + + return buf + offset; +} + +static const char *const * +split_header(struct mail_cache *cache, const char *header) +{ + const char *const *arr, *const *tmp; + const char *null = NULL; + char *str; + buffer_t *buf; + + if (header == NULL) + return NULL; + + arr = t_strsplit(header, "\n"); + buf = buffer_create_dynamic(cache->split_header_pool, 32, (size_t)-1); + for (tmp = arr; *tmp != NULL; tmp++) { + str = p_strdup(cache->split_header_pool, *tmp); + buffer_append(buf, &str, sizeof(str)); + } + buffer_append(buf, &null, sizeof(null)); + + return buffer_get_data(buf, NULL); +} + +const char *const *mail_cache_get_header_fields(struct mail_cache *cache, + unsigned int idx) +{ + const char *str; + int i; + + i_assert(idx < MAIL_CACHE_HEADERS_COUNT); + + /* t_strsplit() is a bit slow, so we cache it */ + if (cache->header->header_offsets[idx] != cache->split_offsets[idx]) { + p_clear(cache->split_header_pool); + + t_push(); + for (i = 0; i < MAIL_CACHE_HEADERS_COUNT; i++) { + cache->split_offsets[i] = + cache->header->header_offsets[i]; + + str = mail_cache_get_header_fields_str(cache, idx); + cache->split_headers[i] = split_header(cache, str); + } + t_pop(); + } + + return cache->split_headers[idx]; +} + +int mail_cache_set_header_fields(struct mail_cache_transaction_ctx *ctx, + unsigned int idx, const char *const headers[]) +{ + struct mail_cache *cache = ctx->cache; + uint32_t offset, update_offset, size; + buffer_t *buffer; + + i_assert(idx < MAIL_CACHE_HEADERS_COUNT); + i_assert(idx >= ctx->next_unused_header_lowwater); + i_assert((nbo_to_uint32(cache->header->header_offsets[idx]) & 2) == 0); + + t_push(); + + buffer = buffer_create_dynamic(data_stack_pool, 512, (size_t)-1); + while (*headers != NULL) { + if (buffer_get_used_size(buffer) != 0) + buffer_append(buffer, "\n", 1); + buffer_append(buffer, *headers, strlen(*headers)); + headers++; + } + buffer_append(buffer, null4, 1); + + size = buffer_get_used_size(buffer); + if ((size & 3) != 0) { + buffer_append(buffer, null4, 4 - (size & 3)); + size += 4 - (size & 3); + } + + offset = mail_cache_append_space(ctx, size + sizeof(uint32_t)); + if (offset != 0) { + memcpy((char *) cache->mmap_base + offset + sizeof(uint32_t), + buffer_get_data(buffer, NULL), size); + + size = uint32_to_nbo(size); + memcpy((char *) cache->mmap_base + offset, + &size, sizeof(uint32_t)); + + cache->header->header_offsets[idx] = uint32_to_nbo(offset); + + /* mark used-bit to be updated later. not really needed for + read-safety, but if transaction get rolled back we can't let + this point to invalid location. */ + update_offset = (char *) &cache->header->header_offsets[idx] - + (char *) cache->mmap_base; + mark_update(&ctx->cache_marks, update_offset, offset | 2); + + /* make sure get_header_fields() still works for this header + while the transaction isn't yet committed. */ + ctx->next_unused_header_lowwater = idx + 1; + } + + t_pop(); + return offset > 0; +} + +static struct mail_cache_record * +cache_get_record(struct mail_cache *cache, uint32_t offset) +{ + struct mail_cache_record *cache_rec; + size_t size; + + offset = nbo_to_uint32(offset); + + if ((offset & 1) != 0) { + mail_cache_set_corrupted(cache, "bit 0 set in data offset"); + return NULL; + } + if ((offset & 2) == 0) + return NULL; + offset &= ~2; + + if (!mmap_update(cache, offset, sizeof(*cache_rec) + 1024)) + return NULL; + + if (offset + sizeof(*cache_rec) > cache->mmap_length) { + mail_cache_set_corrupted(cache, "record points outside file"); + return NULL; + } + cache_rec = CACHE_RECORD(cache, offset); + + size = nbo_to_uint32(cache_rec->size); + if (!mmap_update(cache, offset, sizeof(*cache_rec) + size)) + return NULL; + + if (offset + sizeof(*cache_rec) + size > cache->mmap_length) { + mail_cache_set_corrupted(cache, "record points outside file"); + return NULL; + } + return cache_rec; +} + +static struct mail_cache_record * +cache_get_next_record(struct mail_cache *cache, struct mail_cache_record *rec) +{ + struct mail_cache_record *next; + + next = cache_get_record(cache, rec->next_offset); + if (next != NULL && next <= rec) { + mail_cache_set_corrupted(cache, "next_offset points backwards"); + return NULL; + } + return next; +} + +static int mail_cache_write(struct mail_cache_transaction_ctx *ctx) +{ + struct mail_cache *cache = ctx->cache; + struct mail_cache_record *cache_rec, *next; + struct mail_index_record *rec; + uint32_t write_offset, update_offset; + const void *buf; + size_t size, buf_size; + + buf = buffer_get_data(ctx->cache_data, &buf_size); + + size = sizeof(*cache_rec) + buf_size; + ctx->cache_rec.size = uint32_to_nbo(size); + + write_offset = mail_cache_append_space(ctx, size); + if (write_offset == 0) + return FALSE; + + rec = INDEX_RECORD_AT(ctx->cache->index, ctx->last_idx); + ctx->last_idx = (unsigned int)-1; + + cache_rec = cache_get_record(cache, rec->data_offset); + if (cache_rec == NULL) { + /* first cache record - update offset in index file */ + i_assert(cache->index->lock_type == MAIL_LOCK_EXCLUSIVE); + + rec->data_offset = uint32_to_nbo(write_offset); + + /* mark used-bit to be updated later */ + update_offset = (char *) &rec->data_offset - + (char *) cache->index->mmap_base; + mark_update(&ctx->index_marks, update_offset, write_offset | 2); + } else { + /* find the last cache record */ + while ((next = cache_get_next_record(cache, cache_rec)) != NULL) + cache_rec = next; + + /* set our offset, keep the used-bit still unset */ + cache_rec->next_offset = uint32_to_nbo(write_offset); + + /* mark used-bit to be updated later */ + update_offset = (char *) &cache_rec->next_offset - + (char *) cache->mmap_base; + mark_update(&ctx->cache_marks, update_offset, write_offset | 2); + } + + memcpy((char *) cache->mmap_base + write_offset, + &ctx->cache_rec, sizeof(ctx->cache_rec)); + memcpy((char *) cache->mmap_base + write_offset + + sizeof(ctx->cache_rec), buf, buf_size); + + /* reset the write context */ + memset(&ctx->cache_rec, 0, sizeof(ctx->cache_rec)); + buffer_set_used_size(ctx->cache_data, 0); + return TRUE; +} + +static struct mail_cache_record * +mail_cache_lookup(struct mail_cache *cache, const struct mail_index_record *rec) +{ + struct mail_cache_record *cache_rec; + unsigned int idx; + + if (cache->trans_ctx != NULL && + cache->trans_ctx->first_uid <= rec->uid && + cache->trans_ctx->last_uid >= rec->uid) { + /* we have to auto-commit since we're not capable of looking + into uncommitted records. it would be possible by checking + index_marks and cache_marks, but it's just more trouble + than worth. */ + idx = INDEX_RECORD_INDEX(cache->index, rec); + if (cache->trans_ctx->last_idx == idx) { + if (!mail_cache_write(cache->trans_ctx)) + return NULL; + } + + if (!mail_cache_transaction_commit(cache->trans_ctx)) + return NULL; + } + + cache_rec = cache_get_record(cache, rec->data_offset); + if (cache_rec == NULL) + return NULL; + + if (cache_rec->fields == 0) { + mail_cache_set_corrupted(cache, "record has no fields"); + return NULL; + } + return cache_rec; +} + +static int get_field_num(enum mail_cache_field field) +{ + int i; + + for (i = 0; i < 32; i++) { + if ((field & nbo32_bitmasks[i]) != 0) + return i; + } + + return -1; +} + +static size_t get_insert_offset(struct mail_cache_transaction_ctx *ctx, + enum mail_cache_field field) +{ + const unsigned char *buf; + unsigned int mask; + uint32_t data_size; + size_t offset = 0; + int i; + + buf = buffer_get_data(ctx->cache_data, NULL); + + for (i = 0; i < 32; i++) { + mask = nbo32_bitmasks[i]; + if ((field & mask) != 0) + return offset; + + if ((ctx->cache_rec.fields & mask) != 0) { + if ((mask & MAIL_CACHE_FIXED_MASK) != 0) + data_size = mail_cache_field_sizes[i]; + else { + memcpy(&data_size, buf + offset, + sizeof(data_size)); + data_size = nbo_to_uint32(data_size); + offset += sizeof(data_size); + } + offset += (data_size + 3) & ~3; + } + } + + i_unreached(); + return offset; +} + +int mail_cache_add(struct mail_cache_transaction_ctx *ctx, + struct mail_index_record *rec, enum mail_cache_field field, + const void *data, size_t data_size) +{ + uint32_t nb_data_size; + size_t full_size, offset; + unsigned char *buf; + unsigned int idx; + int field_num; + + i_assert(data_size > 0); + i_assert(data_size < (uint32_t)-1); + + nb_data_size = uint32_to_nbo((uint32_t)data_size); + + if ((field & MAIL_CACHE_FIXED_MASK) != 0) { + field_num = get_field_num(field); + i_assert(field_num != -1); + i_assert(mail_cache_field_sizes[field_num] == data_size); + } else if ((field & MAIL_CACHE_STRING_MASK) != 0) { + i_assert(((char *) data)[data_size-1] == '\0'); + } + + /* NOTE: we use index because the record pointer might not last. */ + idx = INDEX_RECORD_INDEX(ctx->cache->index, rec); + if (ctx->last_idx != idx && ctx->last_idx != (unsigned int)-1) { + if (!mail_cache_write(ctx)) + return FALSE; + } + ctx->last_idx = idx; + + i_assert((ctx->cache_rec.fields & field) == 0); + + full_size = (data_size + 3) & ~3; + if ((field & MAIL_CACHE_FIXED_MASK) == 0) + full_size += sizeof(nb_data_size); + + /* fields must be ordered. find where to insert it. */ + if (field > ctx->cache_rec.fields) + buf = buffer_append_space_unsafe(ctx->cache_data, full_size); + else { + offset = get_insert_offset(ctx, field); + buffer_copy(ctx->cache_data, offset + full_size, + ctx->cache_data, offset, (size_t)-1); + buf = buffer_get_space_unsafe(ctx->cache_data, + offset, full_size); + } + ctx->cache_rec.fields |= field; + + /* @UNSAFE */ + if ((field & MAIL_CACHE_FIXED_MASK) == 0) { + memcpy(buf, &nb_data_size, sizeof(nb_data_size)); + buf += sizeof(nb_data_size); + } + memcpy(buf, data, data_size); buf += data_size; + if ((data_size & 3) != 0) + memset(buf, 0, 4 - (data_size & 3)); + + /* remember the transaction uid range */ + if (rec->uid < ctx->first_uid || ctx->first_uid == 0) + ctx->first_uid = rec->uid; + if (rec->uid > ctx->last_uid) + ctx->last_uid = rec->uid; + + return TRUE; +} + +int mail_cache_delete(struct mail_cache_transaction_ctx *ctx, + struct mail_index_record *rec) +{ + struct mail_cache *cache = ctx->cache; + struct mail_cache_record *cache_rec; + uint32_t deleted_space; + uoff_t max_del_space; + + cache_rec = mail_cache_lookup(cache, rec); + if (cache_rec == NULL) + return TRUE; + + /* NOTE: it would be nice to erase the cached data for the record, + but some other processes might still be using them. So, we just + update the deleted_space in header */ + deleted_space = nbo_to_uint32(cache->header->deleted_space); + + do { + deleted_space -= nbo_to_uint32(cache_rec->size); + cache_rec = cache_get_next_record(cache, cache_rec); + } while (cache_rec != NULL); + + /* see if we've reached the max. deleted space in file */ + max_del_space = cache->used_file_size / 100 * COMPRESS_PERCENTAGE; + if (deleted_space >= max_del_space && + cache->used_file_size >= COMPRESS_MIN_SIZE) + cache->index->set_flags |= MAIL_INDEX_HDR_FLAG_COMPRESS_CACHE; + + cache->header->deleted_space = uint32_to_nbo(deleted_space); + + return TRUE; +} + +enum mail_cache_field +mail_cache_get_fields(struct mail_cache *cache, + const struct mail_index_record *rec) +{ + struct mail_cache_record *cache_rec; + enum mail_cache_field fields = 0; + + cache_rec = mail_cache_lookup(cache, rec); + while (cache_rec != NULL) { + fields |= cache_rec->fields; + cache_rec = cache_get_next_record(cache, cache_rec); + } + + return fields; +} + +static int cache_get_field(struct mail_cache *cache, + struct mail_cache_record *cache_rec, + enum mail_cache_field field, + void **data_r, size_t *size_r) +{ + unsigned char *buf; + unsigned int mask; + uint32_t rec_size, data_size; + size_t offset = 0; + int i; + + rec_size = nbo_to_uint32(cache_rec->size); + buf = (unsigned char *) cache_rec + sizeof(*cache_rec); + + for (i = 0; i < 32; i++) { + mask = nbo32_bitmasks[i]; + if ((cache_rec->fields & mask) == 0) + continue; + + if ((mask & MAIL_CACHE_FIXED_MASK) != 0) + data_size = mail_cache_field_sizes[i]; + else { + memcpy(&data_size, buf + offset, sizeof(data_size)); + data_size = nbo_to_uint32(data_size); + offset += sizeof(data_size); + } + + if (field == mask) { + if (data_size == 0) { + mail_cache_set_corrupted(cache, + "Field size is 0"); + return FALSE; + } + *data_r = buf + offset; + *size_r = data_size; + return TRUE; + } + + offset += (data_size + 3) & ~3; + + if (offset >= rec_size) { + mail_cache_set_corrupted(cache, + "Record continues outside it's allocated size"); + return FALSE; + } + } + + i_unreached(); + return FALSE; +} + +static int cache_lookup_field(struct mail_cache *cache, + const struct mail_index_record *rec, + enum mail_cache_field field, + void **data_r, size_t *size_r) +{ + struct mail_cache_record *cache_rec; + + cache_rec = mail_cache_lookup(cache, rec); + while (cache_rec != NULL) { + if ((cache_rec->fields & field) != 0) { + return cache_get_field(cache, cache_rec, field, + data_r, size_r); + } + cache_rec = cache_get_next_record(cache, cache_rec); + } + + return FALSE; +} + +int mail_cache_lookup_field(struct mail_cache *cache, + const struct mail_index_record *rec, + enum mail_cache_field field, + const void **data_r, size_t *size_r) +{ + void *data; + + if (!cache_lookup_field(cache, rec, field, &data, size_r)) + return FALSE; + + *data_r = data; + return TRUE; +} + +const char *mail_cache_lookup_string_field(struct mail_cache *cache, + const struct mail_index_record *rec, + enum mail_cache_field field) +{ + const void *data; + size_t size; + + i_assert((field & MAIL_CACHE_STRING_MASK) != 0); + + if (!mail_cache_lookup_field(cache, rec, field, &data, &size)) + return NULL; + + if (((const char *) data)[size-1] != '\0') { + mail_cache_set_corrupted(cache, + "String field %x doesn't end with NUL", field); + return NULL; + } + return data; +} + +int mail_cache_copy_fixed_field(struct mail_cache *cache, + const struct mail_index_record *rec, + enum mail_cache_field field, + void *buffer, size_t buffer_size) +{ + const void *data; + size_t size; + + i_assert((field & MAIL_CACHE_FIXED_MASK) != 0); + + if (!mail_cache_lookup_field(cache, rec, field, &data, &size)) + return FALSE; + + if (buffer_size != size) { + i_panic("cache: fixed field %x wrong size " + "(%"PRIuSIZE_T" vs %"PRIuSIZE_T")", + field, size, buffer_size); + } + + memcpy(buffer, data, buffer_size); + return TRUE; +} + +void mail_cache_mark_missing(struct mail_cache *cache, + enum mail_cache_field fields) +{ + // FIXME: count these +} + +enum mail_index_record_flag +mail_cache_get_index_flags(struct mail_cache *cache, + const struct mail_index_record *rec) +{ + enum mail_index_record_flag flags; + + if (!mail_cache_copy_fixed_field(cache, rec, MAIL_CACHE_INDEX_FLAGS, + &flags, sizeof(flags))) + return 0; + + return flags; +} + +int mail_cache_update_index_flags(struct mail_cache *cache, + struct mail_index_record *rec, + enum mail_index_record_flag flags) +{ + void *data; + size_t size; + + i_assert(cache->locks > 0); + + if (!cache_lookup_field(cache, rec, MAIL_CACHE_INDEX_FLAGS, + &data, &size)) { + mail_cache_set_corrupted(cache, + "Missing index flags for record %u", rec->uid); + return FALSE; + } + + memcpy(data, &flags, sizeof(flags)); + return TRUE; +} + +void *mail_cache_get_mmaped(struct mail_cache *cache, size_t *size) +{ + if (!mmap_update(cache, 0, 0)) + return NULL; + + *size = cache->mmap_length; + return cache->mmap_base; +} + +int mail_cache_set_corrupted(struct mail_cache *cache, const char *fmt, ...) +{ + va_list va; + + /* rebuild */ + INDEX_MARK_CORRUPTED(cache->index); + mail_cache_mark_file_deleted(cache); + + va_start(va, fmt); + t_push(); + index_set_error(cache->index, "Corrupted index cache file %s: %s", + cache->filepath, t_strdup_vprintf(fmt, va)); + t_pop(); + va_end(va); + + return FALSE; +}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-index/mail-cache.h Wed Aug 06 23:15:30 2003 +0300 @@ -0,0 +1,172 @@ +#ifndef __MAIL_CACHE_H +#define __MAIL_CACHE_H + +#include "byteorder.h" +#include "mail-index.h" + +#define MAIL_CACHE_FILE_PREFIX ".cache" + +#define MAIL_CACHE_HEADERS_COUNT 4 + +struct mail_cache_transaction_ctx; + +enum mail_cache_field { + /* fixed size fields */ + MAIL_CACHE_INDEX_FLAGS = NBO32_BIT0, + MAIL_CACHE_LOCATION_OFFSET = NBO32_BIT1, + MAIL_CACHE_MD5 = NBO32_BIT2, + MAIL_CACHE_SENT_DATE = NBO32_BIT3, + MAIL_CACHE_RECEIVED_DATE = NBO32_BIT4, + MAIL_CACHE_HEADER_SIZE = NBO32_BIT5, + MAIL_CACHE_BODY_SIZE = NBO32_BIT6, + MAIL_CACHE_VIRTUAL_HEADER_SIZE = NBO32_BIT7, + MAIL_CACHE_VIRTUAL_FULL_SIZE = NBO32_BIT8, + + /* variable sized field */ + MAIL_CACHE_HEADERS1 = NBO32_BIT31, + MAIL_CACHE_HEADERS2 = NBO32_BIT30, + MAIL_CACHE_HEADERS3 = NBO32_BIT29, + MAIL_CACHE_HEADERS4 = NBO32_BIT28, + MAIL_CACHE_LOCATION = NBO32_BIT27, + MAIL_CACHE_BODY = NBO32_BIT26, + MAIL_CACHE_BODYSTRUCTURE = NBO32_BIT25, + MAIL_CACHE_MESSAGEPART = NBO32_BIT24, + + MAIL_CACHE_FIXED_MASK = MAIL_CACHE_INDEX_FLAGS | + MAIL_CACHE_LOCATION_OFFSET | + MAIL_CACHE_MD5 | + MAIL_CACHE_SENT_DATE | + MAIL_CACHE_RECEIVED_DATE | + MAIL_CACHE_HEADER_SIZE | + MAIL_CACHE_BODY_SIZE | + MAIL_CACHE_VIRTUAL_HEADER_SIZE | + MAIL_CACHE_VIRTUAL_FULL_SIZE, + MAIL_CACHE_HEADERS_MASK = MAIL_CACHE_HEADERS1 | + MAIL_CACHE_HEADERS2 | + MAIL_CACHE_HEADERS4 | + MAIL_CACHE_HEADERS4, + MAIL_CACHE_STRING_MASK = MAIL_CACHE_HEADERS_MASK | + MAIL_CACHE_LOCATION | + MAIL_CACHE_BODY | + MAIL_CACHE_BODYSTRUCTURE, + MAIL_CACHE_BODYSTRUCTURE_MASK = MAIL_CACHE_BODY | + MAIL_CACHE_BODYSTRUCTURE | + MAIL_CACHE_MESSAGEPART +}; + +struct mail_sent_date { + time_t time; + int timezone; +}; + +extern enum mail_cache_field mail_cache_header_fields[MAIL_CACHE_HEADERS_COUNT]; + +int mail_cache_open_or_create(struct mail_index *index); +void mail_cache_free(struct mail_cache *cache); + +void mail_cache_set_defaults(struct mail_cache *cache, + enum mail_cache_field default_cache_fields, + enum mail_cache_field never_cache_fields); + +/* Compress cache file. */ +int mail_cache_compress(struct mail_cache *cache); + +/* Truncate the cache file and update it's indexid */ +int mail_cache_truncate(struct mail_cache *cache); + +/* Set indexid to 0 to notify other processes using this file that they should + re-open it. */ +int mail_cache_mark_file_deleted(struct mail_cache *cache); + +/* Explicitly lock the cache file. Returns 1 if ok, 0 if nonblock is TRUE and + we couldn't immediately get a lock, or -1 if error. */ +int mail_cache_lock(struct mail_cache *cache, int nonblock); +int mail_cache_unlock(struct mail_cache *cache); + +/* Mark the lock to be removed when unlocking index file. */ +void mail_cache_unlock_later(struct mail_cache *cache); + +/* Returns TRUE if cache file is locked. */ +int mail_cache_is_locked(struct mail_cache *cache); + +/* Begin transaction. Returns same as mail_cache_lock(). Note that if you + call lookup functions for messages within first and last message in + transaction, the transaction will be automatically committed. */ +int mail_cache_transaction_begin(struct mail_cache *cache, int nonblock, + struct mail_cache_transaction_ctx **ctx_r); +/* End transaction. Single transaction can have multiple commits/rollbacks. + If there's any pending changes, they will be rolled back. */ +int mail_cache_transaction_end(struct mail_cache_transaction_ctx *ctx); + +int mail_cache_transaction_commit(struct mail_cache_transaction_ctx *ctx); +int mail_cache_transaction_rollback(struct mail_cache_transaction_ctx *ctx); + +/* Return NULL-terminated list of headers for given index, or NULL if + header index isn't used. */ +const char *const *mail_cache_get_header_fields(struct mail_cache *cache, + unsigned int idx); +/* Set list of headers for given index. */ +int mail_cache_set_header_fields(struct mail_cache_transaction_ctx *ctx, + unsigned int idx, const char *const headers[]); + +/* Add new field to given record. Updates are not allowed. Fixed size fields + must be exactly the expected size and they're converted to network byte + order in disk. */ +int mail_cache_add(struct mail_cache_transaction_ctx *ctx, + struct mail_index_record *rec, enum mail_cache_field field, + const void *data, size_t data_size); + +/* Mark the given record deleted. */ +int mail_cache_delete(struct mail_cache_transaction_ctx *ctx, + struct mail_index_record *rec); + +/* Return all fields that are currently cached for record. */ +enum mail_cache_field +mail_cache_get_fields(struct mail_cache *cache, + const struct mail_index_record *rec); + +/* Set data_r and size_r to point to wanted field in cache file. + Returns TRUE if field was found. If field contains multiple fields, + first one found is returned. This is mostly useful for finding headers. */ +int mail_cache_lookup_field(struct mail_cache *cache, + const struct mail_index_record *rec, + enum mail_cache_field field, + const void **data_r, size_t *size_r); + +/* Return string field. */ +const char *mail_cache_lookup_string_field(struct mail_cache *cache, + const struct mail_index_record *rec, + enum mail_cache_field field); + + +/* Copy fixed size field to given buffer. buffer_size must be exactly the + expected size. The result will be converted to host byte order. + Returns TRUE if field was found. */ +int mail_cache_copy_fixed_field(struct mail_cache *cache, + const struct mail_index_record *rec, + enum mail_cache_field field, + void *buffer, size_t buffer_size); + +/* Mark given fields as missing, ie. they should be cached when possible. */ +void mail_cache_mark_missing(struct mail_cache *cache, + enum mail_cache_field fields); + +/* Return index flags. */ +enum mail_index_record_flag +mail_cache_get_index_flags(struct mail_cache *cache, + const struct mail_index_record *rec); + +/* Update index flags. The cache file must be locked and the flags must be + already inserted to the record. */ +int mail_cache_update_index_flags(struct mail_cache *cache, + struct mail_index_record *rec, + enum mail_index_record_flag flags); + +/* Return the whole file mmaped. */ +void *mail_cache_get_mmaped(struct mail_cache *cache, size_t *size); + +/* "Error in index cache file %s: ...". */ +int mail_cache_set_corrupted(struct mail_cache *cache, const char *fmt, ...) + __attr_format__(2, 3); + +#endif
--- a/src/lib-index/mail-custom-flags.c Wed Aug 06 23:13:57 2003 +0300 +++ b/src/lib-index/mail-custom-flags.c Wed Aug 06 23:15:30 2003 +0300 @@ -534,7 +534,7 @@ if (!custom_flags_add(mcf, first_empty, flag)) return -1; - mcf->index->set_flags |= MAIL_INDEX_FLAG_DIRTY_CUSTOMFLAGS; + mcf->index->set_flags |= MAIL_INDEX_HDR_FLAG_DIRTY_CUSTOMFLAGS; mcf->custom_flags[first_empty] = i_strdup(flag); return first_empty;
--- a/src/lib-index/mail-index-compress.c Wed Aug 06 23:13:57 2003 +0300 +++ b/src/lib-index/mail-index-compress.c Wed Aug 06 23:15:30 2003 +0300 @@ -3,8 +3,8 @@ #include "lib.h" #include "write-full.h" #include "mail-index.h" -#include "mail-index-data.h" #include "mail-index-util.h" +#include "mail-cache.h" #include <stdio.h> #include <unistd.h> @@ -15,7 +15,8 @@ i_assert(index->lock_type == MAIL_LOCK_EXCLUSIVE); - if (index->mmap_full_length <= INDEX_FILE_MIN_SIZE || index->anon_mmap) + if (index->mmap_full_length <= INDEX_FILE_MIN_SIZE(index) || + index->anon_mmap) return TRUE; /* really truncate the file only when it's almost empty */ @@ -29,11 +30,11 @@ /* keep the size record-aligned */ index->mmap_full_length -= (index->mmap_full_length - - sizeof(struct mail_index_header)) % + index->header_size) % sizeof(struct mail_index_record); - if (index->mmap_full_length < INDEX_FILE_MIN_SIZE) - index->mmap_full_length = INDEX_FILE_MIN_SIZE; + if (index->mmap_full_length < INDEX_FILE_MIN_SIZE(index)) + index->mmap_full_length = INDEX_FILE_MIN_SIZE(index); if (ftruncate(index->fd, (off_t)index->mmap_full_length) < 0) return index_set_syscall_error(index, "ftruncate()"); @@ -44,6 +45,7 @@ return TRUE; } +#if 0 static int mail_index_copy_data(struct mail_index *index, int fd, const char *path) { @@ -69,7 +71,7 @@ /* now we'll begin the actual moving. keep rebuild-flag on while doing it. */ index->header->flags |= MAIL_INDEX_FLAG_REBUILD; - if (!mail_index_fmdatasync(index, sizeof(struct mail_index_header))) + if (!mail_index_fmdatasync(index, index->header_size)) return FALSE; offset = sizeof(data_hdr); @@ -182,3 +184,4 @@ return mail_index_data_open(index); } +#endif
--- a/src/lib-index/mail-index-data.c Wed Aug 06 23:13:57 2003 +0300 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,721 +0,0 @@ -/* Copyright (C) 2002 Timo Sirainen */ - -#include "lib.h" -#include "file-set-size.h" -#include "mmap-util.h" -#include "write-full.h" -#include "mail-index.h" -#include "mail-index-data.h" -#include "mail-index-util.h" - -#include <stdio.h> -#include <fcntl.h> - -#define DATA_FILE_POSITION(data, rec) \ - ((uoff_t) ((char *) (rec) - (char *) ((data)->mmap_base))) - -/* Never compress the file if it's smaller than this */ -#define COMPRESS_MIN_SIZE (1024*50) - -/* Compress the file when deleted space reaches n% of total size */ -#define COMPRESS_PERCENTAGE 20 - -/* Initial size for the file */ -#define INDEX_DATA_INITIAL_SIZE (sizeof(struct mail_index_data_header) + 10240) - -/* When more space is needed, grow the file n% larger than the previous size */ -#define INDEX_DATA_GROW_PERCENTAGE 10 - -struct mail_index_data { - struct mail_index *index; - - int fd; - char *filepath; - - void *mmap_base; - size_t mmap_full_length; - size_t mmap_used_length; - - struct mail_index_data_header *header; - - unsigned int anon_mmap:1; - unsigned int dirty_mmap:1; - unsigned int modified:1; - unsigned int fsynced:1; -}; - -int index_data_set_corrupted(struct mail_index_data *data, const char *fmt, ...) -{ - va_list va; - - INDEX_MARK_CORRUPTED(data->index); - data->index->inconsistent = TRUE; - - va_start(va, fmt); - t_push(); - index_set_error(data->index, "Corrupted index data file %s: %s", - data->filepath, t_strdup_vprintf(fmt, va)); - t_pop(); - va_end(va); - - return FALSE; -} - -static int index_data_set_syscall_error(struct mail_index_data *data, - const char *function) -{ - i_assert(function != NULL); - - if (ENOSPACE(errno)) { - data->index->nodiskspace = TRUE; - return FALSE; - } - - index_set_error(data->index, "%s failed with index data file %s: %m", - function, data->filepath); - return FALSE; -} - -static int mail_index_data_msync(struct mail_index_data *data) -{ - if (!data->modified) - return TRUE; - - if (!data->anon_mmap) { - if (msync(data->mmap_base, data->mmap_used_length, MS_SYNC) < 0) - return index_data_set_syscall_error(data, "msync()"); - } - - data->modified = FALSE; - data->fsynced = FALSE; - return TRUE; -} - -static void mail_index_data_file_close(struct mail_index_data *data) -{ - (void)mail_index_data_msync(data); - - if (data->anon_mmap) { - if (munmap_anon(data->mmap_base, data->mmap_full_length) < 0) - index_data_set_syscall_error(data, "munmap_anon()"); - } else if (data->mmap_base != NULL) { - if (munmap(data->mmap_base, data->mmap_full_length) < 0) - index_data_set_syscall_error(data, "munmap()"); - } - data->mmap_base = NULL; - data->mmap_used_length = 0; - data->mmap_full_length = 0; - - if (data->fd != -1) { - if (close(data->fd) < 0) - index_data_set_syscall_error(data, "close()"); - data->fd = -1; - } -} - -static int data_file_reopen(struct mail_index_data *data) -{ - int fd; - - i_assert(!data->anon_mmap); - - fd = open(data->filepath, O_RDWR); - if (fd == -1) - return index_data_set_syscall_error(data, "open()"); - - mail_index_data_file_close(data); - - data->fd = fd; - return TRUE; -} - -static int mmap_update(struct mail_index_data *data, uoff_t pos, size_t size) -{ - struct mail_index_data_header *hdr; - - if (data->index->mmap_invalidate && data->mmap_base != NULL) { - if (msync(data->mmap_base, data->mmap_used_length, - MS_SYNC | MS_INVALIDATE) < 0) - return index_data_set_syscall_error(data, "msync()"); - } - - if (data->header != NULL && - data->header->indexid != data->index->indexid) { - if (data->header->indexid != 0) { - /* index was just rebuilt. we should have noticed - this before at index->set_lock() though. */ - index_set_error(data->index, - "Warning: Inconsistency - Index " - "%s was rebuilt while we had it open", - data->filepath); - data->index->inconsistent = TRUE; - return FALSE; - } - - /* data file was deleted, reopen it */ - if (!data_file_reopen(data)) - return FALSE; - - /* force mmap refresh */ - size = 0; - } - - if (size != 0) { - debug_mprotect(data->mmap_base, data->mmap_full_length, - data->index); - - if (pos + size <= data->mmap_used_length) - return TRUE; - - if (pos + size <= data->mmap_full_length) { - data->mmap_used_length = data->header->used_file_size; - if (data->mmap_used_length >= - sizeof(struct mail_index_data_header) && - data->mmap_used_length <= data->mmap_full_length) - return TRUE; - - /* file size changed, re-mmap() */ - } - } - - i_assert(!data->anon_mmap); - - if (data->mmap_base != NULL) { - if (!mail_index_data_msync(data)) - return FALSE; - - if (munmap(data->mmap_base, data->mmap_full_length) < 0) - index_data_set_syscall_error(data, "munmap()"); - } - - data->header = NULL; - data->mmap_used_length = 0; - - data->mmap_base = mmap_rw_file(data->fd, &data->mmap_full_length); - if (data->mmap_base == MAP_FAILED) { - data->mmap_base = NULL; - return index_data_set_syscall_error(data, "mmap()"); - } - - if (data->mmap_full_length < sizeof(struct mail_index_data_header)) - return index_data_set_corrupted(data, "File too small"); - - hdr = data->mmap_base; - - if (hdr->used_file_size < sizeof(struct mail_index_data_header)) { - index_data_set_corrupted(data, "used_file_size too small (" - "%"PRIuUOFF_T")", hdr->used_file_size); - return FALSE; - } - - if (hdr->used_file_size > data->mmap_full_length) { - index_data_set_corrupted(data, - "used_file_size larger than real file size " - "(%"PRIuUOFF_T" vs %"PRIuSIZE_T")", - hdr->used_file_size, data->mmap_full_length); - return FALSE; - } - - if ((hdr->used_file_size & (INDEX_ALIGN_SIZE-1)) != 0) { - index_data_set_corrupted(data, - "used_file_size not aligned (%"PRIuUOFF_T")", - hdr->used_file_size); - return FALSE; - } - - data->mmap_used_length = hdr->used_file_size; - data->header = hdr; - debug_mprotect(data->mmap_base, data->mmap_full_length, data->index); - return TRUE; -} - -int mail_index_data_open(struct mail_index *index) -{ - struct mail_index_data *data; - const char *path; - int fd; - - path = t_strconcat(index->filepath, DATA_FILE_PREFIX, NULL); - fd = open(path, O_RDWR); - if (fd == -1) { - if (errno == ENOENT) { - /* doesn't exist, rebuild the index */ - INDEX_MARK_CORRUPTED(index); - } - return index_file_set_syscall_error(index, path, "open()"); - } - - data = i_new(struct mail_index_data, 1); - data->index = index; - data->fd = fd; - data->filepath = i_strdup(path); - - index->data = data; - - if (!mmap_update(data, 0, sizeof(struct mail_index_data_header))) { - mail_index_data_free(data); - return FALSE; - } - - /* verify that this really is the data file for wanted index */ - if (data->header->indexid != index->indexid) { - INDEX_MARK_CORRUPTED(index); - index_set_error(index, "IndexID mismatch for data file %s", - path); - mail_index_data_free(data); - return FALSE; - } - - return TRUE; -} - -static int mail_index_data_init(struct mail_index *index, - struct mail_index_data_header *hdr, - const char *path, int fd) -{ - if (write_full(fd, hdr, sizeof(*hdr)) < 0) { - index_file_set_syscall_error(index, path, "write_full()"); - return FALSE; - } - - if (file_set_size(fd, INDEX_DATA_INITIAL_SIZE) < 0) { - index_file_set_syscall_error(index, path, "file_set_size()"); - return FALSE; - } - - return TRUE; -} - -int mail_index_data_create(struct mail_index *index) -{ - struct mail_index_data_header hdr; - struct mail_index_data *data; - const char *path; - int fd; - - i_assert(index->lock_type == MAIL_LOCK_EXCLUSIVE); - - memset(&hdr, 0, sizeof(struct mail_index_data_header)); - hdr.indexid = index->indexid; - hdr.used_file_size = sizeof(struct mail_index_data_header); - - /* we'll do anon-mmaping only if initially requested. if we fail - because of out of disk space, we'll just let the main index code - know it and fail. */ - if (INDEX_IS_IN_MEMORY(index)) { - fd = -1; - path = NULL; - } else { - path = t_strconcat(index->filepath, DATA_FILE_PREFIX, NULL); - fd = open(path, O_RDWR | O_CREAT, 0660); - if (fd == -1) { - index_file_set_syscall_error(index, path, "open()"); - return FALSE; - } - - if (!mail_index_data_init(index, &hdr, path, fd)) { - (void)close(fd); - return FALSE; - } - } - - data = i_new(struct mail_index_data, 1); - - if (fd == -1) { - data->mmap_full_length = INDEX_DATA_INITIAL_SIZE; - data->mmap_base = mmap_anon(data->mmap_full_length); - if (data->mmap_base == MAP_FAILED) { - i_free(data); - return index_file_set_syscall_error(index, path, - "mmap_anon()"); - } - - memcpy(data->mmap_base, &hdr, sizeof(hdr)); - data->header = data->mmap_base; - data->mmap_used_length = data->header->used_file_size; - - data->anon_mmap = TRUE; - data->filepath = - i_strdup_printf("(in-memory index data index for %s)", - index->mailbox_path); - } else { - data->filepath = i_strdup(path); - } - - data->index = index; - data->fd = fd; - - if (!mmap_update(data, 0, sizeof(struct mail_index_data_header))) { - mail_index_data_free(data); - return FALSE; - } - - index->data = data; - return TRUE; -} - -void mail_index_data_free(struct mail_index_data *data) -{ - data->index->data = NULL; - - mail_index_data_file_close(data); - - i_free(data->filepath); - i_free(data); -} - -int mail_index_data_reset(struct mail_index_data *data) -{ - struct mail_index_data_header hdr; - - memset(&hdr, 0, sizeof(struct mail_index_data_header)); - hdr.indexid = data->index->indexid; - hdr.used_file_size = sizeof(struct mail_index_data_header); - memcpy(data->mmap_base, &hdr, sizeof(hdr)); - - if (data->anon_mmap) - return TRUE; - - if (msync(data->mmap_base, sizeof(hdr), MS_SYNC) < 0) - return index_data_set_syscall_error(data, "msync()"); - - if (file_set_size(data->fd, INDEX_DATA_INITIAL_SIZE) < 0) - return index_data_set_syscall_error(data, "file_set_size()"); - - data->modified = FALSE; - data->fsynced = FALSE; - return mmap_update(data, 0, 0); -} - -int mail_index_data_mark_file_deleted(struct mail_index_data *data) -{ - if (data->anon_mmap) - return TRUE; - - data->header->indexid = 0; - if (msync(data->mmap_base, - sizeof(struct mail_index_data_header), MS_SYNC) < 0) - return index_data_set_syscall_error(data, "msync()"); - - data->fsynced = FALSE; - return TRUE; -} - -void mail_index_data_mark_modified(struct mail_index_data *data) -{ - data->modified = TRUE; -} - -static int mail_index_data_grow(struct mail_index_data *data, size_t size) -{ - void *base; - uoff_t new_fsize; - off_t pos; - - new_fsize = data->header->used_file_size + size; - new_fsize += new_fsize / 100 * INDEX_DATA_GROW_PERCENTAGE; - - if (data->anon_mmap) { - i_assert(new_fsize < SSIZE_T_MAX); - - base = mremap_anon(data->mmap_base, data->mmap_full_length, - (size_t)new_fsize, MREMAP_MAYMOVE); - if (base == MAP_FAILED) { - index_data_set_syscall_error(data, "mremap_anon()"); - return FALSE; - } - - data->mmap_base = base; - data->mmap_full_length = (size_t)new_fsize; - data->header = data->mmap_base; - return TRUE; - } - - pos = lseek(data->fd, 0, SEEK_END); - if (pos < 0) - return index_data_set_syscall_error(data, "lseek()"); - - if (data->header->used_file_size + size <= (uoff_t)pos) { - /* no need to grow, just update mmap */ - if (!mmap_update(data, 0, 0)) - return FALSE; - - i_assert(data->mmap_full_length >= (uoff_t)pos); - return TRUE; - } - - if (pos < (off_t)sizeof(struct mail_index_data_header)) - return index_data_set_corrupted(data, "Header is missing"); - - if (file_set_size(data->fd, (off_t)new_fsize) < 0) - return index_data_set_syscall_error(data, "file_set_size()"); - - return mmap_update(data, 0, 0); -} - -uoff_t mail_index_data_append(struct mail_index_data *data, const void *buffer, - size_t size) -{ - uoff_t offset; - - i_assert((size & (INDEX_ALIGN_SIZE-1)) == 0); - i_assert(data->index->lock_type == MAIL_LOCK_EXCLUSIVE); - - if (!mmap_update(data, 0, sizeof(struct mail_index_data_header))) - return 0; - - if (size > data->mmap_full_length || - data->mmap_full_length - size < data->header->used_file_size) { - if (!mail_index_data_grow(data, size)) - return 0; - } - - offset = data->header->used_file_size; - i_assert(offset + size <= data->mmap_full_length); - - memcpy((char *) data->mmap_base + offset, buffer, size); - data->header->used_file_size += size; - data->mmap_used_length += size; - - data->modified = TRUE; - return offset; -} - -int mail_index_data_delete(struct mail_index_data *data, - struct mail_index_record *index_rec) -{ - struct mail_index_data_record_header *rec_hdr; - uoff_t max_del_space; - - i_assert(data->index->lock_type == MAIL_LOCK_EXCLUSIVE); - - rec_hdr = mail_index_data_lookup_header(data, index_rec); - if (rec_hdr == NULL) - return FALSE; - - /* just mark it deleted. */ - data->header->deleted_space += rec_hdr->data_size; - - /* clear the record data. not really needed, but better not to keep - deleted information lying around.. */ - memset((char *) rec_hdr + sizeof(*rec_hdr), 0, - rec_hdr->data_size - sizeof(*rec_hdr)); - - /* see if we've reached the max. deleted space in file */ - if (data->header->used_file_size >= COMPRESS_MIN_SIZE && - (data->index->header->flags & MAIL_INDEX_FLAG_COMPRESS_DATA) == 0) { - max_del_space = data->header->used_file_size / - 100 * COMPRESS_PERCENTAGE; - if (data->header->deleted_space >= max_del_space) - data->index->set_flags |= MAIL_INDEX_FLAG_COMPRESS_DATA; - } - - data->modified = TRUE; - return TRUE; -} - -int mail_index_data_sync_file(struct mail_index_data *data, int *fsync_fd) -{ - *fsync_fd = -1; - - if (data->anon_mmap) - return TRUE; - - if (!mail_index_data_msync(data)) - return FALSE; - - if (!data->fsynced) { - data->fsynced = TRUE; - *fsync_fd = data->fd; - } - - return TRUE; -} - -struct mail_index_data_record_header * -mail_index_data_lookup_header(struct mail_index_data *data, - struct mail_index_record *index_rec) -{ - uoff_t pos; - - pos = index_rec->data_position; - if (pos == 0) { - /* data not yet written to record */ - return NULL; - } - - if (!mmap_update(data, pos, - sizeof(struct mail_index_data_record_header))) - return NULL; - - if (pos + sizeof(struct mail_index_data_record_header) > - data->mmap_used_length) { - index_data_set_corrupted(data, - "Data position of record %u points outside file " - "(%"PRIuUOFF_T" + %"PRIuSIZE_T" > %"PRIuSIZE_T")", - index_rec->uid, pos, - sizeof(struct mail_index_data_record_header), - data->mmap_used_length); - return NULL; - } - - if ((pos % INDEX_ALIGN_SIZE) != 0) { - index_data_set_corrupted(data, "Data position (%"PRIuUOFF_T") " - "is not memory aligned for record %u", - pos, index_rec->uid); - return NULL; - } - - return (struct mail_index_data_record_header *) - ((char *) data->mmap_base + pos); -} - -struct mail_index_data_record * -mail_index_data_lookup(struct mail_index_data *data, - struct mail_index_record *index_rec, - enum mail_data_field field) -{ - struct mail_index_data_record_header *rec_hdr; - struct mail_index_data_record *rec; - uoff_t pos, max_pos; - - index_reset_error(data->index); - - if (index_rec->data_position == 0) { - /* data not yet written to record */ - return NULL; - } - - rec_hdr = mail_index_data_lookup_header(data, index_rec); - if (rec_hdr == NULL) - return NULL; - - if (!mmap_update(data, index_rec->data_position, rec_hdr->data_size)) - return NULL; - - pos = index_rec->data_position; - max_pos = index_rec->data_position + rec_hdr->data_size; - - if (pos > data->mmap_used_length || - (data->mmap_used_length - pos < rec_hdr->data_size)) { - index_data_set_corrupted(data, - "Given data size larger than file size " - "(%u + %u > %"PRIuSIZE_T") for record %u", - index_rec->data_position, rec_hdr->data_size, - data->mmap_used_length, index_rec->uid); - return NULL; - } - - pos += sizeof(struct mail_index_data_record_header); - do { - rec = (struct mail_index_data_record *) - ((char *) data->mmap_base + pos); - - if (rec->full_field_size > max_pos || - pos + sizeof(struct mail_index_data_record) > max_pos || - pos + DATA_RECORD_SIZE(rec) > max_pos) { - index_data_set_corrupted(data, - "Field %d size points outside file " - "(%"PRIuUOFF_T" / %"PRIuUOFF_T") for record %u", - (int)field, pos, max_pos, index_rec->uid); - break; - } - - if ((rec->full_field_size % INDEX_ALIGN_SIZE) != 0) { - index_data_set_corrupted(data, "Field %d size %u " - "is not memory aligned for record %u", - (int)field, rec->full_field_size, - index_rec->uid); - break; - } - - if (rec->field == field || field == 0) { - /* match */ - return rec; - } else if (rec->field < field) { - /* jump to next record */ - pos += DATA_RECORD_SIZE(rec); - } else { - /* the fields are sorted by field type, so it's not - possible the wanted field could come after this. */ - break; - } - } while (pos < max_pos); - - return NULL; -} - -struct mail_index_data_record * -mail_index_data_next(struct mail_index_data *data, - struct mail_index_record *index_rec, - struct mail_index_data_record *rec) -{ - struct mail_index_data_record_header *rec_hdr; - uoff_t pos, end_pos, max_pos; - - index_reset_error(data->index); - - if (rec == NULL) - return NULL; - - rec_hdr = (struct mail_index_data_record_header *) - ((char *) data->mmap_base + index_rec->data_position); - - /* get position to next record */ - pos = DATA_FILE_POSITION(data, rec) + DATA_RECORD_SIZE(rec); - max_pos = index_rec->data_position + rec_hdr->data_size; - - /* make sure it's within range */ - if (pos >= max_pos) - return NULL; - - rec = (struct mail_index_data_record *) - ((char *) data->mmap_base + pos); - end_pos = pos + DATA_RECORD_SIZE(rec); - if (end_pos < pos || end_pos > max_pos) { - index_data_set_corrupted(data, "Field size points outside file " - "(%"PRIuUOFF_T" + %u > %"PRIuUOFF_T")", - pos, rec->full_field_size, max_pos); - return NULL; - } - - return rec; -} - -int mail_index_data_record_verify(struct mail_index_data *data, - struct mail_index_data_record *rec) -{ - int i; - - if (rec->full_field_size > INT_MAX) { - /* we already checked that the full_field_size is within file, - so this can happen only if the file really is huge.. */ - index_data_set_corrupted(data, "full_field_size (%u) > INT_MAX", - rec->full_field_size); - return FALSE; - } - - /* make sure the data actually contains \0 */ - for (i = (int)rec->full_field_size-1; i >= 0; i--) { - if (rec->data[i] == '\0') { - /* yes, everything ok */ - return TRUE; - } - } - - index_data_set_corrupted(data, "Missing \\0 with field %u " - "(%"PRIuUOFF_T")", rec->field, - DATA_FILE_POSITION(data, rec)); - return FALSE; -} - -void *mail_index_data_get_mmaped(struct mail_index_data *data, size_t *size) -{ - if (!data->anon_mmap) { - if (!mmap_update(data, 0, 0)) - return NULL; - } - - *size = data->mmap_used_length; - return data->mmap_base; -}
--- a/src/lib-index/mail-index-data.h Wed Aug 06 23:13:57 2003 +0300 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,63 +0,0 @@ -#ifndef __MAIL_INDEX_DATA_H -#define __MAIL_INDEX_DATA_H - -#define DATA_FILE_PREFIX ".data" - -int mail_index_data_open(struct mail_index *index); -int mail_index_data_create(struct mail_index *index); -void mail_index_data_free(struct mail_index_data *data); - -/* Truncate the data file and update it's indexid */ -int mail_index_data_reset(struct mail_index_data *data); - -/* Set indexid to 0 to notify other processes using this file that they should - re-open it. */ -int mail_index_data_mark_file_deleted(struct mail_index_data *data); - -/* Mark the file as being modified */ -void mail_index_data_mark_modified(struct mail_index_data *data); - -/* Append new data at the end of the file. Returns the position in file - where the data begins, or 0 if error occured. */ -uoff_t mail_index_data_append(struct mail_index_data *data, const void *buffer, - size_t size); - -/* Mark the given record deleted. */ -int mail_index_data_delete(struct mail_index_data *data, - struct mail_index_record *index_rec); - -/* Synchronize the data into disk */ -int mail_index_data_sync_file(struct mail_index_data *data, int *fsync_fd); - -/* Looks up a record header from data file. Returns NULL if not found or - if error occured. */ -struct mail_index_data_record_header * -mail_index_data_lookup_header(struct mail_index_data *data, - struct mail_index_record *index_rec); - -/* Looks up a field from data file. If field is 0, returns the first field - found. Returns NULL if not found or if error occured. */ -struct mail_index_data_record * -mail_index_data_lookup(struct mail_index_data *data, - struct mail_index_record *index_rec, - enum mail_data_field field); - -/* Returns the next record in data file, or NULL if there's no more. */ -struct mail_index_data_record * -mail_index_data_next(struct mail_index_data *data, - struct mail_index_record *index_rec, - struct mail_index_data_record *rec); - -/* Returns TRUE if rec->data is a valid \0-terminated string */ -int mail_index_data_record_verify(struct mail_index_data *data, - struct mail_index_data_record *rec); - -/* Return the whole data file mmap()ed. */ -void *mail_index_data_get_mmaped(struct mail_index_data *data, size_t *size); - -/* "Error in index data file %s: ...". Also marks the index file as - corrupted. */ -int index_data_set_corrupted(struct mail_index_data *data, const char *fmt, ...) - __attr_format__(2, 3); - -#endif
--- a/src/lib-index/mail-index-fsck.c Wed Aug 06 23:13:57 2003 +0300 +++ b/src/lib-index/mail-index-fsck.c Wed Aug 06 23:15:30 2003 +0300 @@ -46,7 +46,6 @@ i_assert(index->lock_type != MAIL_LOCK_SHARED); - /* Expunge tree can get easily corrupted, compress it away first. */ if (!mail_index_compress(index)) return FALSE;
--- a/src/lib-index/mail-index-open.c Wed Aug 06 23:13:57 2003 +0300 +++ b/src/lib-index/mail-index-open.c Wed Aug 06 23:15:30 2003 +0300 @@ -9,8 +9,8 @@ #include "unlink-lockfiles.h" #include "write-full.h" #include "mail-index.h" -#include "mail-index-data.h" #include "mail-index-util.h" +#include "mail-cache.h" #include "mail-modifylog.h" #include "mail-custom-flags.h" @@ -29,7 +29,7 @@ index->maildir_have_new = (hdr->flags & MAIL_INDEX_FLAG_MAILDIR_NEW) != 0; - if ((hdr->flags & MAIL_INDEX_FLAG_DIRTY_MESSAGES) != 0) + if ((hdr->flags & MAIL_INDEX_HDR_FLAG_DIRTY_MESSAGES) != 0) index->next_dirty_flush = ioloop_time; /* update \Recent message counters */ @@ -52,7 +52,7 @@ if (hdr->next_uid >= MAX_ALLOWED_UID - 1000) { /* UID values are getting too high, rebuild index */ - index->set_flags |= MAIL_INDEX_FLAG_REBUILD; + index->set_flags |= MAIL_INDEX_HDR_FLAG_REBUILD; } if (index->lock_type == MAIL_LOCK_EXCLUSIVE) { @@ -71,36 +71,18 @@ { int rebuilt; - /* open/create the index files */ - if ((flags & _MAIL_INDEX_OPEN_FLAG_CREATING) == 0) { - if (!mail_index_data_open(index)) { - if ((index->set_flags & MAIL_INDEX_FLAG_REBUILD) == 0) - return FALSE; - - /* data file is corrupted, need to rebuild index */ - flags |= _MAIL_INDEX_OPEN_FLAG_CREATING; - index->set_flags = 0; - index->inconsistent = FALSE; - } - } - - if ((flags & _MAIL_INDEX_OPEN_FLAG_CREATING) != 0) { - if (!mail_index_set_lock(index, MAIL_LOCK_EXCLUSIVE)) - return FALSE; - if (!mail_index_data_create(index)) - return FALSE; - } - - /* custom flags file needs to be open before - rebuilding index */ - if (!mail_custom_flags_open_or_create(index)) + if (!mail_cache_open_or_create(index)) return FALSE; - if ((flags & _MAIL_INDEX_OPEN_FLAG_CREATING) != 0 || - (index->header->flags & MAIL_INDEX_FLAG_REBUILD) != 0) { + if ((index->header->flags & MAIL_INDEX_HDR_FLAG_REBUILD) != 0 || + (index->set_flags & MAIL_INDEX_HDR_FLAG_REBUILD) != 0) { + if (!index->rebuild(index)) return FALSE; + if ((index->header->flags & MAIL_INDEX_HDR_FLAG_REBUILD) != 0) + return FALSE; + /* no inconsistency problems since we're still opening the index */ index->inconsistent = FALSE; @@ -109,6 +91,10 @@ rebuilt = FALSE; } + /* custom flags file needs to be open before rebuilding index */ + if (!mail_custom_flags_open_or_create(index)) + return FALSE; + if ((flags & _MAIL_INDEX_OPEN_FLAG_CREATING) == 0) { if (!mail_modifylog_open_or_create(index)) return FALSE; @@ -117,7 +103,7 @@ return FALSE; } - if (index->header->flags & MAIL_INDEX_FLAG_FSCK) { + if (index->header->flags & MAIL_INDEX_HDR_FLAG_FSCK) { /* index needs fscking */ if (!index->fsck(index)) return FALSE; @@ -130,8 +116,6 @@ MAIL_LOCK_SHARED, NULL) && !index->nodiskspace) return FALSE; - - index->inconsistent = FALSE; } /* we never want to keep shared lock if syncing happens to set it. @@ -143,23 +127,15 @@ } if ((flags & MAIL_INDEX_OPEN_FLAG_FAST) == 0) { - if (index->header->flags & MAIL_INDEX_FLAG_COMPRESS) { + if (index->header->flags & MAIL_INDEX_HDR_FLAG_COMPRESS) { /* remove deleted blocks from index file */ if (!mail_index_compress(index)) return FALSE; } - if (index->header->flags & MAIL_INDEX_FLAG_CACHE_FIELDS) { - /* need to update cached fields */ - if (!mail_index_update_cache(index)) - return FALSE; - } - - if (index->header->flags & MAIL_INDEX_FLAG_COMPRESS_DATA) { - /* remove unused space from index data file. - keep after cache updates which may move data - and create unused space */ - if (!mail_index_compress_data(index)) + if (index->header->flags & MAIL_INDEX_HDR_FLAG_COMPRESS_CACHE) { + /* remove unused space from index data file. */ + if (!mail_cache_compress(index->cache)) return FALSE; } } @@ -189,16 +165,6 @@ return 1; } -static int mail_index_is_compatible(const struct mail_index_header *hdr) -{ - return hdr->compat_data[0] == MAIL_INDEX_VERSION && - hdr->compat_data[1] == MAIL_INDEX_COMPAT_FLAGS && - hdr->compat_data[2] == sizeof(unsigned int) && - hdr->compat_data[3] == sizeof(time_t) && - hdr->compat_data[4] == sizeof(uoff_t) && - hdr->compat_data[5] == INDEX_ALIGN_SIZE; -} - static int mail_index_init_file(struct mail_index *index, const struct mail_index_header *hdr) { @@ -224,27 +190,20 @@ return TRUE; } -void mail_index_init_header(struct mail_index *index, - struct mail_index_header *hdr) +void mail_index_init_header(struct mail_index_header *hdr) { + i_assert(sizeof(struct mail_index_header) < 256); + memset(hdr, 0, sizeof(*hdr)); - hdr->compat_data[0] = MAIL_INDEX_VERSION; - hdr->compat_data[1] = MAIL_INDEX_COMPAT_FLAGS; - hdr->compat_data[2] = sizeof(unsigned int); - hdr->compat_data[3] = sizeof(time_t); - hdr->compat_data[4] = sizeof(uoff_t); - hdr->compat_data[5] = INDEX_ALIGN_SIZE; + hdr->major_version = MAIL_INDEX_MAJOR_VERSION; + hdr->minor_version = MAIL_INDEX_MINOR_VERSION; + hdr->header_size = (uint8_t)sizeof(struct mail_index_header); + hdr->indexid = ioloop_time; /* mark the index requiring rebuild - rebuild() removes this flag when it succeeds */ - hdr->flags = MAIL_INDEX_FLAG_REBUILD; - - if (!index->anon_mmap) { - /* set the fields we always want to cache, - but not if we're building into memory */ - hdr->cache_fields |= index->default_cache_fields; - } + hdr->flags = MAIL_INDEX_HDR_FLAG_REBUILD; hdr->used_file_size = sizeof(struct mail_index_header); hdr->uid_validity = ioloop_time; @@ -282,14 +241,15 @@ flags |= _MAIL_INDEX_OPEN_FLAG_CREATING; - index->mmap_full_length = INDEX_FILE_MIN_SIZE; + index->header_size = sizeof(struct mail_index_header); + index->mmap_full_length = INDEX_FILE_MIN_SIZE(index); index->mmap_base = mmap_anon(index->mmap_full_length); if (index->mmap_base == MAP_FAILED) { index->mmap_base = NULL; return index_set_error(index, "mmap_anon() failed: %m"); } - mail_index_init_header(index, index->mmap_base); + mail_index_init_header(index->mmap_base); index->header = index->mmap_base; index->mmap_used_length = index->header->used_file_size; @@ -332,8 +292,8 @@ if ((ret = mail_index_read_header(index, &hdr)) < 0) return FALSE; - if (ret == 0 || !mail_index_is_compatible(&hdr) || - (hdr.flags & MAIL_INDEX_FLAG_REBUILD) != 0) { + if (ret == 0 || hdr.major_version != MAIL_INDEX_MAJOR_VERSION || + (hdr.flags & MAIL_INDEX_HDR_FLAG_REBUILD) != 0) { if ((flags & MAIL_INDEX_OPEN_FLAG_CREATE) == 0) return FALSE; @@ -347,7 +307,7 @@ return mail_index_open_index(index, flags); } - mail_index_init_header(index, &hdr); + mail_index_init_header(&hdr); if (!mail_index_init_file(index, &hdr)) return FALSE; } @@ -367,7 +327,7 @@ } if (!index_open_and_fix(index, flags)) { - if ((index->set_flags & MAIL_INDEX_FLAG_REBUILD) == 0 || + if ((index->set_flags & MAIL_INDEX_HDR_FLAG_REBUILD) == 0 || (flags & _MAIL_INDEX_OPEN_FLAG_CREATING) != 0) return FALSE;
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-index/mail-index-rebuild.c Wed Aug 06 23:15:30 2003 +0300 @@ -0,0 +1,47 @@ +/* Copyright (C) 2002-2003 Timo Sirainen */ + +#include "lib.h" +#include "mail-index.h" +#include "mail-index-util.h" +#include "mail-cache.h" + +#include <sys/mman.h> + +int mail_index_rebuild(struct mail_index *index) +{ + if (!mail_index_set_lock(index, MAIL_LOCK_EXCLUSIVE)) + return FALSE; + + index->set_flags &= ~MAIL_INDEX_HDR_FLAG_REBUILD; + + /* reset the header */ + mail_index_init_header(index->header); + index->mmap_used_length = index->header->used_file_size; + + /* update indexid, which also means that our state has completely + changed */ + index->indexid = index->header->indexid; + index->inconsistent = TRUE; + index->rebuilding = TRUE; + + if (!index->anon_mmap) { + if (msync(index->mmap_base, index->header_size, MS_SYNC) < 0) + return index_set_syscall_error(index, "msync()"); + } + + if (!mail_cache_truncate(index->cache)) + return FALSE; + + /* read the mails by syncing */ + if (!index->sync_and_lock(index, FALSE, MAIL_LOCK_UNLOCK, NULL)) + return FALSE; + + /* rebuild is complete - remove the flag */ + index->header->flags &= ~(MAIL_INDEX_HDR_FLAG_REBUILD | + MAIL_INDEX_HDR_FLAG_FSCK); + index->header->flags |= index->set_flags; + index->set_flags = 0; + + index->rebuilding = FALSE; + return TRUE; +}
--- a/src/lib-index/mail-index-update-cache.c Wed Aug 06 23:13:57 2003 +0300 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,60 +0,0 @@ -/* Copyright (C) 2002 Timo Sirainen */ - -#include "lib.h" -#include "istream.h" -#include "mail-index.h" - -#include <unistd.h> - -static int cache_record(struct mail_index *index, struct mail_index_record *rec, - enum mail_data_field cache_fields) -{ - struct mail_index_update *update; - struct istream *input; - time_t internal_date; - int failed, deleted; - - input = index->open_mail(index, rec, &internal_date, &deleted); - if (input == NULL) - return deleted; - - cache_fields &= ~rec->data_fields; - - update = index->update_begin(index, rec); - index->update_field_raw(update, DATA_HDR_INTERNAL_DATE, - &internal_date, sizeof(internal_date)); - mail_index_update_headers(update, input, cache_fields, NULL, NULL); - failed = !index->update_end(update); - - i_stream_unref(input); - return !failed; -} - -int mail_index_update_cache(struct mail_index *index) -{ - struct mail_index_record *rec; - enum mail_data_field cache_fields; - - if (!index->set_lock(index, MAIL_LOCK_EXCLUSIVE)) - return FALSE; - - cache_fields = index->header->cache_fields; - - rec = index->lookup(index, 1); - while (rec != NULL) { - if (((enum mail_data_field)rec->data_fields & cache_fields) != - cache_fields) { - t_push(); - if (!cache_record(index, rec, cache_fields)) { - t_pop(); - return FALSE; - } - t_pop(); - } - - rec = index->next(index, rec); - } - - index->header->flags &= ~MAIL_INDEX_FLAG_CACHE_FIELDS; - return TRUE; -}
--- a/src/lib-index/mail-index-update.c Wed Aug 06 23:13:57 2003 +0300 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,574 +0,0 @@ -/* Copyright (C) 2002 Timo Sirainen */ - -#include "lib.h" -#include "buffer.h" -#include "str.h" -#include "istream.h" -#include "ioloop.h" -#include "str.h" -#include "message-date.h" -#include "message-parser.h" -#include "message-part-serialize.h" -#include "message-size.h" -#include "imap-envelope.h" -#include "imap-bodystructure.h" -#include "mail-index.h" -#include "mail-index-data.h" -#include "mail-index-util.h" - -struct mail_index_update { - pool_t pool; - - struct mail_index *index; - struct mail_index_record *rec; - struct mail_index_data_record_header data_hdr; - - unsigned int updated_fields; - void *fields[DATA_FIELD_MAX_BITS]; - size_t field_sizes[DATA_FIELD_MAX_BITS]; - size_t field_extra_sizes[DATA_FIELD_MAX_BITS]; -}; - -struct mail_index_update * -mail_index_update_begin(struct mail_index *index, struct mail_index_record *rec) -{ - pool_t pool; - struct mail_index_update *update; - struct mail_index_data_record_header *data_hdr; - - i_assert(index->lock_type == MAIL_LOCK_EXCLUSIVE); - - pool = pool_alloconly_create("mail_index_update", 4096); - - update = p_new(pool, struct mail_index_update, 1); - update->pool = pool; - update->index = index; - update->rec = rec; - - data_hdr = mail_index_data_lookup_header(index->data, rec); - if (data_hdr != NULL) - memcpy(&update->data_hdr, data_hdr, sizeof(*data_hdr)); - return update; -} - -static int mail_field_get_index(enum mail_data_field field) -{ - unsigned int i, mask; - - for (i = 0, mask = 1; i < DATA_FIELD_MAX_BITS; i++, mask <<= 1) { - if (field == mask) - return i; - } - - return -1; -} - -static void get_data_block_sizes(struct mail_index_update *update, - size_t *min_size, size_t *max_size, - int *no_grown_fields) -{ - struct mail_index_data_record *rec; - enum mail_data_field field; - unsigned int field_min_size; - int i, field_exists; - - *min_size = *max_size = sizeof(struct mail_index_data_record_header); - *no_grown_fields = TRUE; - - rec = mail_index_data_lookup(update->index->data, update->rec, 0); - for (i = 0, field = 1; field != DATA_FIELD_LAST; i++, field <<= 1) { - field_exists = rec != NULL && rec->field == field; - - if (update->fields[i] != NULL) { - /* value was modified - use it */ - field_min_size = INDEX_ALIGN(update->field_sizes[i]); - *min_size += SIZEOF_MAIL_INDEX_DATA + field_min_size; - *max_size += SIZEOF_MAIL_INDEX_DATA + - INDEX_ALIGN(update->field_sizes[i] + - update->field_extra_sizes[i]); - - if (!field_exists || - rec->full_field_size < field_min_size) - *no_grown_fields = FALSE; - } else if (field_exists) { - /* use the old value */ - *min_size += SIZEOF_MAIL_INDEX_DATA + - rec->full_field_size; - *max_size += SIZEOF_MAIL_INDEX_DATA + - rec->full_field_size; - } - - if (field_exists) { - rec = mail_index_data_next(update->index->data, - update->rec, rec); - } - } -} - -static size_t get_max_align_size(size_t base, size_t extra, size_t *max_extra) -{ - size_t size; - - size = INDEX_ALIGN(base); - extra -= size - base; - if (*max_extra < INDEX_ALIGN_SIZE || extra == 0) - return size; /* no extra / extra went into alignment */ - - extra = INDEX_ALIGN(extra); - if (extra > *max_extra) { - /* partial */ - extra = *max_extra & ~(size_t)(INDEX_ALIGN_SIZE-1); - i_assert(extra <= *max_extra); - } - - *max_extra -= extra; - return size + extra; -} - -/* extra_size is the amount of data in data_size which can be used for - field_extra_sizes */ -static void *create_data_block(struct mail_index_update *update, - size_t data_size, size_t extra_size) -{ - struct mail_index_data_record *rec, destrec; - enum mail_data_field field; - buffer_t *buf; - const void *src; - size_t src_size, filler_size; - int i; - - i_assert(data_size <= UINT_MAX); - - buf = buffer_create_static_hard(update->pool, data_size); - - /* set header */ - update->data_hdr.data_size = data_size; - buffer_append(buf, &update->data_hdr, sizeof(update->data_hdr)); - - /* set fields */ - rec = mail_index_data_lookup(update->index->data, update->rec, 0); - for (i = 0, field = 1; field != DATA_FIELD_LAST; i++, field <<= 1) { - if (update->fields[i] != NULL) { - /* value was modified - use it */ - destrec.full_field_size = - get_max_align_size(update->field_sizes[i], - update->field_extra_sizes[i], - &extra_size); - src = update->fields[i]; - src_size = update->field_sizes[i]; - } else if (rec != NULL && rec->field == field) { - /* use the old value */ - destrec.full_field_size = rec->full_field_size; - src = rec->data; - src_size = rec->full_field_size; - } else { - /* the field doesn't exist, jump to next */ - continue; - } - i_assert((destrec.full_field_size % INDEX_ALIGN_SIZE) == 0); - - destrec.field = field; - buffer_append(buf, &destrec, SIZEOF_MAIL_INDEX_DATA); - buffer_append(buf, src, src_size); - - filler_size = destrec.full_field_size - src_size; - if (filler_size != 0) { - buffer_set_used_size(buf, buffer_get_used_size(buf) + - filler_size); - } - - if (rec != NULL && rec->field == field) { - rec = mail_index_data_next(update->index->data, - update->rec, rec); - } - } - - return buffer_free_without_data(buf); -} - -/* Append all the data at the end of the data file and update - the index's data position */ -static int update_by_append(struct mail_index_update *update, - size_t data_size, size_t extra_size) -{ - void *mem; - uoff_t fpos; - - mem = create_data_block(update, data_size, extra_size); - - /* append the data at the end of the data file */ - fpos = mail_index_data_append(update->index->data, mem, data_size); - if (fpos == 0) - return FALSE; - - /* the old data is discarded */ - (void)mail_index_data_delete(update->index->data, update->rec); - - /* update index file position - it's mmap()ed so it'll be written - into disk when index is unlocked. */ - update->rec->data_position = fpos; - return TRUE; -} - -/* Replace the whole block - assumes there's enough space to do it */ -static void update_by_replace_block(struct mail_index_update *update, - size_t extra_size) -{ - struct mail_index_data_record_header *data_hdr; - size_t data_size; - void *mem; - - data_hdr = mail_index_data_lookup_header(update->index->data, - update->rec); - - data_size = update->data_hdr.data_size; - i_assert(data_size == data_hdr->data_size); - - mem = create_data_block(update, data_size, extra_size); - memcpy(data_hdr, mem, data_size); - - /* clear the extra space. not really needed. */ - memset((char *) data_hdr + data_size, 0, - data_hdr->data_size - data_size); - - mail_index_data_mark_modified(update->index->data); -} - -/* Replace the modified fields in the file - assumes there's enough - space to do it */ -static void update_by_replace_fields(struct mail_index_update *update) -{ - struct mail_index_data_record_header *data_hdr; - struct mail_index_data_record *rec; - size_t field_size; - int index; - - /* update header */ - data_hdr = mail_index_data_lookup_header(update->index->data, - update->rec); - memcpy(data_hdr, &update->data_hdr, sizeof(*data_hdr)); - - rec = mail_index_data_lookup(update->index->data, update->rec, 0); - while (rec != NULL) { - if (rec->field & update->updated_fields) { - /* field was changed */ - index = mail_field_get_index(rec->field); - i_assert(index >= 0); - - field_size = update->field_sizes[index]; - i_assert(field_size <= rec->full_field_size); - - memcpy(rec->data, update->fields[index], field_size); - - /* clear the extra space. not really needed. */ - memset(rec->data + field_size, 0, - rec->full_field_size - field_size); - } - - rec = mail_index_data_next(update->index->data, - update->rec, rec); - } - - mail_index_data_mark_modified(update->index->data); -} - -int mail_index_update_end(struct mail_index_update *update) -{ - struct mail_index_data_record_header *data_hdr; - size_t min_size, max_size, extra_size; - int no_grown_fields, failed = FALSE; - - i_assert(update->index->lock_type == MAIL_LOCK_EXCLUSIVE); - - if (update->updated_fields != 0) { - /* if fields don't fit to allocated data block, we have - to move it to end of file */ - get_data_block_sizes(update, &min_size, &max_size, - &no_grown_fields); - extra_size = max_size - min_size; - - data_hdr = mail_index_data_lookup_header(update->index->data, - update->rec); - - if (no_grown_fields) - update_by_replace_fields(update); - else if (data_hdr != NULL && min_size <= data_hdr->data_size) - update_by_replace_block(update, extra_size); - else { - if (!update_by_append(update, max_size, extra_size)) - failed = TRUE; - } - - if (!failed) { - /* update cached fields mask */ - update->rec->data_fields |= update->updated_fields; - } - } - - pool_unref(update->pool); - return !failed; -} - -void mail_index_update_abort(struct mail_index_update *update) -{ - pool_unref(update->pool); -} - -static void update_field_full(struct mail_index_update *update, - enum mail_data_field field, - const void *value, size_t size, - size_t extra_space) -{ - int index; - - index = mail_field_get_index(field); - i_assert(index >= 0); - - update->updated_fields |= field; - update->field_sizes[index] = size; - update->field_extra_sizes[index] = extra_space; - update->fields[index] = p_malloc(update->pool, size); - memcpy(update->fields[index], value, size); -} - -static void update_header_field(struct mail_index_update *update, - enum mail_data_field field, - const void *value, size_t size __attr_unused__) -{ - switch (field) { - case DATA_HDR_INTERNAL_DATE: - i_assert(size == sizeof(time_t)); - update->data_hdr.internal_date = *((const time_t *) value); - break; - case DATA_HDR_VIRTUAL_SIZE: - i_assert(size == sizeof(uoff_t)); - update->data_hdr.virtual_size = *((const uoff_t *) value); - break; - case DATA_HDR_HEADER_SIZE: - i_assert(size == sizeof(uoff_t)); - update->data_hdr.header_size = *((const uoff_t *) value); - break; - case DATA_HDR_BODY_SIZE: - i_assert(size == sizeof(uoff_t)); - update->data_hdr.body_size = *((const uoff_t *) value); - break; - default: - i_unreached(); - } - - update->updated_fields |= field; -} - -void mail_index_update_field(struct mail_index_update *update, - enum mail_data_field field, - const char *value, size_t extra_space) -{ - update_field_full(update, field, value, strlen(value) + 1, extra_space); -} - -void mail_index_update_field_raw(struct mail_index_update *update, - enum mail_data_field field, - const void *value, size_t size) -{ - if (field >= DATA_FIELD_LAST) - update_header_field(update, field, value, size); - else - update_field_full(update, field, value, size, 0); -} - -struct header_update_context { - struct mail_index_update *update; - pool_t envelope_pool; - struct message_part_envelope_data *envelope; - - message_header_callback_t *header_cb; - void *context; -}; - -static void update_header_cb(struct message_part *part, - struct message_header_line *hdr, void *context) -{ - struct header_update_context *ctx = context; - - if (part != NULL && part->parent != NULL) - return; - - /* see if we can do anything with this field */ - if (ctx->update->index->header->cache_fields & DATA_FIELD_ENVELOPE) { - if (ctx->envelope_pool == NULL) { - ctx->envelope_pool = - pool_alloconly_create("index envelope", 2048); - } - imap_envelope_parse_header(ctx->envelope_pool, - &ctx->envelope, hdr); - } - - if (ctx->header_cb != NULL) - ctx->header_cb(part, hdr, ctx->context); -} - -void mail_index_update_headers(struct mail_index_update *update, - struct istream *input, - enum mail_data_field cache_fields, - message_header_callback_t *header_cb, - void *context) -{ - struct header_update_context ctx; - struct message_part *part; - struct message_size hdr_size, body_size; - pool_t pool; - buffer_t *buf; - const char *value, *error; - size_t size; - uoff_t start_offset; - - ctx.update = update; - ctx.envelope_pool = NULL; - ctx.envelope = NULL; - ctx.header_cb = header_cb; - ctx.context = context; - - if (cache_fields == 0) - cache_fields = update->index->header->cache_fields; - - if (IS_BODYSTRUCTURE_FIELD(cache_fields)) { - /* for body / bodystructure, we need need to - fully parse the message. unless it's already parsed - and cached. */ - pool = pool_alloconly_create("index message parser", 2048); - - value = update->index->lookup_field_raw(update->index, - update->rec, - DATA_FIELD_MESSAGEPART, - &size); - if (value == NULL) - part = NULL; - else { - part = message_part_deserialize(pool, value, size, - &error); - if (part == NULL) { - /* corrupted, rebuild it */ - index_set_corrupted(update->index, - "Corrupted cached MessagePart data " - "(%s)", error); - } - } - - start_offset = input->v_offset; - - if (part == NULL) { - part = message_parse(pool, input, - update_header_cb, &ctx); - if ((part->flags & MESSAGE_PART_FLAG_HAS_NULS) != 0) { - update->rec->index_flags |= - INDEX_MAIL_FLAG_HAS_NULS; - } else { - update->rec->index_flags |= - INDEX_MAIL_FLAG_HAS_NO_NULS; - } - } else { - /* cached, construct the bodystructure using it. - also we need to parse the header.. */ - i_stream_seek(input, start_offset); - message_parse_header(NULL, input, NULL, - update_header_cb, &ctx); - } - - /* save sizes */ - hdr_size = part->header_size; - body_size = part->body_size; - - /* don't save both BODY + BODYSTRUCTURE since BODY can be - generated from BODYSTRUCTURE. FIXME: However that takes - CPU, maybe this should be configurable (I/O vs. CPU)? */ - if ((cache_fields & DATA_FIELD_BODY) && - ((update->rec->data_fields | cache_fields) & - DATA_FIELD_BODYSTRUCTURE) == 0) { - t_push(); - i_stream_seek(input, start_offset); - value = imap_part_get_bodystructure(pool, &part, - input, FALSE); - update->index->update_field(update, DATA_FIELD_BODY, - value, 0); - t_pop(); - } - - if (cache_fields & DATA_FIELD_BODYSTRUCTURE) { - t_push(); - i_stream_seek(input, start_offset); - value = imap_part_get_bodystructure(pool, &part, - input, TRUE); - update->index->update_field(update, - DATA_FIELD_BODYSTRUCTURE, - value, 0); - t_pop(); - } - - if (cache_fields & DATA_FIELD_MESSAGEPART) { - t_push(); - buf = buffer_create_dynamic(data_stack_pool, 2048, - (size_t)-1); - message_part_serialize(part, buf); - - value = buffer_get_data(buf, &size); - update->index->update_field_raw(update, - DATA_FIELD_MESSAGEPART, - value, size); - t_pop(); - } - - pool_unref(pool); - } else { - message_parse_header(NULL, input, &hdr_size, - update_header_cb, &ctx); - - body_size.physical_size = input->v_limit - input->v_offset; - if (body_size.physical_size == 0) - body_size.virtual_size = 0; - else if (update->data_hdr.virtual_size == 0) - body_size.virtual_size = (uoff_t)-1; - else { - body_size.virtual_size = - update->data_hdr.virtual_size - - hdr_size.virtual_size; - } - } - - if (ctx.envelope != NULL) { - string_t *str; - - t_push(); - str = str_new(data_stack_pool, 2048); - imap_envelope_write_part_data(ctx.envelope, str); - update->index->update_field(update, DATA_FIELD_ENVELOPE, - str_c(str), 0); - t_pop(); - - pool_unref(ctx.envelope_pool); - } - - /* update physical sizes */ - update->index->update_field_raw(update, DATA_HDR_HEADER_SIZE, - &hdr_size.physical_size, - sizeof(hdr_size.physical_size)); - update->index->update_field_raw(update, DATA_HDR_BODY_SIZE, - &body_size.physical_size, - sizeof(body_size.physical_size)); - - /* update virtual size if we know it */ - if (body_size.virtual_size != (uoff_t)-1) { - uoff_t virtual_size; - - virtual_size = hdr_size.virtual_size + body_size.virtual_size; - update->index->update_field_raw(update, DATA_HDR_VIRTUAL_SIZE, - &virtual_size, - sizeof(virtual_size)); - } - - - /* update binary flags. */ - if (hdr_size.virtual_size == hdr_size.physical_size) - update->rec->index_flags |= INDEX_MAIL_FLAG_BINARY_HEADER; - if (body_size.virtual_size == body_size.physical_size) - update->rec->index_flags |= INDEX_MAIL_FLAG_BINARY_BODY; -}
--- a/src/lib-index/mail-index.c Wed Aug 06 23:13:57 2003 +0300 +++ b/src/lib-index/mail-index.c Wed Aug 06 23:15:30 2003 +0300 @@ -6,8 +6,8 @@ #include "file-set-size.h" #include "mmap-util.h" #include "mail-index.h" -#include "mail-index-data.h" #include "mail-index-util.h" +#include "mail-cache.h" #include "mail-modifylog.h" #include "mail-custom-flags.h" @@ -27,7 +27,12 @@ return FALSE; } - extra = (index->mmap_full_length - sizeof(struct mail_index_header)) % + /* keep the header set even if we fail, so we can update the flags */ + hdr = index->mmap_base; + index->header = hdr; + index->header_size = hdr->header_size; + + extra = (index->mmap_full_length - index->header_size) % sizeof(struct mail_index_record); if (extra != 0) { @@ -39,25 +44,20 @@ (void)ftruncate(index->fd, (off_t)index->mmap_full_length); } - /* keep the header set even if we fail, so we can update the flags */ - hdr = index->mmap_base; - index->header = hdr; - if (hdr->used_file_size > index->mmap_full_length) { index_set_corrupted(index, "used_file_size larger than real file size " - "(%"PRIuUOFF_T" vs %"PRIuSIZE_T")", + "(%u vs %"PRIuSIZE_T")", hdr->used_file_size, index->mmap_full_length); return FALSE; } - if (hdr->used_file_size < sizeof(struct mail_index_header) || - (hdr->used_file_size - sizeof(struct mail_index_header)) % + if (hdr->used_file_size < index->header_size || + (hdr->used_file_size - index->header_size) % sizeof(struct mail_index_record) != 0) { index_set_corrupted(index, "Invalid used_file_size in header " - "(%"PRIuUOFF_T")", - hdr->used_file_size); + "(%u)", hdr->used_file_size); return FALSE; } @@ -139,14 +139,12 @@ PROT_READ|PROT_WRITE); #endif index->header->flags |= index->set_flags; - (void)msync(index->mmap_base, - sizeof(struct mail_index_header), MS_SYNC); + (void)msync(index->mmap_base, index->header_size, + MS_SYNC); } index->set_flags = 0; } - index->set_cache_fields = 0; - index->opened = FALSE; index->inconsistent = FALSE; @@ -174,9 +172,9 @@ } index->mmap_base = NULL; - if (index->data != NULL) { - mail_index_data_free(index->data); - index->data = NULL; + if (index->cache != NULL) { + mail_cache_free(index->cache); + index->cache = NULL; } if (index->modifylog != NULL) { @@ -206,9 +204,6 @@ for (i = 0; i < sizeof(fsync_fds)/sizeof(fsync_fds[0]); i++) fsync_fds[i] = -1; - if (!mail_index_data_sync_file(index->data, &fsync_fds[0])) - return FALSE; - if (msync(index->mmap_base, index->mmap_used_length, MS_SYNC) < 0) return index_set_syscall_error(index, "msync()"); @@ -261,11 +256,6 @@ index->header->flags |= index->set_flags; index->set_flags = 0; } - - if (index->set_cache_fields != 0) { - index->header->cache_fields = index->set_cache_fields; - index->set_cache_fields = 0; - } } static int mail_index_write_header_changes(struct mail_index *index) @@ -285,8 +275,7 @@ mail_index_update_header_changes(index); if (!index->anon_mmap) { - if (msync(index->mmap_base, - sizeof(struct mail_index_header), MS_SYNC) < 0) { + if (msync(index->mmap_base, index->header_size, MS_SYNC) < 0) { index_set_syscall_error(index, "msync()"); failed = TRUE; } @@ -305,6 +294,13 @@ static int mail_index_lock_remove(struct mail_index *index) { enum mail_lock_type old_lock_type; + int ret = TRUE; + + while (index->cache_later_locks > 0) { + if (!mail_cache_unlock(index->cache)) + ret = FALSE; + index->cache_later_locks--; + } if (!mail_index_wait_lock(index, F_UNLCK)) return FALSE; @@ -316,18 +312,15 @@ /* releasing shared lock. we may need to update some flags in header. */ unsigned int old_flags; - enum mail_data_field old_cache; old_flags = index->header->flags; - old_cache = index->header->cache_fields; - if ((old_flags | index->set_flags) != old_flags || - (old_cache | index->set_cache_fields) != old_cache) + if ((old_flags | index->set_flags) != old_flags) return mail_index_write_header_changes(index); } debug_mprotect(index->mmap_base, index->mmap_full_length, index); - return TRUE; + return ret; } static int mail_index_lock_change(struct mail_index *index, @@ -335,10 +328,14 @@ { int ret, fd_lock_type; - /* shared -> exclusive isn't allowed without try_lock */ + /* shared -> exclusive can deadlock */ i_assert(try_lock || lock_type != MAIL_LOCK_EXCLUSIVE || index->lock_type != MAIL_LOCK_SHARED); + /* locking index when cache is locked can deadlock */ + i_assert(try_lock || index->lock_type == MAIL_LOCK_EXCLUSIVE || + !mail_cache_is_locked(index->cache)); + if (index->inconsistent) { /* index is in inconsistent state and nothing else than free() is allowed for it. */ @@ -379,7 +376,7 @@ return FALSE; } - if (index->header->flags & MAIL_INDEX_FLAG_FSCK) { + if (index->header->flags & MAIL_INDEX_HDR_FLAG_FSCK) { /* someone just partially updated the index, need to fsck it */ if (lock_type == MAIL_LOCK_SHARED) { /* we need exclusive lock so fsck()'s set_lock() won't @@ -397,7 +394,7 @@ /* check again, in case it was already fscked while we had it unlocked for a while */ - if (index->header->flags & MAIL_INDEX_FLAG_FSCK) { + if (index->header->flags & MAIL_INDEX_HDR_FLAG_FSCK) { if (!index->fsck(index)) return FALSE; } @@ -413,9 +410,8 @@ when the lock is released, the FSCK flag will also be removed. */ index->excl_lock_counter++; - index->header->flags |= MAIL_INDEX_FLAG_FSCK; - if (!mail_index_fmdatasync(index, - sizeof(struct mail_index_header))) { + index->header->flags |= MAIL_INDEX_HDR_FLAG_FSCK; + if (!mail_index_fmdatasync(index, index->header_size)) { (void)index->set_lock(index, MAIL_LOCK_UNLOCK); return FALSE; } @@ -452,15 +448,14 @@ mail_modifylog_notify_lock_drop(index->modifylog); /* dropping exclusive lock (either unlock or to shared) */ - keep_fsck = (index->set_flags & MAIL_INDEX_FLAG_FSCK) != 0; + keep_fsck = (index->set_flags & MAIL_INDEX_HDR_FLAG_FSCK) != 0; mail_index_update_header_changes(index); /* remove the FSCK flag only after successful fsync() */ if (mail_index_sync_file(index) && !keep_fsck) { - index->header->flags &= ~MAIL_INDEX_FLAG_FSCK; + index->header->flags &= ~MAIL_INDEX_HDR_FLAG_FSCK; if (!index->anon_mmap && - msync(index->mmap_base, - sizeof(struct mail_index_header), + msync(index->mmap_base, index->header_size, MS_SYNC) < 0) { /* we only failed to remove the fsck flag, so this isn't fatal. */ @@ -502,115 +497,6 @@ return index->header; } -const char *mail_index_lookup_field(struct mail_index *index, - struct mail_index_record *rec, - enum mail_data_field field) -{ - struct mail_index_data_record *datarec; - - datarec = (rec->data_fields & field) == 0 ? NULL : - mail_index_data_lookup(index->data, rec, field); - if (datarec == NULL) - return NULL; - - if (!mail_index_data_record_verify(index->data, datarec)) { - /* index is corrupted, it will be rebuilt */ - return NULL; - } - - return datarec->data; -} - -const void *mail_index_lookup_field_raw(struct mail_index *index, - struct mail_index_record *rec, - enum mail_data_field field, - size_t *size) -{ - struct mail_index_data_record_header *datahdr; - struct mail_index_data_record *datarec; - - if ((rec->data_fields & field) == 0) { - *size = 0; - return NULL; - } - - if (field < DATA_FIELD_LAST) { - /* read data field */ - datarec = mail_index_data_lookup(index->data, rec, field); - if (datarec == NULL) { - *size = 0; - return NULL; - } - - *size = datarec->full_field_size; - return datarec->data; - } - - /* read header field */ - datahdr = mail_index_data_lookup_header(index->data, rec); - if (datahdr == NULL) { - *size = 0; - return NULL; - } - - switch (field) { - case DATA_HDR_INTERNAL_DATE: - *size = sizeof(datahdr->internal_date); - return &datahdr->internal_date; - case DATA_HDR_VIRTUAL_SIZE: - *size = sizeof(datahdr->virtual_size); - return &datahdr->virtual_size; - case DATA_HDR_HEADER_SIZE: - *size = sizeof(datahdr->header_size); - return &datahdr->header_size; - case DATA_HDR_BODY_SIZE: - *size = sizeof(datahdr->body_size); - return &datahdr->body_size; - default: - *size = 0; - return NULL; - } -} - -void mail_index_cache_fields_later(struct mail_index *index, - enum mail_data_field field) -{ - i_assert(index->lock_type != MAIL_LOCK_UNLOCK); - - field &= ~index->never_cache_fields; - - /* first check if the field even could be in the file */ - if ((index->set_cache_fields & field) != field) { - if ((index->header->cache_fields & field) == 0) { - /* no, but make sure the future records will have it. - we don't immediately mark the index to cache this - field for old messages as some clients never ask - the info again */ - index->set_cache_fields |= field; - } else { - /* this is at least the second time it's being asked, - make sure it'll be cached soon. */ - index->set_flags |= MAIL_INDEX_FLAG_CACHE_FIELDS; - } - } -} - -time_t mail_get_internal_date(struct mail_index *index, - struct mail_index_record *rec) -{ - const time_t *date; - size_t size; - - date = index->lookup_field_raw(index, rec, - DATA_HDR_INTERNAL_DATE, &size); - if (date == NULL) - return (time_t)-1; - else { - i_assert(size == sizeof(*date)); - return *date; - } -} - void mail_index_mark_flag_changes(struct mail_index *index, struct mail_index_record *rec, enum mail_flags old_flags, @@ -691,8 +577,8 @@ } if (index->header->messages_count == 0) { - /* all mail was deleted, truncate data file */ - if (!mail_index_data_reset(index->data)) + /* all mail was deleted, truncate cache file */ + if (!mail_cache_truncate(index->cache)) return FALSE; } @@ -759,12 +645,19 @@ return TRUE; } -struct mail_index_record *mail_index_append_begin(struct mail_index *index) +struct mail_index_record *mail_index_append(struct mail_index *index) { struct mail_index_record *rec; i_assert(index->lock_type == MAIL_LOCK_EXCLUSIVE); + if (index->header->next_uid == MAX_ALLOWED_UID) { + index->set_flags |= MAIL_INDEX_HDR_FLAG_REBUILD; + index_set_error(index, "Reached maximum UID in mailbox %s, " + "rebuilding index", index->filepath); + return NULL; + } + if (index->mmap_used_length == index->mmap_full_length) { if (!mail_index_grow(index)) return NULL; @@ -774,50 +667,19 @@ i_assert(index->mmap_used_length + sizeof(struct mail_index_record) <= index->mmap_full_length); + index->header->messages_count++; + rec = (struct mail_index_record *) ((char *) index->mmap_base + index->mmap_used_length); - memset(rec, 0, sizeof(struct mail_index_record)); + rec->uid = index->header->next_uid++; + rec->msg_flags = 0; - index->header->used_file_size += sizeof(struct mail_index_record); - index->mmap_used_length += sizeof(struct mail_index_record); + index->header->used_file_size += sizeof(*rec); + index->mmap_used_length += sizeof(*rec); return rec; } -int mail_index_append_end(struct mail_index *index, - struct mail_index_record *rec) -{ - i_assert(rec->uid == 0); - - if (index->header->next_uid == MAX_ALLOWED_UID) { - index->set_flags |= MAIL_INDEX_FLAG_REBUILD; - index_set_error(index, "Reached maximum UID in mailbox %s, " - "rebuilding index", index->filepath); - return FALSE; - } - - index->header->messages_count++; - rec->uid = index->header->next_uid++; - - return TRUE; -} - -void mail_index_append_abort(struct mail_index *index, - struct mail_index_record *rec) -{ - i_assert(rec->uid == 0); - - if (INDEX_FILE_POSITION(index, rec) == - index->header->used_file_size - sizeof(*rec)) { - /* we can just rollback */ - index->header->used_file_size -= sizeof(*rec); - index->mmap_used_length -= sizeof(*rec); - } else { - /* mark it deleted */ - (void)mail_index_expunge_record_range(index, rec, rec); - } -} - enum mail_index_error mail_index_get_last_error(struct mail_index *index) { if (index->inconsistent)
--- a/src/lib-index/mail-index.h Wed Aug 06 23:13:57 2003 +0300 +++ b/src/lib-index/mail-index.h Wed Aug 06 23:15:30 2003 +0300 @@ -1,11 +1,13 @@ #ifndef __MAIL_INDEX_H #define __MAIL_INDEX_H +#include "byteorder.h" #include "file-dotlock.h" #include "message-parser.h" #include "imap-util.h" -#define MAIL_INDEX_VERSION 3 +#define MAIL_INDEX_MAJOR_VERSION 3 +#define MAIL_INDEX_MINOR_VERSION 0 #define INDEX_FILE_PREFIX ".imap.index" @@ -31,57 +33,34 @@ /* Rebuild flag is set while index is being rebuilt or when some error is noticed in the index file. If this flag is set, the index shouldn't be used before rebuilding it. */ - MAIL_INDEX_FLAG_REBUILD = 0x0001, - MAIL_INDEX_FLAG_FSCK = 0x0002, - MAIL_INDEX_FLAG_CACHE_FIELDS = 0x0004, - MAIL_INDEX_FLAG_COMPRESS = 0x0008, - MAIL_INDEX_FLAG_COMPRESS_DATA = 0x0010, - MAIL_INDEX_FLAG_DIRTY_MESSAGES = 0x0040, - MAIL_INDEX_FLAG_DIRTY_CUSTOMFLAGS = 0x0080, - MAIL_INDEX_FLAG_MAILDIR_NEW = 0x0100 + MAIL_INDEX_HDR_FLAG_FSCK = NBO32_BIT0, + MAIL_INDEX_HDR_FLAG_REBUILD = NBO32_BIT1, + MAIL_INDEX_HDR_FLAG_COMPRESS = NBO32_BIT2, + MAIL_INDEX_HDR_FLAG_COMPRESS_CACHE = NBO32_BIT3, + MAIL_INDEX_HDR_FLAG_DIRTY_MESSAGES = NBO32_BIT4, + MAIL_INDEX_HDR_FLAG_DIRTY_CUSTOMFLAGS = NBO32_BIT5, + MAIL_INDEX_HDR_FLAG_MAILDIR_NEW = NBO32_BIT6 }; -enum mail_data_field { - DATA_FIELD_LOCATION = 0x00000001, - DATA_FIELD_ENVELOPE = 0x00000002, - DATA_FIELD_BODY = 0x00000004, - DATA_FIELD_BODYSTRUCTURE = 0x00000008, - DATA_FIELD_MD5 = 0x00000010, - DATA_FIELD_MESSAGEPART = 0x00000020, - - DATA_FIELD_LAST = 0x00000040, - DATA_FIELD_MAX_BITS = 6, - - /* separate from above, but in same bitmask */ - DATA_HDR_INTERNAL_DATE = 0x40000000, - DATA_HDR_VIRTUAL_SIZE = 0x20000000, - DATA_HDR_HEADER_SIZE = 0x10000000, - DATA_HDR_BODY_SIZE = 0x08000000 -}; - -#define IS_BODYSTRUCTURE_FIELD(field) \ - (((field) & (DATA_FIELD_BODY | DATA_FIELD_BODYSTRUCTURE | \ - DATA_FIELD_MESSAGEPART)) != 0) - -enum mail_index_mail_flag { +enum mail_index_record_flag { /* If binary flags are set, it's not checked whether mail is missing CRs. So this flag may be set as an optimization for regular non-binary mails as well if it's known that it contains valid CR+LF line breaks. */ - INDEX_MAIL_FLAG_BINARY_HEADER = 0x0001, - INDEX_MAIL_FLAG_BINARY_BODY = 0x0002, + MAIL_INDEX_FLAG_BINARY_HEADER = NBO32_BIT0, + MAIL_INDEX_FLAG_BINARY_BODY = NBO32_BIT1, /* Mail flags have been changed in index, but not written into actual mailbox yet. */ - INDEX_MAIL_FLAG_DIRTY = 0x0004, + MAIL_INDEX_FLAG_DIRTY = NBO32_BIT2, /* Maildir: Mail file is in new/ dir instead of cur/ */ - INDEX_MAIL_FLAG_MAILDIR_NEW = 0x0008, + MAIL_INDEX_FLAG_MAILDIR_NEW = NBO32_BIT3, /* Mail header or body is known to contain NUL characters. */ - INDEX_MAIL_FLAG_HAS_NULS = 0x0010, + MAIL_INDEX_FLAG_HAS_NULS = NBO32_BIT4, /* Mail header or body is known to not contain NUL characters. */ - INDEX_MAIL_FLAG_HAS_NO_NULS = 0x0020 + MAIL_INDEX_FLAG_HAS_NO_NULS = NBO32_BIT5 }; enum mail_lock_type { @@ -122,75 +101,41 @@ unsigned int secs_left, void *context); struct mail_index_header { - unsigned char compat_data[8]; - /* 0 = version - 1 = flags, - 2 = sizeof(unsigned int), - 3 = sizeof(time_t), - 4 = sizeof(uoff_t), - 5 = INDEX_ALIGN_SIZE */ + /* major version is increased only when you can't have backwards + compatibility. minor version is increased when header size is + increased to contain new non-critical fields. */ + uint8_t major_version; + uint8_t minor_version; + uint8_t header_size; + uint8_t reserved; - unsigned int indexid; - unsigned int sync_id; /* re-mmap() when changed, required only - if file size is changed */ + uint32_t indexid; - unsigned int flags; - unsigned int cache_fields; + uint32_t used_file_size; + uint32_t sync_id; /* re-mmap() when changed, required only + if file size is shrinked */ - uoff_t used_file_size; + uint32_t flags; - unsigned int uid_validity; - unsigned int next_uid; + uint32_t uid_validity; + uint32_t next_uid; - unsigned int messages_count; - unsigned int seen_messages_count; - unsigned int deleted_messages_count; - unsigned int last_nonrecent_uid; + uint32_t messages_count; + uint32_t seen_messages_count; + uint32_t deleted_messages_count; + uint32_t last_nonrecent_uid; /* these UIDs may not exist and may not even be unseen */ - unsigned int first_unseen_uid_lowwater; - unsigned int first_deleted_uid_lowwater; -}; - -struct mail_index_data_header { - unsigned int indexid; - unsigned int reserved; /* for alignment mostly */ - - uoff_t used_file_size; - uoff_t deleted_space; + uint32_t first_unseen_uid_lowwater; + uint32_t first_deleted_uid_lowwater; }; struct mail_index_record { - unsigned int uid; - unsigned int msg_flags; - unsigned int data_position; /* first bit must be 0 */ - - unsigned int index_flags; /* enum mail_index_mail_flag */ - unsigned int data_fields; /* enum mail_data_field */ + uint32_t uid; + uint32_t msg_flags; + uint32_t data_offset; }; -struct mail_index_data_record_header { - unsigned int data_size; /* including this header */ - - time_t internal_date; - uoff_t virtual_size; - - uoff_t header_size; - uoff_t body_size; -}; - -struct mail_index_data_record { - unsigned int field; /* enum mail_data_field */ - unsigned int full_field_size; - char data[INDEX_ALIGN_SIZE]; /* variable size */ -}; - -#define SIZEOF_MAIL_INDEX_DATA \ - (sizeof(struct mail_index_data_record) - INDEX_ALIGN_SIZE) - -#define DATA_RECORD_SIZE(rec) \ - (SIZEOF_MAIL_INDEX_DATA + (rec)->full_field_size) - struct mail_index { /* Note that opening same index twice in the same process is a bad idea since they share the same file locks. As soon one of the @@ -274,34 +219,16 @@ unsigned int last_uid, unsigned int *seq_r); - /* Find field from specified record, or NULL if it's not in index. - Makes sure that the field ends with \0. */ - const char *(*lookup_field)(struct mail_index *index, - struct mail_index_record *rec, - enum mail_data_field field); - - /* Find field from specified record, or NULL if it's not in index. */ - const void *(*lookup_field_raw)(struct mail_index *index, - struct mail_index_record *rec, - enum mail_data_field field, - size_t *size); - - /* Mark the fields to be cached later. If any of them is already - set in hdr->cache_fields, mark the caching to happen next time - index is opened. */ - void (*cache_fields_later)(struct mail_index *index, - enum mail_data_field field); - /* Open mail file and return it as mmap()ed IStream. If we fail, we return NULL and set deleted = TRUE if failure was because the - mail was just deleted (ie. not an error). internal_date is set + mail was just deleted (ie. not an error). received_date is set if it's non-NULL. */ struct istream *(*open_mail)(struct mail_index *index, struct mail_index_record *rec, - time_t *internal_date, int *deleted); + time_t *received_date, int *deleted); - /* Returns internal date of message, or (time_t)-1 if error occured. */ - time_t (*get_internal_date)(struct mail_index *index, + /* Returns received date of message, or (time_t)-1 if error occured. */ + time_t (*get_received_date)(struct mail_index *index, struct mail_index_record *rec); /* Expunge mails from index. Modifylog is also updated. The @@ -327,41 +254,8 @@ int external_change); /* Append a new record to index. The index must be exclusively - locked before calling this function. rec->uid is updated in - append_end(). */ - struct mail_index_record *(*append_begin)(struct mail_index *index); - int (*append_end)(struct mail_index *index, - struct mail_index_record *rec); - void (*append_abort)(struct mail_index *index, - struct mail_index_record *rec); - - /* Updating fields happens by calling update_begin(), one or more - update_field()s and finally update_end() which does the actual - updating. The index must be exclusively locked all this time. - update_begin() and update_field() functions cannot fail. - - The extra_space parameter for update_field() specifies the amount - of extra empty space we should leave after the value, so that if - the field grows in future it could be expanded without copying it - to end of file. When the field already exists, the extra_space - is ignored. - - The files may not actually be updated until after you've unlocked - the file. */ - struct mail_index_update * - (*update_begin)(struct mail_index *index, - struct mail_index_record *rec); - int (*update_end)(struct mail_index_update *update); - void (*update_abort)(struct mail_index_update *update); - - void (*update_field)(struct mail_index_update *update, - enum mail_data_field field, - const char *value, size_t extra_space); - /* Just remember that full_field_size will be INDEX_ALIGNed, so - it may differer from the given size parameter. */ - void (*update_field_raw)(struct mail_index_update *update, - enum mail_data_field field, - const void *value, size_t size); + locked before calling this function. */ + struct mail_index_record *(*append)(struct mail_index *index); /* Returns the last error code. */ enum mail_index_error (*get_last_error)(struct mail_index *index); @@ -371,7 +265,7 @@ const char *(*get_last_error_text)(struct mail_index *index); /* private: */ - struct mail_index_data *data; + struct mail_cache *cache; struct mail_modify_log *modifylog; struct mail_custom_flags *custom_flags; @@ -379,7 +273,6 @@ char *filepath; /* index file path */ char *mailbox_path; /* file/directory for mailbox location */ char *control_dir; /* destination for control files */ - enum mail_data_field default_cache_fields, never_cache_fields; unsigned int indexid; unsigned int sync_id; @@ -418,6 +311,7 @@ size_t mmap_full_length; struct mail_index_header *header; + size_t header_size; enum mail_lock_type lock_type; time_t file_sync_stamp; @@ -429,7 +323,7 @@ /* these fields are OR'ed to the fields in index header once we get around grabbing exclusive lock */ unsigned int set_flags; - enum mail_data_field set_cache_fields; + unsigned int cache_later_locks; unsigned int anon_mmap:1; unsigned int mmap_invalidate:1; @@ -444,6 +338,7 @@ unsigned int mailbox_lock_timeout:1; unsigned int maildir_keep_new:1; unsigned int maildir_have_new:1; + unsigned int maildir_synced_once:1; }; #ifdef DEV_T_STRUCT @@ -456,7 +351,7 @@ members.. */ #define MAIL_INDEX_PRIVATE_FILL \ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, \ - 0, 0, 0, 0, 0, 0, { 0, 0, 0 }, 0, \ + 0, 0, 0, 0, { 0, 0, 0 }, 0, 0, 0, \ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, \ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, \ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, \ @@ -481,15 +376,6 @@ struct mail_index_record * mail_index_lookup_uid_range(struct mail_index *index, unsigned int first_uid, unsigned int last_uid, unsigned int *seq_r); -const char *mail_index_lookup_field(struct mail_index *index, - struct mail_index_record *rec, - enum mail_data_field field); -const void *mail_index_lookup_field_raw(struct mail_index *index, - struct mail_index_record *rec, - enum mail_data_field field, - size_t *size); -void mail_index_cache_fields_later(struct mail_index *index, - enum mail_data_field field); int mail_index_expunge(struct mail_index *index, struct mail_index_record *first_rec, struct mail_index_record *last_rec, @@ -499,46 +385,22 @@ struct mail_index_record *rec, unsigned int seq, enum mail_flags flags, int external_change); -struct mail_index_record *mail_index_append_begin(struct mail_index *index); -int mail_index_append_end(struct mail_index *index, - struct mail_index_record *rec); -void mail_index_append_abort(struct mail_index *index, - struct mail_index_record *rec); -struct mail_index_update * -mail_index_update_begin(struct mail_index *index, - struct mail_index_record *rec); -int mail_index_update_end(struct mail_index_update *update); -void mail_index_update_abort(struct mail_index_update *update); -void mail_index_update_field(struct mail_index_update *update, - enum mail_data_field field, - const char *value, size_t extra_space); -void mail_index_update_field_raw(struct mail_index_update *update, - enum mail_data_field field, - const void *value, size_t size); -time_t mail_get_internal_date(struct mail_index *index, - struct mail_index_record *rec); +struct mail_index_record *mail_index_append(struct mail_index *index); enum mail_index_error mail_index_get_last_error(struct mail_index *index); const char *mail_index_get_last_error_text(struct mail_index *index); /* INTERNAL: */ void mail_index_init(struct mail_index *index, const char *dir); int mail_index_mmap_update(struct mail_index *index); -void mail_index_init_header(struct mail_index *index, - struct mail_index_header *hdr); +void mail_index_init_header(struct mail_index_header *hdr); void mail_index_close(struct mail_index *index); int mail_index_fmdatasync(struct mail_index *index, size_t size); void mail_index_mark_flag_changes(struct mail_index *index, struct mail_index_record *rec, enum mail_flags old_flags, enum mail_flags new_flags); -void mail_index_update_headers(struct mail_index_update *update, - struct istream *input, - enum mail_data_field cache_fields, - message_header_callback_t *header_cb, - void *context); -int mail_index_update_cache(struct mail_index *index); +int mail_index_rebuild(struct mail_index *index); int mail_index_compress(struct mail_index *index); -int mail_index_compress_data(struct mail_index *index); int mail_index_truncate(struct mail_index *index); int mail_index_expunge_record_range(struct mail_index *index, struct mail_index_record *first_rec, @@ -577,7 +439,7 @@ /* record for given index */ #define INDEX_RECORD_AT(index, idx) \ ((struct mail_index_record *) \ - ((char *) index->mmap_base + sizeof(struct mail_index_header)) + (idx)) + ((char *) index->mmap_base + (index)->header_size) + (idx)) /* returns the next record after last one */ #define INDEX_END_RECORD(index) \ @@ -585,26 +447,28 @@ ((char *) (index)->mmap_base + (index)->mmap_used_length)) /* index number for uoff_t position */ -#define INDEX_POSITION_INDEX(pos) \ - (((pos) - sizeof(struct mail_index_header)) / \ +#define INDEX_POSITION_INDEX(index, pos) \ + (((pos) - (index)->header_size) / \ sizeof(struct mail_index_record)) /* index number for given record */ #define INDEX_RECORD_INDEX(index, ptr) \ - INDEX_POSITION_INDEX(INDEX_FILE_POSITION(index, ptr)) + INDEX_POSITION_INDEX(index, INDEX_FILE_POSITION(index, ptr)) /* mark the index corrupted */ #define INDEX_MARK_CORRUPTED(index) \ - STMT_START { (index)->set_flags |= MAIL_INDEX_FLAG_REBUILD; } STMT_END + STMT_START { \ + (index)->set_flags |= MAIL_INDEX_HDR_FLAG_REBUILD; \ + } STMT_END /* get number of records in mmaped index */ #define MAIL_INDEX_RECORD_COUNT(index) \ - ((index->mmap_used_length - sizeof(struct mail_index_header)) / \ + ((index->mmap_used_length - (index)->header_size) / \ sizeof(struct mail_index_record)) /* minimum size for index file */ -#define INDEX_FILE_MIN_SIZE \ - (sizeof(struct mail_index_header) + \ +#define INDEX_FILE_MIN_SIZE(index) \ + ((index)->header_size + \ INDEX_MIN_RECORDS_COUNT * sizeof(struct mail_index_record)) /* enum mail_lock_type to fcntl() lock type */
--- a/src/lib-index/maildir/Makefile.am Wed Aug 06 23:13:57 2003 +0300 +++ b/src/lib-index/maildir/Makefile.am Wed Aug 06 23:15:30 2003 +0300 @@ -12,7 +12,6 @@ maildir-clean.c \ maildir-expunge.c \ maildir-open.c \ - maildir-rebuild.c \ maildir-sync.c \ maildir-uidlist.c \ maildir-update-flags.c
--- a/src/lib-index/maildir/maildir-build.c Wed Aug 06 23:13:57 2003 +0300 +++ b/src/lib-index/maildir/maildir-build.c Wed Aug 06 23:15:30 2003 +0300 @@ -1,51 +1,27 @@ -/* Copyright (C) 2002 Timo Sirainen */ +/* Copyright (C) 2002-2003 Timo Sirainen */ #include "lib.h" -#include "istream.h" #include "maildir-index.h" -#include "mail-index-util.h" - -#include <unistd.h> -#include <fcntl.h> -#include <sys/stat.h> +#include "mail-cache.h" -static int maildir_record_update(struct mail_index *index, - struct mail_index_update *update, int fd) +int maildir_index_append_file(struct mail_cache_transaction_ctx **trans_ctx, + struct mail_index *index, const char *fname, + int new_dir) { - struct istream *input; - enum mail_data_field cache_fields; + struct mail_index_record *rec; + enum mail_index_record_flag index_flags; + uoff_t virtual_size; + const char *p; - if (index->mail_read_mmaped) { - input = i_stream_create_mmap(fd, system_pool, - MAIL_MMAP_BLOCK_SIZE, 0, 0, FALSE); - } else { - input = i_stream_create_file(fd, system_pool, - MAIL_READ_BLOCK_SIZE, FALSE); + i_assert(index->lock_type == MAIL_LOCK_EXCLUSIVE); + + if (*trans_ctx == NULL) { + if (mail_cache_transaction_begin(index->cache, + TRUE, trans_ctx) <= 0) + return FALSE; } - cache_fields = index->header->cache_fields & ~DATA_FIELD_LOCATION; - mail_index_update_headers(update, input, cache_fields, NULL, NULL); - - i_stream_unref(input); - return TRUE; -} - -static int maildir_index_append_fd(struct mail_index *index, - int fd, const char *fname, int new_dir) -{ - struct mail_index_record *rec; - struct mail_index_update *update; - struct stat st; - uoff_t virtual_size; - const char *p; - int failed; - - i_assert(fname != NULL); - - if (!index->set_lock(index, MAIL_LOCK_EXCLUSIVE)) - return FALSE; - - rec = index->append_begin(index); + rec = index->append(index); if (rec == NULL) return FALSE; @@ -53,7 +29,11 @@ rec->msg_flags = maildir_filename_get_flags(fname, 0); mail_index_mark_flag_changes(index, rec, 0, rec->msg_flags); - update = index->update_begin(index, rec); + /* always set index flags */ + index_flags = new_dir ? MAIL_INDEX_FLAG_MAILDIR_NEW : 0; + if (!mail_cache_add(*trans_ctx, rec, MAIL_CACHE_INDEX_FLAGS, + &index_flags, sizeof(index_flags))) + return FALSE; /* set virtual size if found from file name */ p = strstr(fname, ",W="); @@ -65,66 +45,19 @@ p++; } - if (*p == ':' || *p == ',' || *p == '\0') { - index->update_field_raw(update, DATA_HDR_VIRTUAL_SIZE, - &virtual_size, - sizeof(virtual_size)); + if (*p == ':' || *p == ',' || *p != '\0') { + if (!mail_cache_add(*trans_ctx, rec, + MAIL_CACHE_VIRTUAL_FULL_SIZE, + &virtual_size, + sizeof(virtual_size))) + return FALSE; } } - /* set internal date */ - if (fd != -1 && fstat(fd, &st) == 0) { - index->update_field_raw(update, DATA_HDR_INTERNAL_DATE, - &st.st_mtime, sizeof(st.st_mtime)); - } - - /* set the location */ - if (new_dir) - rec->index_flags |= INDEX_MAIL_FLAG_MAILDIR_NEW; - index->update_field(update, DATA_FIELD_LOCATION, fname, - MAILDIR_LOCATION_EXTRA_SPACE); - - /* parse the header and update record's fields */ - failed = fd == -1 ? FALSE : !maildir_record_update(index, update, fd); - - if (!index->update_end(update) || failed) { - index->append_abort(index, rec); + /* always set location */ + if (!mail_cache_add(*trans_ctx, rec, MAIL_CACHE_LOCATION, + fname, strlen(fname)+1)) return FALSE; - } - - return index->append_end(index, rec); -} -int maildir_index_append_file(struct mail_index *index, const char *dir, - const char *fname, int new_dir) -{ - const char *path; - int fd, ret; - - i_assert(index->lock_type != MAIL_LOCK_SHARED); - - if ((index->header->cache_fields & ~DATA_FIELD_LOCATION) == 0) { - /* nothing cached, don't bother opening the file */ - return maildir_index_append_fd(index, -1, fname, new_dir); - } - - path = t_strconcat(dir, "/", fname, NULL); - fd = open(path, O_RDONLY); - if (fd == -1) { - if (errno == ENOENT) { - /* it's not found because it's deleted or renamed. - don't try to handle any error cases here, just - save the thing and let the syncing handle it - later */ - return maildir_index_append_fd(index, -1, - fname, new_dir); - } - - return index_file_set_syscall_error(index, path, "open()"); - } - - ret = maildir_index_append_fd(index, fd, fname, new_dir); - if (close(fd) < 0) - return index_file_set_syscall_error(index, path, "close()"); - return ret; + return TRUE; }
--- a/src/lib-index/maildir/maildir-expunge.c Wed Aug 06 23:13:57 2003 +0300 +++ b/src/lib-index/maildir/maildir-expunge.c Wed Aug 06 23:15:30 2003 +0300 @@ -12,8 +12,9 @@ const char **fname) { const char *path; + int new_dir; - *fname = maildir_get_location(index, rec); + *fname = maildir_get_location(index, rec, &new_dir); if (*fname == NULL) return -1; @@ -23,7 +24,7 @@ if (index->next_dirty_flush != 0) index->next_dirty_flush = ioloop_time; - if ((rec->index_flags & INDEX_MAIL_FLAG_MAILDIR_NEW) != 0) { + if (new_dir) { /* probably in new/ dir */ path = t_strconcat(index->mailbox_path, "/new/", *fname, NULL); if (unlink(path) == 0)
--- a/src/lib-index/maildir/maildir-index.c Wed Aug 06 23:13:57 2003 +0300 +++ b/src/lib-index/maildir/maildir-index.c Wed Aug 06 23:15:30 2003 +0300 @@ -6,8 +6,8 @@ #include "hostpid.h" #include "str.h" #include "maildir-index.h" -#include "mail-index-data.h" #include "mail-index-util.h" +#include "mail-cache.h" #include <stdio.h> #include <sys/stat.h> @@ -24,20 +24,39 @@ } const char *maildir_get_location(struct mail_index *index, - struct mail_index_record *rec) + struct mail_index_record *rec, int *new_dir) { const char *fname, *new_fname; - fname = index->lookup_field(index, rec, DATA_FIELD_LOCATION); - if (fname == NULL) { - index_data_set_corrupted(index->data, - "Missing location field for record %u", rec->uid); + if (new_dir != NULL) + *new_dir = FALSE; + + if (index->new_filenames != NULL) { + /* this has the most up-to-date filename */ + new_fname = hash_lookup(index->new_filenames, + POINTER_CAST(rec->uid)); + if (new_fname != NULL) { + if (*new_fname == '/') { + new_fname++; + if (new_dir != NULL) + *new_dir = TRUE; + } + return new_fname; + } } - if (index->new_filenames != NULL) { - new_fname = hash_lookup(index->new_filenames, fname); - if (new_fname != NULL) - return new_fname; + /* index file should give us at least the base name. */ + fname = mail_cache_lookup_string_field(index->cache, rec, + MAIL_CACHE_LOCATION); + if (fname == NULL) { + mail_cache_set_corrupted(index->cache, + "Missing location field for record %u", rec->uid); + return NULL; + } + + if (new_dir != NULL) { + *new_dir = (mail_cache_get_index_flags(index->cache, rec) & + MAIL_INDEX_FLAG_MAILDIR_NEW) != 0; } return fname; @@ -213,6 +232,31 @@ return str_c(flags_str); } +void maildir_index_update_filename(struct mail_index *index, unsigned int uid, + const char *fname, int new_dir) +{ + const char *new_fname, *old_fname; + + if (index->new_filename_pool == NULL) { + index->new_filename_pool = + pool_alloconly_create("Maildir filenames", 10240); + } + if (index->new_filenames == NULL) { + index->new_filenames = + hash_create(system_pool, index->new_filename_pool, 0, + NULL, NULL); + } + + t_push(); + new_fname = !new_dir ? fname : t_strconcat("/", fname, NULL); + old_fname = hash_lookup(index->new_filenames, POINTER_CAST(uid)); + if (old_fname == NULL || strcmp(old_fname, new_fname) != 0) { + hash_insert(index->new_filenames, POINTER_CAST(uid), + p_strdup(index->new_filename_pool, new_fname)); + } + t_pop(); +} + struct mail_index * maildir_index_alloc(const char *maildir, const char *index_dir, const char *control_dir) @@ -247,18 +291,19 @@ i_free(index); } -static int maildir_get_internal_date_file(struct mail_index *index, +static int maildir_get_received_date_file(struct mail_index *index, struct mail_index_record *rec, const char **fname, struct stat *st) { const char *path; + int new_dir; /* stat() gives it */ - *fname = maildir_get_location(index, rec); + *fname = maildir_get_location(index, rec, &new_dir); if (*fname == NULL) return -1; - if ((rec->index_flags & INDEX_MAIL_FLAG_MAILDIR_NEW) != 0) { + if (new_dir) { /* probably in new/ dir */ path = t_strconcat(index->mailbox_path, "/new/", *fname, NULL); if (stat(path, st) < 0 && errno != ENOENT) { @@ -279,7 +324,7 @@ return TRUE; } -static time_t maildir_get_internal_date(struct mail_index *index, +static time_t maildir_get_received_date(struct mail_index *index, struct mail_index_record *rec) { struct stat st; @@ -288,11 +333,12 @@ int ret, i, found; /* try getting it from cache */ - date = mail_get_internal_date(index, rec); - if (date != (time_t)-1) + if (mail_cache_copy_fixed_field(index->cache, rec, + MAIL_CACHE_RECEIVED_DATE, + &date, sizeof(date))) return date; - ret = maildir_get_internal_date_file(index, rec, &fname, &st); + ret = maildir_get_received_date_file(index, rec, &fname, &st); for (i = 0; ret == 0 && i < 10; i++) { /* file is either renamed or deleted. sync the maildir and see which one. if file appears to be renamed constantly, @@ -305,7 +351,7 @@ return (time_t)-1; } - ret = maildir_get_internal_date_file(index, rec, &fname, &st); + ret = maildir_get_received_date_file(index, rec, &fname, &st); } return st.st_mtime; @@ -317,28 +363,18 @@ mail_index_set_lock, mail_index_try_lock, mail_index_set_lock_notify_callback, - maildir_index_rebuild, + mail_index_rebuild, mail_index_fsck, maildir_index_sync, mail_index_get_header, mail_index_lookup, mail_index_next, mail_index_lookup_uid_range, - mail_index_lookup_field, - mail_index_lookup_field_raw, - mail_index_cache_fields_later, maildir_open_mail, - maildir_get_internal_date, + maildir_get_received_date, mail_index_expunge, maildir_index_update_flags, - mail_index_append_begin, - mail_index_append_end, - mail_index_append_abort, - mail_index_update_begin, - mail_index_update_end, - mail_index_update_abort, - mail_index_update_field, - mail_index_update_field_raw, + mail_index_append, mail_index_get_last_error, mail_index_get_last_error_text,
--- a/src/lib-index/maildir/maildir-index.h Wed Aug 06 23:13:57 2003 +0300 +++ b/src/lib-index/maildir/maildir-index.h Wed Aug 06 23:15:30 2003 +0300 @@ -1,13 +1,11 @@ #ifndef __MAILDIR_INDEX_H #define __MAILDIR_INDEX_H +struct mail_cache_transaction_ctx; + #include <sys/time.h> -#include <dirent.h> #include "mail-index.h" -/* ":2,DFRST" - leave the 2 extra for other clients' additions */ -#define MAILDIR_LOCATION_EXTRA_SPACE 10 - /* How often to try to flush dirty flags. */ #define MAILDIR_DIRTY_FLUSH_TIMEOUT (60*5) @@ -21,20 +19,22 @@ const char **path); const char *maildir_get_location(struct mail_index *index, - struct mail_index_record *rec); + struct mail_index_record *rec, int *new_dir); enum mail_flags maildir_filename_get_flags(const char *fname, enum mail_flags default_flags); const char *maildir_filename_set_flags(const char *fname, enum mail_flags flags); +void maildir_index_update_filename(struct mail_index *index, unsigned int uid, + const char *fname, int new_dir); -int maildir_index_rebuild(struct mail_index *index); int maildir_index_sync_readonly(struct mail_index *index, const char *fname, int *found); int maildir_index_sync(struct mail_index *index, int minimal_sync, enum mail_lock_type lock_type, int *changes); -int maildir_index_append_file(struct mail_index *index, const char *dir, - const char *fname, int new_dir); +int maildir_index_append_file(struct mail_cache_transaction_ctx **trans_ctx, + struct mail_index *index, const char *fname, + int new_dir); int maildir_index_update_flags(struct mail_index *index, struct mail_index_record *rec, unsigned int seq, enum mail_flags flags, int external_change); @@ -42,7 +42,7 @@ struct istream *maildir_open_mail(struct mail_index *index, struct mail_index_record *rec, - time_t *internal_date, int *deleted); + time_t *received_date, int *deleted); int maildir_expunge_mail(struct mail_index *index, struct mail_index_record *rec);
--- a/src/lib-index/maildir/maildir-open.c Wed Aug 06 23:13:57 2003 +0300 +++ b/src/lib-index/maildir/maildir-open.c Wed Aug 06 23:15:30 2003 +0300 @@ -3,8 +3,8 @@ #include "lib.h" #include "istream.h" #include "maildir-index.h" -#include "mail-index-data.h" #include "mail-index-util.h" +#include "mail-cache.h" #include <unistd.h> #include <fcntl.h> @@ -15,13 +15,13 @@ const char **fname, int *deleted) { const char *path; - int fd = -1; + int new_dir, fd = -1; - *fname = maildir_get_location(index, rec); + *fname = maildir_get_location(index, rec, &new_dir); if (*fname == NULL) return -1; - if ((rec->index_flags & INDEX_MAIL_FLAG_MAILDIR_NEW) != 0) { + if (new_dir) { /* probably in new/ dir */ path = t_strconcat(index->mailbox_path, "/new/", *fname, NULL); fd = open(path, O_RDONLY); @@ -50,7 +50,7 @@ struct istream *maildir_open_mail(struct mail_index *index, struct mail_index_record *rec, - time_t *internal_date, int *deleted) + time_t *received_date, int *deleted) { struct stat st; const char *fname; @@ -84,13 +84,9 @@ return NULL; } - if (internal_date != NULL) { - *internal_date = mail_get_internal_date(index, rec); - - if (*internal_date == (time_t)-1) { - if (fstat(fd, &st) == 0) - *internal_date = st.st_mtime; - } + if (received_date != NULL) { + if (fstat(fd, &st) == 0) + *received_date = st.st_mtime; } if (index->mail_read_mmaped) {
--- a/src/lib-index/maildir/maildir-rebuild.c Wed Aug 06 23:13:57 2003 +0300 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,48 +0,0 @@ -/* Copyright (C) 2002 Timo Sirainen */ - -#include "lib.h" -#include "maildir-index.h" -#include "mail-index-data.h" -#include "mail-index-util.h" - -#include <unistd.h> -#include <sys/stat.h> -#include <sys/mman.h> - -int maildir_index_rebuild(struct mail_index *index) -{ - if (!mail_index_set_lock(index, MAIL_LOCK_EXCLUSIVE)) - return FALSE; - - /* reset the header */ - mail_index_init_header(index, index->header); - index->mmap_used_length = index->header->used_file_size; - - /* require these fields */ - index->header->cache_fields |= DATA_FIELD_LOCATION; - - /* update indexid, which also means that our state has completely - changed */ - index->indexid = index->header->indexid; - index->inconsistent = TRUE; - index->rebuilding = TRUE; - - if (!index->anon_mmap) { - if (msync(index->mmap_base, - sizeof(struct mail_index_header), MS_SYNC) < 0) - return index_set_syscall_error(index, "msync()"); - } - - /* reset data file */ - if (!mail_index_data_reset(index->data)) - return FALSE; - - /* read the mails by syncing */ - if (!index->sync_and_lock(index, FALSE, MAIL_LOCK_UNLOCK, NULL)) - return FALSE; - - /* rebuild is complete - remove the flag */ - index->header->flags &= ~(MAIL_INDEX_FLAG_REBUILD|MAIL_INDEX_FLAG_FSCK); - index->rebuilding = FALSE; - return TRUE; -}
--- a/src/lib-index/maildir/maildir-sync.c Wed Aug 06 23:13:57 2003 +0300 +++ b/src/lib-index/maildir/maildir-sync.c Wed Aug 06 23:15:30 2003 +0300 @@ -186,8 +186,8 @@ #include "str.h" #include "maildir-index.h" #include "maildir-uidlist.h" -#include "mail-index-data.h" #include "mail-index-util.h" +#include "mail-cache.h" #include <stdio.h> #include <stdlib.h> @@ -202,7 +202,6 @@ enum maildir_file_action { MAILDIR_FILE_ACTION_EXPUNGE, MAILDIR_FILE_ACTION_UPDATE_FLAGS, - MAILDIR_FILE_ACTION_UPDATE_CONTENT, MAILDIR_FILE_ACTION_NEW, MAILDIR_FILE_ACTION_NONE, @@ -236,6 +235,7 @@ struct dirent *new_dent; struct maildir_uidlist *uidlist; + struct mail_cache_transaction_ctx *trans_ctx; unsigned int readonly_check:1; unsigned int flag_updates:1; unsigned int uidlist_rewrite:1; @@ -276,41 +276,6 @@ return *s1 - *s2; } -static void maildir_update_filename_memory(struct mail_index *index, - const char *fname) -{ - char *new_fname; - - if (index->new_filename_pool == NULL) { - index->new_filename_pool = - pool_alloconly_create("Maildir fname", 10240); - } - if (index->new_filenames == NULL) { - index->new_filenames = - hash_create(system_pool, index->new_filename_pool, 0, - maildir_hash, maildir_cmp); - } - - new_fname = p_strdup(index->new_filename_pool, fname); - hash_insert(index->new_filenames, new_fname, new_fname); -} - -static int maildir_update_filename(struct maildir_sync_context *ctx, - struct mail_index_record *rec, - const char *new_fname) -{ - struct mail_index_update *update; - - if (ctx->index->lock_type != MAIL_LOCK_EXCLUSIVE) { - maildir_update_filename_memory(ctx->index, new_fname); - return TRUE; - } - - update = ctx->index->update_begin(ctx->index, rec); - ctx->index->update_field(update, DATA_FIELD_LOCATION, new_fname, 0); - return ctx->index->update_end(update); -} - static int maildir_update_flags(struct maildir_sync_context *ctx, struct mail_index_record *rec, unsigned int seq, const char *new_fname) @@ -373,11 +338,6 @@ return FALSE; } - if (!index->rebuilding) { - index->set_flags |= MAIL_INDEX_FLAG_REBUILD; - return FALSE; - } - index->header->uid_validity = ctx->uidlist->uid_validity; i_assert(index->header->next_uid == 1); } @@ -393,47 +353,6 @@ return TRUE; } -static int is_file_content_changed(struct mail_index *index, - struct mail_index_record *rec, - const char *dir, const char *fname) -{ -#define DATA_HDR_SIZE (DATA_HDR_HEADER_SIZE | DATA_HDR_BODY_SIZE) - struct mail_index_data_record_header *data_hdr; - struct stat st; - const char *path; - - if ((rec->data_fields & DATA_HDR_INTERNAL_DATE) == 0 && - (rec->data_fields & DATA_HDR_SIZE) != DATA_HDR_SIZE) { - /* nothing in cache, we can't know if it's changed */ - return FALSE; - } - - t_push(); - path = t_strdup_printf("%s/%s", dir, fname); - - if (stat(path, &st) < 0) { - if (errno != ENOENT) - index_file_set_syscall_error(index, path, "stat()"); - t_pop(); - return FALSE; - } - t_pop(); - - data_hdr = mail_index_data_lookup_header(index->data, rec); - if (data_hdr == NULL) - return FALSE; - - if ((rec->data_fields & DATA_HDR_INTERNAL_DATE) != 0 && - st.st_mtime != data_hdr->internal_date) - return TRUE; - - if ((rec->data_fields & DATA_HDR_SIZE) == DATA_HDR_SIZE && - (uoff_t)st.st_size != data_hdr->body_size + data_hdr->header_size) - return TRUE; - - return FALSE; -} - static void uidlist_hash_get_filenames(void *key, void *value, void *context) { buffer_t *buf = context; @@ -464,6 +383,53 @@ return t1 < t2 ? -1 : t1 > t2 ? 1 : 0; } +static int maildir_full_sync_finish_new_mails(struct maildir_sync_context *ctx) +{ + const char *dir, **new_files; + buffer_t *buf; + unsigned int i; + int new_dir; + + ctx->uidlist_rewrite = TRUE; + + /* then there's the completely new mails. sort them by the filename + so we should get them to same order as they were created. */ + buf = buffer_create_static_hard(ctx->pool, + ctx->new_count * sizeof(const char *)); + hash_foreach(ctx->files, uidlist_hash_get_filenames, buf); + i_assert(buffer_get_used_size(buf) == + ctx->new_count * sizeof(const char *)); + + new_files = buffer_get_modifyable_data(buf, NULL); + qsort(new_files, ctx->new_count, sizeof(const char *), + maildir_time_cmp); + + if (!ctx->index->maildir_keep_new) { + dir = ctx->cur_dir; + new_dir = FALSE; + } else { + /* this is actually slightly wrong, because we don't really + know if some of the new messages are in cur/ already. + we could know that by saving it into buffer, but that'd + require extra memory. luckily it doesn't really matter if + we say it's in new/, but it's actually in cur/. we have + to deal with such case anyway since another client might + have just moved it. */ + dir = ctx->new_dir; + new_dir = TRUE; + ctx->index->maildir_have_new = TRUE; + } + + for (i = 0; i < ctx->new_count; i++) { + if (!maildir_index_append_file(&ctx->trans_ctx, ctx->index, + new_files[i], new_dir)) + return FALSE; + } + ctx->new_count = 0; + + return TRUE; +} + static int maildir_full_sync_finish(struct maildir_sync_context *ctx) { struct mail_index *index = ctx->index; @@ -472,11 +438,10 @@ struct maildir_hash_rec *hash_rec; struct maildir_uidlist_rec uid_rec; enum maildir_file_action action; - const char *fname, **new_files, *dir; + const char *fname, *dir; void *orig_key, *orig_value; - unsigned int seq, first_seq, last_seq, uid, last_uid, i, new_flag; + unsigned int seq, first_seq, last_seq, uid, last_uid, new_flag; int new_dir, skip_next; - buffer_t *buf; if (ctx->new_count > 0) { /* new mails, either they're already in uidlist or we have @@ -512,7 +477,7 @@ return FALSE; } - fname = maildir_get_location(index, rec); + fname = maildir_get_location(index, rec, NULL); if (fname == NULL) return FALSE; @@ -536,20 +501,16 @@ (ACTION(hash_rec) == MAILDIR_FILE_ACTION_UPDATE_FLAGS || ACTION(hash_rec) == MAILDIR_FILE_ACTION_NONE)) { /* it's UID has changed. shouldn't happen. */ - i_warning("UID changed for %s/%s: %u -> %u", - index->mailbox_path, fname, uid, uid_rec.uid); - hash_rec->action = MAILDIR_FILE_ACTION_UPDATE_CONTENT | - (hash_rec->action & MAILDIR_FILE_FLAGS); + index_set_corrupted(index, + "UID changed for %s/%s: %u -> %u", + index->mailbox_path, fname, + uid, uid_rec.uid); + return FALSE; } action = hash_rec != NULL ? ACTION(hash_rec) : MAILDIR_FILE_ACTION_NONE; switch (action) { - case MAILDIR_FILE_ACTION_UPDATE_CONTENT: - hash_rec->action = MAILDIR_FILE_ACTION_NEW | - (hash_rec->action & MAILDIR_FILE_FLAGS); - ctx->new_count++; - /* fall through */ case MAILDIR_FILE_ACTION_EXPUNGE: if (first_rec == NULL) { first_rec = rec; @@ -559,8 +520,10 @@ last_seq = seq; break; case MAILDIR_FILE_ACTION_UPDATE_FLAGS: - if (!maildir_update_filename(ctx, rec, orig_key)) - return FALSE; + new_dir = (hash_rec->action & + MAILDIR_FILE_FLAG_NEWDIR) != 0; + maildir_index_update_filename(index, rec->uid, + orig_key, new_dir); if (!maildir_update_flags(ctx, rec, seq, fname)) return FALSE; /* fall through */ @@ -651,8 +614,8 @@ ctx->index->maildir_have_new = TRUE; dir = new_flag != 0 ? ctx->new_dir : ctx->cur_dir; - if (!maildir_index_append_file(index, dir, orig_key, - new_flag != 0)) + if (!maildir_index_append_file(&ctx->trans_ctx, index, + orig_key, new_flag != 0)) return FALSE; } @@ -677,48 +640,13 @@ index->last_uidlist_mtime = st.st_mtime; } - if (ctx->new_count == 0 || !INDEX_IS_UIDLIST_LOCKED(index)) { - /* all done (or can't do it since we don't have lock) */ - return TRUE; - } - - ctx->uidlist_rewrite = TRUE; - - /* then there's the completely new mails. sort them by the filename - so we should get them to same order as they were created. */ - buf = buffer_create_static_hard(ctx->pool, - ctx->new_count * sizeof(const char *)); - hash_foreach(ctx->files, uidlist_hash_get_filenames, buf); - i_assert(buffer_get_used_size(buf) == - ctx->new_count * sizeof(const char *)); - - new_files = buffer_get_modifyable_data(buf, NULL); - qsort(new_files, ctx->new_count, sizeof(const char *), - maildir_time_cmp); + if (ctx->new_count > 0 && INDEX_IS_UIDLIST_LOCKED(index)) + maildir_full_sync_finish_new_mails(ctx); - if (!index->maildir_keep_new) { - dir = ctx->cur_dir; - new_dir = FALSE; - } else { - /* this is actually slightly wrong, because we don't really - know if some of the new messages are in cur/ already. - we could know that by saving it into buffer, but that'd - require extra memory. luckily it doesn't really matter if - we say it's in new/, but it's actually in cur/. we have - to deal with such case anyway since another client might - have just moved it. */ - dir = ctx->new_dir; - new_dir = TRUE; - ctx->index->maildir_have_new = TRUE; - } - - for (i = 0; i < ctx->new_count; i++) { - if (!maildir_index_append_file(index, dir, - new_files[i], new_dir)) - return FALSE; - } - - ctx->new_count = 0; + /* all done (or can't do it since we don't have lock) */ + ctx->index->maildir_synced_once = TRUE; + if (ctx->trans_ctx != NULL) + mail_cache_transaction_commit(ctx->trans_ctx); return TRUE; } @@ -730,12 +658,12 @@ struct maildir_hash_rec *hash_rec; const char *fname; size_t size; - int have_new; + int new_dir, have_new; - /* FIXME: kludge. we want to have pointers to data file, so we must - make sure that it's base address doesn't change. this call makes - sure it's fully mmaped in memory even when we begin */ - if (mail_index_data_get_mmaped(index->data, &size) == NULL) + /* kludge. we want to have pointers to data file, so we must make sure + that it's base address doesn't change. this call makes sure it's + fully mmaped in memory even when we begin */ + if (mail_cache_get_mmaped(index->cache, &size) == NULL) return FALSE; if (index->header->messages_count >= INT_MAX/32) { @@ -744,6 +672,19 @@ return FALSE; } + /* we're resyncing everything, so reset the filename hash */ + if (index->new_filenames != NULL) { + hash_destroy(index->new_filenames); + index->new_filenames = NULL; + } + + if (index->new_filename_pool != NULL) + p_clear(index->new_filename_pool); + + /* reset synced-flag too, just in case something fails and we don't + have up-to-date new_filenames */ + ctx->index->maildir_synced_once = FALSE; + /* read current messages in index into hash */ size = nearest_power(index->header->messages_count * sizeof(struct maildir_hash_rec) + 1024); @@ -757,15 +698,14 @@ rec = index->lookup(index, 1); while (rec != NULL) { - fname = maildir_get_location(index, rec); + fname = maildir_get_location(index, rec, &new_dir); if (fname == NULL) return FALSE; - if ((rec->index_flags & INDEX_MAIL_FLAG_MAILDIR_NEW) != 0) + if (new_dir) have_new = TRUE; - if (!only_new || - (rec->index_flags & INDEX_MAIL_FLAG_MAILDIR_NEW) != 0) { + if (!only_new || new_dir) { hash_rec = p_new(ctx->pool, struct maildir_hash_rec, 1); hash_rec->rec = rec; hash_rec->action = MAILDIR_FILE_ACTION_EXPUNGE; @@ -841,20 +781,14 @@ } static int maildir_full_sync_dir(struct maildir_sync_context *ctx, - const char *dir, int new_dir, - DIR *dirp, struct dirent *d) + int new_dir, DIR *dirp, struct dirent *d) { struct maildir_hash_rec *hash_rec; void *orig_key, *orig_value; - int check_content_changes, newflag; + int newflag; newflag = new_dir ? MAILDIR_FILE_FLAG_NEWDIR : 0; - /* Do we want to check changes in file contents? This slows down - things as we need to do extra stat() for all files. */ - check_content_changes = !ctx->readonly_check && - getenv("MAILDIR_CHECK_CONTENT_CHANGES") != NULL; - do { if (d->d_name[0] == '.') continue; @@ -889,25 +823,7 @@ continue; } - if (!new_dir && (hash_rec->rec->index_flags & - INDEX_MAIL_FLAG_MAILDIR_NEW) != 0 && - ctx->index->lock_type == MAIL_LOCK_EXCLUSIVE) { - /* mail was indexed in new/ but it has been - moved to cur/ later */ - hash_rec->rec->index_flags &= - ~INDEX_MAIL_FLAG_MAILDIR_NEW; - } - - if (check_content_changes && - is_file_content_changed(ctx->index, hash_rec->rec, - dir, d->d_name)) { - /* file content changed, treat it as new message */ - hash_rec->action = - MAILDIR_FILE_ACTION_UPDATE_CONTENT | newflag; - - hash_insert(ctx->files, p_strdup(ctx->pool, d->d_name), - hash_rec); - } else if (strcmp(orig_key, d->d_name) != 0) { + if (strcmp(orig_key, d->d_name) != 0) { hash_rec->action = MAILDIR_FILE_ACTION_UPDATE_FLAGS | newflag; @@ -969,8 +885,8 @@ } if (ctx->new_dent != NULL) { - if (!maildir_full_sync_dir(ctx, ctx->new_dir, TRUE, - ctx->new_dirp, ctx->new_dent)) + if (!maildir_full_sync_dir(ctx, TRUE, ctx->new_dirp, + ctx->new_dent)) return FALSE; ctx->new_dent = NULL; } @@ -981,8 +897,7 @@ "opendir()"); } - failed = !maildir_full_sync_dir(ctx, ctx->cur_dir, FALSE, - dirp, readdir(dirp)); + failed = !maildir_full_sync_dir(ctx, FALSE, dirp, readdir(dirp)); if (closedir(dirp) < 0) { return index_file_set_syscall_error(ctx->index, ctx->cur_dir, @@ -1000,8 +915,7 @@ if (!maildir_full_sync_init(ctx, TRUE)) return FALSE; - if (!maildir_full_sync_dir(ctx, ctx->new_dir, TRUE, - ctx->new_dirp, ctx->new_dent)) + if (!maildir_full_sync_dir(ctx, TRUE, ctx->new_dirp, ctx->new_dent)) return FALSE; ctx->new_dent = NULL; @@ -1098,8 +1012,8 @@ ctx->index->maildir_have_new = TRUE; t_push(); - if (!maildir_index_append_file(ctx->index, final_dir, - d->d_name, + if (!maildir_index_append_file(&ctx->trans_ctx, + ctx->index, d->d_name, !move_to_cur)) { t_pop(); return FALSE; @@ -1244,13 +1158,16 @@ void *orig_key, *orig_value; const char *fname; unsigned int seq; + int new_dir; - if (index->lock_type != MAIL_LOCK_EXCLUSIVE || !ctx->flag_updates) + if (!ctx->flag_updates) { + ctx->index->maildir_synced_once = TRUE; return TRUE; + } rec = index->lookup(index, 1); seq = 1; while (rec != NULL) { - fname = maildir_get_location(index, rec); + fname = maildir_get_location(index, rec, NULL); if (fname == NULL) return FALSE; @@ -1261,15 +1178,21 @@ if (hash_rec != NULL && ACTION(hash_rec) == MAILDIR_FILE_ACTION_UPDATE_FLAGS) { - if (!maildir_update_filename(ctx, rec, orig_key)) - return FALSE; - if (!maildir_update_flags(ctx, rec, seq, fname)) - return FALSE; + new_dir = (hash_rec->action & + MAILDIR_FILE_FLAG_NEWDIR) != 0; + maildir_index_update_filename(index, rec->uid, + orig_key, new_dir); + + if (index->lock_type == MAIL_LOCK_EXCLUSIVE) { + if (!maildir_update_flags(ctx, rec, seq, fname)) + return FALSE; + } } rec = index->next(index, rec); seq++; } + ctx->index->maildir_synced_once = TRUE; return TRUE; } @@ -1281,14 +1204,20 @@ i_assert(index->lock_type != MAIL_LOCK_UNLOCK); - if (stat(ctx->cur_dir, &st) < 0) { - index_file_set_syscall_error(index, ctx->cur_dir, "stat()"); - return FALSE; + if (!index->maildir_synced_once) { + /* we haven't synced yet in this session. do it */ + cur_changed = TRUE; + } else { + if (stat(ctx->cur_dir, &st) < 0) { + index_file_set_syscall_error(index, ctx->cur_dir, + "stat()"); + return FALSE; + } + + cur_changed = st.st_mtime != index->file_sync_stamp || + index->maildir_cur_dirty != 0; } - cur_changed = st.st_mtime != index->file_sync_stamp || - index->maildir_cur_dirty != 0; - if (!cur_changed) { if (!index->maildir_have_new) { /* no changes */ @@ -1325,6 +1254,9 @@ static void maildir_index_sync_deinit(struct maildir_sync_context *ctx) { + // FIXME: remove new flags from cache if needed + if (ctx->trans_ctx != NULL) + mail_cache_transaction_end(ctx->trans_ctx); if (ctx->uidlist != NULL) maildir_uidlist_close(ctx->uidlist); if (ctx->files != NULL) @@ -1347,14 +1279,6 @@ { struct maildir_sync_context *ctx; - if (index->new_filenames != NULL) { - hash_destroy(index->new_filenames); - index->new_filenames = NULL; - } - - if (index->new_filename_pool != NULL) - p_clear(index->new_filename_pool); - ctx = t_new(struct maildir_sync_context, 1); ctx->index = index; ctx->new_dir = t_strconcat(index->mailbox_path, "/new", NULL); @@ -1374,7 +1298,7 @@ ret = maildir_index_sync_context_readonly(ctx); - if (!ret) + if (!ret || ctx->files == NULL) *found = FALSE; else { hash_rec = hash_lookup(ctx->files, fname);
--- a/src/lib-index/maildir/maildir-uidlist.c Wed Aug 06 23:13:57 2003 +0300 +++ b/src/lib-index/maildir/maildir-uidlist.c Wed Aug 06 23:15:30 2003 +0300 @@ -155,7 +155,7 @@ rec = index->lookup(index, 1); while (rec != NULL) { - fname = maildir_get_location(index, rec); + fname = maildir_get_location(index, rec, NULL); if (fname == NULL) return FALSE;
--- a/src/lib-index/maildir/maildir-update-flags.c Wed Aug 06 23:13:57 2003 +0300 +++ b/src/lib-index/maildir/maildir-update-flags.c Wed Aug 06 23:15:30 2003 +0300 @@ -1,17 +1,55 @@ /* Copyright (C) 2002 Timo Sirainen */ #include "lib.h" +#include "hash.h" #include "ioloop.h" #include "maildir-index.h" #include "mail-index-util.h" +#include "mail-cache.h" #include <stdio.h> +static int update_filename(struct mail_index *index, + struct mail_index_record *rec) +{ + const char *old_fname, *old_path, *new_fname, *new_path; + enum mail_index_record_flag flags; + + old_fname = maildir_get_location(index, rec, NULL); + if (old_fname == NULL) + return -1; + + flags = mail_cache_get_index_flags(index->cache, rec); + + old_path = t_strconcat(index->mailbox_path, + (flags & MAIL_INDEX_FLAG_MAILDIR_NEW) != 0 ? + "/new/" : "/cur/", old_fname, NULL); + + new_fname = maildir_filename_set_flags(old_fname, rec->msg_flags); + new_path = t_strconcat(index->mailbox_path, "/cur/", new_fname, NULL); + + if (strcmp(old_path, new_path) == 0 || + rename(old_path, new_path) == 0) { + flags &= ~(MAIL_INDEX_FLAG_DIRTY | MAIL_INDEX_FLAG_MAILDIR_NEW); + if (!mail_cache_update_index_flags(index->cache, rec, flags)) + return -1; + return 1; + } else { + if (errno != ENOENT && errno != EACCES && + !ENOSPACE(errno)) { + index_set_error(index, + "rename(%s, %s) failed: %m", + old_path, new_path); + return -1; + } + return 0; + } +} + int maildir_try_flush_dirty_flags(struct mail_index *index, int force) { struct mail_index_record *rec; - const char *old_fname, *old_path, *new_fname, *new_path; - int flag, dirty = FALSE; + int ret, dirty = FALSE; if (index->next_dirty_flush == 0 || (ioloop_time < index->next_dirty_flush && !force)) @@ -20,44 +58,30 @@ if (!index->set_lock(index, MAIL_LOCK_EXCLUSIVE)) return FALSE; + ret = mail_cache_lock(index->cache, !force); + if (ret <= 0) + return ret == 0; + mail_cache_unlock_later(index->cache); + rec = index->lookup(index, 1); while (rec != NULL) { - if ((rec->index_flags & INDEX_MAIL_FLAG_DIRTY) != 0) { - old_fname = maildir_get_location(index, rec); - if (old_fname == NULL) - return FALSE; - - flag = (rec->index_flags & - INDEX_MAIL_FLAG_MAILDIR_NEW) != 0; - old_path = t_strconcat(index->mailbox_path, - flag ? "/new/" : "/cur/", - old_fname, NULL); - - new_fname = maildir_filename_set_flags(old_fname, - rec->msg_flags); - new_path = t_strconcat(index->mailbox_path, - "/cur/", new_fname, NULL); - - if (strcmp(old_path, new_path) == 0 || - rename(old_path, new_path) == 0) - rec->index_flags &= ~INDEX_MAIL_FLAG_DIRTY; - else { + if ((mail_cache_get_index_flags(index->cache, rec) & + MAIL_INDEX_FLAG_DIRTY) != 0) { + ret = update_filename(index, rec); + if (ret < 0) + break; + if (ret == 0) dirty = TRUE; - if (errno != ENOENT && errno != EACCES && - !ENOSPACE(errno)) { - index_set_error(index, - "rename(%s, %s) failed: %m", - old_path, new_path); - return FALSE; - } - } } rec = index->next(index, rec); } + if (ret < 0) + return FALSE; + if (!dirty) { - index->header->flags &= ~MAIL_INDEX_FLAG_DIRTY_MESSAGES; + index->header->flags &= ~MAIL_INDEX_HDR_FLAG_DIRTY_MESSAGES; index->next_dirty_flush = 0; } else { index->next_dirty_flush = @@ -88,13 +112,12 @@ return -1; } -static int maildir_rename_mail_file(struct mail_index *index, - struct mail_index_record *rec, +static int maildir_rename_mail_file(struct mail_index *index, int new_dir, const char *old_fname, const char *new_path) { const char *path; - if ((rec->index_flags & INDEX_MAIL_FLAG_MAILDIR_NEW) != 0) { + if (new_dir) { /* probably in new/ dir */ path = t_strconcat(index->mailbox_path, "/new/", old_fname, NULL); @@ -117,14 +140,15 @@ enum mail_flags flags, const char **new_fname_r) { const char *old_fname, *new_fname, *new_path; - int i, ret, found; + enum mail_index_record_flag index_flags; + int i, ret, found, new_dir; new_fname = new_path = NULL; i = 0; do { /* we need to update the flags in the file name */ - old_fname = maildir_get_location(index, rec); + old_fname = maildir_get_location(index, rec, &new_dir); if (old_fname == NULL) return FALSE; @@ -139,21 +163,17 @@ if (strcmp(old_fname, new_fname) == 0) ret = 1; else { - ret = maildir_rename_mail_file(index, rec, old_fname, - new_path); + ret = maildir_rename_mail_file(index, new_dir, + old_fname, new_path); if (ret == -1) return FALSE; if (ret == 1) { - if (index->maildir_keep_new && - (rec->index_flags & - INDEX_MAIL_FLAG_MAILDIR_NEW) != 0) { + if (index->maildir_keep_new && new_dir) { /* looks like we have some more space again, see if we could move mails from new/ to cur/ again */ index->maildir_keep_new = FALSE; - rec->index_flags &= - ~INDEX_MAIL_FLAG_MAILDIR_NEW; } /* cur/ was updated, set it dirty-synced */ @@ -173,15 +193,26 @@ i++; } while (i < 10 && ret == 0); - if (ret != 1) { - /* we couldn't actually rename() the file now. - leave it's flags dirty so they get changed later. */ - rec->index_flags |= INDEX_MAIL_FLAG_DIRTY; - index->header->flags |= MAIL_INDEX_FLAG_DIRTY_MESSAGES; - index->next_dirty_flush = - ioloop_time + MAILDIR_DIRTY_FLUSH_TIMEOUT; - *new_fname_r = NULL; + if (ret == 1) + return TRUE; + + /* we couldn't actually rename() the file now. + leave it's flags dirty so they get changed later. */ + index_flags = mail_cache_get_index_flags(index->cache, rec); + if ((index_flags & MAIL_INDEX_FLAG_DIRTY) == 0) { + if (mail_cache_lock(index->cache, FALSE) <= 0) + return FALSE; + mail_cache_unlock_later(index->cache); + + index_flags |= MAIL_INDEX_FLAG_DIRTY; + mail_cache_update_index_flags(index->cache, rec, index_flags); + + index->header->flags |= MAIL_INDEX_HDR_FLAG_DIRTY_MESSAGES; } + + index->next_dirty_flush = + ioloop_time + MAILDIR_DIRTY_FLUSH_TIMEOUT; + *new_fname_r = NULL; return TRUE; } @@ -189,7 +220,6 @@ struct mail_index_record *rec, unsigned int seq, enum mail_flags flags, int external_change) { - struct mail_index_update *update; const char *new_fname; int failed = FALSE; @@ -200,11 +230,8 @@ } if (new_fname != NULL) { - /* update the filename in index */ - update = index->update_begin(index, rec); - index->update_field(update, DATA_FIELD_LOCATION, new_fname, 0); - if (!index->update_end(update)) - failed = TRUE; + maildir_index_update_filename(index, rec->uid, + new_fname, FALSE); } if (!failed && !mail_index_update_flags(index, rec, seq, flags,
--- a/src/lib-index/mbox/Makefile.am Wed Aug 06 23:13:57 2003 +0300 +++ b/src/lib-index/mbox/Makefile.am Wed Aug 06 23:15:30 2003 +0300 @@ -12,7 +12,6 @@ mbox-index.c \ mbox-lock.c \ mbox-open.c \ - mbox-rebuild.c \ mbox-rewrite.c \ mbox-sync.c \ mbox-sync-full.c
--- a/src/lib-index/mbox/mbox-append.c Wed Aug 06 23:13:57 2003 +0300 +++ b/src/lib-index/mbox/mbox-append.c Wed Aug 06 23:15:30 2003 +0300 @@ -7,19 +7,21 @@ #include "md5.h" #include "mbox-index.h" #include "mail-index-util.h" +#include "mail-cache.h" static int mbox_index_append_next(struct mail_index *index, + struct mail_index_record *rec, + struct mail_cache_transaction_ctx *trans_ctx, struct istream *input) { - struct mail_index_record *rec; - struct mail_index_update *update; struct mbox_header_context ctx; - time_t internal_date; + enum mail_index_record_flag index_flags; + time_t received_date; uoff_t abs_start_offset, eoh_offset; const unsigned char *data; unsigned char md5_digest[16]; size_t size, pos; - int ret, dirty; + int dirty; /* get the From-line */ pos = 0; @@ -40,14 +42,14 @@ index_set_error(index, "Error indexing mbox file %s: " "From-line not found where expected", index->mailbox_path); - index->set_flags |= MAIL_INDEX_FLAG_FSCK; + index->set_flags |= MAIL_INDEX_HDR_FLAG_FSCK; return -1; } /* parse the From-line */ - internal_date = mbox_from_parse_date(data + 5, size - 5); - if (internal_date == (time_t)-1) - internal_date = ioloop_time; + received_date = mbox_from_parse_date(data + 5, size - 5); + if (received_date == (time_t)-1) + received_date = ioloop_time; i_stream_skip(input, pos+1); abs_start_offset = input->start_offset + input->v_offset; @@ -57,19 +59,16 @@ mbox_skip_header(input); eoh_offset = input->v_offset; - /* add message to index */ - rec = index->append_begin(index); - if (rec == NULL) + index_flags = 0; + + if (!mail_cache_add(trans_ctx, rec, MAIL_CACHE_RECEIVED_DATE, + &received_date, sizeof(received_date))) return -1; - update = index->update_begin(index, rec); - - index->update_field_raw(update, DATA_HDR_INTERNAL_DATE, - &internal_date, sizeof(internal_date)); - - /* location = offset to beginning of headers in message */ - index->update_field_raw(update, DATA_FIELD_LOCATION, - &abs_start_offset, sizeof(uoff_t)); + /* location offset = beginning of headers in message */ + if (!mail_cache_add(trans_ctx, rec, MAIL_CACHE_LOCATION_OFFSET, + &abs_start_offset, sizeof(abs_start_offset))) + return -1; /* parse the header and cache wanted fields. get the message flags from Status and X-Status fields. temporarily limit the stream length @@ -84,12 +83,12 @@ i_stream_seek(input, abs_start_offset - input->start_offset); i_stream_set_read_limit(input, eoh_offset); - mail_index_update_headers(update, input, 0, mbox_header_cb, &ctx); + //FIXME:mail_index_update_headers(update, input, 0, mbox_header_cb, &ctx); i_stream_seek(input, input->v_limit); i_stream_set_read_limit(input, 0); - ret = 1; + dirty = ctx.content_length_broken; if (index->header->messages_count == 0 && ctx.uid_validity != index->header->uid_validity) { /* UID validity is different */ @@ -103,66 +102,53 @@ /* we have to write it to mbox */ if (index->mbox_lock_type != MAIL_LOCK_EXCLUSIVE) { /* try again */ - ret = 0; - } else { - index->header->flags |= - MAIL_INDEX_FLAG_DIRTY_MESSAGES; - rec->index_flags |= INDEX_MAIL_FLAG_DIRTY; + return 0; } + + dirty = TRUE; } } if (ctx.uid >= index->header->next_uid) { /* X-UID header looks ok */ - if (ret != 0) - index->header->next_uid = ctx.uid; - dirty = ctx.content_length_broken; + index->header->next_uid = ctx.uid; } else if (!index->mailbox_readonly) { /* Write X-UID for it */ dirty = TRUE; } else { /* save MD5 */ md5_final(&ctx.md5, md5_digest); - index->update_field_raw(update, DATA_FIELD_MD5, - md5_digest, sizeof(md5_digest)); - dirty = FALSE; + + if (!mail_cache_add(trans_ctx, rec, MAIL_CACHE_MD5, + md5_digest, sizeof(md5_digest))) + return -1; } if (dirty && !index->mailbox_readonly) { if (index->mbox_lock_type != MAIL_LOCK_EXCLUSIVE) { /* try again */ - ret = 0; - } else { - index->header->flags |= MAIL_INDEX_FLAG_DIRTY_MESSAGES; - rec->index_flags |= INDEX_MAIL_FLAG_DIRTY; + return 0; } + + index->header->flags |= MAIL_INDEX_HDR_FLAG_DIRTY_MESSAGES; + index_flags |= MAIL_INDEX_FLAG_DIRTY; } - if (ret <= 0) { - index->update_abort(update); - index->append_abort(index, rec); - } else { - if (!index->update_end(update)) { - index->append_abort(index, rec); - ret = -1; - } else { - /* save message flags */ - rec->msg_flags = ctx.flags; - mail_index_mark_flag_changes(index, rec, 0, - rec->msg_flags); - ret = 1; + /* save message flags */ + rec->msg_flags = ctx.flags; + mail_index_mark_flag_changes(index, rec, 0, rec->msg_flags); - if (!index->append_end(index, rec)) - ret = -1; - } - } + if (!mail_cache_add(trans_ctx, rec, MAIL_CACHE_INDEX_FLAGS, + &index_flags, sizeof(index_flags))) + return -1; - mbox_header_free_context(&ctx); - return ret; + return 1; } -int mbox_index_append(struct mail_index *index, struct istream *input) +int mbox_index_append_stream(struct mail_index *index, struct istream *input) { + struct mail_cache_transaction_ctx *trans_ctx; + struct mail_index_record *rec; uoff_t offset; int ret; @@ -174,6 +160,9 @@ if (!index->set_lock(index, MAIL_LOCK_EXCLUSIVE)) return FALSE; + if (mail_cache_transaction_begin(index->cache, TRUE, &trans_ctx) <= 0) + return FALSE; + do { offset = input->v_offset; if (input->start_offset + input->v_offset != 0) { @@ -185,8 +174,9 @@ "LF not found where expected", index->mailbox_path); - index->set_flags |= MAIL_INDEX_FLAG_FSCK; - return FALSE; + index->set_flags |= MAIL_INDEX_HDR_FLAG_FSCK; + ret = -1; + break; } } @@ -195,8 +185,15 @@ break; } + /* add message to index */ + rec = index->append(index); + if (rec == NULL) { + ret = -1; + break; + } + t_push(); - ret = mbox_index_append_next(index, input); + ret = mbox_index_append_next(index, rec, trans_ctx, input); t_pop(); if (ret == 0) { @@ -206,10 +203,18 @@ } } while (ret > 0); - if (index->mbox_lock_type == MAIL_LOCK_EXCLUSIVE) { + if (ret >= 0 && index->mbox_lock_type == MAIL_LOCK_EXCLUSIVE) { /* Write missing X-IMAPbase and new/changed X-UID headers */ - return mbox_index_rewrite(index); + if (!mbox_index_rewrite(index)) + ret = -1; } + if (ret >= 0) { + if (!mail_cache_transaction_commit(trans_ctx)) + ret = -1; + } + if (!mail_cache_transaction_end(trans_ctx)) + ret = -1; + return ret >= 0; }
--- a/src/lib-index/mbox/mbox-index.c Wed Aug 06 23:13:57 2003 +0300 +++ b/src/lib-index/mbox/mbox-index.c Wed Aug 06 23:15:30 2003 +0300 @@ -6,8 +6,8 @@ #include "mbox-index.h" #include "mbox-lock.h" #include "mail-index-util.h" -#include "mail-index-data.h" #include "mail-custom-flags.h" +#include "mail-cache.h" #include <stdlib.h> #include <unistd.h> @@ -138,10 +138,6 @@ ctx->content_length = (uoff_t)-1; } -void mbox_header_free_context(struct mbox_header_context *ctx __attr_unused__) -{ -} - static enum mail_flags mbox_get_status_flags(const unsigned char *value, size_t len) { @@ -694,52 +690,36 @@ struct mail_index_record *rec, uoff_t *offset, uoff_t *hdr_size, uoff_t *body_size) { - struct mail_index_data_record_header *data_hdr; - const uoff_t *location; - size_t size; - if (offset != NULL) { - location = index->lookup_field_raw(index, rec, - DATA_FIELD_LOCATION, &size); - if (location == NULL) { - index_data_set_corrupted(index->data, + if (!mail_cache_copy_fixed_field(index->cache, rec, + MAIL_CACHE_LOCATION_OFFSET, + offset, sizeof(*offset))) { + mail_cache_set_corrupted(index->cache, "Missing location field for record %u", rec->uid); return FALSE; - } else if (size != sizeof(uoff_t) || *location > OFF_T_MAX) { - index_data_set_corrupted(index->data, - "Invalid location field for record %u", - rec->uid); - return FALSE; } - - *offset = *location; } - if (hdr_size != NULL || body_size != NULL) { - data_hdr = mail_index_data_lookup_header(index->data, rec); - if (data_hdr == NULL) { - index_set_corrupted(index, - "Missing data header for record %u", rec->uid); - return FALSE; - } - - if ((rec->data_fields & DATA_HDR_HEADER_SIZE) == 0) { - index_set_corrupted(index, + if (hdr_size != NULL) { + if (!mail_cache_copy_fixed_field(index->cache, rec, + MAIL_CACHE_HEADER_SIZE, + hdr_size, sizeof(*hdr_size))) { + mail_cache_set_corrupted(index->cache, "Missing header size for record %u", rec->uid); return FALSE; } + } - if ((rec->data_fields & DATA_HDR_BODY_SIZE) == 0) { - index_set_corrupted(index, + if (body_size != NULL) { + if (!mail_cache_copy_fixed_field(index->cache, rec, + MAIL_CACHE_BODY_SIZE, + body_size, + sizeof(*body_size))) { + mail_cache_set_corrupted(index->cache, "Missing body size for record %u", rec->uid); return FALSE; } - - if (hdr_size != NULL) - *hdr_size = data_hdr->header_size; - if (body_size != NULL) - *body_size = data_hdr->body_size; } return TRUE; @@ -805,8 +785,8 @@ if (first_seq == 1) { /* Our message containing X-IMAPbase was deleted. Get it back there. */ - index->header->flags |= MAIL_INDEX_FLAG_DIRTY_MESSAGES | - MAIL_INDEX_FLAG_DIRTY_CUSTOMFLAGS; + index->header->flags |= MAIL_INDEX_HDR_FLAG_DIRTY_MESSAGES | + MAIL_INDEX_HDR_FLAG_DIRTY_CUSTOMFLAGS; } return TRUE; } @@ -816,26 +796,51 @@ unsigned int seq, enum mail_flags flags, int external_change) { + enum mail_index_record_flag index_flags; + if (!mail_index_update_flags(index, rec, seq, flags, external_change)) return FALSE; if (!external_change) { - rec->index_flags |= INDEX_MAIL_FLAG_DIRTY; - index->header->flags |= MAIL_INDEX_FLAG_DIRTY_MESSAGES; + index_flags = mail_cache_get_index_flags(index->cache, rec); + if ((index_flags & MAIL_INDEX_FLAG_DIRTY) == 0) { + if (mail_cache_lock(index->cache, FALSE) <= 0) + return FALSE; + mail_cache_unlock_later(index->cache); + + index_flags |= MAIL_INDEX_FLAG_DIRTY; + mail_cache_update_index_flags(index->cache, rec, + index_flags); + + index->header->flags |= + MAIL_INDEX_HDR_FLAG_DIRTY_MESSAGES; + } } return TRUE; } -static int mbox_index_append_end(struct mail_index *index, - struct mail_index_record *rec) +static struct mail_index_record *mbox_index_append(struct mail_index *index) { - if (!mail_index_append_end(index, rec)) - return FALSE; + /* update last_uid in X-IMAPbase */ + index->header->flags |= MAIL_INDEX_HDR_FLAG_DIRTY_MESSAGES | + MAIL_INDEX_HDR_FLAG_DIRTY_CUSTOMFLAGS; + + return mail_index_append(index); +} - /* update last_uid in X-IMAPbase */ - index->header->flags |= MAIL_INDEX_FLAG_DIRTY_MESSAGES | - MAIL_INDEX_FLAG_DIRTY_CUSTOMFLAGS; - return TRUE; +static time_t mbox_get_received_date(struct mail_index *index, + struct mail_index_record *rec) +{ + time_t date; + + if (mail_cache_copy_fixed_field(index->cache, rec, + MAIL_CACHE_RECEIVED_DATE, + &date, sizeof(date))) + return date; + + mail_cache_set_corrupted(index->cache, + "Missing internal date for record %u", rec->uid); + return (time_t)-1; } struct mail_index mbox_index = { @@ -844,28 +849,18 @@ mbox_index_set_lock, mbox_index_try_lock, mail_index_set_lock_notify_callback, - mbox_index_rebuild, + mail_index_rebuild, mail_index_fsck, mbox_index_sync, mail_index_get_header, mail_index_lookup, mail_index_next, mail_index_lookup_uid_range, - mail_index_lookup_field, - mail_index_lookup_field_raw, - mail_index_cache_fields_later, mbox_open_mail, - mail_get_internal_date, + mbox_get_received_date, mbox_index_expunge, mbox_index_update_flags, - mail_index_append_begin, - mbox_index_append_end, - mail_index_append_abort, - mail_index_update_begin, - mail_index_update_end, - mail_index_update_abort, - mail_index_update_field, - mail_index_update_field_raw, + mbox_index_append, mail_index_get_last_error, mail_index_get_last_error_text,
--- a/src/lib-index/mbox/mbox-index.h Wed Aug 06 23:13:57 2003 +0300 +++ b/src/lib-index/mbox/mbox-index.h Wed Aug 06 23:15:30 2003 +0300 @@ -35,7 +35,6 @@ void mbox_header_init_context(struct mbox_header_context *ctx, struct mail_index *index, struct istream *input); -void mbox_header_free_context(struct mbox_header_context *ctx); void mbox_header_cb(struct message_part *part, struct message_header_line *hdr, void *context); void mbox_keywords_parse(const unsigned char *value, size_t len, @@ -50,20 +49,19 @@ int mbox_verify_end_of_body(struct istream *input, uoff_t end_offset); int mbox_mail_get_location(struct mail_index *index, struct mail_index_record *rec, - uoff_t *offset, uoff_t *hdr_size, uoff_t *body_size); + uoff_t *offset, uoff_t *hdr_size, uoff_t *full_size); struct mail_index * mbox_index_alloc(const char *mbox_path, const char *index_dir, const char *control_dir); -int mbox_index_rebuild(struct mail_index *index); int mbox_index_sync(struct mail_index *index, int minimal_sync, enum mail_lock_type lock_type, int *changes); int mbox_sync_full(struct mail_index *index); struct istream *mbox_open_mail(struct mail_index *index, struct mail_index_record *rec, - time_t *internal_date, int *deleted); + time_t *received_date, int *deleted); -int mbox_index_append(struct mail_index *index, struct istream *input); +int mbox_index_append_stream(struct mail_index *index, struct istream *input); time_t mbox_from_parse_date(const unsigned char *msg, size_t size); const char *mbox_from_create(const char *sender, time_t time);
--- a/src/lib-index/mbox/mbox-open.c Wed Aug 06 23:13:57 2003 +0300 +++ b/src/lib-index/mbox/mbox-open.c Wed Aug 06 23:15:30 2003 +0300 @@ -3,7 +3,6 @@ #include "lib.h" #include "istream.h" #include "mbox-index.h" -#include "mail-index-data.h" #include "mail-index-util.h" #include <stdlib.h> @@ -12,7 +11,7 @@ struct istream *mbox_open_mail(struct mail_index *index, struct mail_index_record *rec, - time_t *internal_date, int *deleted) + time_t *received_date, int *deleted) { struct istream *input; uoff_t offset, hdr_size, body_size; @@ -32,8 +31,8 @@ if (input == NULL) return NULL; - if (internal_date != NULL) - *internal_date = mail_get_internal_date(index, rec); + if (received_date != NULL) + *received_date = index->get_received_date(index, rec); i_assert(index->mbox_sync_counter == index->mbox_lock_counter);
--- a/src/lib-index/mbox/mbox-rebuild.c Wed Aug 06 23:13:57 2003 +0300 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,58 +0,0 @@ -/* Copyright (C) 2002 Timo Sirainen */ - -#include "lib.h" -#include "mbox-index.h" -#include "mail-index-data.h" -#include "mail-index-util.h" - -#include <unistd.h> -#include <sys/stat.h> -#include <sys/mman.h> - -int mbox_index_rebuild(struct mail_index *index) -{ - struct stat st; - - i_assert(index->lock_type != MAIL_LOCK_SHARED); - - if (!mail_index_set_lock(index, MAIL_LOCK_EXCLUSIVE)) - return FALSE; - - /* reset the header */ - mail_index_init_header(index, index->header); - index->mmap_used_length = index->header->used_file_size; - - /* require these fields */ - index->header->cache_fields |= DATA_FIELD_LOCATION | - DATA_FIELD_MESSAGEPART | DATA_FIELD_MD5; - - /* update indexid, which also means that our state has completely - changed */ - index->indexid = index->header->indexid; - if (index->opened) - index->inconsistent = TRUE; - - if (!index->anon_mmap) { - if (msync(index->mmap_base, - sizeof(struct mail_index_header), MS_SYNC) < 0) - return index_set_syscall_error(index, "msync()"); - } - - /* reset data file */ - if (!mail_index_data_reset(index->data)) - return FALSE; - - if (!mbox_sync_full(index)) - return FALSE; - - /* update sync stamp */ - if (stat(index->mailbox_path, &st) < 0) - return mbox_set_syscall_error(index, "fstat()"); - - index->file_sync_stamp = st.st_mtime; - - /* rebuild is complete - remove the flag */ - index->header->flags &= ~(MAIL_INDEX_FLAG_REBUILD|MAIL_INDEX_FLAG_FSCK); - index->set_flags &= ~MAIL_INDEX_FLAG_REBUILD; - return TRUE; -}
--- a/src/lib-index/mbox/mbox-rewrite.c Wed Aug 06 23:13:57 2003 +0300 +++ b/src/lib-index/mbox/mbox-rewrite.c Wed Aug 06 23:15:30 2003 +0300 @@ -11,6 +11,7 @@ #include "mbox-lock.h" #include "mail-index-util.h" #include "mail-custom-flags.h" +#include "mail-cache.h" #include <stdio.h> #include <stdlib.h> @@ -38,18 +39,25 @@ }; /* Remove dirty flag from all messages */ -static void reset_dirty_flags(struct mail_index *index) +static int reset_dirty_flags(struct mail_index *index) { struct mail_index_record *rec; + enum mail_index_record_flag index_flags; rec = index->lookup(index, 1); while (rec != NULL) { - rec->index_flags &= ~INDEX_MAIL_FLAG_DIRTY; + index_flags = mail_cache_get_index_flags(index->cache, rec); + if ((index_flags & MAIL_INDEX_FLAG_DIRTY) != 0) { + if (!mail_cache_update_index_flags(index->cache, rec, index_flags)) + return FALSE; + } + rec = index->next(index, rec); } - index->header->flags &= ~(MAIL_INDEX_FLAG_DIRTY_MESSAGES | - MAIL_INDEX_FLAG_DIRTY_CUSTOMFLAGS); + index->header->flags &= ~(MAIL_INDEX_HDR_FLAG_DIRTY_MESSAGES | + MAIL_INDEX_HDR_FLAG_DIRTY_CUSTOMFLAGS); + return TRUE; } static int mbox_write(struct mail_index *index, struct istream *input, @@ -553,7 +561,8 @@ } #define INDEX_DIRTY_FLAGS \ - (MAIL_INDEX_FLAG_DIRTY_MESSAGES | MAIL_INDEX_FLAG_DIRTY_CUSTOMFLAGS) + (MAIL_INDEX_HDR_FLAG_DIRTY_MESSAGES | \ + MAIL_INDEX_HDR_FLAG_DIRTY_CUSTOMFLAGS) int mbox_index_rewrite(struct mail_index *index) { @@ -567,7 +576,7 @@ uoff_t offset, hdr_size, body_size, dirty_offset, wanted_offset; const char *path; unsigned int seq; - int tmp_fd, failed, dirty_found, rewrite, no_locking; + int tmp_fd, failed, dirty, dirty_found, rewrite, no_locking; i_assert(!index->mailbox_readonly); i_assert(index->lock_type == MAIL_LOCK_UNLOCK || @@ -633,7 +642,7 @@ return !failed; } - if (index->header->flags & MAIL_INDEX_FLAG_DIRTY_CUSTOMFLAGS) { + if (index->header->flags & MAIL_INDEX_HDR_FLAG_DIRTY_CUSTOMFLAGS) { /* need to update X-IMAPbase in first message */ dirty_found = TRUE; } else { @@ -649,7 +658,14 @@ failed = FALSE; seq = 1; rec = index->lookup(index, 1); while (rec != NULL) { - if (dirty_found || (rec->index_flags & INDEX_MAIL_FLAG_DIRTY)) { + if (dirty_found) + dirty = FALSE; + else { + dirty = (mail_cache_get_index_flags(index->cache, rec) & + MAIL_INDEX_FLAG_DIRTY) != 0; + } + + if (dirty_found || dirty) { /* get offset to beginning of mail headers */ if (!mbox_mail_get_location(index, rec, &offset, &hdr_size, &body_size)) { @@ -659,30 +675,27 @@ } if (offset < input->v_offset) { - index_set_corrupted(index, - "Invalid message offset"); + mail_cache_set_corrupted(index->cache, + "Invalid message offset"); failed = TRUE; break; } if (offset + hdr_size + body_size > input->v_size) { - index_set_corrupted(index, - "Invalid message size"); + mail_cache_set_corrupted(index->cache, + "Invalid message size"); failed = TRUE; break; } - } - if (!dirty_found && - (rec->index_flags & INDEX_MAIL_FLAG_DIRTY)) { - /* first dirty message */ - dirty_found = TRUE; - dirty_offset = offset; + if (!dirty_found) { + /* first dirty message */ + dirty_found = TRUE; + dirty_offset = offset; - i_stream_seek(input, dirty_offset); - } + i_stream_seek(input, dirty_offset); + } - if (dirty_found) { /* write the From-line */ if (!mbox_write(index, input, output, offset)) { failed = TRUE; @@ -748,8 +761,10 @@ } } - if (!failed) - reset_dirty_flags(index); + if (!failed) { + if (!reset_dirty_flags(index)) + failed = TRUE; + } i_stream_unref(input); o_stream_unref(output);
--- a/src/lib-index/mbox/mbox-sync-full.c Wed Aug 06 23:13:57 2003 +0300 +++ b/src/lib-index/mbox/mbox-sync-full.c Wed Aug 06 23:15:30 2003 +0300 @@ -8,11 +8,14 @@ #include "mbox-index.h" #include "mbox-lock.h" #include "mail-index-util.h" +#include "mail-cache.h" #include <unistd.h> #include <fcntl.h> #include <sys/stat.h> +#if 0 + static void skip_line(struct istream *input) { const unsigned char *msg; @@ -34,21 +37,25 @@ struct mail_index_record *rec, unsigned int uid, unsigned char current_digest[16]) { - const unsigned char *old_digest; + const void *old_digest; size_t size; - /* MD5 sums must match */ - old_digest = index->lookup_field_raw(index, rec, DATA_FIELD_MD5, &size); - if (old_digest == NULL) + if (uid != 0) { + /* X-UID header - no need to check more */ return uid == rec->uid; + } - return size >= 16 && memcmp(old_digest, current_digest, 16) == 0 && - (uid == 0 || uid == rec->uid); + /* check if MD5 sums match */ + if (!mail_cache_lookup_field(index->cache, rec, MAIL_CACHE_MD5, + &old_digest, &size)) + return FALSE; + + return memcmp(old_digest, current_digest, 16) == 0; } static int mail_update_header_size(struct mail_index *index, struct mail_index_record *rec, - struct mail_index_update *update, + struct mail_cache_transaction_ctx *ctx, struct message_size *hdr_size) { const void *part_data; @@ -105,14 +112,14 @@ if (uid_validity == index->header->uid_validity) return TRUE; - index->header->flags |= MAIL_INDEX_FLAG_DIRTY_MESSAGES | - MAIL_INDEX_FLAG_DIRTY_CUSTOMFLAGS; + index->header->flags |= MAIL_INDEX_HDR_FLAG_DIRTY_MESSAGES | + MAIL_INDEX_HDR_FLAG_DIRTY_CUSTOMFLAGS; if (uid_validity == 0) { /* X-IMAPbase header isn't written yet */ } else { /* UID validity has changed - rebuild whole index */ - index->set_flags |= MAIL_INDEX_FLAG_REBUILD; + index->set_flags |= MAIL_INDEX_HDR_FLAG_REBUILD; return FALSE; } @@ -387,3 +394,9 @@ return !failed; } +#endif + +int mbox_sync_full(struct mail_index *index) +{ + // FIXME +}
--- a/src/lib-index/mbox/mbox-sync.c Wed Aug 06 23:13:57 2003 +0300 +++ b/src/lib-index/mbox/mbox-sync.c Wed Aug 06 23:15:30 2003 +0300 @@ -141,7 +141,8 @@ if (!mbox_lock_and_sync_full(index, data_lock_type)) return FALSE; - if ((index->set_flags & MAIL_INDEX_FLAG_REBUILD) != 0) { + if ((index->set_flags & + MAIL_INDEX_HDR_FLAG_REBUILD) != 0) { /* uidvalidity probably changed, rebuild */ if (!index->rebuild(index)) return FALSE;
--- a/src/lib-storage/index/index-expunge.c Wed Aug 06 23:13:57 2003 +0300 +++ b/src/lib-storage/index/index-expunge.c Wed Aug 06 23:15:30 2003 +0300 @@ -34,7 +34,7 @@ hdr->first_deleted_uid_lowwater); /* fsck should be enough to fix it */ - ibox->index->set_flags |= MAIL_INDEX_FLAG_FSCK; + ibox->index->set_flags |= MAIL_INDEX_HDR_FLAG_FSCK; return FALSE; } } else { @@ -144,7 +144,7 @@ ctx->mail.mail.seq = ctx->seq; ctx->mail.mail.uid = ctx->rec->uid; - if (!index_mail_next(&ctx->mail, ctx->rec, ctx->seq)) { + if (index_mail_next(&ctx->mail, ctx->rec, ctx->seq, FALSE) < 0) { ctx->failed = TRUE; return NULL; }
--- a/src/lib-storage/index/index-fetch.c Wed Aug 06 23:13:57 2003 +0300 +++ b/src/lib-storage/index/index-fetch.c Wed Aug 06 23:15:30 2003 +0300 @@ -23,6 +23,7 @@ struct mail_fetch_context * index_storage_fetch_init(struct mailbox *box, enum mail_fetch_field wanted_fields, + const char *const *wanted_headers, const char *messageset, int uidset) { struct index_mailbox *ibox = (struct index_mailbox *) box; @@ -41,7 +42,7 @@ ctx->ibox = ibox; ctx->index = ibox->index; - index_mail_init(ibox, &ctx->mail, wanted_fields, NULL); + index_mail_init(ibox, &ctx->mail, wanted_fields, wanted_headers); ctx->msgset_ctx = index_messageset_init(ibox, messageset, uidset, TRUE); return ctx; } @@ -55,12 +56,14 @@ if (all_found != NULL) *all_found = ret > 0; + if (ctx->ibox->fetch_mail.pool != NULL) + index_mail_deinit(&ctx->ibox->fetch_mail); + if (ctx->mail.pool != NULL) + index_mail_deinit(&ctx->mail); + if (!index_storage_lock(ctx->ibox, ctx->old_lock)) ret = -1; - if (ctx->ibox->fetch_mail.pool != NULL) - index_mail_deinit(&ctx->ibox->fetch_mail); - index_mail_deinit(&ctx->mail); i_free(ctx); return ret >= 0; } @@ -72,17 +75,25 @@ do { msgset_mail = index_messageset_next(ctx->msgset_ctx); - if (msgset_mail == NULL) - return NULL; + if (msgset_mail == NULL) { + ret = -1; + break; + } ctx->mail.mail.seq = msgset_mail->client_seq; ctx->mail.mail.uid = msgset_mail->rec->uid; ret = index_mail_next(&ctx->mail, msgset_mail->rec, - msgset_mail->idx_seq); + msgset_mail->idx_seq, FALSE); } while (ret == 0); - return ret < 0 ? NULL : &ctx->mail.mail; + if (ret < 0) { + /* error or last record */ + index_mail_deinit(&ctx->mail); + return NULL; + } + + return &ctx->mail.mail; } static struct mail * @@ -93,7 +104,7 @@ index_mail_deinit(&ibox->fetch_mail); index_mail_init(ibox, &ibox->fetch_mail, wanted_fields, NULL); - if (index_mail_next(&ibox->fetch_mail, rec, idx_seq) <= 0) + if (index_mail_next(&ibox->fetch_mail, rec, idx_seq, FALSE) <= 0) return NULL; return &ibox->fetch_mail.mail;
--- a/src/lib-storage/index/index-mail.c Wed Aug 06 23:13:57 2003 +0300 +++ b/src/lib-storage/index/index-mail.c Wed Aug 06 23:15:30 2003 +0300 @@ -1,6 +1,7 @@ /* Copyright (C) 2002 Timo Sirainen */ #include "lib.h" +#include "buffer.h" #include "istream.h" #include "str.h" #include "message-address.h" @@ -11,6 +12,7 @@ #include "mail-index.h" #include "mail-index-util.h" #include "mail-custom-flags.h" +#include "mail-cache.h" #include "index-storage.h" #include "index-expunge.h" #include "index-mail.h" @@ -24,19 +26,25 @@ const char *error; size_t part_size; - part_data = mail->ibox->index-> - lookup_field_raw(mail->ibox->index, mail->data.rec, - DATA_FIELD_MESSAGEPART, &part_size); - if (part_data == NULL) { - mail->ibox->index->cache_fields_later(mail->ibox->index, - DATA_FIELD_MESSAGEPART); + if ((mail->data.cached_fields & MAIL_CACHE_MESSAGEPART) == 0) { + mail_cache_mark_missing(mail->ibox->index->cache, + MAIL_CACHE_MESSAGEPART); + return NULL; + } + + // FIXME: for non-multipart messages we could build it + + if (!mail_cache_lookup_field(mail->ibox->index->cache, mail->data.rec, + MAIL_CACHE_MESSAGEPART, + &part_data, &part_size)) { + /* unexpected - must be an error */ return NULL; } part = message_part_deserialize(mail->pool, part_data, part_size, &error); if (part == NULL) { - index_set_corrupted(mail->ibox->index, + mail_cache_set_corrupted(mail->ibox->index->cache, "Corrupted cached message_part data (%s)", error); return NULL; } @@ -53,103 +61,188 @@ return part; } -static char *get_cached_field(struct index_mail *mail, - enum mail_data_field field) +static char *get_cached_string(struct index_mail *mail, + enum mail_cache_field field) { const char *ret; - ret = mail->ibox->index->lookup_field(mail->ibox->index, - mail->data.rec, field); - if (ret == NULL) - mail->ibox->index->cache_fields_later(mail->ibox->index, field); + if ((mail->data.cached_fields & field) == 0) { + mail_cache_mark_missing(mail->ibox->index->cache, field); + return NULL; + } + + ret = mail_cache_lookup_string_field(mail->ibox->index->cache, + mail->data.rec, field); return p_strdup(mail->pool, ret); } static uoff_t get_cached_uoff_t(struct index_mail *mail, - enum mail_data_field field, - const char *field_name) + enum mail_cache_field field) { - const uoff_t *uoff_p; - size_t size; + uoff_t uoff; - uoff_p = mail->ibox->index-> - lookup_field_raw(mail->ibox->index, mail->data.rec, - field, &size); - - if (uoff_p == NULL) - mail->ibox->index->cache_fields_later(mail->ibox->index, field); - else if (size != sizeof(*uoff_p)) { - index_set_corrupted(mail->ibox->index, - "Corrupted cached %s", field_name); - uoff_p = NULL; + if (!mail_cache_copy_fixed_field(mail->ibox->index->cache, + mail->data.rec, field, + &uoff, sizeof(uoff))) { + mail_cache_mark_missing(mail->ibox->index->cache, field); + uoff = (uoff_t)-1; } - return uoff_p == NULL ? (uoff_t)-1 : *uoff_p; + return uoff; } static uoff_t get_cached_virtual_size(struct index_mail *mail) { - return get_cached_uoff_t(mail, DATA_HDR_VIRTUAL_SIZE, "virtual size"); + return get_cached_uoff_t(mail, MAIL_CACHE_VIRTUAL_FULL_SIZE); } static time_t get_cached_received_date(struct index_mail *mail) { - const time_t *time_p; - size_t size; - - time_p = mail->ibox->index-> - lookup_field_raw(mail->ibox->index, mail->data.rec, - DATA_HDR_INTERNAL_DATE, &size); + time_t t; - if (time_p == NULL) { - mail->ibox->index->cache_fields_later(mail->ibox->index, - DATA_HDR_INTERNAL_DATE); - } else if (size != sizeof(*time_p)) { - index_set_corrupted(mail->ibox->index, - "Corrupted cached received time"); - time_p = NULL; + if (!mail_cache_copy_fixed_field(mail->ibox->index->cache, + mail->data.rec, + MAIL_CACHE_RECEIVED_DATE, + &t, sizeof(t))) { + mail_cache_mark_missing(mail->ibox->index->cache, + MAIL_CACHE_RECEIVED_DATE); + t = (time_t)-1; } - return time_p == NULL ? (time_t)-1 : *time_p; + return t; +} + +static void get_cached_sent_date(struct index_mail *mail, + struct mail_sent_date *sent_date) +{ + if (!mail_cache_copy_fixed_field(mail->ibox->index->cache, + mail->data.rec, MAIL_CACHE_SENT_DATE, + sent_date, sizeof(*sent_date))) { + mail_cache_mark_missing(mail->ibox->index->cache, + MAIL_CACHE_SENT_DATE); + + sent_date->time = (time_t)-1; + sent_date->timezone = 0; + } +} + +static int index_mail_cache_transaction_begin(struct index_mail *mail) +{ + if (mail->ibox->trans_ctx != NULL) + return TRUE; + + if (mail_cache_transaction_begin(mail->ibox->index->cache, TRUE, + &mail->ibox->trans_ctx) <= 0) + return FALSE; + + mail->data.cached_fields = + mail_cache_get_fields(mail->ibox->index->cache, + mail->data.rec); + return TRUE; +} + +static int index_mail_cache_can_add(struct index_mail *mail, + enum mail_cache_field field) +{ + if ((mail->data.cached_fields & field) != 0) + return FALSE; + + // FIXME: check if we really want to cache this + + if (!index_mail_cache_transaction_begin(mail)) + return FALSE; + + /* cached_fields may have changed, recheck */ + if ((mail->data.cached_fields & field) != 0) + return FALSE; + + return TRUE; +} + +static void index_mail_cache_add(struct index_mail *mail, + enum mail_cache_field field, + const void *data, size_t size) +{ + struct index_mailbox *ibox = mail->ibox; + + if (!index_mail_cache_can_add(mail, field)) + return; + + if (!mail_cache_add(ibox->trans_ctx, mail->data.rec, + field, data, size)) + mail_cache_transaction_rollback(ibox->trans_ctx); + + mail->data.cached_fields |= field; } static int open_stream(struct index_mail *mail, uoff_t position) { + struct index_mail_data *data = &mail->data; int deleted; - if (mail->data.stream == NULL) { - mail->data.stream = mail->ibox->index-> - open_mail(mail->ibox->index, mail->data.rec, - &mail->data.received_date, &deleted); + if (data->stream == NULL) { + data->stream = mail->ibox->index-> + open_mail(mail->ibox->index, data->rec, + &data->received_date, &deleted); + data->deleted = deleted; - if (mail->data.stream == NULL) + if (data->stream == NULL) return FALSE; + + if (data->received_date != (time_t)-1) { + index_mail_cache_add(mail, MAIL_CACHE_RECEIVED_DATE, + &data->received_date, + sizeof(data->received_date)); + } } i_stream_seek(mail->data.stream, position); return TRUE; } -static void prepend_cached_header(struct index_mail *mail, const char *name) +static int find_wanted_headers(struct mail_cache *cache, + const char *const wanted_headers[]) +{ + const char *const *headers, *const *tmp, *const *tmp2; + int i; + + if (wanted_headers == NULL || *wanted_headers == NULL) + return -1; + + for (i = MAIL_CACHE_HEADERS_COUNT-1; i >= 0; i--) { + headers = mail_cache_get_header_fields(cache, i); + if (headers == NULL) + continue; + + for (tmp = wanted_headers; *tmp != NULL; tmp++) { + for (tmp2 = headers; *tmp2 != NULL; tmp2++) { + if (strcasecmp(*tmp2, *tmp) == 0) + break; + } + + if (*tmp2 == NULL) + break; + } + + if (*tmp == NULL) + return i; + } + + return -1; +} + +static struct cached_header * +find_cached_header(struct index_mail *mail, const char *name, size_t len) { struct cached_header *hdr; - hdr = p_new(mail->pool, struct cached_header, 1); - hdr->name = name; - hdr->name_len = strlen(name); - - hdr->next = mail->data.headers; - mail->data.headers = hdr; -} + for (hdr = mail->data.headers; hdr != NULL; hdr = hdr->next) { + if (len == hdr->name_len && + memcasecmp(hdr->name, name, len) == 0) + return hdr; + } -void index_mail_init_parse_header(struct index_mail *mail) -{ - const char *const *tmp; - - if (mail->wanted_headers != NULL) { - for (tmp = mail->wanted_headers; *tmp != NULL; tmp++) - prepend_cached_header(mail, *tmp); - } + return NULL; } void index_mail_parse_header(struct message_part *part __attr_unused__, @@ -175,66 +268,201 @@ if (hdr == NULL) { /* end of headers */ - if (data->save_sent_time) { + if (data->save_sent_date) { /* not found */ - data->sent_time = 0; - data->sent_timezone = 0; - data->save_sent_time = FALSE; + data->sent_date.time = 0; + data->sent_date.timezone = 0; + data->save_sent_date = FALSE; + } + if (data->sent_date.time != (time_t)-1) { + index_mail_cache_add(mail, MAIL_CACHE_SENT_DATE, + &data->sent_date, + sizeof(data->sent_date)); + } + + /* mark parsed headers as fully saved */ + cached_hdr = data->headers; + for (; cached_hdr != NULL; cached_hdr = cached_hdr->next) { + if (cached_hdr->parsing) { + cached_hdr->parsing = FALSE; + cached_hdr->fully_saved = TRUE; + } } return; } - if (data->save_sent_time && strcasecmp(hdr->name, "Date") == 0) { + if (data->save_sent_date && strcasecmp(hdr->name, "Date") == 0) { if (hdr->continues) { hdr->use_full_value = TRUE; return; } if (!message_date_parse(hdr->full_value, hdr->full_value_len, - &data->sent_time, - &data->sent_timezone)) { + &data->sent_date.time, + &data->sent_date.timezone)) { /* 0 == parse error */ - data->sent_time = 0; - data->sent_timezone = 0; + data->sent_date.time = 0; + data->sent_date.timezone = 0; } - data->save_sent_time = FALSE; + data->save_sent_date = FALSE; } - cached_hdr = data->headers; - while (cached_hdr != NULL) { - if (cached_hdr->name_len == hdr->name_len && - memcasecmp(hdr->name, hdr->name, hdr->name_len) == 0) { - /* save only the first header */ - if (cached_hdr->value != NULL) - break; + cached_hdr = find_cached_header(mail, hdr->name, hdr->name_len); + if (cached_hdr != NULL && !cached_hdr->fully_saved) { + if (!hdr->continued) { + str_append(data->header_data, hdr->name); + str_append(data->header_data, ": "); + } + if (cached_hdr->value_idx == 0) + cached_hdr->value_idx = str_len(data->header_data); + str_append_n(data->header_data, hdr->value, hdr->value_len); + if (!hdr->no_newline) + str_append(data->header_data, "\n"); + } +} + +static struct cached_header * +add_cached_header(struct index_mail *mail, const char *name) +{ + struct cached_header *hdr; + + i_assert(*name != '\0'); + + hdr = find_cached_header(mail, name, strlen(name)); + if (hdr != NULL) + return hdr; + + hdr = p_new(mail->pool, struct cached_header, 1); + hdr->name = p_strdup(mail->pool, name); + hdr->name_len = strlen(name); + + hdr->next = mail->data.headers; + mail->data.headers = hdr; + + return hdr; +} + +static const char *const *get_header_names(struct cached_header *hdr) +{ + const char *null = NULL; + buffer_t *buffer; - if (hdr->continues) { - hdr->use_full_value = TRUE; - break; - } + buffer = buffer_create_dynamic(data_stack_pool, 128, (size_t)-1); + for (; hdr != NULL; hdr = hdr->next) + buffer_append(buffer, &hdr->name, sizeof(const char *)); + buffer_append(buffer, &null, sizeof(const char *)); + + return buffer_get_data(buffer, NULL); +} + +static int find_unused_header_idx(struct mail_cache *cache) +{ + int i; + + for (i = 0; i < MAIL_CACHE_HEADERS_COUNT; i++) { + if (mail_cache_get_header_fields(cache, i) == NULL) + return i; + } + return -1; +} - cached_hdr->value = p_strndup(mail->pool, - hdr->full_value, - hdr->full_value_len); - break; +void index_mail_parse_header_init(struct index_mail *mail, + const char *const *headers) +{ + struct cached_header *hdr; + const char *const *tmp; + + if (mail->data.header_data == NULL) + mail->data.header_data = str_new(mail->pool, 4096); + + if (headers == NULL) { + /* parsing all headers */ + for (hdr = mail->data.headers; hdr != NULL; hdr = hdr->next) + hdr->parsing = TRUE; + } else { + for (hdr = mail->data.headers; hdr != NULL; hdr = hdr->next) { + for (tmp = headers; *tmp != NULL; tmp++) { + if (strcasecmp(*tmp, hdr->name) == 0) + hdr->parsing = TRUE; + } } - cached_hdr = cached_hdr->next; } } static int parse_header(struct index_mail *mail) { + struct mail_cache *cache = mail->ibox->index->cache; + const char *const *headers, *const *tmp; + int idx; + if (!open_stream(mail, 0)) return FALSE; - index_mail_init_parse_header(mail); + if (mail->data.save_cached_headers) { + /* we want to save some of the headers. that means we'll have + to save all the headers in that group. if we're creating a + new group, save all the headers in previous group in it + too. */ + idx = mail->data.save_header_idx; + if (idx < 0) { + /* can we reuse existing? */ + headers = get_header_names(mail->data.headers); + idx = find_wanted_headers(cache, headers); + if (idx >= 0) + mail->data.save_header_idx = idx; + } + if (idx < 0) { + idx = find_unused_header_idx(cache); + idx--; /* include all previous headers too */ + } + + headers = idx < 0 ? NULL : + mail_cache_get_header_fields(cache, idx); + + if (headers != NULL) { + for (tmp = headers; *tmp != NULL; tmp++) + add_cached_header(mail, *tmp); + } + } + + index_mail_parse_header_init(mail, NULL); message_parse_header(NULL, mail->data.stream, &mail->data.hdr_size, index_mail_parse_header, mail); mail->data.parse_header = FALSE; + mail->data.headers_read = TRUE; mail->data.hdr_size_set = TRUE; return TRUE; } +static int parse_cached_header(struct index_mail *mail, int idx) +{ + struct istream *istream; + const char *str, *const *idx_headers; + + idx_headers = mail_cache_get_header_fields(mail->ibox->index->cache, + idx); + i_assert(idx_headers != NULL); + + str = mail_cache_lookup_string_field(mail->ibox->index->cache, + mail->data.rec, + mail_cache_header_fields[idx]); + if (str == NULL) + return FALSE; + + t_push(); + istream = i_stream_create_from_data(data_stack_pool, str, strlen(str)); + index_mail_parse_header_init(mail, idx_headers); + message_parse_header(NULL, istream, NULL, + index_mail_parse_header, mail); + + i_stream_unref(istream); + t_pop(); + + if (idx == mail->data.header_idx) + mail->data.headers_read = TRUE; + return TRUE; +} + static const struct mail_full_flags *get_flags(struct mail *_mail) { struct index_mail *mail = (struct index_mail *) _mail; @@ -255,6 +483,9 @@ { struct index_mail *mail = (struct index_mail *) _mail; struct index_mail_data *data = &mail->data; + buffer_t *buffer; + const void *buf_data; + size_t buf_size; if (data->parts != NULL) return data->parts; @@ -268,7 +499,6 @@ if (!open_stream(mail, 0)) return NULL; - index_mail_init_parse_header(mail); data->parts = message_parse(mail->pool, data->stream, index_mail_parse_header, mail); @@ -281,6 +511,17 @@ _mail->has_no_nuls = TRUE; } + if (index_mail_cache_can_add(mail, MAIL_CACHE_MESSAGEPART)) { + t_push(); + buffer = buffer_create_dynamic(data_stack_pool, + 1024, (size_t)-1); + message_part_serialize(data->parts, buffer); + + buf_data = buffer_get_data(buffer, &buf_size); + index_mail_cache_add(mail, MAIL_CACHE_MESSAGEPART, + &buf_data, buf_size); + t_pop(); + } return data->parts; } @@ -299,41 +540,51 @@ } data->received_date = mail->ibox->index-> - get_internal_date(mail->ibox->index, mail->data.rec); + get_received_date(mail->ibox->index, mail->data.rec); + if (data->received_date != (time_t)-1) { + index_mail_cache_add(mail, MAIL_CACHE_RECEIVED_DATE, + &data->received_date, + sizeof(data->received_date)); + } return data->received_date; } static time_t get_date(struct mail *_mail, int *timezone) { + static const char *date_headers[] = { "Date", NULL }; struct index_mail *mail = (struct index_mail *) _mail; struct index_mail_data *data = &mail->data; - const char *result; + int idx; - if (data->sent_time != (time_t)-1) { + if (data->sent_date.time != (time_t)-1) { if (timezone != NULL) - *timezone = data->sent_timezone; - return data->sent_time; + *timezone = data->sent_date.timezone; + return data->sent_date.time; } - if (data->parse_header || data->envelope == NULL) { - data->save_sent_time = TRUE; - parse_header(mail); - } else { - if (!imap_envelope_parse(data->envelope, - IMAP_ENVELOPE_DATE, - IMAP_ENVELOPE_RESULT_TYPE_STRING, - &result)) - return (time_t)-1; + if ((mail->wanted_fields & MAIL_FETCH_DATE) == 0) + get_cached_sent_date(mail, &data->sent_date); + + if (data->sent_date.time == (time_t)-1) { + idx = data->parse_header ? -1 : + find_wanted_headers(mail->ibox->index->cache, + date_headers); - if (!message_date_parse((const unsigned char *) result, - (size_t)-1, &data->sent_time, - &data->sent_timezone)) - data->sent_time = 0; + data->save_sent_date = TRUE; + if (idx >= 0) { + if (!parse_cached_header(mail, idx)) + idx = -1; + } + if (idx < 0) + parse_header(mail); + + index_mail_cache_add(mail, MAIL_CACHE_SENT_DATE, + &data->sent_date, sizeof(data->sent_date)); } if (timezone != NULL) - *timezone = data->sent_timezone; - return data->sent_time; + *timezone = data->sent_date.timezone; + return data->sent_date.time; } static int get_msgpart_sizes(struct index_mail *mail) @@ -358,12 +609,15 @@ static void get_binary_sizes(struct index_mail *mail) { + enum mail_index_record_flag index_flags; uoff_t size; + index_flags = mail_cache_get_index_flags(mail->ibox->index->cache, + mail->data. rec); + if (!mail->data.hdr_size_set && - (mail->data.rec->index_flags & INDEX_MAIL_FLAG_BINARY_HEADER)) { - size = get_cached_uoff_t(mail, DATA_HDR_HEADER_SIZE, - "header size"); + (index_flags & MAIL_INDEX_FLAG_BINARY_HEADER) != 0) { + size = get_cached_uoff_t(mail, MAIL_CACHE_HEADER_SIZE); if (size != (uoff_t)-1) { mail->data.hdr_size.physical_size = mail->data.hdr_size.virtual_size = size; @@ -372,8 +626,8 @@ } if (!mail->data.body_size_set && - (mail->data.rec->index_flags & INDEX_MAIL_FLAG_BINARY_BODY)) { - size = get_cached_uoff_t(mail, DATA_HDR_BODY_SIZE, "body size"); + (index_flags & MAIL_INDEX_FLAG_BINARY_BODY) != 0) { + size = get_cached_uoff_t(mail, MAIL_CACHE_BODY_SIZE); if (size != (uoff_t)-1) { mail->data.body_size.physical_size = mail->data.body_size.virtual_size = size; @@ -382,6 +636,20 @@ } } +static void index_mail_cache_add_sizes(struct index_mail *mail) +{ + if (mail->data.hdr_size_set) { + index_mail_cache_add(mail, MAIL_CACHE_HEADER_SIZE, + &mail->data.hdr_size.physical_size, + sizeof(uoff_t)); + } + if (mail->data.body_size_set) { + index_mail_cache_add(mail, MAIL_CACHE_BODY_SIZE, + &mail->data.body_size.physical_size, + sizeof(uoff_t)); + } +} + static uoff_t get_size(struct mail *_mail) { struct index_mail *mail = (struct index_mail *) _mail; @@ -440,48 +708,104 @@ body_size = data->body_size.virtual_size; data->body_size_set = TRUE; } + data->size = hdr_size + body_size; - data->size = hdr_size + body_size; + index_mail_cache_add_sizes(mail); + index_mail_cache_add(mail, MAIL_CACHE_VIRTUAL_FULL_SIZE, + &data->size, sizeof(data->size)); + return data->size; } static const char *get_header(struct mail *_mail, const char *field) { struct index_mail *mail = (struct index_mail *) _mail; - struct index_mail_data *data = &mail->data; struct cached_header *hdr; - enum imap_envelope_field env_field = 0; - const char *str; - char *ret; + size_t field_len; + int idx; + + field_len = strlen(field); + hdr = find_cached_header(mail, field, field_len); + if (hdr == NULL) { + /* not wanted initially, add it and check if we can + get it from cache */ + const char *headers[2]; + + hdr = add_cached_header(mail, field); - for (hdr = data->headers; hdr != NULL; hdr = hdr->next) { - if (strcasecmp(hdr->name, field) == 0) - return hdr->value; + headers[0] = field; headers[1] = NULL; + idx = find_wanted_headers(mail->ibox->index->cache, headers); + } else { + idx = mail->data.header_idx; + } + + if (!hdr->fully_saved) { + if (idx >= 0) { + if (!parse_cached_header(mail, idx)) + idx = -1; + } + if (idx < 0) { + mail->data.save_cached_headers = TRUE; + parse_header(mail); + } + + hdr = find_cached_header(mail, field, field_len); } - if (data->parse_header || data->envelope == NULL || - !imap_envelope_get_field(field, &env_field)) { - /* if we have to parse the header, do it even if we could use - envelope - envelope parsing would just slow up. */ - prepend_cached_header(mail, field); - parse_header(mail); + return hdr->value_idx == 0 ? NULL : + t_strcut(str_c(mail->data.header_data) + hdr->value_idx, '\n'); +} + +static struct istream *get_headers(struct mail *_mail, + const char *const minimum_fields[]) +{ + struct index_mail *mail = (struct index_mail *) _mail; + struct cached_header *hdr; + const char *const *tmp, *str; + int idx, all_exists, all_saved; + + i_assert(*minimum_fields != NULL); - for (hdr = data->headers; hdr != NULL; hdr = hdr->next) { - if (strcasecmp(hdr->name, field) == 0) - return hdr->value; + all_exists = all_saved = TRUE; + for (tmp = minimum_fields; *tmp != NULL; tmp++) { + hdr = find_cached_header(mail, *tmp, strlen(*tmp)); + if (hdr == NULL) { + add_cached_header(mail, *tmp); + all_exists = FALSE; + } else if (!hdr->fully_saved) + all_saved = FALSE; + } + + if (all_exists) { + if (all_saved) { + return i_stream_create_from_data(mail->pool, + str_data(mail->data.header_data), + str_len(mail->data.header_data)); } - return NULL; + idx = mail->data.header_idx; } else { - t_push(); - if (!imap_envelope_parse(data->envelope, env_field, - IMAP_ENVELOPE_RESULT_TYPE_STRING, - &str)) - str = NULL; - ret = p_strdup(mail->pool, str); - t_pop(); - return ret; + idx = find_wanted_headers(mail->ibox->index->cache, + get_header_names(mail->data.headers)); } + + if (idx >= 0) { + /* everything should be cached */ + str = mail_cache_lookup_string_field(mail->ibox->index->cache, + mail->data.rec, mail_cache_header_fields[idx]); + if (str != NULL) { + return i_stream_create_from_data(mail->pool, + str, strlen(str)); + } + } + if (idx < 0) { + mail->data.save_cached_headers = TRUE; + parse_header(mail); + } + + return i_stream_create_from_data(mail->pool, + str_data(mail->data.header_data), + str_len(mail->data.header_data)); } static const struct message_address * @@ -490,8 +814,6 @@ struct index_mail *mail = (struct index_mail *) _mail; const char *str; - /* don't bother with checking envelope - we're most likely - creating it */ str = get_header(_mail, field); if (str == NULL) return NULL; @@ -503,37 +825,16 @@ static const char *get_first_mailbox(struct mail *_mail, const char *field) { struct index_mail *mail = (struct index_mail *) _mail; - struct index_mail_data *data = &mail->data; - enum imap_envelope_field env_field; + struct message_address *addr; const char *str; - const char *ret = NULL; - if (data->envelope != NULL && - imap_envelope_get_field(field, &env_field)) { - /* prefer parsing envelope - faster than having to actually - parse the header field */ - t_push(); - if (!imap_envelope_parse(data->envelope, env_field, - IMAP_ENVELOPE_RESULT_TYPE_FIRST_MAILBOX, - &str)) - str = NULL; - ret = p_strdup(mail->pool, str); - t_pop(); - } else { - struct message_address *addr; + str = get_header(_mail, field); + if (str == NULL) + return NULL; - str = get_header(_mail, field); - if (str == NULL) - return NULL; - - addr = message_address_parse(mail->pool, - (const unsigned char *) str, - (size_t)-1, 1); - if (addr != NULL) - ret = addr->mailbox; - } - - return ret; + addr = message_address_parse(mail->pool, (const unsigned char *) str, + (size_t)-1, 1); + return addr != NULL ? addr->mailbox : NULL; } static struct istream *get_stream(struct mail *_mail, @@ -574,6 +875,7 @@ *body_size = data->body_size; } + index_mail_cache_add_sizes(mail); i_stream_seek(data->stream, 0); return data->stream; } @@ -582,14 +884,27 @@ { struct index_mail *mail = (struct index_mail *) _mail; struct index_mail_data *data = &mail->data; + struct mail_cache *cache = mail->ibox->index->cache; + enum mail_cache_field cache_field; char *str; + int i, idx; switch (field) { case MAIL_FETCH_IMAP_BODY: + if ((data->cached_fields & MAIL_CACHE_BODY) && + data->body == NULL) + data->body = get_cached_string(mail, MAIL_CACHE_BODY); if (data->body != NULL) return data->body; /* fall through */ case MAIL_FETCH_IMAP_BODYSTRUCTURE: + if ((data->cached_fields & MAIL_CACHE_BODYSTRUCTURE) && + data->bodystructure == NULL) { + data->bodystructure = + get_cached_string(mail, + MAIL_CACHE_BODYSTRUCTURE); + } + if (data->bodystructure != NULL) { if (field == MAIL_FETCH_IMAP_BODYSTRUCTURE) return data->bodystructure; @@ -600,9 +915,17 @@ imap_body_parse_from_bodystructure( data->bodystructure)); t_pop(); + + if (data->body == NULL) { + mail_cache_set_corrupted(cache, + "Corrupted BODYSTRUCTURE"); + } return data->body; } + if (!open_stream(mail, 0)) + return NULL; + if (data->parts == NULL) data->parts = get_cached_parts(mail); @@ -612,6 +935,13 @@ field == MAIL_FETCH_IMAP_BODYSTRUCTURE)); t_pop(); + /* should never fail */ + i_assert(str != NULL); + + cache_field = field == MAIL_FETCH_IMAP_BODYSTRUCTURE ? + MAIL_CACHE_BODYSTRUCTURE : MAIL_CACHE_BODY; + index_mail_cache_add(mail, cache_field, str, strlen(str)+1); + if (field == MAIL_FETCH_IMAP_BODYSTRUCTURE) data->bodystructure = str; else @@ -621,15 +951,34 @@ if (data->envelope != NULL) return data->envelope; - if ((mail->wanted_fields & MAIL_FETCH_IMAP_ENVELOPE) == 0) { - data->envelope = p_strdup(mail->pool, - get_cached_field(mail, DATA_FIELD_ENVELOPE)); - if (data->envelope != NULL) - return data->envelope; + if (data->parse_header) { + data->save_envelope = TRUE; + parse_header(mail); + return data->envelope; + } + + if (data->save_envelope) { + /* it was in wanted_fields, header_idx should be + correct */ + idx = data->header_idx; + i_assert(idx >= 0); + } else { + idx = find_wanted_headers(cache, imap_envelope_headers); } data->save_envelope = TRUE; - parse_header(mail); + if (idx >= 0) { + if (!parse_cached_header(mail, idx)) + idx = -1; + } + if (idx < 0) { + for (i = 0; imap_envelope_headers[i] != NULL; i++) { + add_cached_header(mail, + imap_envelope_headers[i]); + } + data->save_cached_headers = TRUE; + parse_header(mail); + } return data->envelope; default: i_unreached(); @@ -646,6 +995,7 @@ get_date, get_size, get_header, + get_headers, get_address, get_first_mailbox, get_stream, @@ -662,7 +1012,10 @@ mail->mail = index_mail; mail->mail.box = &ibox->box; - mail->pool = pool_alloconly_create("index_mail", 4096); + mail->wanted_headers_idx = + find_wanted_headers(ibox->index->cache, wanted_headers); + + mail->pool = pool_alloconly_create("index_mail", 16384); mail->ibox = ibox; mail->wanted_fields = wanted_fields; mail->wanted_headers = wanted_headers; @@ -672,137 +1025,182 @@ ibox->mail_init(mail); } -int index_mail_next(struct index_mail *mail, struct mail_index_record *rec, - unsigned int idx_seq) +static void index_mail_close(struct index_mail *mail) { struct index_mail_data *data = &mail->data; - int ret, open_mail, parse_header, envelope_headers; + const char *const *headers; + + if (data->stream != NULL) + i_stream_unref(data->stream); + + if (!data->save_cached_headers || !data->headers_read) + return; + + /* save cached headers - FIXME: this breaks if fetch_uid() and + fetch/search are both accessing headers from same message. + index_mails should probably be shared.. */ + if (!index_mail_cache_transaction_begin(mail)) + return; + + if (data->save_header_idx < 0) { + data->save_header_idx = + find_unused_header_idx(mail->ibox->index->cache); + if (data->save_header_idx < 0) + return; - i_assert(mail->expunge_counter == mail->ibox->index->expunge_counter); + headers = get_header_names(data->headers); + if (!mail_cache_set_header_fields(mail->ibox->trans_ctx, + data->save_header_idx, + headers)) + return; + } + + mail_cache_add(mail->ibox->trans_ctx, data->rec, + mail_cache_header_fields[data->save_header_idx], + str_c(mail->data.header_data), + str_len(mail->data.header_data)+1); + data->save_cached_headers = FALSE; +} + +int index_mail_next(struct index_mail *mail, struct mail_index_record *rec, + unsigned int idx_seq, int delay_open) +{ + struct mail_index *index = mail->ibox->index; + struct index_mail_data *data = &mail->data; + enum mail_index_record_flag index_flags; + int i, ret, open_mail, only_wanted_headers; + + i_assert(mail->expunge_counter == index->expunge_counter); t_push(); - /* close the old one */ - if (data->stream != NULL) - i_stream_unref(data->stream); - + index_mail_close(mail); memset(data, 0, sizeof(*data)); p_clear(mail->pool); - mail->mail.has_nuls = - (rec->index_flags & INDEX_MAIL_FLAG_HAS_NULS) != 0; + data->cached_fields = mail_cache_get_fields(index->cache, rec); + index_flags = (data->cached_fields & MAIL_CACHE_INDEX_FLAGS) == 0 ? 0 : + mail_cache_get_index_flags(index->cache, rec); + + mail->mail.has_nuls = (index_flags & MAIL_INDEX_FLAG_HAS_NULS) != 0; mail->mail.has_no_nuls = - (rec->index_flags & INDEX_MAIL_FLAG_HAS_NO_NULS) != 0; + (index_flags & MAIL_INDEX_FLAG_HAS_NO_NULS) != 0; data->rec = rec; data->idx_seq = idx_seq; data->size = (uoff_t)-1; - data->received_date = data->sent_time = (time_t)-1; + data->received_date = data->sent_date.time = (time_t)-1; /* if some wanted fields are cached, get them */ if (mail->wanted_fields & MAIL_FETCH_MESSAGE_PARTS) data->parts = get_cached_parts(mail); - if (mail->wanted_fields & MAIL_FETCH_IMAP_ENVELOPE) - data->envelope = get_cached_field(mail, DATA_FIELD_ENVELOPE); if (mail->wanted_fields & MAIL_FETCH_IMAP_BODY) - data->body = get_cached_field(mail, DATA_FIELD_BODY); - if (mail->wanted_fields & MAIL_FETCH_IMAP_BODYSTRUCTURE) { + data->body = get_cached_string(mail, MAIL_CACHE_BODY); + if ((mail->wanted_fields & MAIL_FETCH_IMAP_BODYSTRUCTURE) || + ((mail->wanted_fields & MAIL_FETCH_IMAP_BODY) && + data->body == NULL)) { data->bodystructure = - get_cached_field(mail, DATA_FIELD_BODYSTRUCTURE); + get_cached_string(mail, MAIL_CACHE_BODYSTRUCTURE); } if (mail->wanted_fields & MAIL_FETCH_SIZE) data->size = get_cached_virtual_size(mail); + if (mail->wanted_fields & MAIL_FETCH_DATE) + get_cached_sent_date(mail, &data->sent_date); /* see if we have to parse the message */ - open_mail = parse_header = FALSE; + open_mail = FALSE; if ((mail->wanted_fields & MAIL_FETCH_MESSAGE_PARTS) && data->parts == NULL) - parse_header = TRUE; - else if ((mail->wanted_fields & MAIL_FETCH_IMAP_ENVELOPE) && - data->envelope == NULL) - parse_header = TRUE; + data->parse_header = TRUE; else if ((mail->wanted_fields & MAIL_FETCH_IMAP_BODYSTRUCTURE) && data->bodystructure == NULL) { if (data->parts == NULL) data->parts = get_cached_parts(mail); - /* FIXME: this isn't helping really.. */ open_mail = TRUE; - parse_header = data->parts == NULL; + data->parse_header = data->parts == NULL; } else if ((mail->wanted_fields & MAIL_FETCH_IMAP_BODY) && data->body == NULL && data->bodystructure == NULL) { if (data->parts == NULL) data->parts = get_cached_parts(mail); open_mail = TRUE; - parse_header = data->parts == NULL; + data->parse_header = data->parts == NULL; } else if (mail->wanted_fields & (MAIL_FETCH_STREAM_HEADER | MAIL_FETCH_STREAM_BODY)) open_mail = TRUE; - else if ((mail->wanted_fields & MAIL_FETCH_SIZE) && data->size == 0) - open_mail = TRUE; - - envelope_headers = FALSE; - if (!parse_header && mail->wanted_headers != NULL) { - const char *const *tmp; - enum imap_envelope_field env_field; - for (tmp = mail->wanted_headers; *tmp != NULL; tmp++) { - if (imap_envelope_get_field(*tmp, &env_field)) - envelope_headers = TRUE; - else { - open_mail = TRUE; - parse_header = TRUE; - break; - } - } + /* check headers */ + if (mail->wanted_fields & MAIL_FETCH_IMAP_ENVELOPE) { + for (i = 0; imap_envelope_headers[i] != NULL; i++) + add_cached_header(mail, imap_envelope_headers[i]); + } else if ((mail->wanted_fields & MAIL_FETCH_DATE) && + data->sent_date.time == (time_t)-1) + add_cached_header(mail, "Date"); + + only_wanted_headers = mail->data.headers == NULL; + if (mail->wanted_headers != NULL) { + const char *const *tmp; + + for (tmp = mail->wanted_headers; *tmp != NULL; tmp++) + add_cached_header(mail, *tmp); } - if (!parse_header && (mail->wanted_fields & MAIL_FETCH_DATE)) - envelope_headers = TRUE; + data->save_header_idx = -1; + if (data->headers != NULL && + (mail->wanted_fields & (MAIL_FETCH_STREAM_HEADER | + MAIL_FETCH_STREAM_BODY)) == 0) { + /* we're not explicitly opening the file, caching the headers + could be a good idea if they're not already cached */ + if (only_wanted_headers) { + /* no extra headers, we already know if it's indexed */ + data->header_idx = mail->wanted_headers_idx; + } else { + const char *const *headers; - if (!parse_header && envelope_headers && - data->envelope == NULL) { - data->envelope = - get_cached_field(mail, DATA_FIELD_ENVELOPE); - if (data->envelope == NULL) - parse_header = TRUE; + headers = get_header_names(data->headers); + data->header_idx = + find_wanted_headers(index->cache, headers); + } + if (data->header_idx == -1 || + (data->cached_fields & + mail_cache_header_fields[data->header_idx]) == 0) { + data->save_cached_headers = TRUE; + data->parse_header = TRUE; + data->save_header_idx = data->header_idx; + } + } else { + data->header_idx = -1; } - if (open_mail || parse_header) { - int deleted; - - data->stream = mail->ibox->index-> - open_mail(mail->ibox->index, data->rec, - &data->received_date, &deleted); - if (data->stream == NULL) - ret = deleted ? 0 : -1; + if ((open_mail || data->parse_header) && !delay_open) { + if (!open_stream(mail, 0)) + ret = data->deleted ? 0 : -1; else ret = 1; } else { - if ((mail->wanted_fields & MAIL_FETCH_RECEIVED_DATE) && - data->received_date == (time_t)-1) { + if (mail->wanted_fields & MAIL_FETCH_RECEIVED_DATE) { /* check this only after open_mail() */ data->received_date = get_cached_received_date(mail); } - - if (mail->wanted_fields & MAIL_FETCH_DATE) - data->save_sent_time = TRUE; - if (mail->wanted_fields & MAIL_FETCH_IMAP_ENVELOPE) - data->save_envelope = TRUE; - - data->parse_header = parse_header; ret = 1; } + + if ((mail->wanted_fields & MAIL_FETCH_DATE) && + data->sent_date.time == (time_t)-1) + data->save_sent_date = TRUE; + + if (mail->wanted_fields & MAIL_FETCH_IMAP_ENVELOPE) + data->save_envelope = TRUE; + t_pop(); return ret; } void index_mail_deinit(struct index_mail *mail) { - if (mail->data.stream != NULL) { - i_stream_unref(mail->data.stream); - mail->data.stream = NULL; - } + t_push(); + index_mail_close(mail); + t_pop(); pool_unref(mail->pool); - mail->pool = NULL; + memset(mail, 0, sizeof(*mail)); }
--- a/src/lib-storage/index/index-mail.h Wed Aug 06 23:13:57 2003 +0300 +++ b/src/lib-storage/index/index-mail.h Wed Aug 06 23:15:30 2003 +0300 @@ -2,6 +2,7 @@ #define __INDEX_MAIL_H #include "message-size.h" +#include "mail-cache.h" struct message_header_line; @@ -9,18 +10,25 @@ struct cached_header *next; size_t name_len; - const char *name, *value; + const char *name; + size_t value_idx; /* in header_data */ + + unsigned int parsing:1; + unsigned int fully_saved:1; }; struct index_mail_data { struct mail_full_flags flags; - time_t received_date; + time_t date, received_date; uoff_t size; - time_t sent_time; - int sent_timezone; + enum mail_cache_field cached_fields; + struct mail_sent_date sent_date; struct cached_header *headers; + string_t *header_data; + int header_idx, save_header_idx; + struct message_part *parts; const char *envelope, *body, *bodystructure; struct message_part_envelope_data *envelope_data; @@ -32,10 +40,13 @@ struct message_size hdr_size, body_size; unsigned int parse_header:1; - unsigned int save_sent_time:1; + unsigned int headers_read:1; + unsigned int save_cached_headers:1; + unsigned int save_sent_date:1; unsigned int save_envelope:1; unsigned int hdr_size_set:1; unsigned int body_size_set:1; + unsigned int deleted:1; }; struct index_mail { @@ -49,16 +60,18 @@ enum mail_fetch_field wanted_fields; const char *const *wanted_headers; + int wanted_headers_idx; }; void index_mail_init(struct index_mailbox *ibox, struct index_mail *mail, enum mail_fetch_field wanted_fields, const char *const wanted_headers[]); int index_mail_next(struct index_mail *mail, struct mail_index_record *rec, - unsigned int idx_seq); + unsigned int idx_seq, int delay_open); void index_mail_deinit(struct index_mail *mail); -void index_mail_init_parse_header(struct index_mail *mail); +void index_mail_parse_header_init(struct index_mail *mail, + const char *const *headers); void index_mail_parse_header(struct message_part *part, struct message_header_line *hdr, void *context);
--- a/src/lib-storage/index/index-search.c Wed Aug 06 23:13:57 2003 +0300 +++ b/src/lib-storage/index/index-search.c Wed Aug 06 23:15:30 2003 +0300 @@ -8,7 +8,6 @@ #include "message-body-search.h" #include "message-header-search.h" #include "imap-date.h" -#include "imap-envelope.h" #include "index-storage.h" #include "index-messageset.h" #include "index-mail.h" @@ -216,16 +215,17 @@ enum mail_search_arg_type type, const char *value) { - time_t internal_date, search_time; + time_t date, search_time; uoff_t virtual_size, search_size; + int timezone_offset; switch (type) { /* internal dates */ case SEARCH_BEFORE: case SEARCH_ON: case SEARCH_SINCE: - internal_date = ctx->mail->get_received_date(ctx->mail); - if (internal_date == (time_t)-1) + date = ctx->mail->get_received_date(ctx->mail); + if (date == (time_t)-1) return -1; if (!imap_parse_date(value, &search_time)) @@ -233,12 +233,39 @@ switch (type) { case SEARCH_BEFORE: - return internal_date < search_time; + return date < search_time; case SEARCH_ON: - return internal_date >= search_time && - internal_date < search_time + 3600*24; + return date >= search_time && + date < search_time + 3600*24; case SEARCH_SINCE: - return internal_date >= search_time; + return date >= search_time; + default: + /* unreachable */ + break; + } + + /* sent dates */ + case SEARCH_SENTBEFORE: + case SEARCH_SENTON: + case SEARCH_SENTSINCE: + /* NOTE: RFC-3501 specifies that timezone is ignored + in searches. date is returned as UTC, so change it. */ + date = ctx->mail->get_date(ctx->mail, &timezone_offset); + if (date == (time_t)-1) + return -1; + date += timezone_offset * 60; + + if (!imap_parse_date(value, &search_time)) + return 0; + + switch (type) { + case SEARCH_SENTBEFORE: + return date < search_time; + case SEARCH_SENTON: + return date >= search_time && + date < search_time + 3600*24; + case SEARCH_SENTSINCE: + return date >= search_time; default: /* unreachable */ break; @@ -339,120 +366,6 @@ return arg->context; } -/* Returns >0 = matched, 0 = not matched, -1 = unknown */ -static int search_arg_match_envelope(struct mail_search_context *ctx, - struct mail_search_arg *arg) -{ - struct mail_index *index = ctx->ibox->index; - enum imap_envelope_field env_field; - struct header_search_context *hdr_search_ctx; - const char *envelope, *field; - int ret; - - switch (arg->type) { - case SEARCH_SENTBEFORE: - case SEARCH_SENTON: - case SEARCH_SENTSINCE: - env_field = IMAP_ENVELOPE_DATE; - break; - - case SEARCH_FROM: - env_field = IMAP_ENVELOPE_FROM; - break; - case SEARCH_TO: - env_field = IMAP_ENVELOPE_TO; - break; - case SEARCH_CC: - env_field = IMAP_ENVELOPE_CC; - break; - case SEARCH_BCC: - env_field = IMAP_ENVELOPE_BCC; - break; - case SEARCH_SUBJECT: - env_field = IMAP_ENVELOPE_SUBJECT; - break; - - case SEARCH_IN_REPLY_TO: - env_field = IMAP_ENVELOPE_IN_REPLY_TO; - break; - case SEARCH_MESSAGE_ID: - env_field = IMAP_ENVELOPE_MESSAGE_ID; - break; - default: - return -1; - } - - t_push(); - - /* get field from hopefully cached envelope */ - envelope = index->lookup_field(index, ctx->imail.data.rec, - DATA_FIELD_ENVELOPE); - if (envelope != NULL) { - ret = imap_envelope_parse(envelope, env_field, - IMAP_ENVELOPE_RESULT_TYPE_STRING, - &field) ? 1 : -1; - } else { - index->cache_fields_later(index, DATA_FIELD_ENVELOPE); - field = NULL; - ret = -1; - } - - if (ret != -1) { - switch (arg->type) { - case SEARCH_SENTBEFORE: - case SEARCH_SENTON: - case SEARCH_SENTSINCE: - ret = search_sent(arg->type, arg->value.str, - (const unsigned char *) field, - (size_t)-1); - break; - default: - if (arg->value.str[0] == '\0') { - /* we're just testing existence of the field. - assume it matches with non-NIL values. */ - ret = field != NULL ? 1 : 0; - break; - } - - if (field == NULL) { - /* doesn't exist */ - ret = 0; - break; - } - - hdr_search_ctx = search_header_context(ctx, arg); - if (hdr_search_ctx == NULL) { - ret = 0; - break; - } - - ret = message_header_search( - (const unsigned char *) field, - strlen(field), - hdr_search_ctx) ? 1 : 0; - } - } - t_pop(); - return ret; -} - -static void search_envelope_arg(struct mail_search_arg *arg, void *context) -{ - struct mail_search_context *ctx = context; - - switch (search_arg_match_envelope(ctx, arg)) { - case -1: - /* unknown */ - break; - case 0: - ARG_SET_RESULT(arg, 0); - break; - default: - ARG_SET_RESULT(arg, 1); - break; - } -} - static void search_header_arg(struct mail_search_arg *arg, void *context) { struct search_header_context *ctx = context; @@ -477,35 +390,8 @@ } return; - case SEARCH_FROM: - if (strcasecmp(ctx->hdr->name, "From") != 0) - return; - break; - case SEARCH_TO: - if (strcasecmp(ctx->hdr->name, "To") != 0) - return; - break; - case SEARCH_CC: - if (strcasecmp(ctx->hdr->name, "Cc") != 0) - return; - break; - case SEARCH_BCC: - if (strcasecmp(ctx->hdr->name, "Bcc") != 0) - return; - break; - case SEARCH_SUBJECT: - if (strcasecmp(ctx->hdr->name, "Subject") != 0) - return; - break; - case SEARCH_IN_REPLY_TO: - if (strcasecmp(ctx->hdr->name, "In-Reply-To") != 0) - return; - break; - case SEARCH_MESSAGE_ID: - if (strcasecmp(ctx->hdr->name, "Message-ID") != 0) - return; - break; case SEARCH_HEADER: + case SEARCH_HEADER_ADDRESS: ctx->custom_header = TRUE; if (strcasecmp(ctx->hdr->name, arg->hdr_field_name) != 0) @@ -532,8 +418,7 @@ hdr_search_ctx = search_header_context(ctx->index_context, arg); if (hdr_search_ctx == NULL) ret = 0; - else if (arg->type == SEARCH_FROM || arg->type == SEARCH_TO || - arg->type == SEARCH_CC || arg->type == SEARCH_BCC) { + else if (arg->type == SEARCH_HEADER_ADDRESS) { /* we have to match against normalized address */ struct message_address *addr; string_t *str; @@ -574,14 +459,8 @@ ARG_SET_RESULT(arg, 0); } break; - case SEARCH_FROM: - case SEARCH_TO: - case SEARCH_CC: - case SEARCH_BCC: - case SEARCH_SUBJECT: case SEARCH_HEADER: - case SEARCH_IN_REPLY_TO: - case SEARCH_MESSAGE_ID: + case SEARCH_HEADER_ADDRESS: ARG_SET_RESULT(arg, 0); break; default: @@ -605,15 +484,7 @@ index_mail_parse_header(part, hdr, ctx->index_context->mail); - if (ctx->custom_header || - strcasecmp(hdr->name, "Date") == 0 || - strcasecmp(hdr->name, "From") == 0 || - strcasecmp(hdr->name, "To") == 0 || - strcasecmp(hdr->name, "Cc") == 0 || - strcasecmp(hdr->name, "Bcc") == 0 || - strcasecmp(hdr->name, "Subject") == 0 || - strcasecmp(hdr->name, "In-Reply-To") == 0 || - strcasecmp(hdr->name, "Message-ID") == 0) { + if (ctx->custom_header || strcasecmp(hdr->name, "Date") == 0) { ctx->hdr = hdr; ctx->custom_header = FALSE; @@ -649,17 +520,23 @@ struct mail_search_context *ctx) { struct istream *input; - int have_headers, have_body, have_text; + const char *const *headers; + int have_headers, have_body; /* first check what we need to use */ - mail_search_args_analyze(args, &have_headers, &have_body, &have_text); - if (!have_headers && !have_body && !have_text) + headers = mail_search_args_analyze(args, &have_headers, &have_body); + if (!have_headers && !have_body) return TRUE; - if (have_headers || have_text) { + if (have_headers) { struct search_header_context hdr_ctx; - input = ctx->mail->get_stream(ctx->mail, NULL, NULL); + if (have_body) + headers = NULL; + + input = headers == NULL ? + ctx->mail->get_stream(ctx->mail, NULL, NULL) : + ctx->mail->get_headers(ctx->mail, headers); if (input == NULL) return FALSE; @@ -668,7 +545,7 @@ hdr_ctx.custom_header = TRUE; hdr_ctx.args = args; - index_mail_init_parse_header(&ctx->imail); + index_mail_parse_header_init(&ctx->imail, headers); message_parse_header(NULL, input, NULL, search_header, &hdr_ctx); } else { @@ -681,7 +558,7 @@ i_stream_seek(input, hdr_size.physical_size); } - if (have_text || have_body) { + if (have_body) { struct search_body_context body_ctx; memset(&body_ctx, 0, sizeof(body_ctx)); @@ -967,6 +844,11 @@ ret = FALSE; } + if (ctx->ibox->fetch_mail.pool != NULL) + index_mail_deinit(&ctx->ibox->fetch_mail); + if (ctx->imail.pool != NULL) + index_mail_deinit(&ctx->imail); + if (!index_storage_lock(ctx->ibox, MAIL_LOCK_UNLOCK)) ret = FALSE; @@ -978,18 +860,42 @@ if (ctx->hdr_pool != NULL) pool_unref(ctx->hdr_pool); - if (ctx->ibox->fetch_mail.pool != NULL) - index_mail_deinit(&ctx->ibox->fetch_mail); - index_mail_deinit(&ctx->imail); i_free(ctx); return ret; } +static int search_match_next(struct mail_search_context *ctx) +{ + struct mail_search_arg *arg; + int ret; + + /* check the index matches first */ + mail_search_args_reset(ctx->args); + ret = mail_search_args_foreach(ctx->args, search_index_arg, ctx); + if (ret >= 0) + return ret > 0; + + /* next search only from cached arguments */ + ret = mail_search_args_foreach(ctx->args, search_cached_arg, ctx); + if (ret >= 0) + return ret > 0; + + /* open the mail file and check the rest */ + if (!search_arg_match_text(ctx->args, ctx)) + return FALSE; + + for (arg = ctx->args; arg != NULL; arg = arg->next) { + if (arg->result != 1) + return FALSE; + } + + return TRUE; +} + struct mail *index_storage_search_next(struct mail_search_context *ctx) { const struct messageset_mail *msgset_mail; - struct mail_search_arg *arg; - int found, ret; + int ret; if (ctx->msgset_ctx == NULL) { /* initialization failed or didn't found any messages */ @@ -998,47 +904,35 @@ do { msgset_mail = index_messageset_next(ctx->msgset_ctx); - if (msgset_mail == NULL) - return NULL; + if (msgset_mail == NULL) { + ret = -1; + break; + } ctx->mail->seq = msgset_mail->client_seq; ctx->mail->uid = msgset_mail->rec->uid; + ret = index_mail_next(&ctx->imail, msgset_mail->rec, - msgset_mail->idx_seq); - - if (ret < 0) - return NULL; - - if (ret == 0) - found = FALSE; - else { - mail_search_args_reset(ctx->args); - - t_push(); - - mail_search_args_foreach(ctx->args, search_index_arg, - ctx); - mail_search_args_foreach(ctx->args, search_cached_arg, - ctx); - mail_search_args_foreach(ctx->args, search_envelope_arg, - ctx); - found = search_arg_match_text(ctx->args, ctx); - - t_pop(); - - if (ctx->error != NULL) - return NULL; + msgset_mail->idx_seq, TRUE); + if (ret <= 0) { + if (ret < 0) + break; + continue; } - if (found) { - for (arg = ctx->args; arg != NULL; arg = arg->next) { - if (arg->result != 1) { - found = FALSE; - break; - } - } - } - } while (!found); + t_push(); + ret = search_match_next(ctx); + t_pop(); + + if (ctx->error != NULL) + ret = -1; + } while (ret == 0); + + if (ret < 0) { + /* error or last record */ + index_mail_deinit(&ctx->imail); + return NULL; + } return ctx->mail; }
--- a/src/lib-storage/index/index-storage.c Wed Aug 06 23:13:57 2003 +0300 +++ b/src/lib-storage/index/index-storage.c Wed Aug 06 23:15:30 2003 +0300 @@ -149,20 +149,28 @@ destroy_unrefed(TRUE); } -static enum mail_data_field get_data_fields(const char *fields) +static enum mail_cache_field get_cache_fields(const char *fields) { + static enum mail_cache_field field_masks[] = { + MAIL_CACHE_SENT_DATE, + MAIL_CACHE_RECEIVED_DATE, + MAIL_CACHE_VIRTUAL_FULL_SIZE, + MAIL_CACHE_BODY, + MAIL_CACHE_BODYSTRUCTURE, + MAIL_CACHE_MESSAGEPART, + }; static const char *field_names[] = { - "Location", - "Envelope", - "Body", - "Bodystructure", - "MD5", - "MessagePart", + "sent_date", + "received_date", + "virtual_size", + "body", + "bodystructure", + "messagepart", NULL }; const char *const *arr; - enum mail_data_field ret; + enum mail_cache_field ret; int i; if (fields == NULL || *fields == '\0') @@ -175,7 +183,7 @@ for (i = 0; field_names[i] != NULL; i++) { if (strcasecmp(field_names[i], *arr) == 0) { - ret |= 1 << i; + ret |= field_masks[i]; break; } } @@ -188,28 +196,28 @@ return ret; } -static enum mail_data_field get_default_cache_fields(void) +static enum mail_cache_field get_default_cache_fields(void) { - static enum mail_data_field ret = 0; + static enum mail_cache_field ret = 0; static int ret_set = FALSE; if (ret_set) return ret; - ret = get_data_fields(getenv("MAIL_CACHE_FIELDS")); + ret = get_cache_fields(getenv("MAIL_CACHE_FIELDS")); ret_set = TRUE; return ret; } -static enum mail_data_field get_never_cache_fields(void) +static enum mail_cache_field get_never_cache_fields(void) { - static enum mail_data_field ret = 0; + static enum mail_cache_field ret = 0; static int ret_set = FALSE; if (ret_set) return ret; - ret = get_data_fields(getenv("MAIL_NEVER_CACHE_FIELDS")); + ret = get_cache_fields(getenv("MAIL_NEVER_CACHE_FIELDS")); ret_set = TRUE; return ret; } @@ -282,9 +290,16 @@ int index_storage_lock(struct index_mailbox *ibox, enum mail_lock_type lock_type) { - int ret; + int ret = TRUE; if (lock_type == MAIL_LOCK_UNLOCK) { + if (ibox->trans_ctx != NULL) { + if (!mail_cache_transaction_commit(ibox->trans_ctx)) + ret = FALSE; + if (!mail_cache_transaction_end(ibox->trans_ctx)) + ret = FALSE; + ibox->trans_ctx = NULL; + } if (ibox->lock_type != MAILBOX_LOCK_UNLOCK) return TRUE; } else { @@ -295,7 +310,8 @@ /* we have to set/reset this every time, because the same index may be used by multiple IndexMailboxes. */ index_storage_init_lock_notify(ibox); - ret = ibox->index->set_lock(ibox->index, lock_type); + if (!ibox->index->set_lock(ibox->index, lock_type)) + ret = FALSE; ibox->index->set_lock_notify_callback(ibox->index, NULL, NULL); if (!ret) @@ -337,14 +353,13 @@ if (!index->opened) { /* open the index first */ - index->default_cache_fields = - get_default_cache_fields(); - index->never_cache_fields = - get_never_cache_fields(); - if (!index->open(index, index_flags)) break; + mail_cache_set_defaults(index->cache, + get_default_cache_fields(), + get_never_cache_fields()); + if (INDEX_IS_IN_MEMORY(index) && storage->index_dir != NULL) { storage->callbacks->notify_no(&ibox->box,
--- a/src/lib-storage/index/index-storage.h Wed Aug 06 23:13:57 2003 +0300 +++ b/src/lib-storage/index/index-storage.h Wed Aug 06 23:15:30 2003 +0300 @@ -21,6 +21,7 @@ struct mail_index *index; enum mailbox_lock_type lock_type; + struct mail_cache_transaction_ctx *trans_ctx; struct timeout *autosync_to; struct index_autosync_file *autosync_files; @@ -90,6 +91,7 @@ struct mail_fetch_context * index_storage_fetch_init(struct mailbox *box, enum mail_fetch_field wanted_fields, + const char *const *wanted_headers, const char *messageset, int uidset); int index_storage_fetch_deinit(struct mail_fetch_context *ctx, int *all_found); struct mail *index_storage_fetch_next(struct mail_fetch_context *ctx);
--- a/src/lib-storage/index/maildir/maildir-copy.c Wed Aug 06 23:13:57 2003 +0300 +++ b/src/lib-storage/index/maildir/maildir-copy.c Wed Aug 06 23:15:30 2003 +0300 @@ -31,12 +31,13 @@ const char **fname, const char *new_path) { const char *path; + int new_dir; - *fname = maildir_get_location(index, rec); + *fname = maildir_get_location(index, rec, &new_dir); if (*fname == NULL) return -1; - if ((rec->index_flags & INDEX_MAIL_FLAG_MAILDIR_NEW) != 0) { + if (new_dir) { /* probably in new/ dir */ path = t_strconcat(index->mailbox_path, "/new/", *fname, NULL); if (link(path, new_path) == 0)
--- a/src/lib-storage/mail-search.c Wed Aug 06 23:13:57 2003 +0300 +++ b/src/lib-storage/mail-search.c Wed Aug 06 23:15:30 2003 +0300 @@ -1,6 +1,7 @@ /* Copyright (C) 2002 Timo Sirainen */ #include "lib.h" +#include "buffer.h" #include "mail-search.h" void mail_search_args_reset(struct mail_search_arg *args) @@ -93,9 +94,11 @@ return result; } -static void search_arg_analyze(struct mail_search_arg *arg, int *have_headers, - int *have_body, int *have_text) +static void +search_arg_analyze(struct mail_search_arg *arg, buffer_t *headers, + int *have_headers, int *have_body, int *have_text) { + static const char *date_hdr = "Date"; struct mail_search_arg *subarg; if (arg->result != -1) @@ -107,8 +110,9 @@ subarg = arg->value.subargs; while (subarg != NULL) { if (subarg->result == -1) { - search_arg_analyze(subarg, have_headers, - have_body, have_text); + search_arg_analyze(subarg, headers, + have_headers, have_body, + have_text); } subarg = subarg->next; @@ -117,14 +121,13 @@ case SEARCH_SENTBEFORE: case SEARCH_SENTON: case SEARCH_SENTSINCE: - case SEARCH_FROM: - case SEARCH_TO: - case SEARCH_CC: - case SEARCH_BCC: - case SEARCH_SUBJECT: - case SEARCH_IN_REPLY_TO: - case SEARCH_MESSAGE_ID: + *have_headers = TRUE; + buffer_append(headers, &date_hdr, sizeof(const char *)); + break; case SEARCH_HEADER: + case SEARCH_HEADER_ADDRESS: + buffer_append(headers, &arg->hdr_field_name, + sizeof(const char *)); *have_headers = TRUE; break; case SEARCH_BODY: @@ -138,12 +141,25 @@ } } -void mail_search_args_analyze(struct mail_search_arg *args, int *have_headers, - int *have_body, int *have_text) +const char *const * +mail_search_args_analyze(struct mail_search_arg *args, + int *have_headers, int *have_body) { - *have_headers = *have_body = *have_text = FALSE; + const char *null = NULL; + buffer_t *headers; + int have_text; + + *have_headers = *have_body = have_text = FALSE; - for (; args != NULL; args = args->next) - search_arg_analyze(args, have_headers, have_body, have_text); + headers = buffer_create_dynamic(data_stack_pool, 128, (size_t)-1); + for (; args != NULL; args = args->next) { + search_arg_analyze(args, headers, have_headers, + have_body, &have_text); + } + + if (!have_headers || have_text) + return NULL; + + buffer_append(headers, &null, sizeof(const char *)); + return buffer_get_data(headers, NULL); } -
--- a/src/lib-storage/mail-search.h Wed Aug 06 23:13:57 2003 +0300 +++ b/src/lib-storage/mail-search.h Wed Aug 06 23:15:30 2003 +0300 @@ -32,20 +32,12 @@ SEARCH_LARGER, /* headers */ - SEARCH_FROM, - SEARCH_TO, - SEARCH_CC, - SEARCH_BCC, - SEARCH_SUBJECT, SEARCH_HEADER, + SEARCH_HEADER_ADDRESS, /* body */ SEARCH_BODY, - SEARCH_TEXT, - - /* our shortcuts for headers */ - SEARCH_IN_REPLY_TO, - SEARCH_MESSAGE_ID + SEARCH_TEXT }; struct mail_search_arg { @@ -58,7 +50,7 @@ } value; void *context; - const char *hdr_field_name; /* for SEARCH_HEADER */ + const char *hdr_field_name; /* for SEARCH_HEADER* */ unsigned int not:1; int result; /* -1 = unknown, 0 = unmatched, 1 = matched */ @@ -82,9 +74,11 @@ mail_search_foreach_callback_t callback, void *context); -/* Fills have_headers, have_body and have_text based on if such search - argument exists that needs to be checked. */ -void mail_search_args_analyze(struct mail_search_arg *args, int *have_headers, - int *have_body, int *have_text); +/* Fills have_headers and have_body based on if such search argument exists + that needs to be checked. Returns the headers that we're searching for, or + NULL if we're searching for TEXT. */ +const char *const * +mail_search_args_analyze(struct mail_search_arg *args, + int *have_headers, int *have_body); #endif
--- a/src/lib-storage/mail-storage.h Wed Aug 06 23:13:57 2003 +0300 +++ b/src/lib-storage/mail-storage.h Wed Aug 06 23:15:30 2003 +0300 @@ -267,6 +267,7 @@ struct mail_fetch_context * (*fetch_init)(struct mailbox *box, enum mail_fetch_field wanted_fields, + const char *const *wanted_headers, const char *messageset, int uidset); /* Deinitialize fetch request. all_found is set to TRUE if all of the fetched messages were found (ie. not just deleted). */ @@ -377,6 +378,10 @@ /* Get value for single header field */ const char *(*get_header)(struct mail *mail, const char *field); + /* Returns partial headers which contain _at least_ the given fields, + but it may contain others as well. */ + struct istream *(*get_headers)(struct mail *mail, + const char *const minimum_fields[]); /* Returns the parsed address for given header field. */ const struct message_address *(*get_address)(struct mail *mail,
--- a/src/lib-storage/proxy-mailbox.c Wed Aug 06 23:13:57 2003 +0300 +++ b/src/lib-storage/proxy-mailbox.c Wed Aug 06 23:15:30 2003 +0300 @@ -56,11 +56,13 @@ static struct mail_fetch_context * _fetch_init(struct mailbox *box, enum mail_fetch_field wanted_fields, + const char *const *wanted_headers, const char *messageset, int uidset) { struct proxy_mailbox *p = (struct proxy_mailbox *) box; - return p->box->fetch_init(p->box, wanted_fields, messageset, uidset); + return p->box->fetch_init(p->box, wanted_fields, wanted_headers, + messageset, uidset); } static struct mail *_fetch_uid(struct mailbox *box, unsigned int uid,
--- a/src/pop3/client.c Wed Aug 06 23:13:57 2003 +0300 +++ b/src/pop3/client.c Wed Aug 06 23:15:30 2003 +0300 @@ -62,7 +62,7 @@ messageset = t_strdup_printf("1:%u", client->messages_count); for (i = 0; i < 2; i++) { ctx = client->mailbox->fetch_init(client->mailbox, - MAIL_FETCH_SIZE, + MAIL_FETCH_SIZE, NULL, messageset, FALSE); if (ctx == NULL) { client_send_storage_error(client);
--- a/src/pop3/commands.c Wed Aug 06 23:13:57 2003 +0300 +++ b/src/pop3/commands.c Wed Aug 06 23:15:30 2003 +0300 @@ -261,7 +261,7 @@ ctx = client->mailbox->fetch_init(client->mailbox, MAIL_FETCH_STREAM_HEADER | - MAIL_FETCH_STREAM_BODY, + MAIL_FETCH_STREAM_BODY, NULL, dec2str(msgnum+1), FALSE); if (ctx == NULL) { client_send_storage_error(client); @@ -345,7 +345,7 @@ t_strdup_printf("1:%u", client->messages_count) : t_strdup_printf("%u", message); - ctx = client->mailbox->fetch_init(client->mailbox, 0, + ctx = client->mailbox->fetch_init(client->mailbox, 0, NULL, messageset, FALSE); if (ctx == NULL) { client_send_storage_error(client);