Mercurial > dovecot > original-hg > dovecot-1.2
changeset 3956:a52c36c51ff2 HEAD
Support storing keywords in dbox files. Doesn't yet work while saving.
author | Timo Sirainen <tss@iki.fi> |
---|---|
date | Sun, 29 Jan 2006 00:05:48 +0200 |
parents | 295af5c1cce6 |
children | 8f0ff62befd3 |
files | src/lib-storage/index/dbox/Makefile.am src/lib-storage/index/dbox/dbox-file.c src/lib-storage/index/dbox/dbox-file.h src/lib-storage/index/dbox/dbox-keywords.c src/lib-storage/index/dbox/dbox-keywords.h src/lib-storage/index/dbox/dbox-mail.c src/lib-storage/index/dbox/dbox-storage.h src/lib-storage/index/dbox/dbox-sync-full.c src/lib-storage/index/dbox/dbox-sync.c src/lib-storage/index/dbox/dbox-uidlist.c |
diffstat | 10 files changed, 623 insertions(+), 116 deletions(-) [+] |
line wrap: on
line diff
--- a/src/lib-storage/index/dbox/Makefile.am Sat Jan 28 23:47:00 2006 +0200 +++ b/src/lib-storage/index/dbox/Makefile.am Sun Jan 29 00:05:48 2006 +0200 @@ -10,6 +10,7 @@ libstorage_dbox_a_SOURCES = \ dbox-file.c \ + dbox-keywords.c \ dbox-list.c \ dbox-mail.c \ dbox-save.c \ @@ -22,6 +23,7 @@ noinst_HEADERS = \ dbox-file.h \ + dbox-keywords.h \ dbox-storage.h \ dbox-sync.h \ dbox-uidlist.h
--- a/src/lib-storage/index/dbox/dbox-file.c Sat Jan 28 23:47:00 2006 +0200 +++ b/src/lib-storage/index/dbox/dbox-file.c Sun Jan 29 00:05:48 2006 +0200 @@ -1,6 +1,8 @@ -/* Copyright (C) 2005 Timo Sirainen */ +/* Copyright (C) 2005-2006 Timo Sirainen */ #include "lib.h" +#include "array.h" +#include "bsearch-insert-pos.h" #include "hex-dec.h" #include "istream.h" #include "ostream.h" @@ -38,12 +40,18 @@ void dbox_file_close(struct dbox_file *file) { + if (array_is_created(&file->file_idx_keywords)) { + array_free(&file->idx_file_keywords); + array_free(&file->file_idx_keywords); + } + if (file->input != NULL) i_stream_unref(&file->input); if (file->fd != -1) { if (close(file->fd) < 0) i_error("close(dbox) failed: %m"); } + i_free(file->seeked_keywords); i_free(file->path); i_free(file); } @@ -56,6 +64,7 @@ const unsigned char *data; size_t size; + /* read the header */ i_stream_seek(file->input, offset); (void)i_stream_read_data(file->input, &data, &size, file->mail_header_size-1); @@ -68,10 +77,15 @@ "read(%s) failed: %m", file->path); return -1; } + memcpy(&file->seeked_mail_header, data, sizeof(file->seeked_mail_header)); + /* @UNSAFE */ + memcpy(file->seeked_keywords, data + sizeof(file->seeked_mail_header), + file->keyword_count); file->seeked_offset = offset; + /* parse the header */ hdr = &file->seeked_mail_header; file->seeked_mail_size = hex2dec(hdr->mail_size_hex, sizeof(hdr->mail_size_hex)); @@ -83,15 +97,13 @@ "Corrupted mail header in dbox file %s", file->path); return -1; } - if (file->seeked_mail_size == 0 || file->seeked_uid == 0) { - /* could be legitimately just not written yet. we're at EOF. */ - return 0; - } return 1; } int dbox_file_seek(struct dbox_mailbox *mbox, uint32_t file_seq, uoff_t offset) { + int ret; + if (mbox->file != NULL && mbox->file->file_seq != file_seq) { dbox_file_close(mbox->file); mbox->file = NULL; @@ -129,39 +141,99 @@ if (offset == 0) offset = mbox->file->header_size; - return dbox_file_read_mail_header(mbox, mbox->file, offset); + if ((ret = dbox_file_read_mail_header(mbox, mbox->file, offset)) <= 0) + return ret; + + if (mbox->file->seeked_mail_size == 0 || mbox->file->seeked_uid == 0) { + /* could be legitimately just not written yet. we're at EOF. */ + return 0; + } + return 1; } int dbox_file_seek_next_nonexpunged(struct dbox_mailbox *mbox) { + const struct dbox_mail_header *hdr; uoff_t offset; int ret; - offset = mbox->file->seeked_offset + - mbox->file->mail_header_size + mbox->file->seeked_mail_size; + for (;;) { + offset = mbox->file->seeked_offset + + mbox->file->mail_header_size + + mbox->file->seeked_mail_size; + + ret = dbox_file_seek(mbox, mbox->file->file_seq, offset); + if (ret <= 0) + return ret; - while ((ret = dbox_file_seek(mbox, mbox->file->file_seq, offset)) > 0) { - if (mbox->file->seeked_mail_header.expunged != '1') + hdr = &mbox->file->seeked_mail_header; + if (hdr->expunged != '1') { + /* non-expunged mail found */ break; + } + } - /* marked expunged, get to next mail. */ - } - return ret; + return 1; } void dbox_file_header_init(struct dbox_file_header *hdr) { - uint16_t header_size = sizeof(*hdr); + uint16_t base_header_size = sizeof(*hdr); + uint32_t header_size = + base_header_size + DBOX_KEYWORD_NAMES_RESERVED_SPACE; uint32_t append_offset = header_size; - uint16_t mail_header_size = sizeof(struct dbox_mail_header); + uint16_t keyword_count = DBOX_KEYWORD_COUNT; + uint16_t mail_header_size = + sizeof(struct dbox_mail_header) + keyword_count; uint32_t create_time = ioloop_time; memset(hdr, '0', sizeof(*hdr)); + DEC2HEX(hdr->base_header_size_hex, base_header_size); DEC2HEX(hdr->header_size_hex, header_size); DEC2HEX(hdr->append_offset_hex, append_offset); DEC2HEX(hdr->create_time_hex, create_time); DEC2HEX(hdr->mail_header_size_hex, mail_header_size); - // FIXME: set keyword_count + DEC2HEX(hdr->keyword_list_offset_hex, base_header_size); + DEC2HEX(hdr->keyword_count_hex, keyword_count); +} + +int dbox_file_header_parse(struct dbox_mailbox *mbox, struct dbox_file *file, + const struct dbox_file_header *hdr) +{ + file->base_header_size = hex2dec(hdr->base_header_size_hex, + sizeof(hdr->base_header_size_hex)); + file->header_size = hex2dec(hdr->header_size_hex, + sizeof(hdr->header_size_hex)); + file->append_offset = hex2dec(hdr->append_offset_hex, + sizeof(hdr->append_offset_hex)); + file->create_time = hex2dec(hdr->create_time_hex, + sizeof(hdr->create_time_hex)); + file->mail_header_size = hex2dec(hdr->mail_header_size_hex, + sizeof(hdr->mail_header_size_hex)); + file->mail_header_align = + hex2dec(hdr->mail_header_align_hex, + sizeof(hdr->mail_header_align_hex)); + file->keyword_count = hex2dec(hdr->keyword_count_hex, + sizeof(hdr->keyword_count_hex)); + file->keyword_list_offset = + hex2dec(hdr->keyword_list_offset_hex, + sizeof(hdr->keyword_list_offset_hex)); + + if (file->base_header_size == 0 || + file->header_size < file->base_header_size || + file->append_offset < file->header_size || + file->keyword_list_offset < file->base_header_size || + file->mail_header_size < sizeof(struct dbox_mail_header) || + file->keyword_count > file->mail_header_size - + sizeof(struct dbox_mail_header)) { + mail_storage_set_critical(STORAGE(mbox->storage), + "dbox %s: broken file header", file->path); + return -1; + } + + i_free(file->seeked_keywords); + file->seeked_keywords = i_malloc(file->keyword_count); + return 0; } int dbox_file_read_header(struct dbox_mailbox *mbox, struct dbox_file *file) @@ -170,6 +242,7 @@ const unsigned char *data; size_t size; + /* read the file header */ i_stream_seek(file->input, 0); (void)i_stream_read_data(file->input, &data, &size, sizeof(hdr)-1); if (size < sizeof(hdr)) { @@ -187,25 +260,46 @@ memcpy(&hdr, data, sizeof(hdr)); /* parse the header */ - file->header_size = hex2dec(hdr.header_size_hex, - sizeof(hdr.header_size_hex)); - file->append_offset = hex2dec(hdr.append_offset_hex, - sizeof(hdr.append_offset_hex)); - file->create_time = hex2dec(hdr.create_time_hex, - sizeof(hdr.create_time_hex)); - file->mail_header_size = hex2dec(hdr.mail_header_size_hex, - sizeof(hdr.mail_header_size_hex)); - file->mail_header_padding = - hex2dec(hdr.mail_header_padding_hex, - sizeof(hdr.mail_header_padding_hex)); - file->keyword_count = hex2dec(hdr.keyword_count_hex, - sizeof(hdr.keyword_count_hex)); + if (dbox_file_header_parse(mbox, file, &hdr) < 0) + return -1; - if (file->header_size == 0 || file->append_offset < sizeof(hdr) || - file->mail_header_size < sizeof(struct dbox_mail_header)) { - mail_storage_set_critical(STORAGE(mbox->storage), - "dbox %s: broken file header", file->path); - return -1; + /* keywords may not be up to date anymore */ + if (array_is_created(&file->idx_file_keywords)) { + array_free(&file->idx_file_keywords); + array_free(&file->file_idx_keywords); } return 0; } + +int dbox_file_write_header(struct dbox_mailbox *mbox, struct dbox_file *file) +{ + struct dbox_file_header hdr; + char buf[1024]; + int ret; + + dbox_file_header_init(&hdr); + ret = dbox_file_header_parse(mbox, file, &hdr); + i_assert(ret == 0); + + /* write header + LF to mark end-of-keywords list */ + if (o_stream_send(file->output, &hdr, sizeof(hdr)) < 0 || + o_stream_send_str(file->output, "\n") < 0) { + mail_storage_set_critical(STORAGE(mbox->storage), + "write(%s) failed: %m", file->path); + return -1; + } + + /* fill the rest of the header with spaces */ + memset(buf, ' ', sizeof(buf)); + while (file->output->offset < file->header_size) { + unsigned int size = I_MIN(sizeof(buf), file->header_size - + file->output->offset); + + if (o_stream_send(file->output, buf, size) < 0) { + mail_storage_set_critical(STORAGE(mbox->storage), + "write(%s) failed: %m", file->path); + return -1; + } + } + return 0; +}
--- a/src/lib-storage/index/dbox/dbox-file.h Sat Jan 28 23:47:00 2006 +0200 +++ b/src/lib-storage/index/dbox/dbox-file.h Sun Jan 29 00:05:48 2006 +0200 @@ -13,10 +13,14 @@ void dbox_file_close(struct dbox_file *file); /* Returns -1 = error, 0 = EOF (mail was just moved / file broken), 1 = ok */ -int dbox_file_seek(struct dbox_mailbox *mbox, uint32_t file_seq, uoff_t offset); +int dbox_file_seek(struct dbox_mailbox *mbox, + uint32_t file_seq, uoff_t offset); int dbox_file_seek_next_nonexpunged(struct dbox_mailbox *mbox); void dbox_file_header_init(struct dbox_file_header *hdr); +int dbox_file_header_parse(struct dbox_mailbox *mbox, struct dbox_file *file, + const struct dbox_file_header *hdr); int dbox_file_read_header(struct dbox_mailbox *mbox, struct dbox_file *file); +int dbox_file_write_header(struct dbox_mailbox *mbox, struct dbox_file *file); #endif
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-storage/index/dbox/dbox-keywords.c Sun Jan 29 00:05:48 2006 +0200 @@ -0,0 +1,212 @@ +/* Copyright (C) 2006 Timo Sirainen */ + +#include "lib.h" +#include "array.h" +#include "str.h" +#include "istream.h" +#include "write-full.h" +#include "seq-range-array.h" +#include "bsearch-insert-pos.h" +#include "dbox-file.h" +#include "dbox-storage.h" +#include "dbox-keywords.h" + +#include <stdlib.h> + +static int dbox_keyword_map_compare(const void *p1, const void *p2) +{ + const struct keyword_map *map1 = p1, *map2 = p2; + + return map1->index_idx < map2->index_idx ? -1 : + map1->index_idx > map2->index_idx ? 1 : 0; +} + +int dbox_file_read_keywords(struct dbox_mailbox *mbox, struct dbox_file *file) +{ + struct keyword_map *map, *pos, kw; + const char *line; + unsigned int idx, count; + uoff_t last_offset; + + if (array_is_created(&file->idx_file_keywords)) { + array_clear(&file->idx_file_keywords); + array_clear(&file->file_idx_keywords); + } else { + ARRAY_CREATE(&file->idx_file_keywords, default_pool, + struct keyword_map, file->keyword_count); + ARRAY_CREATE(&file->file_idx_keywords, default_pool, + unsigned int, file->keyword_count); + } + + /* currently we assume that all extra space at the end of header + belongs to keyword list. */ + file->keyword_list_size_alloc = + file->header_size - file->keyword_list_offset; + + i_stream_seek(file->input, file->keyword_list_offset); + idx = 0; + last_offset = file->input->v_offset; + while ((line = i_stream_read_next_line(file->input)) != NULL) { + if (*line == '\0') { + /* end of list */ + break; + } + last_offset = file->input->v_offset; + + /* set up map record for the keyword */ + (void)mail_index_keyword_lookup(mbox->ibox.index, line, TRUE, + &kw.index_idx); + kw.file_idx = idx; + + /* look up the position where to insert it */ + map = array_get_modifyable(&file->idx_file_keywords, &count); + pos = idx == 0 ? map : + bsearch_insert_pos(line, map, count, sizeof(*map), + dbox_keyword_map_compare); + array_insert(&file->idx_file_keywords, pos - map, &kw, 1); + array_append(&file->file_idx_keywords, &kw.index_idx, 1); + + if (++idx == file->keyword_count) + break; + } + + if (line == NULL || file->input->v_offset > file->header_size) { + /* unexpected end of list, or list continues outside its + allocated area */ + mail_storage_set_critical(STORAGE(mbox->storage), + "Corrupted keyword list offset in dbox file %s", + file->path); + array_clear(&file->idx_file_keywords); + return 0; + } + + file->keyword_list_size_used = + last_offset - file->keyword_list_offset; + return 1; +} + +static int keyword_lookup_cmp(const void *key, const void *obj) +{ + const unsigned int *index_idx = key; + const struct keyword_map *map = obj; + + return *index_idx < map->index_idx ? -1 : + *index_idx > map->index_idx ? 1 : 0; +} + +bool dbox_file_lookup_keyword(struct dbox_mailbox *mbox, struct dbox_file *file, + unsigned int index_idx, unsigned int *idx_r) +{ + const struct keyword_map *map, *pos; + unsigned int count; + + if (!array_is_created(&file->idx_file_keywords)) { + /* Read the keywords, if there are any */ + if (dbox_file_read_keywords(mbox, file) <= 0) + return FALSE; + } + + map = array_get(&file->idx_file_keywords, &count); + pos = bsearch(&index_idx, map, count, sizeof(*map), + keyword_lookup_cmp); + if (pos != NULL && idx_r != NULL) + *idx_r = pos->file_idx; + return pos != NULL; +} + +int dbox_file_append_keywords(struct dbox_mailbox *mbox, struct dbox_file *file, + const struct seq_range *idx_range, + unsigned int count) +{ + const array_t *idx_keywords; + ARRAY_SET_TYPE(idx_keywords, const char *); + string_t *keyword_str; + const char *const *idx_keyword_names; + unsigned int i, idx_keyword_count, new_pos; + int ret; + + t_push(); + keyword_str = t_str_new(2048); + idx_keywords = mail_index_get_keywords(mbox->ibox.index); + idx_keyword_names = array_get(idx_keywords, &idx_keyword_count); + + /* make sure we've read the existing keywords */ + if (!array_is_created(&file->idx_file_keywords)) { + ret = dbox_file_read_keywords(mbox, file); + if (ret < 0) + return -1; + + if (ret == 0) { + /* broken keywords list. */ + file->keyword_list_size_used = 0; + } + } + + /* append existing keywords */ + if (array_count(&file->idx_file_keywords) > 0) { + const unsigned int *file_idx; + unsigned int file_count; + + file_idx = array_get(&file->file_idx_keywords, &file_count); + for (i = 0; i < file_count; i++) { + i_assert(file_idx[i] < idx_keyword_count); + + str_append(keyword_str, idx_keyword_names[file_idx[i]]); + str_append_c(keyword_str, '\n'); + } + } + + /* append new keywords */ + if (file->keyword_list_size_used == 0) + new_pos = 0; + else { + new_pos = str_len(keyword_str); + i_assert(new_pos == file->keyword_list_size_used); + } + for (i = 0; i < count; i++) { + unsigned int idx; + + for (idx = idx_range[i].seq1; idx <= idx_range[i].seq2; idx++) { + size_t prev_len; + + i_assert(idx < idx_keyword_count); + i_assert(!dbox_file_lookup_keyword(mbox, file, + idx, NULL)); + + prev_len = str_len(keyword_str); + str_append(keyword_str, idx_keyword_names[idx]); + str_append_c(keyword_str, '\n'); + + if (str_len(keyword_str) >= + file->keyword_list_size_alloc) { + /* FIXME: keyword list doesn't fit to the + space allocated for it. create a new file + where there's more space for keywords and + move the mails there. + + for now we'll just ignore the problem. */ + str_truncate(keyword_str, prev_len); + break; + } + } + } + + str_append_c(keyword_str, '\n'); + i_assert(str_len(keyword_str) <= file->keyword_list_size_alloc); + + /* we can reuse the existing keyword list position */ + if (pwrite_full(file->fd, str_data(keyword_str) + new_pos, + str_len(keyword_str) - new_pos, + file->keyword_list_offset + new_pos) < 0) { + mail_storage_set_critical(STORAGE(mbox->storage), + "pwrite_full(%s) failed: %m", file->path); + } + + /* FIXME: we could do this faster than by reading them.. */ + ret = 0; + if (dbox_file_read_keywords(mbox, file) <= 0) + ret = -1; + + t_pop(); + return ret; +}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-storage/index/dbox/dbox-keywords.h Sun Jan 29 00:05:48 2006 +0200 @@ -0,0 +1,17 @@ +#ifndef __DBOX_KEYWORDS_H +#define __DBOX_KEYWORDS_H + +struct seq_range; + +/* Read keywords from file into memory. Returns 1 if ok, 0 if the list is + broken or -1 if I/O error. */ +int dbox_file_read_keywords(struct dbox_mailbox *mbox, struct dbox_file *file); +/* Index file -> dbox file keyword index lookup. Returns TRUE if found. */ +bool dbox_file_lookup_keyword(struct dbox_mailbox *mbox, struct dbox_file *file, + unsigned int index_idx, unsigned int *idx_r); +/* Save keywords to dbox file. Returns -1 if error, 0 if ok. */ +int dbox_file_append_keywords(struct dbox_mailbox *mbox, struct dbox_file *file, + const struct seq_range *idx_range, + unsigned int count); + +#endif
--- a/src/lib-storage/index/dbox/dbox-mail.c Sat Jan 28 23:47:00 2006 +0200 +++ b/src/lib-storage/index/dbox/dbox-mail.c Sun Jan 29 00:05:48 2006 +0200 @@ -21,9 +21,7 @@ const struct dbox_mail_header *hdr = &file->seeked_mail_header; uint32_t hdr_uid = hex2dec(hdr->uid_hex, sizeof(hdr->uid_hex)); - if (hdr_uid != mail->mail.mail.uid || - memcmp(hdr->magic, DBOX_MAIL_HEADER_MAGIC, - sizeof(hdr->magic)) != 0) { + if (hdr_uid != mail->mail.mail.uid) { mail_storage_set_critical(STORAGE(mbox->storage), "dbox %s: Cached file offset broken", mbox->file->path); @@ -144,9 +142,8 @@ data->received_date = 0; } - mail_cache_add(mail->trans->cache_trans, mail->data.seq, - MAIL_CACHE_RECEIVED_DATE, - &data->received_date, sizeof(data->received_date)); + index_mail_cache_add(mail, MAIL_CACHE_RECEIVED_DATE, + &data->received_date, sizeof(data->received_date)); return data->received_date; } @@ -163,9 +160,8 @@ if (dbox_mail_open(mail, &offset) <= 0) return (uoff_t)-1; - mail_cache_add(mail->trans->cache_trans, mail->data.seq, - MAIL_CACHE_PHYSICAL_FULL_SIZE, - &data->physical_size, sizeof(data->physical_size)); + index_mail_cache_add(mail, MAIL_CACHE_PHYSICAL_FULL_SIZE, + &data->physical_size, sizeof(data->physical_size)); return data->physical_size; }
--- a/src/lib-storage/index/dbox/dbox-storage.h Sat Jan 28 23:47:00 2006 +0200 +++ b/src/lib-storage/index/dbox/dbox-storage.h Sun Jan 29 00:05:48 2006 +0200 @@ -6,6 +6,9 @@ #define DBOX_MAILDIR_NAME "Mails" #define DBOX_MAIL_FILE_PREFIX "msg." +#define DBOX_KEYWORD_COUNT 64 +#define DBOX_KEYWORD_NAMES_RESERVED_SPACE (2048-sizeof(struct dbox_file_header)) + #include "index-storage.h" #define STORAGE(mbox_storage) \ @@ -21,17 +24,37 @@ struct dbox_uidlist; struct dbox_file_header { + /* Size of the base header. sizeof(struct dbox_file_header) */ + unsigned char base_header_size_hex[4]; + /* Size of the full header, including keywords list and padding */ unsigned char header_size_hex[8]; + /* Offset where to store the next mail. note that a mail may already + have been fully written here and added to uidlist, but this offset + just wasn't updated. In that case the append_offset should be + updated instead of overwriting the mail. */ unsigned char append_offset_hex[16]; + /* Initial file creation time as UNIX timestamp. */ unsigned char create_time_hex[8]; + /* Size of each message's header. */ unsigned char mail_header_size_hex[4]; - unsigned char mail_header_padding_hex[4]; + /* If set, mail headers start always at given alignmentation. + Currently not supported. */ + unsigned char mail_header_align_hex[4]; + /* Number of keywords allocated for each mail (not necessarily used) */ unsigned char keyword_count_hex[4]; - /* possible padding to fill header_size */ + /* Offset for the keyword list inside the file header. */ + unsigned char keyword_list_offset_hex[8]; + + /* space reserved for keyword list and possible other future + extensions. */ + /* unsigned char [header_size - header_base_size]; */ }; #define DBOX_MAIL_HEADER_MAGIC "\001\003" struct dbox_mail_header { + /* This field acts as kind of a verification marker to make sure that + seeked offset is valid. So the magic value should be something that + normally doesn't occur in mails. */ unsigned char magic[2]; unsigned char uid_hex[8]; unsigned char mail_size_hex[16]; @@ -42,13 +65,18 @@ unsigned char seen; unsigned char draft; unsigned char expunged; - /* unsigned char keywords[]; */ + /* unsigned char keywords[keywords_count]; */ }; struct dbox_storage { struct index_storage storage; }; +struct keyword_map { + unsigned int index_idx; + unsigned int file_idx; +}; + struct dbox_file { uint32_t file_seq; char *path; @@ -57,17 +85,27 @@ struct istream *input; struct ostream *output; /* while appending mails */ + uint16_t base_header_size; uint32_t header_size; time_t create_time; uint64_t append_offset; uint16_t mail_header_size; - uint16_t mail_header_padding; + uint16_t mail_header_align; uint16_t keyword_count; + uint64_t keyword_list_offset; + uint32_t keyword_list_size_alloc; + uint32_t keyword_list_size_used; uoff_t seeked_offset; uoff_t seeked_mail_size; uint32_t seeked_uid; - struct dbox_mail_header seeked_mail_header; + struct dbox_mail_header seeked_mail_header; + unsigned char *seeked_keywords; + + /* Keywords list, sorted by index_idx. */ + array_t ARRAY_DEFINE(idx_file_keywords, struct keyword_map); + /* idx -> index_idx array */ + array_t ARRAY_DEFINE(file_idx_keywords, unsigned int); }; struct dbox_mailbox {
--- a/src/lib-storage/index/dbox/dbox-sync-full.c Sat Jan 28 23:47:00 2006 +0200 +++ b/src/lib-storage/index/dbox/dbox-sync-full.c Sun Jan 29 00:05:48 2006 +0200 @@ -6,6 +6,7 @@ #include "dbox-storage.h" #include "dbox-uidlist.h" #include "dbox-file.h" +#include "dbox-keywords.h" #include "dbox-sync.h" #include <stdlib.h> @@ -13,11 +14,36 @@ #include <fcntl.h> #include <dirent.h> +static int +dbox_mail_get_keywords(struct dbox_mailbox *mbox, struct dbox_file *file, + array_t *keywords) +{ + const unsigned int *map; + unsigned int i, count; + + ARRAY_SET_TYPE(keywords, unsigned int); + + if (!array_is_created(&file->file_idx_keywords)) { + if (dbox_file_read_keywords(mbox, file) < 0) + return -1; + } + + map = array_get(&file->file_idx_keywords, &count); + for (i = 0; i < count; i++) { + if (file->seeked_keywords[i] != '0') + array_append(keywords, &map[i], 1); + } + + return 0; +} + static int dbox_sync_full_mail(struct dbox_sync_context *ctx, uint32_t *seq_r) { struct dbox_mailbox *mbox = ctx->mbox; const struct dbox_mail_header *hdr = &mbox->file->seeked_mail_header; enum mail_flags flags; + struct mail_keywords *keywords; + array_t ARRAY_DEFINE(keywords_arr, unsigned int); uint32_t seq; uint64_t hdr_offset = mbox->file->seeked_offset; @@ -60,7 +86,20 @@ if (hdr->draft == '1') flags |= MAIL_DRAFT; mail_index_update_flags(ctx->trans, seq, MODIFY_REPLACE, flags); - // FIXME: keywords + + t_push(); + ARRAY_CREATE(&keywords_arr, pool_datastack_create(), + unsigned int, mbox->file->keyword_count); + if (dbox_mail_get_keywords(mbox, mbox->file, &keywords_arr) < 0) { + t_pop(); + return -1; + } + keywords = mail_index_keywords_create_from_indexes(ctx->trans, + &keywords_arr); + mail_index_update_keywords(ctx->trans, seq, MODIFY_REPLACE, keywords); + mail_index_keywords_free(&keywords); + t_pop(); + mail_index_update_ext(ctx->trans, seq, mbox->dbox_file_ext_idx, &mbox->file->file_seq, NULL); mail_index_update_ext(ctx->trans, seq, mbox->dbox_offset_ext_idx,
--- a/src/lib-storage/index/dbox/dbox-sync.c Sat Jan 28 23:47:00 2006 +0200 +++ b/src/lib-storage/index/dbox/dbox-sync.c Sun Jan 29 00:05:48 2006 +0200 @@ -4,8 +4,10 @@ #include "ioloop.h" #include "array.h" #include "hash.h" +#include "seq-range-array.h" #include "write-full.h" #include "dbox-file.h" +#include "dbox-keywords.h" #include "dbox-sync.h" #include "dbox-uidlist.h" #include "dbox-storage.h" @@ -120,45 +122,18 @@ return 0; } -int dbox_sync_update_flags(struct dbox_sync_context *ctx, - const struct dbox_sync_rec *sync_rec) +static int +dbox_sync_write_mask(struct dbox_sync_context *ctx, + const struct dbox_sync_rec *sync_rec, + unsigned int first_flag_offset, unsigned int flag_count, + const unsigned char *array, const unsigned char *mask) { - static enum mail_flags dbox_flag_list[] = { - MAIL_ANSWERED, - MAIL_FLAGGED, - MAIL_DELETED, - MAIL_SEEN, - MAIL_DRAFT, - 0 /* expunged */ - }; -#define DBOX_FLAG_COUNT (sizeof(dbox_flag_list)/sizeof(dbox_flag_list[0])) struct dbox_mailbox *mbox = ctx->mbox; - unsigned char dbox_flag_array[DBOX_FLAG_COUNT]; - unsigned char dbox_flag_mask[DBOX_FLAG_COUNT]; uint32_t file_seq, uid2; uoff_t offset; - unsigned int i, start, first_flag_offset; + unsigned int i, start; int ret; - /* first build flag array and mask */ - if (sync_rec->type == MAIL_INDEX_SYNC_TYPE_EXPUNGE) { - memset(dbox_flag_array, '0', sizeof(dbox_flag_array)); - memset(dbox_flag_mask, 0, sizeof(dbox_flag_mask)); - dbox_flag_mask[5] = 1; - dbox_flag_array[5] = '1'; - } else { - i_assert(sync_rec->type == MAIL_INDEX_SYNC_TYPE_FLAGS); - for (i = 0; i < DBOX_FLAG_COUNT; i++) { - dbox_flag_array[i] = - (sync_rec->value.flags.add & - dbox_flag_list[i]) != 0 ? '1' : '0'; - dbox_flag_mask[i] = dbox_flag_array[i] || - (sync_rec->value.flags.remove & - dbox_flag_list[i]) != 0; - } - } - first_flag_offset = offsetof(struct dbox_mail_header, answered); - if (dbox_sync_get_file_offset(ctx, sync_rec->seq1, &file_seq, &offset) < 0) return -1; @@ -172,18 +147,18 @@ return ret; while (mbox->file->seeked_uid <= uid2) { - for (i = 0; i < DBOX_FLAG_COUNT; ) { - if (!dbox_flag_mask[i]) + for (i = 0; i < flag_count; ) { + if (!mask[i]) continue; start = i; - while (i < DBOX_FLAG_COUNT) { - if (!dbox_flag_mask[i]) + while (i < flag_count) { + if (!mask[i]) break; i++; } ret = pwrite_full(ctx->mbox->file->fd, - dbox_flag_array+start, i - start, + array + start, i - start, offset + first_flag_offset + start); if (ret < 0) { mail_storage_set_critical( @@ -205,11 +180,143 @@ return 0; } +int dbox_sync_update_flags(struct dbox_sync_context *ctx, + const struct dbox_sync_rec *sync_rec) +{ + static enum mail_flags dbox_flag_list[] = { + MAIL_ANSWERED, + MAIL_FLAGGED, + MAIL_DELETED, + MAIL_SEEN, + MAIL_DRAFT, + 0 /* expunged */ + }; +#define DBOX_FLAG_COUNT (sizeof(dbox_flag_list)/sizeof(dbox_flag_list[0])) + unsigned char dbox_flag_array[DBOX_FLAG_COUNT]; + unsigned char dbox_flag_mask[DBOX_FLAG_COUNT]; + unsigned int i, first_flag_offset; + + /* first build flag array and mask */ + if (sync_rec->type == MAIL_INDEX_SYNC_TYPE_EXPUNGE) { + memset(dbox_flag_array, '0', sizeof(dbox_flag_array)); + memset(dbox_flag_mask, 0, sizeof(dbox_flag_mask)); + dbox_flag_mask[5] = 1; + dbox_flag_array[5] = '1'; + } else { + i_assert(sync_rec->type == MAIL_INDEX_SYNC_TYPE_FLAGS); + for (i = 0; i < DBOX_FLAG_COUNT; i++) { + dbox_flag_array[i] = + (sync_rec->value.flags.add & + dbox_flag_list[i]) != 0 ? '1' : '0'; + dbox_flag_mask[i] = dbox_flag_array[i] || + (sync_rec->value.flags.remove & + dbox_flag_list[i]) != 0; + } + } + first_flag_offset = offsetof(struct dbox_mail_header, answered); + + return dbox_sync_write_mask(ctx, sync_rec, + first_flag_offset, DBOX_FLAG_COUNT, + dbox_flag_array, dbox_flag_mask); +} + +static int +dbox_sync_update_keyword(struct dbox_sync_context *ctx, + const struct dbox_sync_rec *sync_rec, bool set) +{ + unsigned char keyword_array, keyword_mask = 1; + unsigned int file_idx, first_flag_offset; + + keyword_array = set ? '1' : '0'; + + if (!dbox_file_lookup_keyword(ctx->mbox, ctx->mbox->file, + sync_rec->value.keyword_idx, &file_idx)) { + /* not found. if removing, just ignore. + + if adding, it currently happens only if the maximum keyword + count was reached. once we support moving mails to new file + to grow keywords count, this should never happen. + for now, just ignore this. */ + return 0; + } + + first_flag_offset = sizeof(struct dbox_mail_header) + file_idx; + return dbox_sync_write_mask(ctx, sync_rec, first_flag_offset, 1, + &keyword_array, &keyword_mask); +} + +static int +dbox_sync_reset_keyword(struct dbox_sync_context *ctx, + const struct dbox_sync_rec *sync_rec) +{ + unsigned char *keyword_array, *keyword_mask; + unsigned int first_flag_offset; + int ret; + + t_push(); + keyword_array = t_malloc(ctx->mbox->file->keyword_count); + keyword_mask = t_malloc(ctx->mbox->file->keyword_count); + memset(keyword_array, '0', ctx->mbox->file->keyword_count); + memset(keyword_mask, 1, ctx->mbox->file->keyword_count); + + first_flag_offset = sizeof(struct dbox_mail_header); + ret = dbox_sync_write_mask(ctx, sync_rec, first_flag_offset, + ctx->mbox->file->keyword_count, + keyword_array, keyword_mask); + t_pop(); + return ret; +} + +static int +dbox_sync_file_add_keywords(struct dbox_sync_context *ctx, + const struct dbox_sync_file_entry *entry, + unsigned int i) +{ + array_t ARRAY_DEFINE(keywords, struct seq_range); + const struct dbox_sync_rec *sync_recs; + const struct seq_range *range; + unsigned int count, file_idx, keyword_idx; + int ret = 0; + + if (dbox_file_seek(ctx->mbox, entry->file_seq, 0) <= 0) + return -1; + + /* Get a list of all new keywords. Using seq_range is the easiest + way to do this and should be pretty fast too. */ + t_push(); + ARRAY_CREATE(&keywords, pool_datastack_create(), struct seq_range, 16); + sync_recs = array_get(&entry->sync_recs, &count); + for (; i < count; i++) { + if (sync_recs[i].type != MAIL_INDEX_SYNC_TYPE_KEYWORD_ADD) + continue; + + /* check if it's already in the file */ + keyword_idx = sync_recs[i].value.keyword_idx; + if (dbox_file_lookup_keyword(ctx->mbox, ctx->mbox->file, + keyword_idx, &file_idx)) + continue; + + /* add it. if it already exists, it's handled internally. */ + seq_range_array_add(&keywords, 0, keyword_idx); + } + + /* now, write them to file */ + range = array_get(&keywords, &count); + if (count > 0) { + ret = dbox_file_append_keywords(ctx->mbox, ctx->mbox->file, + range, count); + } + + t_pop(); + return ret; +} + static int dbox_sync_file(struct dbox_sync_context *ctx, const struct dbox_sync_file_entry *entry) { const struct dbox_sync_rec *sync_recs; unsigned int i, count; + bool first_keyword = TRUE; int ret; sync_recs = array_get(&entry->sync_recs, &count); @@ -232,9 +339,25 @@ return -1; break; case MAIL_INDEX_SYNC_TYPE_KEYWORD_ADD: + if (first_keyword) { + /* add all new keywords in one go */ + first_keyword = FALSE; + if (dbox_sync_file_add_keywords(ctx, entry, + i) < 0) + return -1; + } + if (dbox_sync_update_keyword(ctx, &sync_recs[i], + TRUE) < 0) + return -1; + break; case MAIL_INDEX_SYNC_TYPE_KEYWORD_REMOVE: + if (dbox_sync_update_keyword(ctx, &sync_recs[i], + FALSE) < 0) + return -1; + break; case MAIL_INDEX_SYNC_TYPE_KEYWORD_RESET: - /* FIXME */ + if (dbox_sync_reset_keyword(ctx, &sync_recs[i]) < 0) + return -1; break; case MAIL_INDEX_SYNC_TYPE_APPEND: i_unreached();
--- a/src/lib-storage/index/dbox/dbox-uidlist.c Sat Jan 28 23:47:00 2006 +0200 +++ b/src/lib-storage/index/dbox/dbox-uidlist.c Sun Jan 29 00:05:48 2006 +0200 @@ -794,26 +794,6 @@ return 0; } -static int dbox_file_write_header(struct dbox_mailbox *mbox, - struct dbox_file *file) -{ - struct dbox_file_header hdr; - - // FIXME: code duplication - file->header_size = sizeof(hdr); - file->append_offset = file->header_size; - file->create_time = ioloop_time; - file->mail_header_size = sizeof(struct dbox_mail_header); - - dbox_file_header_init(&hdr); - if (o_stream_send(file->output, &hdr, sizeof(hdr)) < 0) { - mail_storage_set_critical(STORAGE(mbox->storage), - "write(%s) failed: %m", file->path); - return -1; - } - return 0; -} - static int dbox_uidlist_files_lookup(struct dbox_uidlist_append_ctx *ctx, uint32_t file_seq) { @@ -951,12 +931,7 @@ file->input = i_stream_create_file(file->fd, default_pool, 65536, FALSE); - - /* we'll be using CRLF linefeeds always */ - output = o_stream_create_file(file->fd, default_pool, 0, FALSE); - file->output = o_stream_create_crlf(default_pool, output); - o_stream_unref(&output); - + file->output = o_stream_create_file(file->fd, default_pool, 0, FALSE); if ((uoff_t)st.st_size < sizeof(struct dbox_file_header)) { if (dbox_file_write_header(mbox, file) < 0) { dbox_file_close(file); @@ -967,9 +942,16 @@ dbox_file_close(file); return -1; } - o_stream_seek(file->output, file->append_offset); } + /* we'll always use CRLF linefeeds for mails (but not the header, + so don't do this before dbox_file_write_header()) */ + output = o_stream_create_crlf(default_pool, file->output); + o_stream_unref(&file->output); + file->output = output; + + o_stream_seek(file->output, file->append_offset); + save_file->dev = st.st_dev; save_file->ino = st.st_ino; ARRAY_CREATE(&save_file->seqs, ctx->pool, unsigned int, 8);