Mercurial > dovecot > core-2.2
changeset 8910:fef8259e7277 HEAD
dbox: Removed code that is no longer necessary with the redesign.
- Maildir code now uses dovecot-uidlist directly instead of dbox.index.
- Flags and keywords are no longer written to metadata.
- Removed metadata modifying code entirely.
- dbox.index is no longer read or written. The code will be removed soon.
| author | Timo Sirainen <tss@iki.fi> |
|---|---|
| date | Mon, 16 Feb 2009 20:30:15 -0500 |
| parents | b39e85979c6a |
| children | 0b626cf2dd10 |
| files | src/lib-storage/index/dbox/dbox-file-maildir.c src/lib-storage/index/dbox/dbox-file.c src/lib-storage/index/dbox/dbox-file.h src/lib-storage/index/dbox/dbox-index.c src/lib-storage/index/dbox/dbox-index.h src/lib-storage/index/dbox/dbox-save.c src/lib-storage/index/dbox/dbox-storage.c src/lib-storage/index/dbox/dbox-storage.h src/lib-storage/index/dbox/dbox-sync-file.c src/lib-storage/index/dbox/dbox-sync-rebuild.c src/lib-storage/index/dbox/dbox-sync.c src/lib-storage/index/dbox/dbox-sync.h src/lib-storage/index/maildir/maildir-uidlist.c src/lib-storage/index/maildir/maildir-uidlist.h |
| diffstat | 14 files changed, 294 insertions(+), 983 deletions(-) [+] |
line wrap: on
line diff
--- a/src/lib-storage/index/dbox/dbox-file-maildir.c Fri Feb 13 17:44:00 2009 -0500 +++ b/src/lib-storage/index/dbox/dbox-file-maildir.c Mon Feb 16 20:30:15 2009 -0500 @@ -3,62 +3,24 @@ #include "lib.h" #include "array.h" #include "str.h" +#include "maildir/maildir-storage.h" +#include "maildir/maildir-uidlist.h" +#include "maildir/maildir-filename.h" #include "dbox-storage.h" -#include "../maildir/maildir-storage.h" -#include "../maildir/maildir-filename.h" -#include "dbox-index.h" #include "dbox-file.h" #include "dbox-file-maildir.h" #include <stdlib.h> static const char * -dbox_file_maildir_get_flags(struct dbox_file *file, enum dbox_metadata_key key) +dbox_maildir_file_get_ext(struct dbox_file *file, + enum maildir_uidlist_rec_ext_key key) { - ARRAY_TYPE(keyword_indexes) keyword_indexes; - struct mail_keywords *keywords; - enum mail_flags flags; - string_t *str; - - if (file->mbox->maildir_sync_keywords == NULL) - return NULL; + uint32_t uid; - t_array_init(&keyword_indexes, 32); - maildir_filename_get_flags(file->mbox->maildir_sync_keywords, - file->fname, &flags, &keyword_indexes); - str = t_str_new(64); - if (key == DBOX_METADATA_FLAGS) - dbox_mail_metadata_flags_append(str, flags); - else { - keywords = mail_index_keywords_create_from_indexes( - file->mbox->ibox.index, &keyword_indexes); - dbox_mail_metadata_keywords_append(file->mbox, str, keywords); - mail_index_keywords_free(&keywords); - } - return str_c(str); -} - -static const char * -dbox_file_maildir_get_old_metadata(struct dbox_file *file, char key) -{ - struct dbox_index_record *rec; - const char *p, *end; - - rec = dbox_index_record_lookup(file->mbox->dbox_index, file->file_id); - if (rec == NULL) - return NULL; - - for (p = strchr(rec->data, ' '); *p != '\0'; p++) { - if (*p == ' ') { - if (p[1] == key) { - end = strchr(p+2, ' '); - return t_strdup_until(p+2, end); - } - if (p[1] == ':') - break; - } - } - return NULL; + uid = file->file_id & ~DBOX_FILE_ID_FLAG_UID; + return maildir_uidlist_lookup_ext(file->mbox->maildir_uidlist, + uid, key); } const char *dbox_file_maildir_metadata_get(struct dbox_file *file, @@ -69,10 +31,6 @@ const char *p, *value = NULL; switch (key) { - case DBOX_METADATA_FLAGS: - case DBOX_METADATA_KEYWORDS: - value = dbox_file_maildir_get_flags(file, key); - break; case DBOX_METADATA_GUID: p = strchr(file->fname, MAILDIR_INFO_SEP); value = p == NULL ? file->fname : @@ -101,7 +59,8 @@ if (!maildir_filename_get_size(file->fname, MAILDIR_EXTRA_VIRTUAL_SIZE, &size)) { - value = dbox_file_maildir_get_old_metadata(file, 'W'); + value = dbox_maildir_file_get_ext(file, + MAILDIR_UIDLIST_REC_EXT_VSIZE); if (value == NULL) break; size = strtoull(value, NULL, 10); @@ -109,7 +68,8 @@ value = t_strdup_printf("%llx", (unsigned long long)size); break; case DBOX_METADATA_POP3_UIDL: - value = dbox_file_maildir_get_old_metadata(file, 'P'); + value = dbox_maildir_file_get_ext(file, + MAILDIR_UIDLIST_REC_EXT_POP3_UIDL); if (value != NULL && *value == '\0') { /* special case: use base filename */ p = strchr(file->fname, MAILDIR_INFO_SEP); @@ -119,12 +79,12 @@ value = t_strdup_until(file->fname, p); } break; - case DBOX_METADATA_EXPUNGED: + case DBOX_METADATA_OLD_EXPUNGED: + case DBOX_METADATA_OLD_FLAGS: + case DBOX_METADATA_OLD_KEYWORDS: case DBOX_METADATA_EXT_REF: case DBOX_METADATA_SPACE: break; } - if (value != NULL) - dbox_file_metadata_set(file, key, value); return value; }
--- a/src/lib-storage/index/dbox/dbox-file.c Fri Feb 13 17:44:00 2009 -0500 +++ b/src/lib-storage/index/dbox/dbox-file.c Mon Feb 16 20:30:15 2009 -0500 @@ -11,8 +11,8 @@ #include "fdatasync-path.h" #include "write-full.h" #include "str.h" +#include "maildir/maildir-uidlist.h" #include "dbox-storage.h" -#include "dbox-index.h" #include "dbox-file.h" #include "dbox-file-maildir.h" @@ -126,26 +126,33 @@ } } +static bool +dbox_maildir_uid_get_fname(struct dbox_mailbox *mbox, uint32_t uid, + const char **fname_r) +{ + enum maildir_uidlist_rec_flag flags; + + *fname_r = maildir_uidlist_lookup(mbox->maildir_uidlist, uid, &flags); + return *fname_r != NULL; +} + static char * dbox_file_id_get_fname(struct dbox_mailbox *mbox, unsigned int file_id, bool *maildir_file_r) { - struct dbox_index_record *rec; - const char *p; + const char *fname; + uint32_t uid; *maildir_file_r = FALSE; if ((file_id & DBOX_FILE_ID_FLAG_UID) != 0) { - file_id &= ~DBOX_FILE_ID_FLAG_UID; - return i_strdup_printf(DBOX_MAIL_FILE_UID_FORMAT, file_id); - } - - rec = dbox_index_record_lookup(mbox->dbox_index, file_id); - if (rec != NULL && rec->status == DBOX_INDEX_FILE_STATUS_MAILDIR) { - /* data contains <uid> [<fields>] :<filename> */ - *maildir_file_r = TRUE; - p = strstr(rec->data, " :"); - i_assert(p != NULL); - return i_strdup_printf("%s", p + 2); + uid = file_id & ~DBOX_FILE_ID_FLAG_UID; + if (uid <= mbox->highest_maildir_uid && + dbox_maildir_uid_get_fname(mbox, uid, &fname)) { + *maildir_file_r = TRUE; + return i_strdup(fname); + } else { + return i_strdup_printf(DBOX_MAIL_FILE_UID_FORMAT, uid); + } } return i_strdup_printf(DBOX_MAIL_FILE_MULTI_FORMAT, file_id); @@ -192,17 +199,6 @@ return file; } -struct dbox_file * -dbox_file_init_new_maildir(struct dbox_mailbox *mbox, const char *fname) -{ - struct dbox_file *file; - - file = dbox_file_init(mbox, 0); - file->maildir_file = TRUE; - file->fname = i_strdup(fname); - return file; -} - int dbox_file_assign_id(struct dbox_file *file, unsigned int file_id) { struct dbox_mailbox *mbox = file->mbox; @@ -551,22 +547,27 @@ dbox_file_get_maildir_data(struct dbox_file *file, uint32_t *uid_r, uoff_t *physical_size_r) { - struct dbox_index_record *rec; struct stat st; +#if 0 //FIXME + uint32_t uid; + + if ((file->file_id & DBOX_FILE_ID_FLAG_UID) == 0) { + i_assert(file->file_id == 0); + if (maildir_uidlist_get_uid(file->mbox->maildir_uidlist, + file->fname, &uid)) + file->file_id = uid | DBOX_FILE_ID_FLAG_UID; + } +#else + i_assert((file->file_id & DBOX_FILE_ID_FLAG_UID) != 0); +#endif + if (fstat(file->fd, &st) < 0) { dbox_file_set_syscall_error(file, "fstat"); return -1; } - rec = dbox_index_record_lookup(file->mbox->dbox_index, file->file_id); - if (rec == NULL) { - /* should happen only when we're rebuilding the index */ - *uid_r = 0; - } else { - i_assert(rec->status == DBOX_INDEX_FILE_STATUS_MAILDIR); - *uid_r = strtoul(rec->data, NULL, 10); - } + *uid_r = file->file_id & ~DBOX_FILE_ID_FLAG_UID; *physical_size_r = st.st_size; return 1; } @@ -854,11 +855,9 @@ *expunged_r = FALSE; - if (file->metadata_pool != NULL) { - if (array_is_created(&file->metadata_changes)) - array_free(&file->metadata_changes); + if (file->metadata_pool != NULL) p_clear(file->metadata_pool); - } else { + else { file->metadata_pool = pool_alloconly_create("dbox metadata", 1024); } @@ -894,8 +893,6 @@ if (*line == DBOX_METADATA_SPACE || *line == '\0') { /* end of metadata */ - file->metadata_space_pos = - prev_offset - metadata_data_offset; *expunged_r = FALSE; break; } @@ -903,9 +900,6 @@ array_append(&file->metadata, &line, 1); } file->metadata_read_offset = metadata_offset; - file->metadata_len = file->input->v_offset - metadata_data_offset; - if (*expunged_r) - file->metadata_space_pos = file->metadata_len; return 1; } @@ -943,206 +937,11 @@ return NULL; } -void dbox_file_metadata_set(struct dbox_file *file, enum dbox_metadata_key key, - const char *value) -{ - const char **changes, *data; - unsigned int i, count; - - data = file->maildir_file ? NULL : dbox_file_metadata_get(file, key); - if (data != NULL && strcmp(data, value) == 0) { - /* value didn't change */ - return; - } - - data = p_strdup_printf(file->metadata_pool, "%c%s", (char)key, value); - - if (!array_is_created(&file->metadata_changes)) - p_array_init(&file->metadata_changes, file->metadata_pool, 16); - else { - /* see if we have already changed this metadata */ - changes = array_get_modifiable(&file->metadata_changes, &count); - for (i = 0; i < count; i++) { - if (*changes[i] == (char)key) { - changes[i] = data; - return; - } - } - } - - array_append(&file->metadata_changes, &data, 1); -} - -static int dbox_file_metadata_is_at_eof(struct dbox_file *file) -{ - uoff_t size; - uint32_t uid; - uoff_t offset; - int ret; - - if ((file->file_id & DBOX_FILE_ID_FLAG_UID) != 0) - return 1; - - offset = file->metadata_read_offset; - ret = dbox_file_seek_next_at_metadata(file, &offset, &uid, &size); - return ret <= 0 ? ret : uid == 0; -} - -static int dbox_file_write_empty_block(struct dbox_file *file, uoff_t offset, - unsigned int len) -{ - char space[256]; - - i_assert(len > 0); - - len--; - memset(space, DBOX_METADATA_SPACE, I_MIN(sizeof(space), len)); - while (len >= sizeof(space)) { - if (pwrite_full(file->fd, space, sizeof(space), offset) < 0) { - dbox_file_set_syscall_error(file, "pwrite"); - return -1; - } - } - /* @UNSAFE: last block ends with LF */ - space[len++] = '\n'; - if (pwrite_full(file->fd, space, len, offset) < 0) { - dbox_file_set_syscall_error(file, "pwrite"); - return -1; - } - file->metadata_len += len; - return 1; -} - -static int dbox_file_grow_metadata(struct dbox_file *file, unsigned int len) -{ - enum dbox_index_file_lock_status lock_status; - uoff_t offset; - int ret; - - ret = dbox_index_try_lock_file(file->mbox->dbox_index, file->file_id, - &lock_status); - if (ret <= 0 || (ret = dbox_file_metadata_is_at_eof(file)) <= 0) - return ret; - - offset = file->metadata_read_offset + - sizeof(struct dbox_metadata_header) + file->metadata_len; - i_stream_seek(file->input, offset); - (void)i_stream_read(file->input); - if (!i_stream_have_bytes_left(file->input)) { - len = len - file->metadata_len + DBOX_EXTRA_SPACE; - ret = dbox_file_write_empty_block(file, offset, len); - } else { - i_error("%s: Metadata changed unexpectedly", - dbox_file_get_path(file)); - ret = -1; - } - - dbox_index_unlock_file(file->mbox->dbox_index, file->file_id); - return ret; -} - -static int dbox_file_metadata_write_real(struct dbox_file *file) -{ - const char *const *metadata, *const *changes; - unsigned int i, j, count, changes_count, space_needed, skip_pos; - char space[DBOX_EXTRA_SPACE]; - string_t *str; - uoff_t offset; - int ret; - - if (!array_is_created(&file->metadata_changes)) { - /* nothing to write */ - return 1; - } - if (file->maildir_file) - return 0; - - offset = file->metadata_read_offset + - sizeof(struct dbox_metadata_header); - metadata = array_get(&file->metadata, &count); - changes = array_get(&file->metadata_changes, &changes_count); - - /* skip as many metadata fields from beginning as we can */ - for (i = skip_pos = 0; i < count; i++) { - for (j = 0; j < changes_count; j++) { - if (*changes[j] == *metadata[i]) - break; - } - if (j != changes_count) - break; - skip_pos += strlen(metadata[i]) + 1; - } - - str = t_str_new(512); - /* overwrite existing metadata fields */ - for (; i < count; i++) { - for (j = 0; j < changes_count; j++) { - if (*changes[j] == *metadata[i]) - break; - } - if (j != changes_count) { - str_append(str, changes[j]); - str_append_c(str, '\n'); - } else { - str_append(str, metadata[i]); - str_append_c(str, '\n'); - } - } - /* add new metadata */ - for (j = 0; j < changes_count; j++) { - for (i = 0; i < count; i++) { - if (*changes[j] == *metadata[i]) - break; - } - if (i == count) { - str_append(str, changes[j]); - str_append_c(str, '\n'); - } - } - if (skip_pos + str_len(str) >= file->metadata_len) { - if ((ret = dbox_file_grow_metadata(file, skip_pos + - str_len(str))) <= 0) - return ret; - } - - memset(space, DBOX_METADATA_SPACE, sizeof(space)); - while (skip_pos + str_len(str) < file->metadata_space_pos) { - space_needed = file->metadata_space_pos - - (skip_pos + str_len(str)); - str_append_n(str, space, I_MIN(sizeof(space), space_needed)); - } - i_assert(skip_pos + str_len(str) <= file->metadata_len); - - if (file->metadata_space_pos < skip_pos + str_len(str)) { - /* metadata was grown, update space position */ - file->metadata_space_pos = skip_pos + str_len(str); - } - - ret = pwrite_full(file->fd, str_data(str), str_len(str), - offset + skip_pos); - if (ret < 0) { - dbox_file_set_syscall_error(file, "pwrite"); - return -1; - } - return 1; -} - -int dbox_file_metadata_write(struct dbox_file *file) -{ - int ret; - - T_BEGIN { - ret = dbox_file_metadata_write_real(file); - } T_END; - return ret; -} - int dbox_file_metadata_write_to(struct dbox_file *file, struct ostream *output) { struct dbox_metadata_header metadata_hdr; - char space[DBOX_EXTRA_SPACE]; - const char *const *metadata, *const *changes; - unsigned int i, j, count, changes_count; + const char *const *metadata; + unsigned int i, count; memset(&metadata_hdr, 0, sizeof(metadata_hdr)); memcpy(metadata_hdr.magic_post, DBOX_MAGIC_POST, @@ -1151,38 +950,13 @@ return -1; metadata = array_get(&file->metadata, &count); - if (!array_is_created(&file->metadata_changes)) { - for (i = 0; i < count; i++) { - if (o_stream_send_str(output, metadata[i]) < 0 || - o_stream_send(output, "\n", 1) < 0) - return -1; - } - } else { - changes = array_get(&file->metadata_changes, &changes_count); - /* write unmodified metadata */ - for (i = 0; i < count; i++) { - for (j = 0; j < changes_count; j++) { - if (*changes[j] == *metadata[i]) - break; - } - if (j == changes_count) { - if (o_stream_send_str(output, metadata[i]) < 0) - return -1; - if (o_stream_send(output, "\n", 1) < 0) - return -1; - } - } - /* write modified metadata */ - for (i = 0; i < changes_count; i++) { - if (o_stream_send_str(output, changes[i]) < 0 || - o_stream_send(output, "\n", 1) < 0) - return -1; - } + for (i = 0; i < count; i++) { + if (o_stream_send_str(output, metadata[i]) < 0 || + o_stream_send(output, "\n", 1) < 0) + return -1; } - memset(space, ' ', sizeof(space)); - if (o_stream_send(output, space, sizeof(space)) < 0 || - o_stream_send(output, "\n", 1) < 0) + if (o_stream_send(output, "\n", 1) < 0) return -1; return 0; } @@ -1204,6 +978,8 @@ mail_index_lookup_uid(view, seq, &uid); if ((uid & DBOX_FILE_ID_FLAG_UID) != 0) { /* something's broken, we can't handle this high UIDs */ + mail_storage_set_critical(mbox->ibox.box.storage, + "found too high uid=%u", uid); return FALSE; } *file_id_r = DBOX_FILE_ID_FLAG_UID | uid; @@ -1322,41 +1098,6 @@ return 0; } -void dbox_mail_metadata_flags_append(string_t *str, enum mail_flags flags) -{ - unsigned int i; - - for (i = 0; i < DBOX_METADATA_FLAGS_COUNT; i++) { - if ((flags & dbox_mail_flags_map[i]) != 0) - str_append_c(str, dbox_mail_flag_chars[i]); - else - str_append_c(str, '0'); - } -} - -void dbox_mail_metadata_keywords_append(struct dbox_mailbox *mbox, - string_t *str, - const struct mail_keywords *keywords) -{ - const ARRAY_TYPE(keywords) *keyword_names_list; - const char *const *keyword_names; - unsigned int i, keyword_names_count; - - if (keywords == NULL || keywords->count == 0) - return; - - keyword_names_list = mail_index_get_keywords(mbox->ibox.index); - keyword_names = array_get(keyword_names_list, &keyword_names_count); - - for (i = 0; i < keywords->count; i++) { - i_assert(keywords->idx[i] < keyword_names_count); - - str_append(str, keyword_names[keywords->idx[i]]); - str_append_c(str, ' '); - } - str_truncate(str, str_len(str)-1); -} - void dbox_msg_header_fill(struct dbox_message_header *dbox_msg_hdr, uint32_t uid, uoff_t message_size) {
--- a/src/lib-storage/index/dbox/dbox-file.h Fri Feb 13 17:44:00 2009 -0500 +++ b/src/lib-storage/index/dbox/dbox-file.h Mon Feb 16 20:30:15 2009 -0500 @@ -20,7 +20,6 @@ #define DBOX_MAGIC_PRE "\001\002" #define DBOX_MAGIC_POST "\n\001\003\n" -#define DBOX_EXTRA_SPACE 64 /* If file_id has this flag set, the file is a single file with file_id=UID. */ #define DBOX_FILE_ID_FLAG_UID 0x80000000 @@ -47,16 +46,14 @@ }; enum dbox_metadata_key { - /* Message is marked as expunged. '0' = no, '1' = yes */ - DBOX_METADATA_EXPUNGED = 'E', - /* Message flags in dbox_metadata_flags order. '0' = not set, anything - else = set. Unknown flags should be preserved. */ - DBOX_METADATA_FLAGS = 'F', + /* metadata used by old Dovecot versions */ + DBOX_METADATA_OLD_EXPUNGED = 'E', + DBOX_METADATA_OLD_FLAGS = 'F', + DBOX_METADATA_OLD_KEYWORDS = 'K', + /* Globally unique identifier for the message. Preserved when copying. */ DBOX_METADATA_GUID = 'G', - /* Space separated list of keywords */ - DBOX_METADATA_KEYWORDS = 'K', /* POP3 UIDL overriding the default format */ DBOX_METADATA_POP3_UIDL = 'P', /* Received UNIX timestamp in hex */ @@ -107,7 +104,7 @@ unsigned int append_offset_header_pos; unsigned int append_count; - uint32_t last_append_uid, maildir_append_seq; + uint32_t last_append_uid; uoff_t append_offset; time_t create_time; @@ -127,11 +124,7 @@ /* Metadata for the currently seeked metadata block. */ pool_t metadata_pool; ARRAY_DEFINE(metadata, const char *); - ARRAY_DEFINE(metadata_changes, const char *); uoff_t metadata_read_offset; - unsigned int metadata_space_pos; - /* Includes the trailing LF that shouldn't be used */ - unsigned int metadata_len; unsigned int alt_path:1; unsigned int maildir_file:1; @@ -144,8 +137,6 @@ struct dbox_file * dbox_file_init(struct dbox_mailbox *mbox, unsigned int file_id); -struct dbox_file * -dbox_file_init_new_maildir(struct dbox_mailbox *mbox, const char *fname); void dbox_file_unref(struct dbox_file **file); /* Free all currently opened files. */ @@ -209,13 +200,6 @@ /* Return wanted metadata value, or NULL if not found. */ const char *dbox_file_metadata_get(struct dbox_file *file, enum dbox_metadata_key key); -/* Add key=value metadata update (not written yet, not visible to _get()). - The changes are reset by dbox_file_metadata_seek() call. */ -void dbox_file_metadata_set(struct dbox_file *file, enum dbox_metadata_key key, - const char *value); -/* Write all metadata updates to disk. Returns 1 if ok, 0 if metadata doesn't - fit to its reserved space and message isn't last in file, -1 if I/O error. */ -int dbox_file_metadata_write(struct dbox_file *file); /* Write all metadata to output stream. Returns 0 if ok, -1 if I/O error. */ int dbox_file_metadata_write_to(struct dbox_file *file, struct ostream *output); @@ -226,13 +210,6 @@ /* Move the file to alt path or back. */ int dbox_file_move(struct dbox_file *file, bool alt_path); -/* Append flags as metadata value to given string */ -void dbox_mail_metadata_flags_append(string_t *str, enum mail_flags flags); -/* Append keywords as metadata value to given string */ -void dbox_mail_metadata_keywords_append(struct dbox_mailbox *mbox, - string_t *str, - const struct mail_keywords *keywords); - /* Fill dbox_message_header with given uid/size. */ void dbox_msg_header_fill(struct dbox_message_header *dbox_msg_hdr, uint32_t uid, uoff_t message_size);
--- a/src/lib-storage/index/dbox/dbox-index.c Fri Feb 13 17:44:00 2009 -0500 +++ b/src/lib-storage/index/dbox/dbox-index.c Mon Feb 16 20:30:15 2009 -0500 @@ -322,17 +322,6 @@ return 0; } -int dbox_index_get_uid_validity(struct dbox_index *index, - uint32_t *uid_validity_r) -{ - if (index->fd == -1) { - if (dbox_index_refresh(index) < 0) - return -1; - } - *uid_validity_r = index->uid_validity; - return 0; -} - static int dbox_index_record_cmp(const void *key, const void *data) { const unsigned int *file_id = key; @@ -464,28 +453,6 @@ rec->locked = FALSE; } -int dbox_index_try_lock_recreate(struct dbox_index *index) -{ - int i, ret; - - if (index->fd == -1) { - if (dbox_index_refresh(index) < 0) - return 1; - } - - for (i = 0; i < DBOX_INDEX_LOCK_RETRY_COUNT; i++) { - /* lock the whole file */ - ret = dbox_index_lock_range(index, F_SETLK, F_WRLCK, 0, 0); - if (ret <= 0) - return ret; - if ((ret = dbox_index_refresh(index)) <= 0) - return ret < 0 ? -1 : 1; - } - - i_warning("dbox index keeps getting recreated: %s", index->path); - return 0; -} - static int dbox_index_lock_header(struct dbox_index *index) { int i, ret; @@ -761,38 +728,10 @@ return 0; } -void dbox_index_append_file(struct dbox_index_append_context *ctx, - struct dbox_file *file) -{ - file->refcount++; - array_append(&ctx->files, &file, 1); -} - static const char *dbox_file_maildir_get_index_data(struct dbox_file *file) { - const char *pop3_uidl = NULL, *const *changes; - unsigned int i, count; - - if (array_is_created(&file->metadata_changes)) - changes = array_get(&file->metadata_changes, &count); - else { - changes = NULL; - count = 0; - } - for (i = 0; i < count; i++) { - if (*changes[i] == DBOX_METADATA_POP3_UIDL) { - pop3_uidl = changes[i] + 1; - break; - } - } - - if (pop3_uidl == NULL) { - return t_strdup_printf("%u :%s", file->last_append_uid, - file->fname); - } else { - return t_strdup_printf("%u P%s :%s", file->last_append_uid, - pop3_uidl, file->fname); - } + return t_strdup_printf("%u :%s", file->last_append_uid, + file->fname); } static int dbox_index_append_commit_new(struct dbox_index_append_context *ctx,
--- a/src/lib-storage/index/dbox/dbox-index.h Fri Feb 13 17:44:00 2009 -0500 +++ b/src/lib-storage/index/dbox/dbox-index.h Mon Feb 16 20:30:15 2009 -0500 @@ -92,10 +92,6 @@ struct dbox_index *dbox_index_init(struct dbox_mailbox *mbox); void dbox_index_deinit(struct dbox_index **index); -/* Get the current UIDVALIDITY. Returns 0 if ok, -1 if I/O error. */ -int dbox_index_get_uid_validity(struct dbox_index *index, - uint32_t *uid_validity_r); - struct dbox_index_record * dbox_index_record_lookup(struct dbox_index *index, unsigned int file_id); @@ -106,10 +102,6 @@ enum dbox_index_file_lock_status *lock_status_r); void dbox_index_unlock_file(struct dbox_index *index, unsigned int file_id); -/* Try to lock index file for recreating. Returns 1 if ok, 0 if file already - contains locks, -1 if error. */ -int dbox_index_try_lock_recreate(struct dbox_index *index); - struct dbox_index_append_context * dbox_index_append_begin(struct dbox_index *index); /* Request file for saving a new message with given size. If an existing file @@ -119,16 +111,10 @@ uoff_t mail_size, struct dbox_file **file_r, struct ostream **output_r); -void dbox_index_append_file(struct dbox_index_append_context *ctx, - struct dbox_file *file); /* Assign file_ids to all appended files. */ int dbox_index_append_assign_file_ids(struct dbox_index_append_context *ctx); /* Returns 0 if ok, -1 if error. */ int dbox_index_append_commit(struct dbox_index_append_context **ctx); void dbox_index_append_rollback(struct dbox_index_append_context **ctx); -/* Mark */ -void dbox_index_mark_expunges(struct dbox_index *index, unsigned int file_id); -void dbox_index_mark_dirty(struct dbox_index *index, unsigned int file_id); - #endif
--- a/src/lib-storage/index/dbox/dbox-save.c Fri Feb 13 17:44:00 2009 -0500 +++ b/src/lib-storage/index/dbox/dbox-save.c Mon Feb 16 20:30:15 2009 -0500 @@ -48,17 +48,6 @@ unsigned int finished:1; }; -static void dbox_save_keywords(struct dbox_save_context *ctx, - struct mail_keywords *keywords) -{ - if (ctx->cur_keywords == NULL) - ctx->cur_keywords = str_new(default_pool, 128); - else - str_truncate(ctx->cur_keywords, 0); - dbox_mail_metadata_keywords_append(ctx->mbox, ctx->cur_keywords, - keywords); -} - struct mail_save_context * dbox_save_alloc(struct mailbox_transaction_context *_t) { @@ -141,7 +130,6 @@ if (_ctx->received_date == (time_t)-1) _ctx->received_date = ioloop_time; - dbox_save_keywords(ctx, _ctx->keywords); return ctx->failed ? -1 : 0; } @@ -177,7 +165,6 @@ static void dbox_save_write_metadata(struct dbox_save_context *ctx) { struct dbox_metadata_header metadata_hdr; - char space[DBOX_EXTRA_SPACE]; const char *guid; string_t *str; uoff_t vsize; @@ -201,23 +188,9 @@ guid = ctx->ctx.guid != NULL ? ctx->ctx.guid : mail_generate_guid_string(); str_printfa(str, "%c%s\n", DBOX_METADATA_GUID, guid); - - /* flags */ - str_append_c(str, DBOX_METADATA_FLAGS); - dbox_mail_metadata_flags_append(str, ctx->ctx.flags); str_append_c(str, '\n'); - /* keywords */ - if (ctx->cur_keywords != NULL && str_len(ctx->cur_keywords) > 0) { - str_append_c(str, DBOX_METADATA_KEYWORDS); - str_append_str(str, ctx->cur_keywords); - str_append_c(str, '\n'); - } - o_stream_send(ctx->cur_output, str_data(str), str_len(str)); - memset(space, ' ', sizeof(space)); - o_stream_send(ctx->cur_output, space, sizeof(space)); - o_stream_send(ctx->cur_output, "\n", 1); } static int dbox_save_mail_write_header(struct dbox_save_mail *mail) @@ -436,7 +409,7 @@ i_assert(ctx->finished); - if (dbox_sync_begin(ctx->mbox, &ctx->sync_ctx, FALSE, TRUE) < 0) { + if (dbox_sync_begin(ctx->mbox, TRUE, &ctx->sync_ctx) < 0) { ctx->failed = TRUE; dbox_transaction_save_rollback(ctx); return -1;
--- a/src/lib-storage/index/dbox/dbox-storage.c Fri Feb 13 17:44:00 2009 -0500 +++ b/src/lib-storage/index/dbox/dbox-storage.c Mon Feb 16 20:30:15 2009 -0500 @@ -8,6 +8,7 @@ #include "unlink-old-files.h" #include "index-mail.h" #include "mail-copy.h" +#include "maildir/maildir-uidlist.h" #include "dbox-sync.h" #include "dbox-index.h" #include "dbox-file.h" @@ -214,7 +215,6 @@ mbox->path = p_strdup(pool, path); mbox->alt_path = p_strdup(pool, dbox_get_alt_path(storage, path)); mbox->storage = storage; - mbox->last_interactive_change = ioloop_time; value = getenv("DBOX_ROTATE_SIZE"); if (value != NULL) @@ -252,6 +252,7 @@ mbox->dbox_index = dbox_index_init(mbox); index_storage_mailbox_init(&mbox->ibox, name, flags, FALSE); + mbox->maildir_uidlist = maildir_uidlist_init_readonly(&mbox->ibox); return &mbox->ibox.box; } @@ -319,18 +320,13 @@ static int dbox_storage_mailbox_close(struct mailbox *box) { struct dbox_mailbox *mbox = (struct dbox_mailbox *)box; - int ret = 0; - if (box->opened) { - /* see if we want to flush dirty flags */ - ret = dbox_sync(mbox, TRUE); - } - + maildir_uidlist_deinit(&mbox->maildir_uidlist); dbox_index_deinit(&mbox->dbox_index); dbox_files_free(mbox); array_free(&mbox->open_files); - return index_storage_mailbox_close(box) < 0 ? -1 : ret; + return index_storage_mailbox_close(box); } static int dbox_mailbox_create(struct mail_storage *_storage,
--- a/src/lib-storage/index/dbox/dbox-storage.h Fri Feb 13 17:44:00 2009 -0500 +++ b/src/lib-storage/index/dbox/dbox-storage.h Mon Feb 16 20:30:15 2009 -0500 @@ -31,7 +31,8 @@ #define DBOX_INDEX_FLAG_ALT MAIL_INDEX_MAIL_FLAG_BACKEND struct dbox_index_header { - uint32_t last_dirty_flush_stamp; + uint32_t unused; /* for backwards compatibility */ + uint32_t highest_maildir_uid; }; struct dbox_storage { @@ -49,12 +50,11 @@ struct index_mailbox ibox; struct dbox_storage *storage; + struct maildir_uidlist *maildir_uidlist; + uint32_t highest_maildir_uid; + struct dbox_index *dbox_index; uint32_t dbox_ext_id, dbox_hdr_ext_id; - /* timestamp when the mailbox was last modified interactively */ - time_t last_interactive_change; - /* set while rebuilding indexes with converted maildir files */ - struct maildir_keywords_sync_ctx *maildir_sync_keywords; uoff_t rotate_size, rotate_min_size; unsigned int rotate_days;
--- a/src/lib-storage/index/dbox/dbox-sync-file.c Fri Feb 13 17:44:00 2009 -0500 +++ b/src/lib-storage/index/dbox/dbox-sync-file.c Mon Feb 16 20:30:15 2009 -0500 @@ -13,13 +13,10 @@ static int dbox_sync_file_unlink(struct dbox_file *file) { const char *path; - int i; + int i = 0; path = t_strdup_printf("%s/%s", file->mbox->path, file->fname); - for (i = 0;; i++) { - if (unlink(path) == 0) - break; - + while (unlink(path) < 0) { if (errno != ENOENT) { mail_storage_set_critical(file->mbox->ibox.box.storage, "unlink(%s) failed: %m", path); @@ -29,58 +26,15 @@ /* not found */ i_warning("dbox: File unexpectedly lost: %s/%s", file->mbox->path, file->fname); - break; + return 0; } /* try the alternative path */ path = t_strdup_printf("%s/%s", file->mbox->alt_path, file->fname); + i++; } - return 0; -} - -static void -dbox_sync_update_metadata(struct dbox_sync_context *ctx, struct dbox_file *file, - const struct dbox_sync_file_entry *entry, - uint32_t seq) -{ - const struct mail_index_record *rec; - ARRAY_TYPE(keyword_indexes) keyword_indexes; - struct mail_keywords *keywords; - string_t *value; - const char *old_value; - - value = t_str_new(256); - - /* flags */ - rec = mail_index_lookup(ctx->sync_view, seq); - dbox_mail_metadata_flags_append(value, rec->flags); - dbox_file_metadata_set(file, DBOX_METADATA_FLAGS, str_c(value)); - - /* keywords */ - t_array_init(&keyword_indexes, 32); - mail_index_lookup_keywords(ctx->sync_view, seq, &keyword_indexes); - old_value = dbox_file_metadata_get(file, DBOX_METADATA_KEYWORDS); - if (array_count(&keyword_indexes) > 0 || - (old_value != NULL && *old_value != '\0' && - array_count(&keyword_indexes) == 0)) { - str_truncate(value, 0); - keywords = mail_index_keywords_create_from_indexes( - ctx->mbox->ibox.index, &keyword_indexes); - dbox_mail_metadata_keywords_append(ctx->mbox, value, keywords); - mail_index_keywords_free(&keywords); - - dbox_file_metadata_set(file, DBOX_METADATA_KEYWORDS, - str_c(value)); - } - - /* expunge state */ - if (entry != NULL && - array_is_created(&entry->expunges) && - seq_range_exists(&entry->expunges, seq)) { - dbox_file_metadata_set(file, DBOX_METADATA_EXPUNGED, "1"); - mail_index_expunge(ctx->trans, seq); - } + return 1; } static int @@ -152,9 +106,6 @@ /* write metadata */ (void)dbox_file_metadata_seek_mail_offset(file, offset, &expunged); - T_BEGIN { - dbox_sync_update_metadata(ctx, file, entry, seq); - } T_END; if ((ret = dbox_file_metadata_write_to(file, output)) < 0) break; @@ -192,6 +143,7 @@ return ret; } +#if 0 static int dbox_sync_file_split(struct dbox_sync_context *ctx, struct dbox_file *in_file, uoff_t offset, uint32_t seq) @@ -236,9 +188,6 @@ } append_offset = output->offset; dbox_msg_header_fill(&dbox_msg_hdr, uid, size); - T_BEGIN { - dbox_sync_update_metadata(ctx, out_file, NULL, seq); - } T_END; /* set static metadata */ for (i = 0; i < N_ELEMENTS(maildir_metadata_keys); i++) { @@ -288,103 +237,19 @@ } return ret < 0 ? -1 : 1; } - -static int -dbox_sync_file_changes(struct dbox_sync_context *ctx, struct dbox_file *file, - const struct dbox_sync_file_entry *entry, uint32_t seq) -{ - uint32_t file_id; - uoff_t offset; - bool expunged; - int ret; - - if (!dbox_file_lookup(ctx->mbox, ctx->sync_view, seq, - &file_id, &offset)) - return 0; - i_assert(file_id == file->file_id); - - ret = dbox_file_metadata_seek_mail_offset(file, offset, &expunged); - if (ret <= 0) - return ret; - if (expunged) { - mail_index_expunge(ctx->trans, seq); - return 1; - } - - T_BEGIN { - dbox_sync_update_metadata(ctx, file, entry, seq); - } T_END; - ret = dbox_file_metadata_write(file); - if (ret <= 0) { - return ret < 0 ? -1 : - dbox_sync_file_split(ctx, file, offset, seq); - } - - mail_index_update_flags(ctx->trans, seq, MODIFY_REMOVE, - (enum mail_flags)MAIL_INDEX_MAIL_FLAG_DIRTY); - return 1; -} - -static int -dbox_sync_file_int(struct dbox_sync_context *ctx, struct dbox_file *file, - const struct dbox_sync_file_entry *entry, bool full_expunge) -{ - const struct seq_range *seqs; - unsigned int i, count; - uint32_t seq, first_expunge_seq; - int ret; - - if (array_is_created(&entry->expunges) && full_expunge) { - seqs = array_idx(&entry->expunges, 0); - first_expunge_seq = seqs->seq1; - } else { - first_expunge_seq = (uint32_t)-1; - } - - if (array_is_created(&entry->changes)) { - seqs = array_get(&entry->changes, &count); - } else { - seqs = NULL; - count = 0; - } - for (i = 0; i < count; ) { - for (seq = seqs[i].seq1; seq <= seqs[i].seq2; seq++) { - if (seq >= first_expunge_seq) - return dbox_sync_file_expunge(ctx, file, entry); - - ret = dbox_sync_file_changes(ctx, file, entry, seq); - if (ret <= 0) - return ret; - } - i++; - } - if (first_expunge_seq != (uint32_t)-1) - return dbox_sync_file_expunge(ctx, file, entry); - return 1; -} +#endif static void -dbox_sync_file_move_if_needed(struct dbox_sync_context *ctx, - struct dbox_file *file, +dbox_sync_file_move_if_needed(struct dbox_file *file, const struct dbox_sync_file_entry *entry) { - const struct seq_range *seq; - const struct mail_index_record *rec; - bool new_alt_path; - - if (!array_is_created(&entry->changes)) + if (!entry->move_to_alt && !entry->move_from_alt) return; - /* check if we want to move the file to alt path or back. - FIXME: change this check somehow when a file may contain - multiple messages. */ - seq = array_idx(&entry->changes, 0); - rec = mail_index_lookup(ctx->sync_view, seq[0].seq1); - new_alt_path = (rec->flags & DBOX_INDEX_FLAG_ALT) != 0; - if (new_alt_path != file->alt_path) { + if (entry->move_to_alt != file->alt_path) { /* move the file. if it fails, nothing broke so don't worry about it. */ - (void)dbox_file_move(file, new_alt_path); + (void)dbox_file_move(file, !file->alt_path); } } @@ -436,17 +301,17 @@ status == DBOX_INDEX_FILE_STATUS_MAILDIR) && array_is_created(&entry->expunges)) { /* fast path to expunging the whole file */ - if (dbox_sync_file_unlink(file) < 0) - ret = -1; - else { + if ((ret = dbox_sync_file_unlink(file)) == 0) { + /* file was lost, delete it */ dbox_sync_mark_single_file_expunged(ctx, entry); ret = 1; } } else { ret = dbox_file_open_or_create(file, TRUE, &deleted); if (ret > 0 && !deleted) { - dbox_sync_file_move_if_needed(ctx, file, entry); - ret = dbox_sync_file_int(ctx, file, entry, locked); + dbox_sync_file_move_if_needed(file, entry); + if (array_is_created(&entry->expunges) && locked) + ret = dbox_sync_file_expunge(ctx, file, entry); } } dbox_file_unref(&file);
--- a/src/lib-storage/index/dbox/dbox-sync-rebuild.c Fri Feb 13 17:44:00 2009 -0500 +++ b/src/lib-storage/index/dbox/dbox-sync-rebuild.c Mon Feb 16 20:30:15 2009 -0500 @@ -1,11 +1,12 @@ /* Copyright (c) 2007-2009 Dovecot authors, see the included COPYING file */ #include "lib.h" +#include "ioloop.h" #include "array.h" #include "dbox-storage.h" -#include "../maildir/maildir-uidlist.h" -#include "../maildir/maildir-keywords.h" -#include "dbox-index.h" +#include "maildir/maildir-uidlist.h" +#include "maildir/maildir-keywords.h" +#include "maildir/maildir-filename.h" #include "dbox-file.h" #include "dbox-sync.h" @@ -14,34 +15,34 @@ struct dbox_sync_rebuild_context { struct dbox_mailbox *mbox; - struct dbox_index_append_context *append_ctx; struct mail_index_view *view; struct mail_index_transaction *trans; uint32_t cache_ext_id; uint32_t cache_reset_id; - struct maildir_uidlist *maildir_uidlist; + struct maildir_uidlist_sync_ctx *maildir_sync_ctx; struct maildir_keywords *mk; + struct maildir_keywords_sync_ctx *maildir_sync_keywords; - ARRAY_DEFINE(maildir_new_files, char *); - uint32_t maildir_new_uid; + uint32_t highest_uid; unsigned int cache_used:1; }; -static int dbox_sync_set_uidvalidity(struct dbox_sync_rebuild_context *ctx) +static void dbox_sync_set_uidvalidity(struct dbox_sync_rebuild_context *ctx) { + uint32_t uid_validity; - if (dbox_index_get_uid_validity(ctx->mbox->dbox_index, - &uid_validity) < 0) - return -1; + /* if uidvalidity is set in the old index, use it */ + uid_validity = mail_index_get_header(ctx->view)->uid_validity; + if (uid_validity == 0) + uid_validity = ioloop_time; mail_index_update_header(ctx->trans, offsetof(struct mail_index_header, uid_validity), &uid_validity, sizeof(uid_validity), TRUE); - return 0; } static void @@ -103,39 +104,40 @@ } static void +dbox_sync_index_copy_from_maildir(struct dbox_sync_rebuild_context *ctx, + struct dbox_file *file, uint32_t seq) +{ + struct dbox_mailbox *mbox = file->mbox; + ARRAY_TYPE(keyword_indexes) keyword_indexes; + struct mail_keywords *keywords; + enum mail_flags flags; + + t_array_init(&keyword_indexes, 32); + maildir_filename_get_flags(ctx->maildir_sync_keywords, + file->fname, &flags, &keyword_indexes); + mail_index_update_flags(ctx->trans, seq, MODIFY_REPLACE, flags); + + keywords = mail_index_keywords_create_from_indexes(mbox->ibox.index, + &keyword_indexes); + mail_index_update_keywords(ctx->trans, seq, MODIFY_REPLACE, keywords); + mail_index_keywords_free(&keywords); +} + +static void dbox_sync_index_metadata(struct dbox_sync_rebuild_context *ctx, struct dbox_file *file, uint32_t seq, uint32_t uid) { - const char *value; - struct mail_keywords *keywords; - enum mail_flags flags = 0; uint32_t old_seq; - unsigned int i; if (mail_index_lookup_seq(ctx->view, uid, &old_seq)) { /* the message exists in the old index. copy the metadata from it. */ dbox_sync_index_copy_from_old(ctx, old_seq, seq); - return; + } else if (file->maildir_file) { + /* we're probably doing initial sync after migration from + maildir. preserve the old flags. */ + dbox_sync_index_copy_from_maildir(ctx, file, seq); } - - value = dbox_file_metadata_get(file, DBOX_METADATA_FLAGS); - if (value != NULL) { - for (i = 0; value[i] != '\0'; i++) { - if (value[i] != '0' && i < DBOX_METADATA_FLAGS_COUNT) - flags |= dbox_mail_flags_map[i]; - } - mail_index_update_flags(ctx->trans, seq, MODIFY_REPLACE, flags); - } - - value = dbox_file_metadata_get(file, DBOX_METADATA_KEYWORDS); - if (value != NULL) T_BEGIN { - keywords = mail_index_keywords_create(ctx->mbox->ibox.index, - t_strsplit_spaces(value, " ")); - mail_index_update_keywords(ctx->trans, seq, MODIFY_REPLACE, - keywords); - mail_index_keywords_free(&keywords); - } T_END; } static int dbox_sync_index_file_next(struct dbox_sync_rebuild_context *ctx, @@ -143,7 +145,7 @@ { uint32_t seq, uid; uoff_t physical_size; - const char *path, *value; + const char *path; bool expunged; int ret; @@ -167,18 +169,8 @@ return 0; } if (file->maildir_file) { - i_assert(uid == 0); - if (!maildir_uidlist_get_uid(ctx->maildir_uidlist, file->fname, - &uid)) { - if (ctx->maildir_new_uid == 0) { - /* not in uidlist, give it an uid later */ - char *fname = i_strdup(file->fname); - array_append(&ctx->maildir_new_files, - &fname, 1); - return 0; - } - uid = ctx->maildir_new_uid++; - } + i_assert(uid != 0); + file->append_count = 1; file->last_append_uid = uid; } @@ -190,18 +182,8 @@ i_warning("%s: Ignoring broken file (metadata)", path); return 0; } - if (file->maildir_file) { - /* preserve POP3 UIDL */ - value = maildir_uidlist_lookup_ext(ctx->maildir_uidlist, uid, - MAILDIR_UIDLIST_REC_EXT_POP3_UIDL); - if (value != NULL) { - dbox_file_metadata_set(file, DBOX_METADATA_POP3_UIDL, - value); - } - } if (!expunged) { mail_index_append(ctx->trans, uid, &seq); - file->maildir_append_seq = seq; dbox_sync_index_metadata(ctx, file, seq, uid); } return 1; @@ -225,6 +207,9 @@ return 0; } + if (ctx->highest_uid < uid) + ctx->highest_uid = uid; + file = dbox_file_init(ctx->mbox, uid | DBOX_FILE_ID_FLAG_UID); file->current_path = i_strdup_printf("%s/%s", dir, fname); @@ -246,31 +231,33 @@ dbox_sync_index_maildir_file(struct dbox_sync_rebuild_context *ctx, const char *fname) { - struct dbox_file *file; - uoff_t offset = 0; int ret; - if (ctx->mbox->maildir_sync_keywords == NULL) { - ctx->maildir_uidlist = - maildir_uidlist_init_readonly(&ctx->mbox->ibox); + if (ctx->maildir_sync_ctx == NULL) { + i_assert(ctx->mk == NULL); + ctx->mk = maildir_keywords_init_readonly(&ctx->mbox->ibox.box); - ctx->mbox->maildir_sync_keywords = + ctx->maildir_sync_keywords = maildir_keywords_sync_init(ctx->mk, ctx->mbox->ibox.index); - if (maildir_uidlist_refresh(ctx->maildir_uidlist) < 0) + ret = maildir_uidlist_sync_init(ctx->mbox->maildir_uidlist, + MAILDIR_UIDLIST_SYNC_NOLOCK, + &ctx->maildir_sync_ctx); + if (ret <= 0) { + i_assert(ret < 0); return -1; + } } - file = dbox_file_init_new_maildir(ctx->mbox, fname); - if ((ret = dbox_sync_index_file_next(ctx, file, &offset)) > 0) { - dbox_index_append_file(ctx->append_ctx, file); - /* appending referenced the file, so make sure it gets closed - so we don't have too many open files. */ - dbox_file_close(file); + /* sync all maildir files first and let maildir uidlist code assign + UIDs for unseen files. */ + ret = maildir_uidlist_sync_next(ctx->maildir_sync_ctx, fname, 0); + if (ret == 0) { + i_warning("%s: Ignoring duplicate maildir file: %s", + ctx->mbox->path, fname); } - dbox_file_unref(&file); - return ret < 0 ? -1 : 0; + return ret; } static int @@ -312,7 +299,7 @@ "opendir(%s) failed: %m", path); return -1; } - for (;;) { + do { errno = 0; if ((d = readdir(dir)) == NULL) break; @@ -321,7 +308,7 @@ ret = dbox_sync_index_file(ctx, path, d->d_name, primary); } T_END; - } + } while (ret >= 0); if (errno != 0) { mail_storage_set_critical(storage, "readdir(%s) failed: %m", path); @@ -336,34 +323,81 @@ return ret; } -static int dbox_sync_new_maildir(struct dbox_sync_rebuild_context *ctx) +static int dbox_sync_maildir_finish(struct dbox_sync_rebuild_context *ctx) { + struct dbox_mailbox *mbox = ctx->mbox; + struct maildir_uidlist_iter_ctx *iter; struct mail_index_view *trans_view; - char *const *fnames; - unsigned int i, count; + struct dbox_file *file; + const char *fname; + enum maildir_uidlist_rec_flag flags; + uint32_t uid, next_uid; + uoff_t offset; int ret = 0; - fnames = array_get(&ctx->maildir_new_files, &count); - if (count == 0) - return 0; + /* we'll need the uidlist to contain the latest filenames. + since there's no easy way to figure out if they changed, just + recreate the uidlist always. */ + maildir_uidlist_sync_recreate(ctx->maildir_sync_ctx); + /* update the maildir uidlist's next_uid if we have seen higher + dbox UIDs */ trans_view = mail_index_transaction_open_updated_view(ctx->trans); - ctx->maildir_new_uid = mail_index_get_header(trans_view)->next_uid; + next_uid = mail_index_get_header(trans_view)->next_uid; mail_index_view_close(&trans_view); + maildir_uidlist_set_next_uid(mbox->maildir_uidlist, next_uid, FALSE); + maildir_uidlist_set_next_uid(mbox->maildir_uidlist, + ctx->highest_uid + 1, FALSE); + /* assign UIDs for new maildir mails before iterating */ + maildir_uidlist_sync_finish(ctx->maildir_sync_ctx); - for (i = 0; i < count && ret == 0; i++) { - T_BEGIN { - ret = dbox_sync_index_maildir_file(ctx, fnames[i]); - } T_END; + mbox->highest_maildir_uid = + maildir_uidlist_get_next_uid(mbox->maildir_uidlist); + + iter = maildir_uidlist_iter_init(mbox->maildir_uidlist); + while (maildir_uidlist_iter_next(iter, &uid, &flags, &fname)) { + file = dbox_file_init(mbox, uid | DBOX_FILE_ID_FLAG_UID); + file->current_path = + i_strdup_printf("%s/%s", ctx->mbox->path, fname); + + offset = 0; + ret = dbox_sync_index_file_next(ctx, file, &offset); + dbox_file_unref(&file); + if (ret < 0) + break; } - return ret; + maildir_uidlist_iter_deinit(&iter); + return ret < 0 ? -1 : 0; +} + +static void dbox_sync_update_header(struct dbox_sync_rebuild_context *ctx) +{ + const struct dbox_index_header *hdr; + struct dbox_index_header new_hdr; + const void *data; + size_t data_size; + + mail_index_get_header_ext(ctx->mbox->ibox.view, + ctx->mbox->dbox_hdr_ext_id, + &data, &data_size); + hdr = data; + if (data_size == sizeof(*hdr)) { + if (hdr->highest_maildir_uid >= ctx->mbox->highest_maildir_uid) { + /* nothing to change */ + return; + } + new_hdr = *hdr; + } else { + memset(&new_hdr, 0, sizeof(new_hdr)); + } + new_hdr.highest_maildir_uid = ctx->mbox->highest_maildir_uid; + mail_index_update_header_ext(ctx->trans, ctx->mbox->dbox_hdr_ext_id, 0, + &new_hdr, sizeof(new_hdr)); } static int dbox_sync_index_rebuild_ctx(struct dbox_sync_rebuild_context *ctx) { - if (dbox_sync_set_uidvalidity(ctx) < 0) - return -1; - + dbox_sync_set_uidvalidity(ctx); if (dbox_sync_index_rebuild_dir(ctx, ctx->mbox->path, TRUE) < 0) return -1; @@ -373,27 +407,10 @@ return -1; } - /* finally give UIDs to newly seen maildir files */ - return dbox_sync_new_maildir(ctx); -} - -static void dbox_sync_update_maildir_ids(struct dbox_sync_rebuild_context *ctx) -{ - struct dbox_mail_index_record rec; - struct dbox_file *const *files; - unsigned int i, count; - - memset(&rec, 0, sizeof(rec)); - files = array_get(&ctx->mbox->open_files, &count); - for (i = 0; i < count; i++) { - if (!files[i]->maildir_file) - continue; - - i_assert(files[i]->file_id != 0); - rec.file_id = files[i]->file_id; - mail_index_update_ext(ctx->trans, files[i]->maildir_append_seq, - ctx->mbox->dbox_ext_id, &rec, NULL); - } + if (dbox_sync_maildir_finish(ctx) < 0) + return -1; + dbox_sync_update_header(ctx); + return 0; } int dbox_sync_index_rebuild(struct dbox_mailbox *mbox) @@ -401,48 +418,30 @@ struct dbox_sync_rebuild_context ctx; uint32_t seq; uoff_t offset; - char **fnames; - unsigned int i, count; int ret; memset(&ctx, 0, sizeof(ctx)); ctx.mbox = mbox; - ctx.append_ctx = dbox_index_append_begin(mbox->dbox_index); ctx.view = mail_index_view_open(mbox->ibox.index); ctx.trans = mail_index_transaction_begin(ctx.view, MAIL_INDEX_TRANSACTION_FLAG_EXTERNAL); - i_array_init(&ctx.maildir_new_files, 8); mail_index_reset(ctx.trans); index_mailbox_reset_uidvalidity(&mbox->ibox); mail_index_ext_lookup(mbox->ibox.index, "cache", &ctx.cache_ext_id); if ((ret = dbox_sync_index_rebuild_ctx(&ctx)) < 0) mail_index_transaction_rollback(&ctx.trans); - else { - ret = dbox_index_append_assign_file_ids(ctx.append_ctx); - if (ret == 0) { - dbox_sync_update_maildir_ids(&ctx); - ret = mail_index_transaction_commit(&ctx.trans, - &seq, &offset); - } - } + else + ret = mail_index_transaction_commit(&ctx.trans, &seq, &offset); mail_index_view_close(&ctx.view); - fnames = array_get_modifiable(&ctx.maildir_new_files, &count); - for (i = 0; i < count; i++) - i_free(fnames[i]); - array_free(&ctx.maildir_new_files); - - if (ret == 0) - ret = dbox_index_append_commit(&ctx.append_ctx); - else - dbox_index_append_rollback(&ctx.append_ctx); - - if (mbox->maildir_sync_keywords != NULL) - maildir_keywords_sync_deinit(&mbox->maildir_sync_keywords); + if (ctx.maildir_sync_ctx != NULL) { + if (maildir_uidlist_sync_deinit(&ctx.maildir_sync_ctx) < 0) + ret = -1; + } + if (ctx.maildir_sync_keywords != NULL) + maildir_keywords_sync_deinit(&ctx.maildir_sync_keywords); if (ctx.mk != NULL) maildir_keywords_deinit(&ctx.mk); - if (ctx.maildir_uidlist != NULL) - maildir_uidlist_deinit(&ctx.maildir_uidlist); return ret; }
--- a/src/lib-storage/index/dbox/dbox-sync.c Fri Feb 13 17:44:00 2009 -0500 +++ b/src/lib-storage/index/dbox/dbox-sync.c Mon Feb 16 20:30:15 2009 -0500 @@ -6,14 +6,9 @@ #include "str.h" #include "hash.h" #include "dbox-storage.h" -#include "dbox-index.h" #include "dbox-file.h" #include "dbox-sync.h" -#define DBOX_FLUSH_SECS_INTERACTIVE (4*60*60) -#define DBOX_FLUSH_SECS_CLOSE (4*60*60) -#define DBOX_FLUSH_SECS_IMMEDIATE (24*60*60) - #define DBOX_REBUILD_COUNT 3 static int dbox_sync_add_seq(struct dbox_sync_context *ctx, @@ -23,7 +18,10 @@ struct dbox_sync_file_entry *entry; uint32_t file_id; uoff_t offset; - bool uid_file, add; + bool uid_file; + + i_assert(sync_rec->type == MAIL_INDEX_SYNC_TYPE_EXPUNGE || + sync_rec->type == MAIL_INDEX_SYNC_TYPE_FLAGS); if (!dbox_file_lookup(ctx->mbox, ctx->sync_view, seq, &file_id, &offset)) @@ -31,27 +29,6 @@ entry = hash_table_lookup(ctx->syncs, POINTER_CAST(file_id)); if (entry == NULL) { - if (sync_rec->type == MAIL_INDEX_SYNC_TYPE_EXPUNGE || - ctx->flush_dirty_flags) { - /* expunges / flushing dirty flags */ - add = TRUE; - } else if (sync_rec->type != MAIL_INDEX_SYNC_TYPE_FLAGS) { - /* keywords, not flushing dirty flags */ - add = FALSE; - } else { - /* add if we're moving from/to alternative storage - and we actually have an alt directory specified */ - add = ((sync_rec->add_flags | sync_rec->remove_flags) & - DBOX_INDEX_FLAG_ALT) != 0 && - ctx->mbox->alt_path != NULL; - } - - if (!add) { - mail_index_update_flags(ctx->trans, seq, MODIFY_ADD, - (enum mail_flags)MAIL_INDEX_MAIL_FLAG_DIRTY); - return 0; - } - entry = p_new(ctx->pool, struct dbox_sync_file_entry, 1); entry->file_id = file_id; hash_table_insert(ctx->syncs, POINTER_CAST(file_id), entry); @@ -62,15 +39,13 @@ if (!array_is_created(&entry->expunges)) { p_array_init(&entry->expunges, ctx->pool, uid_file ? 1 : 3); - seq_range_array_add(&ctx->expunge_files, 0, file_id); } seq_range_array_add(&entry->expunges, 0, seq); } else { - if (!array_is_created(&entry->changes)) { - p_array_init(&entry->changes, ctx->pool, - uid_file ? 1 : 8); - } - seq_range_array_add(&entry->changes, 0, seq); + if ((sync_rec->add_flags & DBOX_INDEX_FLAG_ALT) != 0) + entry->move_to_alt = TRUE; + else + entry->move_from_alt = TRUE; } return 0; } @@ -80,14 +55,18 @@ { uint32_t seq, seq1, seq2; - if (sync_rec->type == MAIL_INDEX_SYNC_TYPE_APPEND) { - /* don't care about appends */ + if (sync_rec->type == MAIL_INDEX_SYNC_TYPE_EXPUNGE) { + /* we're interested */ + } else if (sync_rec->type == MAIL_INDEX_SYNC_TYPE_FLAGS) { + /* we care only about alt flag changes */ + if ((sync_rec->add_flags & DBOX_INDEX_FLAG_ALT) == 0 && + (sync_rec->remove_flags & DBOX_INDEX_FLAG_ALT) == 0) + return 0; + } else { + /* not interested */ return 0; } - /* we assume that anything else than appends are interactive changes */ - ctx->mbox->last_interactive_change = ioloop_time; - if (!mail_index_lookup_seq_range(ctx->sync_view, sync_rec->uid1, sync_rec->uid2, &seq1, &seq2)) { @@ -102,84 +81,6 @@ return 0; } -static int -dbox_sync_lock_expunge_file(struct dbox_sync_context *ctx, unsigned int file_id) -{ - struct dbox_index_record *rec; - enum dbox_index_file_lock_status lock_status; - int ret; - - ret = dbox_index_try_lock_file(ctx->mbox->dbox_index, file_id, - &lock_status); - if (ret < 0) - return -1; - - rec = dbox_index_record_lookup(ctx->mbox->dbox_index, file_id); - switch (lock_status) { - case DBOX_INDEX_FILE_LOCKED: - seq_range_array_add(&ctx->locked_files, 0, file_id); - rec->status = DBOX_INDEX_FILE_STATUS_NONAPPENDABLE; - break; - case DBOX_INDEX_FILE_LOCK_NOT_NEEDED: - case DBOX_INDEX_FILE_LOCK_UNLINKED: - i_assert(rec == NULL || - rec->status != DBOX_INDEX_FILE_STATUS_APPENDABLE); - break; - case DBOX_INDEX_FILE_LOCK_TRY_AGAIN: - rec->expunges = TRUE; - break; - } - return 0; -} - -static int dbox_sync_lock_expunge_files(struct dbox_sync_context *ctx) -{ - const struct seq_range *range; - unsigned int i, count, id; - - range = array_get(&ctx->expunge_files, &count); - for (i = 0; i < count; i++) { - for (id = range[i].seq1; id <= range[i].seq2; id++) { - if (dbox_sync_lock_expunge_file(ctx, id) < 0) - return -1; - } - } - return 0; -} - -static void dbox_sync_unlock_files(struct dbox_sync_context *ctx) -{ - const struct seq_range *range; - unsigned int i, count, id; - - range = array_get(&ctx->locked_files, &count); - for (i = 0; i < count; i++) { - for (id = range[i].seq1; id <= range[i].seq2; id++) - dbox_index_unlock_file(ctx->mbox->dbox_index, id); - } -} - -static void dbox_sync_update_header(struct dbox_sync_context *ctx) -{ - struct dbox_index_header hdr; - const void *data; - size_t data_size; - - if (!ctx->flush_dirty_flags) { - /* write the header if it doesn't exist yet */ - mail_index_get_header_ext(ctx->mbox->ibox.view, - ctx->mbox->dbox_hdr_ext_id, - &data, &data_size); - if (data_size != 0) - return; - } - - hdr.last_dirty_flush_stamp = ioloop_time; - mail_index_update_header_ext(ctx->trans, ctx->mbox->dbox_hdr_ext_id, 0, - &hdr.last_dirty_flush_stamp, - sizeof(hdr.last_dirty_flush_stamp)); -} - static int dbox_sync_index(struct dbox_sync_context *ctx) { struct mailbox *box = &ctx->mbox->ibox.box; @@ -206,8 +107,6 @@ /* read all changes and sort them to file_id order */ ctx->pool = pool_alloconly_create("dbox sync pool", 1024*32); ctx->syncs = hash_table_create(default_pool, ctx->pool, 0, NULL, NULL); - i_array_init(&ctx->expunge_files, 32); - i_array_init(&ctx->locked_files, 32); for (;;) { if (!mail_index_sync_next(ctx->index_sync_ctx, &sync_rec)) @@ -217,11 +116,6 @@ break; } } - if (ret > 0) { - if (dbox_sync_lock_expunge_files(ctx) < 0) - ret = -1; - } - array_free(&ctx->expunge_files); if (ret > 0) { /* now sync each file separately */ @@ -235,35 +129,26 @@ hash_table_iterate_deinit(&iter); } - if (ret > 0) - dbox_sync_update_header(ctx); - if (box->v.sync_notify != NULL) box->v.sync_notify(box, 0, 0); - dbox_sync_unlock_files(ctx); - array_free(&ctx->locked_files); hash_table_destroy(&ctx->syncs); pool_unref(&ctx->pool); return ret; } -static int dbox_sync_want_flush_dirty(struct dbox_mailbox *mbox, - bool close_flush_dirty_flags) +static int dbox_refresh_header(struct dbox_mailbox *mbox) { const struct dbox_index_header *hdr; const void *data; size_t data_size; - if (mbox->last_interactive_change < - ioloop_time - DBOX_FLUSH_SECS_INTERACTIVE) - return 1; - mail_index_get_header_ext(mbox->ibox.view, mbox->dbox_hdr_ext_id, &data, &data_size); if (data_size != sizeof(*hdr)) { - /* data_size=0 means it's never been synced as dbox */ - if (data_size != 0) { + /* data_size=0 means it's never been synced as dbox. + data_size=4 is for backwards compatibility */ + if (data_size != 0 && data_size != 4) { i_warning("dbox %s: Invalid dbox header size", mbox->path); } @@ -271,46 +156,24 @@ } hdr = data; - if (!close_flush_dirty_flags) { - if ((time_t)hdr->last_dirty_flush_stamp < - ioloop_time - DBOX_FLUSH_SECS_IMMEDIATE) - return 1; - } else { - if ((time_t)hdr->last_dirty_flush_stamp < - ioloop_time - DBOX_FLUSH_SECS_CLOSE) - return 1; - } + mbox->highest_maildir_uid = hdr->highest_maildir_uid; return 0; } -int dbox_sync_begin(struct dbox_mailbox *mbox, - struct dbox_sync_context **ctx_r, - bool close_flush_dirty_flags, bool force) +int dbox_sync_begin(struct dbox_mailbox *mbox, bool force, + struct dbox_sync_context **ctx_r) { struct mail_storage *storage = mbox->ibox.box.storage; struct dbox_sync_context *ctx; enum mail_index_sync_flags sync_flags = 0; unsigned int i; int ret; - bool rebuild = FALSE; + bool rebuild; - ret = dbox_sync_want_flush_dirty(mbox, close_flush_dirty_flags); - if (ret > 0) - sync_flags |= MAIL_INDEX_SYNC_FLAG_FLUSH_DIRTY; - else if (ret < 0) - rebuild = TRUE; - else { - if (close_flush_dirty_flags) { - /* no need to sync */ - *ctx_r = NULL; - return 0; - } - } + rebuild = dbox_refresh_header(mbox) < 0; ctx = i_new(struct dbox_sync_context, 1); ctx->mbox = mbox; - ctx->flush_dirty_flags = - (sync_flags & MAIL_INDEX_SYNC_FLAG_FLUSH_DIRTY) != 0; if (!mbox->ibox.keep_recent) sync_flags |= MAIL_INDEX_SYNC_FLAG_DROP_RECENT; @@ -332,7 +195,7 @@ return ret; } - if (rebuild && dbox_sync_want_flush_dirty(mbox, FALSE) >= 0) { + if (rebuild && dbox_refresh_header(mbox) < 0) { /* another process rebuilt it already */ rebuild = FALSE; } @@ -389,12 +252,11 @@ return 0; } -int dbox_sync(struct dbox_mailbox *mbox, bool close_flush_dirty_flags) +int dbox_sync(struct dbox_mailbox *mbox) { struct dbox_sync_context *sync_ctx; - if (dbox_sync_begin(mbox, &sync_ctx, - close_flush_dirty_flags, FALSE) < 0) + if (dbox_sync_begin(mbox, FALSE, &sync_ctx) < 0) return -1; if (sync_ctx == NULL) @@ -412,7 +274,7 @@ index_storage_mailbox_open(&mbox->ibox); if (index_mailbox_want_full_sync(&mbox->ibox, flags)) - ret = dbox_sync(mbox, FALSE); + ret = dbox_sync(mbox); return index_mailbox_sync_init(box, flags, ret < 0); }
--- a/src/lib-storage/index/dbox/dbox-sync.h Fri Feb 13 17:44:00 2009 -0500 +++ b/src/lib-storage/index/dbox/dbox-sync.h Mon Feb 16 20:30:15 2009 -0500 @@ -6,7 +6,8 @@ struct dbox_sync_file_entry { uint32_t file_id; - ARRAY_TYPE(seq_range) changes; + unsigned int move_from_alt:1; + unsigned int move_to_alt:1; ARRAY_TYPE(seq_range) expunges; }; @@ -21,17 +22,12 @@ pool_t pool; struct hash_table *syncs; /* struct dbox_sync_file_entry */ - ARRAY_TYPE(seq_range) expunge_files; - ARRAY_TYPE(seq_range) locked_files; - - unsigned int flush_dirty_flags:1; }; -int dbox_sync_begin(struct dbox_mailbox *mbox, - struct dbox_sync_context **ctx_r, - bool close_flush_dirty_flags, bool force); +int dbox_sync_begin(struct dbox_mailbox *mbox, bool force, + struct dbox_sync_context **ctx_r); int dbox_sync_finish(struct dbox_sync_context **ctx, bool success); -int dbox_sync(struct dbox_mailbox *mbox, bool close_flush_dirty_flags); +int dbox_sync(struct dbox_mailbox *mbox); int dbox_sync_file(struct dbox_sync_context *ctx, const struct dbox_sync_file_entry *entry);
--- a/src/lib-storage/index/maildir/maildir-uidlist.c Fri Feb 13 17:44:00 2009 -0500 +++ b/src/lib-storage/index/maildir/maildir-uidlist.c Mon Feb 16 20:30:15 2009 -0500 @@ -51,6 +51,8 @@ #define UIDLIST_IS_LOCKED(uidlist) \ ((uidlist)->lock_count > 0) +#define UIDLIST_ALLOW_WRITING(uidlist) \ + (UIDLIST_IS_LOCKED(uidlist) || (uidlist)->mbox == NULL) struct maildir_uidlist_rec { uint32_t uid; @@ -1341,7 +1343,7 @@ *sync_ctx_r = ctx = i_new(struct maildir_uidlist_sync_ctx, 1); ctx->uidlist = uidlist; ctx->sync_flags = sync_flags; - ctx->partial = !locked || + ctx->partial = (!locked && ctx->uidlist->mbox != NULL) || (sync_flags & MAILDIR_UIDLIST_SYNC_PARTIAL) != 0; ctx->locked = locked; ctx->first_unwritten_pos = (unsigned int)-1; @@ -1432,15 +1434,20 @@ { struct maildir_uidlist *uidlist = ctx->uidlist; struct maildir_uidlist_rec *rec, *old_rec; - const char *p; + const char *p, *dir; if (ctx->failed) return -1; for (p = filename; *p != '\0'; p++) { if (*p == 13 || *p == 10) { + struct mailbox *box = &uidlist->ibox->box; + + dir = mailbox_list_get_path(box->storage->list, + box->name, + MAILBOX_LIST_PATH_TYPE_MAILBOX); i_warning("Maildir %s: Ignoring a file with #0x%x: %s", - uidlist->mbox->path, *p, filename); + dir, *p, filename); return 1; } } @@ -1465,7 +1472,7 @@ MAILDIR_UIDLIST_REC_FLAG_MOVED); } else { old_rec = hash_table_lookup(uidlist->files, filename); - i_assert(old_rec != NULL || UIDLIST_IS_LOCKED(uidlist)); + i_assert(old_rec != NULL || UIDLIST_ALLOW_WRITING(uidlist)); rec = p_new(ctx->record_pool, struct maildir_uidlist_rec, 1); @@ -1563,7 +1570,7 @@ struct maildir_uidlist_rec **recs; unsigned int dest, count; - i_assert(UIDLIST_IS_LOCKED(ctx->uidlist)); + i_assert(UIDLIST_ALLOW_WRITING(ctx->uidlist)); i_assert(ctx->first_nouid_pos != (unsigned int)-1); if (ctx->first_unwritten_pos == (unsigned int)-1) @@ -1619,6 +1626,11 @@ } } +void maildir_uidlist_sync_recreate(struct maildir_uidlist_sync_ctx *ctx) +{ + ctx->uidlist->recreate = TRUE; +} + void maildir_uidlist_sync_finish(struct maildir_uidlist_sync_ctx *ctx) { if (!ctx->partial) { @@ -1632,13 +1644,17 @@ ctx->finished = TRUE; ctx->uidlist->initial_sync = TRUE; - i_assert(ctx->locked || !ctx->changed); + /* mbox=NULL means we're coming from dbox rebuilding code. + the dbox is already locked, so allow uidlist recreation */ + i_assert(ctx->locked || !ctx->changed || ctx->uidlist->mbox == NULL); if ((ctx->changed || ctx->uidlist->recreate || maildir_uidlist_want_compress(ctx)) && - !ctx->failed && ctx->locked) T_BEGIN { - if (maildir_uidlist_sync_update(ctx) < 0) - ctx->failed = TRUE; - } T_END; + !ctx->failed && (ctx->locked || ctx->uidlist->mbox == NULL)) { + T_BEGIN { + if (maildir_uidlist_sync_update(ctx) < 0) + ctx->failed = TRUE; + } T_END; + } } int maildir_uidlist_sync_deinit(struct maildir_uidlist_sync_ctx **_ctx)
--- a/src/lib-storage/index/maildir/maildir-uidlist.h Fri Feb 13 17:44:00 2009 -0500 +++ b/src/lib-storage/index/maildir/maildir-uidlist.h Mon Feb 16 20:30:15 2009 -0500 @@ -106,6 +106,7 @@ const char * maildir_uidlist_sync_get_full_filename(struct maildir_uidlist_sync_ctx *ctx, const char *filename); +void maildir_uidlist_sync_recreate(struct maildir_uidlist_sync_ctx *ctx); void maildir_uidlist_sync_finish(struct maildir_uidlist_sync_ctx *ctx); int maildir_uidlist_sync_deinit(struct maildir_uidlist_sync_ctx **ctx);