Mercurial > dovecot > core-2.2
changeset 1675:b5dc9acbbc9b HEAD
Maildir syncing works now without requiring base filenames to be in index
cache file. Also message flag updates with +FLAGS and -FLAGS works correctly
now if another client had just changed it's flags.
author | Timo Sirainen <tss@iki.fi> |
---|---|
date | Mon, 11 Aug 2003 02:56:22 +0300 |
parents | b20ecf0cf8f6 |
children | 3b8c4a370d86 |
files | src/lib-imap/imap-util.h src/lib-index/mail-index.c src/lib-index/mail-index.h src/lib-index/maildir/maildir-build.c src/lib-index/maildir/maildir-expunge.c src/lib-index/maildir/maildir-index.c src/lib-index/maildir/maildir-index.h src/lib-index/maildir/maildir-open.c src/lib-index/maildir/maildir-sync.c src/lib-index/maildir/maildir-uidlist.c src/lib-index/maildir/maildir-update-flags.c src/lib-index/mbox/mbox-index.c src/lib-storage/index/index-update-flags.c src/lib-storage/index/maildir/maildir-copy.c src/lib-storage/mail-storage.h |
diffstat | 15 files changed, 441 insertions(+), 465 deletions(-) [+] |
line wrap: on
line diff
--- a/src/lib-imap/imap-util.h Sun Aug 10 21:14:39 2003 +0300 +++ b/src/lib-imap/imap-util.h Mon Aug 11 02:56:22 2003 +0300 @@ -1,6 +1,12 @@ #ifndef __IMAP_UTIL_H #define __IMAP_UTIL_H +enum modify_type { + MODIFY_ADD, + MODIFY_REMOVE, + MODIFY_REPLACE +}; + enum mail_flags { MAIL_ANSWERED = 0x0000001, MAIL_FLAGGED = 0x0000002,
--- a/src/lib-index/mail-index.c Sun Aug 10 21:14:39 2003 +0300 +++ b/src/lib-index/mail-index.c Mon Aug 11 02:56:22 2003 +0300 @@ -586,19 +586,36 @@ } int mail_index_update_flags(struct mail_index *index, - struct mail_index_record *rec, - unsigned int seq, enum mail_flags flags, + struct mail_index_record *rec, unsigned int seq, + enum modify_type modify_type, enum mail_flags flags, int external_change) { + enum mail_flags new_flags; + i_assert(index->lock_type == MAIL_LOCK_EXCLUSIVE); i_assert(seq != 0); - if (flags == rec->msg_flags) + switch (modify_type) { + case MODIFY_ADD: + new_flags = rec->msg_flags | flags; + break; + case MODIFY_REMOVE: + new_flags = rec->msg_flags & ~flags; + break; + case MODIFY_REPLACE: + new_flags = flags; + break; + default: + new_flags = 0; + i_unreached(); + } + + if (new_flags == rec->msg_flags) return TRUE; /* no changes */ - mail_index_mark_flag_changes(index, rec, rec->msg_flags, flags); + mail_index_mark_flag_changes(index, rec, rec->msg_flags, new_flags); - rec->msg_flags = flags; + rec->msg_flags = new_flags; return index->modifylog == NULL ? TRUE : mail_modifylog_add_flags(index->modifylog, seq, rec->uid, external_change);
--- a/src/lib-index/mail-index.h Sun Aug 10 21:14:39 2003 +0300 +++ b/src/lib-index/mail-index.h Mon Aug 11 02:56:22 2003 +0300 @@ -246,11 +246,10 @@ int external_change); /* Update mail flags. The index must be exclusively locked before - calling this function. This shouldn't be called in the middle of - update_begin() as it may modify location field. */ + calling this function. */ int (*update_flags)(struct mail_index *index, - struct mail_index_record *rec, - unsigned int seq, enum mail_flags flags, + struct mail_index_record *rec, unsigned int seq, + enum modify_type modify_type, enum mail_flags flags, int external_change); /* Append a new record to index. The index must be exclusively @@ -382,8 +381,8 @@ unsigned int first_seq, unsigned int last_seq, int external_change); int mail_index_update_flags(struct mail_index *index, - struct mail_index_record *rec, - unsigned int seq, enum mail_flags flags, + struct mail_index_record *rec, unsigned int seq, + enum modify_type modify_type, enum mail_flags flags, int external_change); struct mail_index_record *mail_index_append(struct mail_index *index); enum mail_index_error mail_index_get_last_error(struct mail_index *index);
--- a/src/lib-index/maildir/maildir-build.c Sun Aug 10 21:14:39 2003 +0300 +++ b/src/lib-index/maildir/maildir-build.c Mon Aug 11 02:56:22 2003 +0300 @@ -4,40 +4,34 @@ #include "maildir-index.h" #include "mail-cache.h" -int maildir_index_append_file(struct mail_cache_transaction_ctx **trans_ctx, - struct mail_index *index, const char *fname, +int maildir_cache_update_file(struct mail_cache_transaction_ctx **trans_ctx, + struct mail_index *index, + struct mail_index_record *rec, const char *fname, int new_dir) { - struct mail_index_record *rec; + enum mail_cache_field cached_fields; enum mail_index_record_flag index_flags; uoff_t virtual_size; const char *p; - i_assert(index->lock_type == MAIL_LOCK_EXCLUSIVE); - if (*trans_ctx == NULL) { if (mail_cache_transaction_begin(index->cache, TRUE, trans_ctx) <= 0) return FALSE; } - rec = index->append(index); - if (rec == NULL) - return FALSE; - - /* set message flags from file name */ - rec->msg_flags = maildir_filename_get_flags(fname, 0); - mail_index_mark_flag_changes(index, rec, 0, rec->msg_flags); - - /* always set index flags */ - index_flags = new_dir ? MAIL_INDEX_FLAG_MAILDIR_NEW : 0; - if (!mail_cache_add(*trans_ctx, rec, MAIL_CACHE_INDEX_FLAGS, - &index_flags, sizeof(index_flags))) - return FALSE; + cached_fields = mail_cache_get_fields(index->cache, rec); + if ((cached_fields & MAIL_CACHE_INDEX_FLAGS) == 0) { + /* always set index flags */ + index_flags = new_dir ? MAIL_INDEX_FLAG_MAILDIR_NEW : 0; + if (!mail_cache_add(*trans_ctx, rec, MAIL_CACHE_INDEX_FLAGS, + &index_flags, sizeof(index_flags))) + return FALSE; + } /* set virtual size if found from file name */ p = strstr(fname, ",W="); - if (p != NULL) { + if (p != NULL && (cached_fields & MAIL_CACHE_VIRTUAL_FULL_SIZE) == 0) { p += 3; virtual_size = 0; while (*p >= '0' && *p <= '9') { @@ -54,10 +48,29 @@ } } - /* always set location */ - if (!mail_cache_add(*trans_ctx, rec, MAIL_CACHE_LOCATION, - fname, strlen(fname)+1)) - return FALSE; + if ((cached_fields & MAIL_CACHE_LOCATION) == 0) { + /* always set location */ + if (!mail_cache_add(*trans_ctx, rec, MAIL_CACHE_LOCATION, + fname, strlen(fname)+1)) + return FALSE; + } return TRUE; } + +int maildir_index_append_file(struct mail_cache_transaction_ctx **trans_ctx, + struct mail_index *index, const char *fname, + int new_dir) +{ + struct mail_index_record *rec; + + rec = index->append(index); + if (rec == NULL) + return FALSE; + + /* set message flags from file name */ + rec->msg_flags = maildir_filename_get_flags(fname, 0); + mail_index_mark_flag_changes(index, rec, 0, rec->msg_flags); + + return maildir_cache_update_file(trans_ctx, index, rec, fname, new_dir); +}
--- a/src/lib-index/maildir/maildir-expunge.c Sun Aug 10 21:14:39 2003 +0300 +++ b/src/lib-index/maildir/maildir-expunge.c Mon Aug 11 02:56:22 2003 +0300 @@ -7,82 +7,44 @@ #include <unistd.h> -static int maildir_expunge_mail_file(struct mail_index *index, - struct mail_index_record *rec, - const char **fname) +static int do_expunge(struct mail_index *index, const char *path, void *context) { - const char *path; - int new_dir; - - *fname = maildir_get_location(index, rec, &new_dir); - if (*fname == NULL) - return -1; - - /* if we're in out-of-space condition, reset it since we'll probably - have enough space now. */ - index->maildir_keep_new = FALSE; - if (index->next_dirty_flush != 0) - index->next_dirty_flush = ioloop_time; + int *found = context; - if (new_dir) { - /* probably in new/ dir */ - path = t_strconcat(index->mailbox_path, "/new/", *fname, NULL); - if (unlink(path) == 0) + if (unlink(path) < 0) { + if (errno == ENOENT) + return 0; + if (errno == EACCES) { + index->mailbox_readonly = TRUE; return 1; + } - if (errno == EACCES) - return -1; - if (errno != ENOENT) { - index_set_error(index, "unlink(%s) failed: %m", path); - return -1; - } - } - - path = t_strconcat(index->mailbox_path, "/cur/", *fname, NULL); - if (unlink(path) == 0) - return 1; - - if (errno == EACCES) - return -1; - - if (errno != ENOENT) { index_set_error(index, "unlink(%s) failed: %m", path); return -1; } - return 0; + *found = TRUE; + return 1; } int maildir_expunge_mail(struct mail_index *index, struct mail_index_record *rec) { - const char *fname; - int i, ret, found; + int found = FALSE; - for (i = 0;; i++) { - ret = maildir_expunge_mail_file(index, rec, &fname); - if (ret > 0) - break; - if (ret < 0) - return FALSE; + if (!maildir_file_do(index, rec, do_expunge, &found)) + return FALSE; - if (i == 10) { - index_set_error(index, "Filename keeps changing, " - "expunge failed: %s", fname); - return FALSE; - } - - if (!maildir_index_sync_readonly(index, fname, &found)) - return FALSE; + if (found) { + /* if we're in out-of-space condition, reset it since we'll + probably have enough space now. */ + index->maildir_keep_new = FALSE; + if (index->next_dirty_flush != 0) + index->next_dirty_flush = ioloop_time; - if (!found) { - /* syncing didn't find it, it's already deleted */ - return TRUE; - } + /* cur/ was updated, set it dirty-synced */ + index->maildir_cur_dirty = ioloop_time; + index->file_sync_stamp = ioloop_time; } - - /* cur/ was updated, set it dirty-synced */ - index->maildir_cur_dirty = ioloop_time; - index->file_sync_stamp = ioloop_time; return TRUE; }
--- a/src/lib-index/maildir/maildir-index.c Sun Aug 10 21:14:39 2003 +0300 +++ b/src/lib-index/maildir/maildir-index.c Mon Aug 11 02:56:22 2003 +0300 @@ -45,12 +45,11 @@ } } - /* index file should give us at least the base name. */ + /* cache file file should give us at least the base name. */ fname = mail_cache_lookup_string_field(index->cache, rec, MAIL_CACHE_LOCATION); if (fname == NULL) { - mail_cache_set_corrupted(index->cache, - "Missing location field for record %u", rec->uid); + /* Not cached, we'll have to resync the directory. */ return NULL; } @@ -62,6 +61,54 @@ return fname; } +static int +maildir_file_do_try(struct mail_index *index, struct mail_index_record *rec, + const char **fname, + maildir_file_do_func *func, void *context) +{ + const char *path; + int ret, new_dir; + + *fname = maildir_get_location(index, rec, &new_dir); + if (*fname == NULL) + return 0; + + if (new_dir) { + /* probably in new/ dir */ + path = t_strconcat(index->mailbox_path, "/new/", *fname, NULL); + ret = func(index, path, context); + if (ret != 0) + return ret; + } + + path = t_strconcat(index->mailbox_path, "/cur/", *fname, NULL); + return func(index, path, context); +} + +int maildir_file_do(struct mail_index *index, struct mail_index_record *rec, + maildir_file_do_func *func, void *context) +{ + const char *fname; + int i, ret, found; + + ret = maildir_file_do_try(index, rec, &fname, func, context); + for (i = 0; i < 10 && ret == 0; i++) { + /* file is either renamed or deleted. sync the maildir and + see which one. if file appears to be renamed constantly, + don't try to open it more than 10 times. */ + fname = t_strdup(fname); + if (!maildir_index_sync_readonly(index, fname, &found)) + return FALSE; + + if (!found && fname != NULL) + return TRUE; + + ret = maildir_file_do_try(index, rec, &fname, func, context); + } + + return ret >= 0; +} + const char *maildir_generate_tmp_filename(const struct timeval *tv) { static unsigned int create_count = 0; @@ -291,46 +338,27 @@ i_free(index); } -static int maildir_get_received_date_file(struct mail_index *index, - struct mail_index_record *rec, - const char **fname, struct stat *st) +static int do_get_received_date(struct mail_index *index, + const char *path, void *context) { - const char *path; - int new_dir; - - /* stat() gives it */ - *fname = maildir_get_location(index, rec, &new_dir); - if (*fname == NULL) - return -1; + time_t *date = context; + struct stat st; - if (new_dir) { - /* probably in new/ dir */ - path = t_strconcat(index->mailbox_path, "/new/", *fname, NULL); - if (stat(path, st) < 0 && errno != ENOENT) { - index_file_set_syscall_error(index, path, "stat()"); - return -1; - } - } - - path = t_strconcat(index->mailbox_path, "/cur/", *fname, NULL); - if (stat(path, st) < 0) { + if (stat(path, &st) < 0) { if (errno == ENOENT) return 0; - index_file_set_syscall_error(index, path, "stat()"); return -1; } - return TRUE; + *date = st.st_mtime; + return 1; } static time_t maildir_get_received_date(struct mail_index *index, struct mail_index_record *rec) { - struct stat st; - const char *fname; time_t date; - int ret, i, found; /* try getting it from cache */ if (mail_cache_copy_fixed_field(index->cache, rec, @@ -338,23 +366,11 @@ &date, sizeof(date))) return date; - ret = maildir_get_received_date_file(index, rec, &fname, &st); - for (i = 0; ret == 0 && i < 10; i++) { - /* file is either renamed or deleted. sync the maildir and - see which one. if file appears to be renamed constantly, - don't try to open it more than 10 times. */ - if (!maildir_index_sync_readonly(index, fname, &found)) - return FALSE; + date = (time_t)-1; + if (!maildir_file_do(index, rec, do_get_received_date, &date)) + return (time_t)-1; - if (!found) { - /* syncing didn't find it, it's deleted */ - return (time_t)-1; - } - - ret = maildir_get_received_date_file(index, rec, &fname, &st); - } - - return st.st_mtime; + return date; } struct mail_index maildir_index = {
--- a/src/lib-index/maildir/maildir-index.h Sun Aug 10 21:14:39 2003 +0300 +++ b/src/lib-index/maildir/maildir-index.h Mon Aug 11 02:56:22 2003 +0300 @@ -9,6 +9,10 @@ /* How often to try to flush dirty flags. */ #define MAILDIR_DIRTY_FLUSH_TIMEOUT (60*5) +/* Return -1 = error, 0 = file not found, 1 = ok */ +typedef int maildir_file_do_func(struct mail_index *index, + const char *path, void *context); + struct mail_index * maildir_index_alloc(const char *maildir, const char *index_dir, const char *control_dir); @@ -20,6 +24,8 @@ const char *maildir_get_location(struct mail_index *index, struct mail_index_record *rec, int *new_dir); +int maildir_file_do(struct mail_index *index, struct mail_index_record *rec, + maildir_file_do_func *func, void *context); enum mail_flags maildir_filename_get_flags(const char *fname, enum mail_flags default_flags); const char *maildir_filename_set_flags(const char *fname, @@ -32,11 +38,16 @@ int maildir_index_sync(struct mail_index *index, int minimal_sync, enum mail_lock_type lock_type, int *changes); +int maildir_cache_update_file(struct mail_cache_transaction_ctx **trans_ctx, + struct mail_index *index, + struct mail_index_record *rec, const char *fname, + int new_dir); int maildir_index_append_file(struct mail_cache_transaction_ctx **trans_ctx, struct mail_index *index, const char *fname, int new_dir); int maildir_index_update_flags(struct mail_index *index, struct mail_index_record *rec, unsigned int seq, + enum modify_type modify_type, enum mail_flags flags, int external_change); int maildir_try_flush_dirty_flags(struct mail_index *index, int force);
--- a/src/lib-index/maildir/maildir-open.c Sun Aug 10 21:14:39 2003 +0300 +++ b/src/lib-index/maildir/maildir-open.c Mon Aug 11 02:56:22 2003 +0300 @@ -10,42 +10,18 @@ #include <fcntl.h> #include <sys/stat.h> -static int maildir_open_mail_file(struct mail_index *index, - struct mail_index_record *rec, - const char **fname, int *deleted) +static int do_open(struct mail_index *index, const char *path, void *context) { - const char *path; - int new_dir, fd = -1; - - *fname = maildir_get_location(index, rec, &new_dir); - if (*fname == NULL) - return -1; + int *fd = context; - if (new_dir) { - /* probably in new/ dir */ - path = t_strconcat(index->mailbox_path, "/new/", *fname, NULL); - fd = open(path, O_RDONLY); - if (fd == -1 && errno != ENOENT) { - index_set_error(index, "open(%s) failed: %m", path); - return -1; - } - } + *fd = open(path, O_RDONLY); + if (*fd != -1) + return 1; + if (errno == ENOENT) + return 0; - if (fd == -1) { - path = t_strconcat(index->mailbox_path, "/cur/", *fname, NULL); - fd = open(path, O_RDONLY); - if (fd == -1) { - if (errno == ENOENT) { - *deleted = TRUE; - return -1; - } - - index_set_error(index, "open(%s) failed: %m", path); - return -1; - } - } - - return fd; + index_file_set_syscall_error(index, path, "open()"); + return -1; } struct istream *maildir_open_mail(struct mail_index *index, @@ -53,8 +29,7 @@ time_t *received_date, int *deleted) { struct stat st; - const char *fname; - int i, found, fd; + int fd; i_assert(index->lock_type != MAIL_LOCK_UNLOCK); @@ -64,24 +39,13 @@ if (index->inconsistent) return NULL; - fd = maildir_open_mail_file(index, rec, &fname, deleted); - for (i = 0; fd == -1 && *deleted && i < 10; i++) { - /* file is either renamed or deleted. sync the maildir and - see which one. if file appears to be renamed constantly, - don't try to open it more than 10 times. */ - if (!maildir_index_sync_readonly(index, fname, &found)) { - *deleted = FALSE; - return NULL; - } + fd = -1; + if (!maildir_file_do(index, rec, do_open, &fd)) + return NULL; - if (!found) { - /* syncing didn't find it, it's deleted */ - return NULL; - } - - fd = maildir_open_mail_file(index, rec, &fname, deleted); - if (fd == -1) - return NULL; + if (fd == -1) { + *deleted = TRUE; + return NULL; } if (received_date != NULL) {
--- a/src/lib-index/maildir/maildir-sync.c Sun Aug 10 21:14:39 2003 +0300 +++ b/src/lib-index/maildir/maildir-sync.c Mon Aug 11 02:56:22 2003 +0300 @@ -241,6 +241,7 @@ unsigned int uidlist_rewrite:1; unsigned int new_mails_new:1; unsigned int new_mails_cur:1; + unsigned int have_uncached_filenames:1; }; static int maildir_sync_cur_dir(struct maildir_sync_context *ctx); @@ -288,7 +289,7 @@ flags = maildir_filename_get_flags(new_fname, rec->msg_flags); if (flags != rec->msg_flags) { if (!ctx->index->update_flags(ctx->index, rec, - seq, flags, TRUE)) + seq, MODIFY_REPLACE, flags, TRUE)) return FALSE; } @@ -304,7 +305,8 @@ if (ctx->uidlist != NULL) return TRUE; - /* open it only if it's changed since we last synced it. */ + /* open it only if it's changed since we last synced it, + or if we have uncached filenames. */ path = t_strconcat(index->control_dir, "/" MAILDIR_UIDLIST_NAME, NULL); if (stat(path, &st) < 0) { if (errno == ENOENT) { @@ -323,7 +325,8 @@ } /* FIXME: last_uidlist_mtime should be in index headers */ - if (st.st_mtime == index->last_uidlist_mtime) + if (st.st_mtime == index->last_uidlist_mtime && + !ctx->have_uncached_filenames) return TRUE; ctx->uidlist = maildir_uidlist_open(index); @@ -478,15 +481,24 @@ } fname = maildir_get_location(index, rec, NULL); - if (fname == NULL) - return FALSE; + if (fname == NULL) { + /* filename not cached, it must be in uidlist or + it's expunged */ + fname = uid_rec.uid == rec->uid ? + uid_rec.filename : NULL; + } - if (!hash_lookup_full(ctx->files, fname, - &orig_key, &orig_value)) { + if (fname == NULL) { + hash_rec = NULL; + action = MAILDIR_FILE_ACTION_EXPUNGE; + } else if (hash_lookup_full(ctx->files, fname, + &orig_key, &orig_value)) { + hash_rec = orig_value; + action = ACTION(hash_rec); + } else { /* none action */ hash_rec = NULL; - } else { - hash_rec = orig_value; + action = MAILDIR_FILE_ACTION_NONE; } if (uid_rec.uid == uid && @@ -498,8 +510,8 @@ } if (uid_rec.uid > uid && hash_rec != NULL && - (ACTION(hash_rec) == MAILDIR_FILE_ACTION_UPDATE_FLAGS || - ACTION(hash_rec) == MAILDIR_FILE_ACTION_NONE)) { + (action == MAILDIR_FILE_ACTION_UPDATE_FLAGS || + action == MAILDIR_FILE_ACTION_NONE)) { /* it's UID has changed. shouldn't happen. */ index_set_corrupted(index, "UID changed for %s/%s: %u -> %u", @@ -508,8 +520,6 @@ return FALSE; } - action = hash_rec != NULL ? - ACTION(hash_rec) : MAILDIR_FILE_ACTION_NONE; switch (action) { case MAILDIR_FILE_ACTION_EXPUNGE: if (first_rec == NULL) { @@ -519,12 +529,22 @@ last_rec = rec; last_seq = seq; break; + case MAILDIR_FILE_ACTION_NEW: + /* filename wasn't cached */ + new_flag = hash_rec->action & MAILDIR_FILE_FLAG_NEWDIR; + hash_rec->action = MAILDIR_FILE_ACTION_NONE | new_flag; + ctx->new_count--; + + if (!maildir_cache_update_file(&ctx->trans_ctx, index, + rec, fname, new_flag)) + return FALSE; + /* fall through */ case MAILDIR_FILE_ACTION_UPDATE_FLAGS: new_dir = (hash_rec->action & MAILDIR_FILE_FLAG_NEWDIR) != 0; maildir_index_update_filename(index, rec->uid, orig_key, new_dir); - if (!maildir_update_flags(ctx, rec, seq, fname)) + if (!maildir_update_flags(ctx, rec, seq, orig_key)) return FALSE; /* fall through */ case MAILDIR_FILE_ACTION_NONE: @@ -540,8 +560,7 @@ } break; default: - i_panic("BUG: %s/%s suddenly appeared as UID %u", - index->mailbox_path, (char *) orig_key, uid); + i_unreached(); } if (uid_rec.uid == uid) { @@ -565,9 +584,9 @@ } if (seq-1 != index->header->messages_count) { - index_set_corrupted(index, "Wrong messages_count in header " - "(%u != %u)", seq, - index->header->messages_count); + index_set_corrupted(index, + "Wrong messages_count in header (%u != %u)", + seq, index->header->messages_count); return FALSE; } @@ -660,12 +679,6 @@ size_t size; int new_dir, have_new; - /* kludge. we want to have pointers to data file, so we must make sure - that it's base address doesn't change. this call makes sure it's - fully mmaped in memory even when we begin */ - if (mail_cache_get_mmaped(index->cache, &size) == NULL) - return FALSE; - if (index->header->messages_count >= INT_MAX/32) { index_set_corrupted(index, "Header says %u messages", index->header->messages_count); @@ -696,16 +709,26 @@ have_new = FALSE; + /* Now we'll fill the hash with cached filenames. This is done mostly + just to save some memory since we can use pointers to mmaped cache + file. Note that all records may not have the filename cached. + + WARNING: Cache file must not be modified as long as these pointers + exist, as modifying might change the mmap base address. The call + below makes sure that cache file is initially fully mmaped. */ + if (mail_cache_get_mmaped(index->cache, &size) == NULL) + return FALSE; + rec = index->lookup(index, 1); while (rec != NULL) { fname = maildir_get_location(index, rec, &new_dir); if (fname == NULL) - return FALSE; + ctx->have_uncached_filenames = TRUE; if (new_dir) have_new = TRUE; - if (!only_new || new_dir) { + if ((!only_new || new_dir) && fname != NULL) { hash_rec = p_new(ctx->pool, struct maildir_hash_rec, 1); hash_rec->rec = rec; hash_rec->action = MAILDIR_FILE_ACTION_EXPUNGE; @@ -716,9 +739,6 @@ return FALSE; } - /* WARNING: index must not be modified as long as - these hash keys exist. Modifying might change the - mmap base address. */ hash_insert(ctx->files, (void *) fname, hash_rec); } @@ -756,7 +776,7 @@ return ret; } -static void uidlist_hash_fix_allocs(void *key, void *value, void *context) +static void maildir_sync_hash_fix_allocs(void *key, void *value, void *context) { struct maildir_sync_context *ctx = context; struct maildir_hash_rec *hash_rec = value; @@ -808,7 +828,8 @@ if (hash_rec->rec == NULL) { /* new message */ - if (ctx->readonly_check) + if (ctx->readonly_check && + !ctx->have_uncached_filenames) continue; if (new_dir) @@ -836,9 +857,9 @@ } while ((d = readdir(dirp)) != NULL); /* records that are left to hash must not have any (filename) pointers - to index file. So remove none actions, and p_strdup() expunge + to cache file. So remove none actions, and p_strdup() expunge actions. */ - hash_foreach(ctx->files, uidlist_hash_fix_allocs, ctx); + hash_foreach(ctx->files, maildir_sync_hash_fix_allocs, ctx); return TRUE; } @@ -1108,8 +1129,8 @@ return FALSE; /* this will set maildir_cur_dirty. it may actually be - different from cur/'s mtime if we're unlucky, but that - doesn't really matter and it's not worth the extra stat() */ + different from cur/'s mtime if we're unlucky, but that only + causes extra sync and it's not worth the extra stat() */ if (ctx->new_dent == NULL && (ctx->new_count == 0 || !ctx->new_mails_new)) cur_mtime = time(NULL); @@ -1155,41 +1176,74 @@ struct mail_index *index = ctx->index; struct mail_index_record *rec; struct maildir_hash_rec *hash_rec; + struct maildir_uidlist *uidlist; + struct maildir_uidlist_rec uid_rec; void *orig_key, *orig_value; const char *fname; unsigned int seq; - int new_dir; + int new_dir, tried_uidlist; - if (!ctx->flag_updates) { + if (!ctx->flag_updates && !ctx->have_uncached_filenames) { ctx->index->maildir_synced_once = TRUE; return TRUE; } - rec = index->lookup(index, 1); seq = 1; - while (rec != NULL) { - fname = maildir_get_location(index, rec, NULL); - if (fname == NULL) - return FALSE; + memset(&uid_rec, 0, sizeof(uid_rec)); + uidlist = ctx->uidlist; + tried_uidlist = FALSE; - if (hash_lookup_full(ctx->files, fname, &orig_key, &orig_value)) - hash_rec = orig_value; - else - hash_rec = NULL; + rec = index->lookup(index, 1); seq = 1; + for (; rec != NULL; rec = index->next(index, rec), seq++) { + fname = maildir_get_location(index, rec, NULL); + if (fname == NULL) { + /* not cached, get it from uidlist */ + if (uidlist == NULL && !tried_uidlist) { + ctx->have_uncached_filenames = TRUE; + if (!maildir_sync_open_uidlist(ctx)) + return FALSE; - if (hash_rec != NULL && - ACTION(hash_rec) == MAILDIR_FILE_ACTION_UPDATE_FLAGS) { - new_dir = (hash_rec->action & - MAILDIR_FILE_FLAG_NEWDIR) != 0; - maildir_index_update_filename(index, rec->uid, - orig_key, new_dir); + uidlist = ctx->uidlist; + tried_uidlist = TRUE; - if (index->lock_type == MAIL_LOCK_EXCLUSIVE) { - if (!maildir_update_flags(ctx, rec, seq, fname)) + /* get the initial record */ + if (uidlist != NULL && + maildir_uidlist_next(uidlist, &uid_rec) < 0) return FALSE; } + + if (uidlist == NULL) { + /* uidlist doesn't exist? shouldn't happen */ + continue; + } + + while (uid_rec.uid != 0 && uid_rec.uid < rec->uid) { + if (maildir_uidlist_next(uidlist, &uid_rec) < 0) + return FALSE; + } + + if (uid_rec.uid != rec->uid) { + /* not in uidlist, it's expunged */ + continue; + } + + fname = uid_rec.filename; } - rec = index->next(index, rec); seq++; + if (!hash_lookup_full(ctx->files, fname, + &orig_key, &orig_value)) + continue; + + hash_rec = orig_value; + if (ACTION(hash_rec) != MAILDIR_FILE_ACTION_UPDATE_FLAGS && + ACTION(hash_rec) != MAILDIR_FILE_ACTION_NEW) + continue; + + new_dir = (hash_rec->action & MAILDIR_FILE_FLAG_NEWDIR) != 0; + maildir_index_update_filename(index, rec->uid, + orig_key, new_dir); + + if (!maildir_update_flags(ctx, rec, seq, orig_key)) + return FALSE; } ctx->index->maildir_synced_once = TRUE; @@ -1298,7 +1352,7 @@ ret = maildir_index_sync_context_readonly(ctx); - if (!ret || ctx->files == NULL) + if (!ret || ctx->files == NULL || fname == NULL) *found = FALSE; else { hash_rec = hash_lookup(ctx->files, fname);
--- a/src/lib-index/maildir/maildir-uidlist.c Sun Aug 10 21:14:39 2003 +0300 +++ b/src/lib-index/maildir/maildir-uidlist.c Mon Aug 11 02:56:22 2003 +0300 @@ -156,8 +156,8 @@ rec = index->lookup(index, 1); while (rec != NULL) { fname = maildir_get_location(index, rec, NULL); - if (fname == NULL) - return FALSE; + /* maildir should be synced, so above call should never fail */ + i_assert(fname != NULL); p = strchr(fname, ':'); len = p == NULL ? strlen(fname) : (size_t)(p-fname);
--- a/src/lib-index/maildir/maildir-update-flags.c Sun Aug 10 21:14:39 2003 +0300 +++ b/src/lib-index/maildir/maildir-update-flags.c Mon Aug 11 02:56:22 2003 +0300 @@ -8,6 +8,15 @@ #include "mail-cache.h" #include <stdio.h> +#include <sys/stat.h> + +struct update_flags_ctx { + const char *new_fname; + int found; + + enum modify_type modify_type; + enum mail_flags flags; +}; static int update_filename(struct mail_index *index, struct mail_index_record *rec) @@ -91,153 +100,132 @@ return TRUE; } -static int handle_error(struct mail_index *index, - const char *path, const char *new_path) +static int do_rename(struct mail_index *index, const char *path, void *context) { - if (errno == ENOENT) - return 0; + struct update_flags_ctx *ctx = context; + const char *fname, *new_path; + enum mail_flags old_flags, new_flags; + int new_dir; - if (ENOSPACE(errno)) { - index->nodiskspace = TRUE; - return -2; - } - - if (errno == EACCES) - index->mailbox_readonly = TRUE; - else { - index_set_error(index, "rename(%s, %s) failed: %m", - path, new_path); + old_flags = maildir_filename_get_flags(path, 0); + switch (ctx->modify_type) { + case MODIFY_ADD: + new_flags = old_flags | ctx->flags; + break; + case MODIFY_REMOVE: + new_flags = old_flags & ~ctx->flags; + break; + case MODIFY_REPLACE: + new_flags = ctx->flags; + break; + default: + new_flags = 0; + i_unreached(); } - return -1; -} + fname = strrchr(path, '/'); + ctx->new_fname = maildir_filename_set_flags(fname != NULL ? + fname+1 : path, new_flags); -static int maildir_rename_mail_file(struct mail_index *index, int new_dir, - const char *old_fname, const char *new_path) -{ - const char *path; + if (old_flags == new_flags) { + /* it's what we wanted. verify that the file exists. */ + struct stat st; - if (new_dir) { - /* probably in new/ dir */ - path = t_strconcat(index->mailbox_path, "/new/", - old_fname, NULL); - if (rename(path, new_path) == 0) - return 1; - - if (errno != ENOENT) - return handle_error(index, path, new_path); + if (stat(path, &st) < 0) { + if (errno == ENOENT) + return 0; + index_file_set_syscall_error(index, path, "stat()"); + return -1; + } + ctx->found = TRUE; + return 1; } - path = t_strconcat(index->mailbox_path, "/cur/", old_fname, NULL); - if (rename(path, new_path) == 0) - return 1; - - return handle_error(index, path, new_path); -} - -static int maildir_rename_mail(struct mail_index *index, - struct mail_index_record *rec, - enum mail_flags flags, const char **new_fname_r) -{ - const char *old_fname, *new_fname, *new_path; - enum mail_index_record_flag index_flags; - int i, ret, found, new_dir; + new_dir = fname != NULL && path + 4 <= fname && + strncmp(fname-4, "/new", 4) == 0; + if (new_dir) { + /* move from new/ to cur/ */ + new_path = t_strconcat(t_strdup_until(path, fname-4), + "/cur/", ctx->new_fname, NULL); + } else { + new_path = maildir_filename_set_flags(path, new_flags); + } - new_fname = new_path = NULL; + if (rename(path, new_path) < 0) { + if (errno == ENOENT) + return 0; - i = 0; - do { - /* we need to update the flags in the file name */ - old_fname = maildir_get_location(index, rec, &new_dir); - if (old_fname == NULL) - return FALSE; - - if (new_path == NULL) { - new_fname = maildir_filename_set_flags(old_fname, - flags); - *new_fname_r = new_fname; - new_path = t_strconcat(index->mailbox_path, - "/cur/", new_fname, NULL); + if (ENOSPACE(errno)) { + index->nodiskspace = TRUE; + return 1; } - if (strcmp(old_fname, new_fname) == 0) - ret = 1; - else { - ret = maildir_rename_mail_file(index, new_dir, - old_fname, new_path); - if (ret == -1) - return FALSE; - - if (ret == 1) { - if (index->maildir_keep_new && new_dir) { - /* looks like we have some more space - again, see if we could move mails - from new/ to cur/ again */ - index->maildir_keep_new = FALSE; - } - - /* cur/ was updated, set it dirty-synced */ - index->file_sync_stamp = ioloop_time; - index->maildir_cur_dirty = ioloop_time; - } - - } - if (ret == 0) { - if (!maildir_index_sync_readonly(index, old_fname, - &found)) - return FALSE; - if (!found) - break; + if (errno == EACCES) { + index->mailbox_readonly = TRUE; + return 1; } - i++; - } while (i < 10 && ret == 0); - - if (ret == 1) - return TRUE; - - /* we couldn't actually rename() the file now. - leave it's flags dirty so they get changed later. */ - index_flags = mail_cache_get_index_flags(index->cache, rec); - if ((index_flags & MAIL_INDEX_FLAG_DIRTY) == 0) { - if (mail_cache_lock(index->cache, FALSE) <= 0) - return FALSE; - mail_cache_unlock_later(index->cache); - - index_flags |= MAIL_INDEX_FLAG_DIRTY; - mail_cache_update_index_flags(index->cache, rec, index_flags); - - index->header->flags |= MAIL_INDEX_HDR_FLAG_DIRTY_MESSAGES; + index_set_error(index, "rename(%s, %s) failed: %m", + path, new_path); + return -1; } - index->next_dirty_flush = - ioloop_time + MAILDIR_DIRTY_FLUSH_TIMEOUT; - *new_fname_r = NULL; - return TRUE; + if (index->maildir_keep_new && new_dir) { + /* looks like we have some more space again, see if we could + move mails from new/ to cur/ again */ + index->maildir_keep_new = FALSE; + } + + /* cur/ was updated, set it dirty-synced */ + index->file_sync_stamp = ioloop_time; + index->maildir_cur_dirty = ioloop_time; + ctx->found = TRUE; + return 1; } int maildir_index_update_flags(struct mail_index *index, struct mail_index_record *rec, unsigned int seq, + enum modify_type modify_type, enum mail_flags flags, int external_change) { - const char *new_fname; - int failed = FALSE; + struct update_flags_ctx ctx; + enum mail_index_record_flag index_flags; + + memset(&ctx, 0, sizeof(ctx)); + ctx.modify_type = modify_type; + ctx.flags = flags; t_push(); - if (!maildir_rename_mail(index, rec, flags, &new_fname)) { + if (!maildir_file_do(index, rec, do_rename, &ctx)) { t_pop(); return FALSE; } - if (new_fname != NULL) { + if (!ctx.found) { + /* we couldn't actually rename() the file now. + leave it's flags dirty so they get changed later. */ + index_flags = mail_cache_get_index_flags(index->cache, rec); + if ((index_flags & MAIL_INDEX_FLAG_DIRTY) == 0) { + if (mail_cache_lock(index->cache, FALSE) <= 0) + return FALSE; + mail_cache_unlock_later(index->cache); + + index_flags |= MAIL_INDEX_FLAG_DIRTY; + mail_cache_update_index_flags(index->cache, rec, + index_flags); + + index->header->flags |= + MAIL_INDEX_HDR_FLAG_DIRTY_MESSAGES; + } + + index->next_dirty_flush = + ioloop_time + MAILDIR_DIRTY_FLUSH_TIMEOUT; + } else if (ctx.new_fname != NULL) { maildir_index_update_filename(index, rec->uid, - new_fname, FALSE); + ctx.new_fname, FALSE); } - - if (!failed && !mail_index_update_flags(index, rec, seq, flags, - external_change)) - failed = TRUE; t_pop(); - return !failed; + return mail_index_update_flags(index, rec, seq, + modify_type, flags, external_change); }
--- a/src/lib-index/mbox/mbox-index.c Sun Aug 10 21:14:39 2003 +0300 +++ b/src/lib-index/mbox/mbox-index.c Mon Aug 11 02:56:22 2003 +0300 @@ -793,15 +793,19 @@ static int mbox_index_update_flags(struct mail_index *index, struct mail_index_record *rec, - unsigned int seq, enum mail_flags flags, + unsigned int seq, + enum modify_type modify_type, + enum mail_flags flags, int external_change) { enum mail_index_record_flag index_flags; - if (!mail_index_update_flags(index, rec, seq, flags, external_change)) + if (!mail_index_update_flags(index, rec, seq, + modify_type, flags, external_change)) return FALSE; if (!external_change) { + /* we'll just mark the message as dirty */ index_flags = mail_cache_get_index_flags(index->cache, rec); if ((index_flags & MAIL_INDEX_FLAG_DIRTY) == 0) { if (mail_cache_lock(index->cache, FALSE) <= 0)
--- a/src/lib-storage/index/index-update-flags.c Sun Aug 10 21:14:39 2003 +0300 +++ b/src/lib-storage/index/index-update-flags.c Mon Aug 11 02:56:22 2003 +0300 @@ -12,7 +12,7 @@ struct index_mail *imail = (struct index_mail *) mail; struct index_mailbox *ibox = imail->ibox; struct mail_storage *storage = mail->box->storage; - enum mail_flags modify_flags, new_flags; + enum mail_flags modify_flags; if (mail->box->is_readonly(mail->box)) { if (ibox->sent_readonly_flags_warning) @@ -34,22 +34,9 @@ flags->custom_flags_count)) return FALSE; - switch (modify_type) { - case MODIFY_ADD: - new_flags = imail->data.rec->msg_flags | modify_flags; - break; - case MODIFY_REMOVE: - new_flags = imail->data.rec->msg_flags & ~modify_flags; - break; - case MODIFY_REPLACE: - new_flags = modify_flags; - break; - default: - i_unreached(); - } - if (!ibox->index->update_flags(ibox->index, imail->data.rec, - imail->data.idx_seq, new_flags, FALSE)) + imail->data.idx_seq, + modify_type, modify_flags, FALSE)) return FALSE; if (mail_custom_flags_has_changes(ibox->index->custom_flags)) {
--- a/src/lib-storage/index/maildir/maildir-copy.c Sun Aug 10 21:14:39 2003 +0300 +++ b/src/lib-storage/index/maildir/maildir-copy.c Mon Aug 11 02:56:22 2003 +0300 @@ -21,112 +21,73 @@ struct mail_copy_context *ctx; }; +struct hardlink_ctx { + const char *dest_path; + int found; +}; + struct rollback { struct rollback *next; const char *fname; }; -static int maildir_hardlink_file(struct mail_index *index, - struct mail_index_record *rec, - const char **fname, const char *new_path) +static int do_hardlink(struct mail_index *index, const char *path, + void *context) { - const char *path; - int new_dir; + struct hardlink_ctx *ctx = context; - *fname = maildir_get_location(index, rec, &new_dir); - if (*fname == NULL) - return -1; - - if (new_dir) { - /* probably in new/ dir */ - path = t_strconcat(index->mailbox_path, "/new/", *fname, NULL); - if (link(path, new_path) == 0) - return 1; + if (link(path, ctx->dest_path) < 0) { + if (errno == ENOENT) + return 0; if (ENOSPACE(errno)) { index->nodiskspace = TRUE; return -1; } if (errno == EACCES || errno == EXDEV) - return -1; - if (errno != ENOENT) { - index_set_error(index, "link(%s, %s) failed: %m", - path, new_path); - return -1; - } - } + return 1; - path = t_strconcat(index->mailbox_path, "/cur/", *fname, NULL); - if (link(path, new_path) == 0) - return 1; - - if (ENOSPACE(errno)) { - index->nodiskspace = TRUE; - return -1; - } - if (errno == EACCES || errno == EXDEV) - return -1; - if (errno != ENOENT) { index_set_error(index, "link(%s, %s) failed: %m", - path, new_path); + path, ctx->dest_path); return -1; } - return 0; + ctx->found = TRUE; + return 1; } static int maildir_copy_hardlink(struct mail *mail, struct maildir_copy_context *ctx) { struct index_mail *imail = (struct index_mail *) mail; - struct rollback *rb; - const char *fname, *dest_fname, *dest_path; - enum mail_flags flags; - int i, ret, found; + struct hardlink_ctx do_ctx; + struct rollback *rb; + const char *dest_fname; - flags = mail->get_flags(mail)->flags; - - /* link the file */ dest_fname = maildir_generate_tmp_filename(&ioloop_timeval); - dest_fname = maildir_filename_set_flags(dest_fname, flags); - dest_path = t_strconcat(ctx->ibox->index->mailbox_path, "/new/", - dest_fname, NULL); + dest_fname = maildir_filename_set_flags(dest_fname, + mail->get_flags(mail)->flags); - for (i = 0;; i++) { - ret = maildir_hardlink_file(imail->ibox->index, imail->data.rec, - &fname, dest_path); - if (ret != 0) - break; + memset(&do_ctx, 0, sizeof(do_ctx)); + do_ctx.dest_path = t_strconcat(ctx->ibox->index->mailbox_path, "/new/", + dest_fname, NULL); - if (i == 10) { - mail_storage_set_error(mail->box->storage, - "File name keeps changing, copy failed"); - break; - } + if (!maildir_file_do(imail->ibox->index, imail->data.rec, + do_hardlink, &do_ctx)) + return -1; - if (!maildir_index_sync_readonly(imail->ibox->index, fname, - &found)) { - ret = -1; - break; - } + if (!do_ctx.found) + return 0; - if (!found) - break; - } + if (ctx->pool == NULL) + ctx->pool = pool_alloconly_create("hard copy rollbacks", 2048); - if (ret > 0) { - if (ctx->pool == NULL) { - ctx->pool = pool_alloconly_create("hard copy rollbacks", - 2048); - } + rb = p_new(ctx->pool, struct rollback, 1); + rb->fname = p_strdup(ctx->pool, dest_fname); - rb = p_new(ctx->pool, struct rollback, 1); - rb->fname = p_strdup(ctx->pool, dest_fname); - rb->next = ctx->rollbacks; - ctx->rollbacks = rb; - } - - return ret; + rb->next = ctx->rollbacks; + ctx->rollbacks = rb; + return 1; } struct mail_copy_context *maildir_storage_copy_init(struct mailbox *box)
--- a/src/lib-storage/mail-storage.h Sun Aug 10 21:14:39 2003 +0300 +++ b/src/lib-storage/mail-storage.h Mon Aug 11 02:56:22 2003 +0300 @@ -61,12 +61,6 @@ MAILBOX_LOCK_SAVE = 0x08 }; -enum modify_type { - MODIFY_ADD, - MODIFY_REMOVE, - MODIFY_REPLACE -}; - enum mail_sort_type { /* Maximum size for sort program, 2x for reverse + END */ #define MAX_SORT_PROGRAM_SIZE (2*7 + 1)