Mercurial > dovecot > original-hg > dovecot-1.2
changeset 4092:ae38c59ddf60 HEAD
Merged save-copying and hardlink-copying code so that hardlink-copying updates indexes immediately.
author | Timo Sirainen <timo.sirainen@movial.fi> |
---|---|
date | Mon, 06 Mar 2006 20:16:12 +0200 |
parents | f4cd3e942678 |
children | 840d6403ef09 |
files | src/lib-storage/index/maildir/maildir-copy.c src/lib-storage/index/maildir/maildir-save.c src/lib-storage/index/maildir/maildir-storage.c src/lib-storage/index/maildir/maildir-storage.h src/lib-storage/index/maildir/maildir-transaction.c |
diffstat | 5 files changed, 184 insertions(+), 184 deletions(-) [+] |
line wrap: on
line diff
--- a/src/lib-storage/index/maildir/maildir-copy.c Mon Mar 06 20:15:37 2006 +0200 +++ b/src/lib-storage/index/maildir/maildir-copy.c Mon Mar 06 20:16:12 2006 +0200 @@ -12,25 +12,9 @@ #include <stdlib.h> #include <unistd.h> -struct maildir_copy_context { - struct maildir_mailbox *mbox; - bool hardlink; - - struct maildir_uidlist_sync_ctx *uidlist_sync_ctx; - struct maildir_keywords_sync_ctx *keywords_sync_ctx; - - pool_t pool; - struct rollback *rollbacks; -}; - struct hardlink_ctx { const char *dest_path; - bool found; -}; - -struct rollback { - struct rollback *next; - const char *fname; + bool success; }; static int do_hardlink(struct maildir_mailbox *mbox, const char *path, @@ -56,118 +40,88 @@ return -1; } - ctx->found = TRUE; + ctx->success = TRUE; return 1; } static int -maildir_copy_hardlink(struct mail *mail, +maildir_copy_hardlink(struct maildir_transaction_context *t, struct mail *mail, enum mail_flags flags, struct mail_keywords *keywords, - struct maildir_copy_context *ctx) + struct mail *dest_mail) { - struct index_mail *imail = (struct index_mail *)mail; - struct maildir_mailbox *dest_mbox = ctx->mbox; + struct maildir_mailbox *dest_mbox = + (struct maildir_mailbox *)t->ictx.ibox; struct maildir_mailbox *src_mbox = - (struct maildir_mailbox *)imail->ibox; + (struct maildir_mailbox *)mail->box; + struct maildir_save_context *ctx; struct hardlink_ctx do_ctx; - struct rollback *rb; const char *dest_fname; - unsigned int keywords_count; - array_t ARRAY_DEFINE(keywords_arr, unsigned int); - - dest_fname = maildir_generate_tmp_filename(&ioloop_timeval); + uint32_t seq; - keywords_count = keywords == NULL ? 0 : keywords->count; - if (keywords_count > 0) { - ARRAY_CREATE(&keywords_arr, pool_datastack_create(), - unsigned int, keywords->count); - array_append(&keywords_arr, keywords->idx, keywords->count); + i_assert((t->ictx.flags & MAILBOX_TRANSACTION_FLAG_EXTERNAL) != 0); - if (ctx->keywords_sync_ctx == NULL) { - /* uidlist must be locked while accessing - keywords files */ - if (maildir_uidlist_sync_init(dest_mbox->uidlist, TRUE, - &ctx->uidlist_sync_ctx) <= 0) { - /* error or timeout */ - return -1; - } + if (t->save_ctx == NULL) + t->save_ctx = maildir_save_transaction_init(t); + ctx = t->save_ctx; - ctx->keywords_sync_ctx = - maildir_keywords_sync_init(dest_mbox->keywords, - dest_mbox->ibox.index); - } - } - + /* don't allow caller to specify recent flag */ flags &= ~MAIL_RECENT; if (dest_mbox->ibox.keep_recent) flags |= MAIL_RECENT; - dest_fname = maildir_filename_set_flags(ctx->keywords_sync_ctx, - dest_fname, flags, - keywords_count != 0 ? - &keywords_arr : NULL); + memset(&do_ctx, 0, sizeof(do_ctx)); + + /* the generated filename is _always_ unique, so we don't bother + trying to check if it already exists */ + dest_fname = maildir_generate_tmp_filename(&ioloop_timeval); + if (keywords == NULL || keywords->count == 0) { + /* no keywords, hardlink directly to destination */ + if (flags == MAIL_RECENT) { + do_ctx.dest_path = + t_strconcat(dest_mbox->path, "/new/", + dest_fname, NULL); + } else { + const char *fname; - if (keywords_count == 0 && flags == MAIL_RECENT) - dest_fname = t_strconcat("new/", dest_fname, NULL); - else - dest_fname = t_strconcat("cur/", dest_fname, NULL); + fname = maildir_filename_set_flags(NULL, dest_fname, + flags, NULL); - memset(&do_ctx, 0, sizeof(do_ctx)); - do_ctx.dest_path = - t_strconcat(dest_mbox->path, "/", dest_fname, NULL); - - if (maildir_file_do(src_mbox, imail->mail.mail.uid, - do_hardlink, &do_ctx) < 0) + do_ctx.dest_path = + t_strconcat(dest_mbox->path, "/cur/", + fname, NULL); + } + } else { + /* keywords, hardlink to tmp/ with basename and later when we + have uidlist locked, move it to new/cur. */ + do_ctx.dest_path = + t_strconcat(dest_mbox->path, "/tmp/", dest_fname, NULL); + } + if (maildir_file_do(src_mbox, mail->uid, do_hardlink, &do_ctx) < 0) return -1; - if (!do_ctx.found) + if (!do_ctx.success) { + /* couldn't copy with hardlinking, fallback to copying */ return 0; - - rb = p_new(ctx->pool, struct rollback, 1); - rb->fname = p_strdup(ctx->pool, dest_fname); - - rb->next = ctx->rollbacks; - ctx->rollbacks = rb; - return 1; -} - -static struct maildir_copy_context * -maildir_copy_init(struct maildir_mailbox *mbox) -{ - struct maildir_copy_context *ctx; - pool_t pool; - - pool = pool_alloconly_create("maildir_copy_context", 2048); - - ctx = p_new(pool, struct maildir_copy_context, 1); - ctx->pool = pool; - ctx->hardlink = getenv("MAILDIR_COPY_WITH_HARDLINKS") != NULL; - ctx->mbox = mbox; - return ctx; -} - -int maildir_transaction_copy_commit(struct maildir_copy_context *ctx) -{ - if (ctx->keywords_sync_ctx != NULL) { - maildir_keywords_sync_deinit(ctx->keywords_sync_ctx); - maildir_uidlist_sync_deinit(ctx->uidlist_sync_ctx); - } - pool_unref(ctx->pool); - return 0; -} - -void maildir_transaction_copy_rollback(struct maildir_copy_context *ctx) -{ - struct rollback *rb; - - for (rb = ctx->rollbacks; rb != NULL; rb = rb->next) { - t_push(); - (void)unlink(t_strconcat(ctx->mbox->path, "/", - rb->fname, NULL)); - t_pop(); } - pool_unref(ctx->pool); + if (keywords == NULL || keywords->count == 0) { + /* hardlinked to destination, set hardlinked-flag */ + seq = maildir_save_add(t, dest_fname, + flags | MAILDIR_SAVE_FLAG_HARDLINK, NULL, + dest_mail != NULL); + } else { + /* hardlinked to tmp/, treat as normal copied mail */ + seq = maildir_save_add(t, dest_fname, flags, keywords, + dest_mail != NULL); + } + + if (dest_mail != NULL) { + i_assert(seq != 0); + + if (mail_set_seq(dest_mail, seq) < 0) + return -1; + } + return 1; } int maildir_copy(struct mailbox_transaction_context *_t, struct mail *mail, @@ -177,18 +131,13 @@ struct maildir_transaction_context *t = (struct maildir_transaction_context *)_t; struct maildir_mailbox *mbox = (struct maildir_mailbox *)t->ictx.ibox; - struct maildir_copy_context *ctx; int ret; - if (t->copy_ctx == NULL) - t->copy_ctx = maildir_copy_init(mbox); - ctx = t->copy_ctx; - - if (ctx->hardlink && - mail->box->storage == STORAGE(ctx->mbox->storage)) { - // FIXME: handle dest_mail + if (mbox->storage->copy_with_hardlinks && + mail->box->storage == mbox->ibox.box.storage) { t_push(); - ret = maildir_copy_hardlink(mail, flags, keywords, ctx); + ret = maildir_copy_hardlink(t, mail, flags, + keywords, dest_mail); t_pop(); if (ret > 0)
--- a/src/lib-storage/index/maildir/maildir-save.c Mon Mar 06 20:15:37 2006 +0200 +++ b/src/lib-storage/index/maildir/maildir-save.c Mon Mar 06 20:16:12 2006 +0200 @@ -90,7 +90,7 @@ return ret; } -static struct maildir_save_context * +struct maildir_save_context * maildir_save_transaction_init(struct maildir_transaction_context *t) { struct maildir_mailbox *mbox = (struct maildir_mailbox *)t->ictx.ibox; @@ -113,9 +113,62 @@ ctx->keywords_buffer = buffer_create_const_data(pool, NULL, 0); array_create_from_buffer(&ctx->keywords_array, ctx->keywords_buffer, sizeof(unsigned int)); + ctx->finished = TRUE; return ctx; } +uint32_t maildir_save_add(struct maildir_transaction_context *t, + const char *base_fname, enum mail_flags flags, + struct mail_keywords *keywords, bool want_mail) +{ + struct maildir_save_context *ctx = t->save_ctx; + struct maildir_mailbox *mbox = (struct maildir_mailbox *)t->ictx.ibox; + struct maildir_filename *mf; + + /* now, we want to be able to rollback the whole append session, + so we'll just store the name of this temp file and move it later + into new/ or cur/. */ + /* @UNSAFE */ + mf = p_malloc(ctx->pool, sizeof(*mf) + + sizeof(unsigned int) * (keywords == NULL ? 0 : + keywords->count)); + mf->next = ctx->files; + mf->basename = p_strdup(ctx->pool, base_fname); + mf->flags = flags; + ctx->files = mf; + + if (keywords != NULL) { + i_assert(sizeof(keywords->idx[0]) == sizeof(unsigned int)); + + /* @UNSAFE */ + mf->keywords_count = keywords->count; + memcpy(mf + 1, keywords->idx, + sizeof(unsigned int) * keywords->count); + } + + if (!ctx->synced && want_mail) { + if (maildir_storage_sync_force(mbox) < 0) + ctx->failed = TRUE; + else + ctx->synced = TRUE; + } + + if (ctx->synced) { + /* insert into index */ + mail_index_append(ctx->trans, 0, &ctx->seq); + mail_index_update_flags(ctx->trans, ctx->seq, + MODIFY_REPLACE, flags); + if (keywords != NULL) { + mail_index_update_keywords(ctx->trans, ctx->seq, + MODIFY_REPLACE, keywords); + } + } else { + ctx->seq = 0; + } + + return ctx->seq; +} + int maildir_save_init(struct mailbox_transaction_context *_t, enum mail_flags flags, struct mail_keywords *keywords, time_t received_date, int timezone_offset __attr_unused__, @@ -127,7 +180,6 @@ (struct maildir_transaction_context *)_t; struct maildir_save_context *ctx; struct maildir_mailbox *mbox = (struct maildir_mailbox *)t->ictx.ibox; - struct maildir_filename *mf; struct ostream *output; const char *fname, *path; @@ -166,46 +218,9 @@ if (mbox->ibox.keep_recent) flags |= MAIL_RECENT; - /* now, we want to be able to rollback the whole append session, - so we'll just store the name of this temp file and move it later - into new/ or cur/. */ - /* @UNSAFE */ - mf = p_malloc(ctx->pool, sizeof(*mf) + - sizeof(unsigned int) * (keywords == NULL ? 0 : - keywords->count)); - mf->next = ctx->files; - mf->basename = p_strdup(ctx->pool, fname); - mf->flags = flags; - ctx->files = mf; - - if (keywords != NULL) { - i_assert(sizeof(keywords->idx[0]) == sizeof(unsigned int)); + maildir_save_add(t, fname, flags, keywords, want_mail); - /* @UNSAFE */ - mf->keywords_count = keywords->count; - memcpy(mf + 1, keywords->idx, - sizeof(unsigned int) * keywords->count); - } - - if (!ctx->synced && want_mail) { - if (maildir_storage_sync_force(mbox) < 0) - ctx->failed = TRUE; - else - ctx->synced = TRUE; - } - - if (ctx->synced) { - /* insert into index */ - mail_index_append(ctx->trans, 0, &ctx->seq); - mail_index_update_flags(ctx->trans, ctx->seq, - MODIFY_REPLACE, flags); - if (keywords != NULL) { - mail_index_update_keywords(ctx->trans, ctx->seq, - MODIFY_REPLACE, keywords); - } - } t_pop(); - *ctx_r = &ctx->ctx; return ctx->failed ? -1 : 0; } @@ -293,6 +308,7 @@ t_pop(); return -1; } + t_pop(); if (dest_mail != NULL) { i_assert(ctx->seq != 0); @@ -301,7 +317,6 @@ return -1; } - t_pop(); return 0; } @@ -317,25 +332,34 @@ maildir_get_updated_filename(struct maildir_save_context *ctx, struct maildir_filename *mf) { - if (mf->flags == MAIL_RECENT && mf->keywords_count == 0) - return NULL; + if (mf->keywords_count == 0) { + if ((mf->flags & MAIL_FLAGS_MASK) == MAIL_RECENT) + return NULL; + return maildir_filename_set_flags(NULL, mf->basename, + mf->flags & MAIL_FLAGS_MASK, + NULL); + } buffer_update_const_data(ctx->keywords_buffer, mf + 1, mf->keywords_count * sizeof(unsigned int)); return maildir_filename_set_flags( maildir_sync_get_keywords_sync_ctx(ctx->sync_ctx), - mf->basename, mf->flags, &ctx->keywords_array); + mf->basename, mf->flags & MAIL_FLAGS_MASK, + &ctx->keywords_array); } static void -maildir_save_commit_abort(struct maildir_save_context *ctx, - struct maildir_filename *pos) +maildir_transaction_unlink_copied_files(struct maildir_save_context *ctx, + struct maildir_filename *pos) { struct maildir_filename *mf; const char *path, *dest; /* try to unlink the mails already moved */ for (mf = ctx->files; mf != pos; mf = mf->next) { + if ((mf->flags & MAILDIR_SAVE_FLAG_DELETED) != 0) + continue; + t_push(); dest = maildir_get_updated_filename(ctx, mf); if (dest != NULL) @@ -345,10 +369,9 @@ ctx->newdir, mf->basename); } (void)unlink(path); + t_pop(); } ctx->files = pos; - - maildir_transaction_save_rollback(ctx); } int maildir_transaction_save_commit_pre(struct maildir_save_context *ctx) @@ -365,7 +388,7 @@ /* Start syncing so that keywords_sync_ctx gets set.. */ ctx->sync_ctx = maildir_sync_index_begin(ctx->mbox); if (ctx->sync_ctx == NULL) { - maildir_save_commit_abort(ctx, ctx->files); + maildir_transaction_save_rollback(ctx); return -1; } @@ -373,7 +396,7 @@ &ctx->uidlist_sync_ctx) <= 0) { /* error or timeout - our transaction is broken */ maildir_sync_index_abort(ctx->sync_ctx); - maildir_save_commit_abort(ctx, ctx->files); + maildir_transaction_save_rollback(ctx); return -1; } @@ -385,28 +408,41 @@ flags = MAILDIR_UIDLIST_REC_FLAG_NEW_DIR | MAILDIR_UIDLIST_REC_FLAG_RECENT; - /* move them into new/ */ + /* move them into new/ and/or cur/ */ ret = 0; - for (mf = ctx->files; mf != NULL; mf = mf->next) { + for (mf = ctx->files; mf != NULL && ret == 0; mf = mf->next) { t_push(); dest = maildir_get_updated_filename(ctx, mf); fname = dest != NULL ? dest : mf->basename; - if (maildir_file_move(ctx, mf->basename, dest) < 0 || - maildir_uidlist_sync_next(ctx->uidlist_sync_ctx, - fname, flags) < 0) { - maildir_save_commit_abort(ctx, mf); - t_pop(); - ret = -1; - break; + /* if hardlink-flag is set, the file is already in destination. + if the hardlinked mail contained keywords, it was linked + into tmp/ and it doesn't have the hardlink-flag set, so it's + treated as any other saved mail. */ + if ((mf->flags & MAILDIR_SAVE_FLAG_HARDLINK) == 0) + ret = maildir_file_move(ctx, mf->basename, dest); + if (ret == 0) { + ret = maildir_uidlist_sync_next(ctx->uidlist_sync_ctx, + fname, flags); } t_pop(); } + if (ret < 0) { + /* unlink the files we just moved in an attempt to rollback + the transaction. uidlist is still locked, so at least other + Dovecot instances haven't yet seen the files. */ + maildir_transaction_unlink_copied_files(ctx, mf); + } + if (maildir_uidlist_sync_deinit(ctx->uidlist_sync_ctx) < 0) ret = -1; ctx->uidlist_sync_ctx = NULL; + if (ret < 0) { + /* returning failure finishes the save_context */ + maildir_transaction_save_rollback(ctx); + } return ret; } @@ -429,6 +465,7 @@ struct maildir_filename *mf; string_t *str; size_t dir_len; + bool hardlinks = FALSE; i_assert(ctx->output == NULL); @@ -444,10 +481,19 @@ /* clean up the temp files */ for (mf = ctx->files; mf != NULL; mf = mf->next) { - str_truncate(str, dir_len); - str_append(str, mf->basename); - (void)unlink(str_c(str)); + if ((mf->flags & MAILDIR_SAVE_FLAG_HARDLINK) == 0) { + mf->flags |= MAILDIR_SAVE_FLAG_DELETED; + str_truncate(str, dir_len); + str_append(str, mf->basename); + (void)unlink(str_c(str)); + } else { + hardlinks = TRUE; + } } + + if (hardlinks) + maildir_transaction_unlink_copied_files(ctx, NULL); + t_pop(); pool_unref(ctx->pool);
--- a/src/lib-storage/index/maildir/maildir-storage.c Mon Mar 06 20:15:37 2006 +0200 +++ b/src/lib-storage/index/maildir/maildir-storage.c Mon Mar 06 20:16:12 2006 +0200 @@ -126,6 +126,8 @@ pool = pool_alloconly_create("storage", 512); storage = p_new(pool, struct maildir_storage, 1); storage->control_dir = p_strdup(pool, home_expand(control_dir)); + storage->copy_with_hardlinks = + getenv("MAILDIR_COPY_WITH_HARDLINKS") != NULL; istorage = INDEX_STORAGE(storage); istorage->storage = maildir_storage;
--- a/src/lib-storage/index/maildir/maildir-storage.h Mon Mar 06 20:15:37 2006 +0200 +++ b/src/lib-storage/index/maildir/maildir-storage.h Mon Mar 06 20:16:12 2006 +0200 @@ -31,6 +31,9 @@ calculating file's virtual size (added missing CRs). */ #define MAILDIR_EXTRA_VIRTUAL_SIZE "W" +#define MAILDIR_SAVE_FLAG_HARDLINK 0x10000000 +#define MAILDIR_SAVE_FLAG_DELETED 0x20000000 + #include "index-storage.h" #define STORAGE(maildir_storage) \ @@ -47,6 +50,7 @@ struct index_storage storage; const char *control_dir; + unsigned int copy_with_hardlinks:1; }; struct maildir_mailbox { @@ -70,7 +74,6 @@ struct maildir_transaction_context { struct index_transaction_context ictx; struct maildir_save_context *save_ctx; - struct maildir_copy_context *copy_ctx; }; extern struct mail_vfuncs maildir_mail_vfuncs; @@ -121,6 +124,12 @@ int maildir_save_finish(struct mail_save_context *ctx, struct mail *dest_mail); void maildir_save_cancel(struct mail_save_context *ctx); +struct maildir_save_context * +maildir_save_transaction_init(struct maildir_transaction_context *t); +uint32_t maildir_save_add(struct maildir_transaction_context *t, + const char *base_fname, enum mail_flags flags, + struct mail_keywords *keywords, bool want_mail); + int maildir_transaction_save_commit_pre(struct maildir_save_context *ctx); void maildir_transaction_save_commit_post(struct maildir_save_context *ctx); void maildir_transaction_save_rollback(struct maildir_save_context *ctx);
--- a/src/lib-storage/index/maildir/maildir-transaction.c Mon Mar 06 20:15:37 2006 +0200 +++ b/src/lib-storage/index/maildir/maildir-transaction.c Mon Mar 06 20:16:12 2006 +0200 @@ -30,10 +30,6 @@ ret = -1; } } - if (t->copy_ctx != NULL) { - if (maildir_transaction_copy_commit(t->copy_ctx) < 0) - ret = -1; - } save_ctx = t->save_ctx; @@ -56,7 +52,5 @@ if (t->save_ctx != NULL) maildir_transaction_save_rollback(t->save_ctx); - if (t->copy_ctx != NULL) - maildir_transaction_copy_rollback(t->copy_ctx); index_transaction_rollback(_t); }