# HG changeset patch # User Timo Sirainen # Date 1204604784 -7200 # Node ID 969656b58e7f28c87e2999310ab7281a122ac30b # Parent ae8180a4febd798db8cbb0a69f5301891c8ee2a8 Added wildcard support to expire plugin. Added a new expire_altmove setting which moves mails to alt directory with dbox instead of expunging them. Both settings can be used simultaneously. Added --test parameter to expire-tool. Fixed several bugs and did some optimizations. diff -r ae8180a4febd -r 969656b58e7f src/plugins/expire/Makefile.am --- a/src/plugins/expire/Makefile.am Tue Mar 04 06:24:44 2008 +0200 +++ b/src/plugins/expire/Makefile.am Tue Mar 04 06:26:24 2008 +0200 @@ -4,6 +4,7 @@ -I$(top_srcdir)/src/lib \ -I$(top_srcdir)/src/lib-dict \ -I$(top_srcdir)/src/lib-mail \ + -I$(top_srcdir)/src/lib-imap \ -I$(top_srcdir)/src/lib-index \ -I$(top_srcdir)/src/lib-storage \ -I$(top_srcdir)/src/lib-storage/index \ diff -r ae8180a4febd -r 969656b58e7f src/plugins/expire/expire-env.c --- a/src/plugins/expire/expire-env.c Tue Mar 04 06:24:44 2008 +0200 +++ b/src/plugins/expire/expire-env.c Tue Mar 04 06:26:24 2008 +0200 @@ -2,42 +2,70 @@ #include "lib.h" #include "array.h" +#include "imap-match.h" #include "expire-env.h" #include +enum expire_type { + EXPIRE_TYPE_EXPUNGE, + EXPIRE_TYPE_ALTMOVE +}; + +struct expire_box { + const char *pattern; + struct imap_match_glob *glob; + + enum expire_type type; + unsigned int expire_secs; +}; + struct expire_env { pool_t pool; ARRAY_DEFINE(expire_boxes, struct expire_box); }; -struct expire_env *expire_env_init(const char *str) +static void expire_env_parse(struct expire_env *env, const char *str, + enum expire_type type) { - struct expire_env *env; struct expire_box box; - pool_t pool; char *const *names; unsigned int len; - pool = pool_alloconly_create("Expire pool", 512); - env = p_new(pool, struct expire_env, 1); - env->pool = pool; + if (str == NULL) + return; - names = p_strsplit(pool, str, " "); + names = p_strsplit(env->pool, str, " "); len = str_array_length((const char *const *)names); - p_array_init(&env->expire_boxes, pool, len / 2); + p_array_init(&env->expire_boxes, env->pool, len / 2); for (; *names != NULL; names += 2) { if (names[1] == NULL) { i_fatal("expire: Missing expire days for mailbox '%s'", *names); } - box.name = *names; - box.expire_secs = strtoul(names[1], NULL, 10) * 3600 * 24; + box.pattern = *names; + /* FIXME: hardcoded separator isn't very good */ + box.glob = imap_match_init(env->pool, box.pattern, TRUE, '/'); + box.type = type; + box.expire_secs = strtoul(names[1], NULL, 10)/* * 3600 * 24*/ * 10;//FIXME + array_append(&env->expire_boxes, &box, 1); } +} +struct expire_env *expire_env_init(const char *expunges, const char *altmoves) +{ + struct expire_env *env; + pool_t pool; + + pool = pool_alloconly_create("Expire pool", 512); + env = p_new(pool, struct expire_env, 1); + env->pool = pool; + + expire_env_parse(env, expunges, EXPIRE_TYPE_EXPUNGE); + expire_env_parse(env, altmoves, EXPIRE_TYPE_ALTMOVE); return env; } @@ -46,16 +74,41 @@ pool_unref(&env->pool); } -const struct expire_box *expire_box_find(struct expire_env *env, - const char *name) +bool expire_box_find(struct expire_env *env, const char *name, + unsigned int *expunge_secs_r, + unsigned int *altmove_secs_r) { const struct expire_box *expire_boxes; unsigned int i, count; + unsigned int secs, expunge_min = 0, altmove_min = 0; expire_boxes = array_get(&env->expire_boxes, &count); for (i = 0; i < count; i++) { - if (strcmp(name, expire_boxes[i].name) == 0) - return &expire_boxes[i]; + if (imap_match(expire_boxes[i].glob, name) == IMAP_MATCH_YES) { + secs = expire_boxes[i].expire_secs; + i_assert(secs > 0); + + switch (expire_boxes[i].type) { + case EXPIRE_TYPE_EXPUNGE: + if (expunge_min == 0 || expunge_min > secs) + expunge_min = secs; + break; + case EXPIRE_TYPE_ALTMOVE: + if (altmove_min == 0 || altmove_min > secs) + altmove_min = secs; + break; + } + } } - return NULL; + *expunge_secs_r = expunge_min; + *altmove_secs_r = altmove_min; + return expunge_min > 0 || altmove_min > 0; } + +unsigned int expire_box_find_min_secs(struct expire_env *env, const char *name) +{ + unsigned int secs1, secs2; + + (void)expire_box_find(env, name, &secs1, &secs2); + return secs1 < secs2 && secs1 != 0 ? secs1 : secs2; +} diff -r ae8180a4febd -r 969656b58e7f src/plugins/expire/expire-env.h --- a/src/plugins/expire/expire-env.h Tue Mar 04 06:24:44 2008 +0200 +++ b/src/plugins/expire/expire-env.h Tue Mar 04 06:26:24 2008 +0200 @@ -3,15 +3,13 @@ struct expire_env; -struct expire_box { - const char *name; - time_t expire_secs; -}; - -struct expire_env *expire_env_init(const char *str); +struct expire_env *expire_env_init(const char *expunges, const char *altmoves); void expire_env_deinit(struct expire_env *env); -const struct expire_box *expire_box_find(struct expire_env *env, - const char *name); +bool expire_box_find(struct expire_env *env, const char *name, + unsigned int *expunge_secs_r, + unsigned int *altmove_secs_r); + +unsigned int expire_box_find_min_secs(struct expire_env *env, const char *name); #endif diff -r ae8180a4febd -r 969656b58e7f src/plugins/expire/expire-plugin.c --- a/src/plugins/expire/expire-plugin.c Tue Mar 04 06:24:44 2008 +0200 +++ b/src/plugins/expire/expire-plugin.c Tue Mar 04 06:26:24 2008 +0200 @@ -3,6 +3,7 @@ #include "lib.h" #include "ioloop.h" #include "array.h" +#include "str.h" #include "dict.h" #include "mail-namespace.h" #include "index-mail.h" @@ -33,9 +34,7 @@ struct expire_transaction_context { union mailbox_transaction_module_context module_ctx; - struct mail *mail; - time_t first_save_time; - + unsigned int saves:1; unsigned int first_expunged:1; }; @@ -56,7 +55,6 @@ t = xpr_box->module_ctx.super.transaction_begin(box, flags); xt = i_new(struct expire_transaction_context, 1); - xt->mail = mail_alloc(t, 0, NULL); MODULE_CONTEXT_SET(t, expire_storage_module, xt); return t; @@ -67,21 +65,26 @@ { struct index_transaction_context *t = (struct index_transaction_context *)_t; - struct expire_transaction_context *xt = EXPIRE_CONTEXT(_t); struct mail_index_view *view = t->trans_view; const struct mail_index_header *hdr; + struct mail *mail; uint32_t seq; + mail = mail_alloc(_t, 0, NULL); + /* find the first non-expunged mail. we're here because the first mail was expunged, so don't bother checking it. */ hdr = mail_index_get_header(view); for (seq = 2; seq <= hdr->messages_count; seq++) { if (!mail_index_is_expunged(view, seq)) { - mail_set_seq(xt->mail, seq); - if (mail_get_save_date(xt->mail, stamp_r) == 0) + mail_set_seq(mail, seq); + if (mail_get_save_date(mail, stamp_r) == 0) { + mail_free(&mail); return; + } } } + mail_free(&mail); /* everything expunged */ *stamp_r = 0; @@ -106,7 +109,6 @@ update_dict = TRUE; } - mail_free(&xt->mail); if (xpr_box->module_ctx.super. transaction_commit(t, uid_validity_r, first_saved_uid_r, last_saved_uid_r) < 0) { @@ -114,27 +116,33 @@ return -1; } - T_BEGIN { + if (xt->first_expunged || xt->saves) T_BEGIN { const char *key, *value; key = t_strconcat(DICT_PATH_SHARED, expire.username, "/", mailbox_name, NULL); - if (!xt->first_expunged) { + if (!xt->first_expunged && xt->saves) { /* saved new mails. dict needs to be updated only if this is the first mail in the database */ ret = dict_lookup(expire.db, pool_datastack_create(), key, &value); update_dict = ret == 0 || strtoul(value, NULL, 10) == 0; - new_stamp = xt->first_save_time; + /* may not be exactly the first message's save time + but a few second difference doesn't matter */ + new_stamp = ioloop_time; } if (update_dict) { struct dict_transaction_context *dctx; - new_stamp += xpr_box->expire_secs; - dctx = dict_transaction_begin(expire.db); - dict_set(dctx, key, dec2str(new_stamp)); + if (new_stamp == 0) { + /* everything expunged */ + dict_unset(dctx, key); + } else { + new_stamp += xpr_box->expire_secs; + dict_set(dctx, key, dec2str(new_stamp)); + } dict_transaction_commit(dctx); } } T_END; @@ -148,8 +156,6 @@ struct expire_mailbox *xpr_box = EXPIRE_CONTEXT(t->box); struct expire_transaction_context *xt = EXPIRE_CONTEXT(t); - mail_free(&xt->mail); - xpr_box->module_ctx.super.transaction_rollback(t); i_free(xt); } @@ -190,41 +196,14 @@ return _mail; } -static void -mail_set_save_time(struct mailbox_transaction_context *t, uint32_t seq) +static int expire_save_finish(struct mail_save_context *ctx) { - struct expire_transaction_context *xt = EXPIRE_CONTEXT(t); - struct index_transaction_context *it = - (struct index_transaction_context *)t; - - if (xt->first_save_time == 0) - xt->first_save_time = ioloop_time; - - mail_cache_add(it->cache_trans, seq, MAIL_CACHE_SAVE_DATE, - &ioloop_time, sizeof(ioloop_time)); -} + struct expire_transaction_context *xt = + EXPIRE_CONTEXT(ctx->transaction); + struct expire_mailbox *xpr_box = EXPIRE_CONTEXT(ctx->transaction->box); -static int -expire_save_init(struct mailbox_transaction_context *t, - enum mail_flags flags, struct mail_keywords *keywords, - time_t received_date, int timezone_offset, - const char *from_envelope, struct istream *input, - struct mail *dest_mail, struct mail_save_context **ctx_r) -{ - struct expire_transaction_context *xt = EXPIRE_CONTEXT(t); - struct expire_mailbox *xpr_box = EXPIRE_CONTEXT(t->box); - int ret; - - if (dest_mail == NULL) - dest_mail = xt->mail; - - ret = xpr_box->module_ctx.super. - save_init(t, flags, keywords, received_date, - timezone_offset, from_envelope, input, - dest_mail, ctx_r); - if (ret >= 0) - mail_set_save_time(t, dest_mail->seq); - return ret; + xt->saves = TRUE; + return xpr_box->module_ctx.super.save_finish(ctx); } static int @@ -234,16 +213,10 @@ { struct expire_transaction_context *xt = EXPIRE_CONTEXT(t); struct expire_mailbox *xpr_box = EXPIRE_CONTEXT(t->box); - int ret; - if (dest_mail == NULL) - dest_mail = xt->mail; - - ret = xpr_box->module_ctx.super. + xt->saves = TRUE; + return xpr_box->module_ctx.super. copy(t, mail, flags, keywords, dest_mail); - if (ret >= 0) - mail_set_save_time(t, dest_mail->seq); - return ret; } static void mailbox_expire_hook(struct mailbox *box, time_t expire_secs) @@ -257,7 +230,7 @@ box->v.transaction_commit = expire_mailbox_transaction_commit; box->v.transaction_rollback = expire_mailbox_transaction_rollback; box->v.mail_alloc = expire_mail_alloc; - box->v.save_init = expire_save_init; + box->v.save_finish = expire_save_finish; box->v.copy = expire_copy; xpr_box->expire_secs = expire_secs; @@ -272,15 +245,17 @@ union mail_storage_module_context *xpr_storage = EXPIRE_CONTEXT(storage); struct mailbox *box; - const struct expire_box *expire_box; - const char *full_name; + string_t *vname; + unsigned int secs; box = xpr_storage->super.mailbox_open(storage, name, input, flags); if (box != NULL) { - full_name = t_strconcat(storage->ns->prefix, name, NULL); - expire_box = expire_box_find(expire.env, full_name); - if (expire_box != NULL) - mailbox_expire_hook(box, expire_box->expire_secs); + vname = t_str_new(128); + (void)mail_namespace_get_vname(storage->ns, vname, name); + + secs = expire_box_find_min_secs(expire.env, str_c(vname)); + if (secs != 0) + mailbox_expire_hook(box, secs); } return box; } @@ -302,15 +277,16 @@ void expire_plugin_init(void) { - const char *env, *dict_uri; + const char *expunge_env, *altmove_env, *dict_uri; - env = getenv("EXPIRE"); - if (env != NULL) { + expunge_env = getenv("EXPIRE"); + altmove_env = getenv("EXPIRE_ALTMOVE"); + if (expunge_env != NULL || altmove_env != NULL) { dict_uri = getenv("EXPIRE_DICT"); if (dict_uri == NULL) i_fatal("expire plugin: expire_dict setting missing"); - expire.env = expire_env_init(env); + expire.env = expire_env_init(expunge_env, altmove_env); expire.db = dict_init(dict_uri, DICT_DATA_TYPE_UINT32, NULL); expire.username = getenv("USER"); diff -r ae8180a4febd -r 969656b58e7f src/plugins/expire/expire-tool.c --- a/src/plugins/expire/expire-tool.c Tue Mar 04 06:24:44 2008 +0200 +++ b/src/plugins/expire/expire-tool.c Tue Mar 04 06:26:24 2008 +0200 @@ -6,6 +6,7 @@ #include "randgen.h" #include "lib-signals.h" #include "dict-client.h" +#include "mail-index.h" #include "mail-search.h" #include "mail-storage.h" #include "mail-namespace.h" @@ -26,6 +27,7 @@ char *user; pool_t namespace_pool; struct mail_namespace *ns; + bool testrun; }; static int user_init(struct expire_context *ctx, const char *user) @@ -54,7 +56,8 @@ static int mailbox_delete_old_mails(struct expire_context *ctx, const char *user, - const char *mailbox, time_t expire_secs, + const char *mailbox, + unsigned int expunge_secs, unsigned int altmove_secs, time_t *oldest_r) { struct mail_namespace *ns; @@ -63,7 +66,10 @@ struct mailbox_transaction_context *t; struct mail_search_arg search_arg; struct mail *mail; + const char *ns_mailbox; time_t now, save_time; + enum mail_error error; + enum mail_flags flags; int ret; *oldest_r = 0; @@ -71,8 +77,11 @@ if (ctx->user != NULL && strcmp(user, ctx->user) != 0) user_deinit(ctx); if (ctx->user == NULL) { - if ((ret = user_init(ctx, user)) <= 0) + if ((ret = user_init(ctx, user)) <= 0) { + if (ctx->testrun) + i_info("User lookup failed: %s", user); return ret; + } ctx->user = i_strdup(user); } @@ -80,11 +89,25 @@ search_arg.type = SEARCH_ALL; search_arg.next = NULL; - ns = mail_namespace_find(ctx->ns, &mailbox); - if (ns == NULL) - return -1; + ns_mailbox = mailbox; + ns = mail_namespace_find(ctx->ns, &ns_mailbox); + if (ns == NULL) { + /* entire namespace no longer exists, remove the entry */ + if (ctx->testrun) + i_info("Namespace lookup failed: %s", mailbox); + return 0; + } - box = mailbox_open(ns->storage, mailbox, NULL, 0); + box = mailbox_open(ns->storage, ns_mailbox, NULL, 0); + if (box == NULL) { + (void)mail_storage_get_last_error(ns->storage, &error); + if (error != MAIL_ERROR_NOTFOUND) + return -1; + + /* mailbox no longer exists, remove the entry */ + return 0; + } + t = mailbox_transaction_begin(box, 0); search_ctx = mailbox_search_init(t, NULL, &search_arg, NULL); mail = mail_alloc(t, 0, NULL); @@ -93,14 +116,36 @@ while ((ret = mailbox_search_next(search_ctx, mail)) > 0) { if (mail_get_save_date(mail, &save_time) < 0) { /* maybe just got expunged. anyway try again later. */ + if (ctx->testrun) { + i_info("%s: seq=%u uid=%u: " + "Save date lookup failed", + mailbox, mail->seq, mail->uid); + } ret = -1; break; } - if (save_time + expire_secs <= now) - mail_expunge(mail); - else { - /* first non-expunged one. */ + if (save_time + expunge_secs <= now && expunge_secs != 0) { + if (!ctx->testrun) + mail_expunge(mail); + else { + i_info("%s: seq=%u uid=%u: Expunge", + mailbox, mail->seq, mail->uid); + } + } else if (save_time + altmove_secs <= now && altmove_secs != 0) { + /* works only with dbox */ + flags = mail_get_flags(mail); + if ((flags & MAIL_INDEX_MAIL_FLAG_BACKEND) != 0) { + /* alread moved */ + } else if (!ctx->testrun) { + mail_update_flags(mail, MODIFY_ADD, + MAIL_INDEX_MAIL_FLAG_BACKEND); + } else { + i_info("%s: seq=%u uid=%u: Move to alt dir", + mailbox, mail->seq, mail->uid); + } + } else { + /* first non-expired one. */ *oldest_r = save_time; break; } @@ -109,21 +154,29 @@ if (mailbox_search_deinit(&search_ctx) < 0) ret = -1; - if (mailbox_transaction_commit(&t) < 0) + if (!ctx->testrun) { + if (mailbox_transaction_commit(&t) < 0) + ret = -1; + } else { + mailbox_transaction_rollback(&t); + } + + if (mailbox_sync(box, MAILBOX_SYNC_FLAG_FAST, 0, NULL) < 0) ret = -1; + mailbox_close(&box); return ret < 0 ? -1 : 0; } -static void expire_run(void) +static void expire_run(bool testrun) { struct expire_context ctx; struct dict *dict = NULL; struct dict_transaction_context *trans; struct dict_iterate_context *iter; struct expire_env *env; - const struct expire_box *expire_box; time_t oldest; + unsigned int expunge_secs, altmove_secs; const char *auth_socket, *p, *key, *value; const char *userp, *mailbox; int ret; @@ -133,8 +186,8 @@ mail_storage_register_all(); mailbox_list_register_all(); - if (getenv("EXPIRE") == NULL) - i_fatal("expire setting not set"); + if (getenv("EXPIRE") == NULL && getenv("EXPIRE_ALTMOVE") == NULL) + i_fatal("expire and expire_altmove settings not set"); if (getenv("EXPIRE_DICT") == NULL) i_fatal("expire_dict setting not set"); @@ -143,9 +196,10 @@ auth_socket = DEFAULT_AUTH_SOCKET_PATH; memset(&ctx, 0, sizeof(ctx)); + ctx.testrun = testrun; ctx.auth_conn = auth_connection_init(auth_socket); ctx.namespace_pool = pool_alloconly_create("namespaces", 1024); - env = expire_env_init(getenv("EXPIRE")); + env = expire_env_init(getenv("EXPIRE"), getenv("EXPIRE_ALTMOVE")); dict = dict_init(getenv("EXPIRE_DICT"), DICT_DATA_TYPE_UINT32, ""); trans = dict_transaction_begin(dict); iter = dict_iterate_init(dict, DICT_PATH_SHARED, @@ -164,43 +218,64 @@ } mailbox = p + 1; - expire_box = expire_box_find(env, mailbox); - if (expire_box == NULL) { + if (!expire_box_find(env, mailbox, + &expunge_secs, &altmove_secs)) { /* we're no longer expunging old messages from here */ - dict_unset(trans, key); - } else if (time(NULL) < (time_t)strtoul(value, NULL, 10)) { + if (!testrun) + dict_unset(trans, key); + else + i_info("%s: removed from config", mailbox); + continue; + } + if (time(NULL) < (time_t)strtoul(value, NULL, 10)) { /* this and the rest of the timestamps are in future, so stop processing */ + if (testrun) { + i_info("%s: stop, expire time in future: %s", + mailbox, value); + } break; - } else { - T_BEGIN { - const char *username; + } + + T_BEGIN { + const char *username; + + username = t_strdup_until(userp, p); + ret = mailbox_delete_old_mails(&ctx, username, + mailbox, expunge_secs, + altmove_secs, &oldest); + } T_END; - username = t_strdup_until(userp, p); - ret = mailbox_delete_old_mails(&ctx, username, - mailbox, - expire_box->expire_secs, - &oldest); - } T_END; - if (ret < 0) { - /* failed to update */ - } else if (oldest == 0) { - /* no more messages or we're no longer - expunging messages from here */ + if (ret < 0) { + /* failed to update */ + } else if (oldest == 0) { + /* no more messages or mailbox deleted */ + if (!testrun) dict_unset(trans, key); - } else { - char new_value[MAX_INT_STRLEN]; + else + i_info("%s: no messages left", mailbox); + } else { + char new_value[MAX_INT_STRLEN]; - oldest += expire_box->expire_secs; - i_snprintf(new_value, sizeof(new_value), "%lu", - (unsigned long)oldest); - if (strcmp(value, new_value) != 0) - dict_set(trans, key, new_value); + oldest += altmove_secs != 0 ? + altmove_secs : expunge_secs; + i_snprintf(new_value, sizeof(new_value), "%lu", + (unsigned long)oldest); + if (strcmp(value, new_value) == 0) { + /* no change */ + } else if (!testrun) + dict_set(trans, key, new_value); + else { + i_info("%s: timestamp %s -> %s", + mailbox, value, new_value); } } } dict_iterate_deinit(iter); - dict_transaction_commit(trans); + if (!testrun) + dict_transaction_commit(trans); + else + dict_transaction_rollback(trans); dict_deinit(&dict); if (ctx.user != NULL) @@ -211,16 +286,19 @@ dict_driver_unregister(&dict_driver_client); } -int main(void) +int main(int argc, const char *argv[]) { struct ioloop *ioloop; + bool test; lib_init(); lib_signals_init(); random_init(); + test = argc > 1 && strcmp(argv[1], "--test") == 0; + ioloop = io_loop_create(); - expire_run(); + expire_run(test); io_loop_destroy(&ioloop); lib_signals_deinit();