Mercurial > dovecot > core-2.2
changeset 1915:79790750c349 HEAD
importing new index code. mbox still broken.
line wrap: on
line diff
--- a/configure.in Tue Apr 27 23:14:15 2004 +0300 +++ b/configure.in Tue Apr 27 23:25:52 2004 +0300 @@ -1,7 +1,7 @@ AC_INIT(src) AM_CONFIG_HEADER(config.h) -AM_INIT_AUTOMAKE(dovecot, 0.99.11-test4) +AM_INIT_AUTOMAKE(dovecot, 1.0-test1) AM_MAINTAINER_MODE @@ -195,7 +195,7 @@ AC_ARG_WITH(storages, [ --with-storages Build specified mail storage formats (maildir,mbox)], [ mail_storages=`echo "$withval"|sed 's/,/ /g'` ], - mail_storages="maildir mbox") + mail_storages="maildir") dnl * gcc specific options if test "x$ac_cv_prog_gcc" = "xyes"; then @@ -1116,8 +1116,8 @@ dnl ** storage classes dnl ** -maildir_libs="../lib-storage/index/maildir/libstorage_maildir.a ../lib-index/maildir/libindex_maildir.a" -mbox_libs="../lib-storage/index/mbox/libstorage_mbox.a ../lib-index/mbox/libindex_mbox.a" +maildir_libs="../lib-storage/index/maildir/libstorage_maildir.a" +mbox_libs="../lib-storage/index/mbox/libstorage_mbox.a" index_libs="../lib-storage/index/libstorage_index.a ../lib-index/libindex.a" STORAGE_LIBS= @@ -1146,8 +1146,6 @@ src/lib-charset/Makefile src/lib-imap/Makefile src/lib-index/Makefile -src/lib-index/maildir/Makefile -src/lib-index/mbox/Makefile src/lib-mail/Makefile src/lib-settings/Makefile src/lib-storage/Makefile
--- a/src/imap/Makefile.am Tue Apr 27 23:14:15 2004 +0300 +++ b/src/imap/Makefile.am Tue Apr 27 23:25:52 2004 +0300 @@ -65,6 +65,7 @@ imap-expunge.c \ imap-fetch.c \ imap-fetch-body-section.c \ + imap-messageset.c \ imap-search.c \ imap-sort.c \ imap-thread.c \ @@ -80,6 +81,7 @@ common.h \ imap-expunge.h \ imap-fetch.h \ + imap-messageset.h \ imap-search.h \ imap-sort.h \ imap-thread.h \
--- a/src/imap/client.c Tue Apr 27 23:14:15 2004 +0300 +++ b/src/imap/client.c Tue Apr 27 23:25:52 2004 +0300 @@ -69,9 +69,8 @@ client->namespaces = namespaces; while (namespaces != NULL) { - namespaces->storage->set_callbacks(namespaces->storage, - &mail_storage_callbacks, - client); + mail_storage_set_callbacks(namespaces->storage, + &mail_storage_callbacks, client); namespaces = namespaces->next; } @@ -88,7 +87,7 @@ o_stream_flush(client->output); if (client->mailbox != NULL) - client->mailbox->close(client->mailbox); + mailbox_close(client->mailbox); namespace_deinit(client->namespaces); imap_parser_destroy(client->parser);
--- a/src/imap/cmd-append.c Tue Apr 27 23:14:15 2004 +0300 +++ b/src/imap/cmd-append.c Tue Apr 27 23:25:52 2004 +0300 @@ -48,7 +48,7 @@ struct mail_storage *storage; struct mailbox *box; struct mailbox_status status; - struct mail_save_context *ctx; + struct mailbox_transaction_context *t; struct imap_parser *save_parser; struct imap_arg *args; struct imap_arg_list *flags_list; @@ -72,16 +72,16 @@ if (storage == NULL) return TRUE; - box = storage->open_mailbox(storage, mailbox, - mailbox_open_flags | MAILBOX_OPEN_FAST); + box = mailbox_open(storage, mailbox, + mailbox_open_flags | MAILBOX_OPEN_FAST); if (box == NULL) { client_send_storage_error(client, storage); return TRUE; } - if (!box->get_status(box, STATUS_CUSTOM_FLAGS, &status)) { + if (mailbox_get_status(box, STATUS_CUSTOM_FLAGS, &status) < 0) { client_send_storage_error(client, storage); - box->close(box); + mailbox_close(box); return TRUE; } memset(&old_flags, 0, sizeof(old_flags)); @@ -89,11 +89,7 @@ client_save_custom_flags(&old_flags, status.custom_flags, status.custom_flags_count); - ctx = box->save_init(box, TRUE); - if (ctx == NULL) { - client_send_storage_error(client, storage); - return TRUE; - } + t = mailbox_transaction_begin(box, FALSE); /* if error occurs, the CRLF is already read. */ client->input_skip_line = FALSE; @@ -159,8 +155,8 @@ if (internal_date_str == NULL) { /* no time given, default to now. */ - internal_date = ioloop_time; - timezone_offset = ioloop_timezone.tz_minuteswest; + internal_date = (time_t)-1; + timezone_offset = 0; } else if (!imap_parse_datetime(internal_date_str, &internal_date, &timezone_offset)) { @@ -184,8 +180,8 @@ input = i_stream_create_limit(default_pool, client->input, client->input->v_offset, msg_size); - if (!box->save_next(ctx, &flags, internal_date, - timezone_offset, input)) { + if (mailbox_save(t, &flags, internal_date, timezone_offset, + NULL, input) < 0) { i_stream_unref(input); client_send_storage_error(client, storage); break; @@ -199,15 +195,19 @@ } imap_parser_destroy(save_parser); - if (!box->save_deinit(ctx, failed)) { - failed = TRUE; - client_send_storage_error(client, storage); + if (failed) + mailbox_transaction_rollback(t); + else { + if (mailbox_transaction_commit(t) < 0) { + failed = TRUE; + client_send_storage_error(client, storage); + } } - box->close(box); + mailbox_close(box); if (!failed) { - client_sync_full_fast(client); + client_sync_full(client); client_send_tagline(client, "OK Append completed."); } return TRUE;
--- a/src/imap/cmd-close.c Tue Apr 27 23:14:15 2004 +0300 +++ b/src/imap/cmd-close.c Tue Apr 27 23:25:52 2004 +0300 @@ -7,21 +7,20 @@ int cmd_close(struct client *client) { struct mailbox *mailbox = client->mailbox; + struct mail_storage *storage = mailbox_get_storage(mailbox); if (!client_verify_open_mailbox(client)) return TRUE; client->mailbox = NULL; - if (!mailbox->is_readonly(mailbox)) { - if (!imap_expunge(mailbox, FALSE)) { - client_send_untagged_storage_error(client, - mailbox->storage); - } + if (!mailbox_is_readonly(mailbox)) { + if (!imap_expunge(mailbox)) + client_send_untagged_storage_error(client, storage); } - if (!mailbox->close(mailbox)) - client_send_untagged_storage_error(client, mailbox->storage); + if (mailbox_close(mailbox) < 0) + client_send_untagged_storage_error(client, storage); client_send_tagline(client, "OK Close completed."); return TRUE;
--- a/src/imap/cmd-copy.c Tue Apr 27 23:14:15 2004 +0300 +++ b/src/imap/cmd-copy.c Tue Apr 27 23:25:52 2004 +0300 @@ -4,42 +4,53 @@ #include "commands.h" #include "imap-search.h" -static int fetch_and_copy(struct mail_copy_context *copy_ctx, - struct mailbox *srcbox, struct mailbox *destbox, - const char *messageset, int uidset, int *all_found) +static int fetch_and_copy(struct mailbox_transaction_context *t, + struct mailbox *srcbox, + struct mail_search_arg *search_args) { - struct mail_search_arg *search_arg; struct mail_search_context *search_ctx; + struct mailbox_transaction_context *src_trans; struct mail *mail; - int failed = FALSE; + int ret; - search_arg = imap_search_get_msgset_arg(messageset, uidset); - search_ctx = srcbox->search_init(srcbox, NULL, search_arg, NULL, + src_trans = mailbox_transaction_begin(srcbox, FALSE); + search_ctx = mailbox_search_init(src_trans, NULL, search_args, NULL, MAIL_FETCH_STREAM_HEADER | MAIL_FETCH_STREAM_BODY, NULL); - if (search_ctx == NULL) - return FALSE; + if (search_ctx == NULL) { + mailbox_transaction_rollback(src_trans); + return -1; + } - while ((mail = srcbox->search_next(search_ctx)) != NULL) { - if (!destbox->copy(mail, copy_ctx)) { - failed = TRUE; + ret = 1; + while ((mail = mailbox_search_next(search_ctx)) != NULL) { + if (mail->expunged) { + ret = 0; + break; + } + if (mailbox_copy(t, mail) < 0) { + ret = -1; break; } } - if (!srcbox->search_deinit(search_ctx, all_found)) - return FALSE; + if (mailbox_search_deinit(search_ctx) < 0) + ret = -1; - return !failed; + if (mailbox_transaction_commit(src_trans) < 0) + ret = -1; + + return ret; } int cmd_copy(struct client *client) { struct mail_storage *storage; struct mailbox *destbox; - struct mail_copy_context *copy_ctx; + struct mailbox_transaction_context *t; + struct mail_search_arg *search_arg; const char *messageset, *mailbox; - int failed = FALSE, all_found = TRUE; + int ret; /* <message set> <mailbox> */ if (!client_read_string_args(client, 2, &messageset, &mailbox)) @@ -55,42 +66,34 @@ if (!client_verify_mailbox_name(client, mailbox, TRUE, FALSE)) return TRUE; + search_arg = imap_search_get_arg(client, messageset, client->cmd_uid); + if (search_arg == NULL) + return TRUE; + storage = client_find_storage(client, mailbox); if (storage == NULL) return TRUE; - destbox = storage->open_mailbox(storage, mailbox, - mailbox_open_flags | MAILBOX_OPEN_FAST); + destbox = mailbox_open(storage, mailbox, + mailbox_open_flags | MAILBOX_OPEN_FAST); if (destbox == NULL) { client_send_storage_error(client, storage); return TRUE; } - if (destbox == client->mailbox) { - /* copying inside same mailbox, make sure we get the - locking right */ - if (!destbox->lock(destbox, MAILBOX_LOCK_READ | - MAILBOX_LOCK_SAVE)) - failed = TRUE; + t = mailbox_transaction_begin(destbox, FALSE); + ret = fetch_and_copy(t, client->mailbox, search_arg); + + if (ret <= 0) + mailbox_transaction_rollback(t); + else { + if (mailbox_transaction_commit(t) < 0) + ret = -1; } - copy_ctx = failed ? NULL : destbox->copy_init(destbox); - if (copy_ctx == NULL) - failed = TRUE; - else { - if (!fetch_and_copy(copy_ctx, client->mailbox, destbox, - messageset, client->cmd_uid, &all_found)) - failed = TRUE; - - if (!destbox->copy_deinit(copy_ctx, failed || !all_found)) - failed = TRUE; - } - - (void)destbox->lock(destbox, MAILBOX_LOCK_UNLOCK); - - if (failed) + if (ret < 0) client_send_storage_error(client, storage); - else if (!all_found) { + else if (ret == 0) { /* some messages were expunged, sync them */ client_sync_full(client); client_send_tagline(client, @@ -103,6 +106,6 @@ client_send_tagline(client, "OK Copy completed."); } - destbox->close(destbox); + mailbox_close(destbox); return TRUE; }
--- a/src/imap/cmd-create.c Tue Apr 27 23:14:15 2004 +0300 +++ b/src/imap/cmd-create.c Tue Apr 27 23:25:52 2004 +0300 @@ -19,7 +19,7 @@ return TRUE; len = strlen(mailbox); - if (mailbox[len-1] != storage->hierarchy_sep) + if (mailbox[len-1] != mail_storage_get_hierarchy_sep(storage)) directory = FALSE; else { /* name ends with hierarchy separator - client is just @@ -32,11 +32,9 @@ if (!client_verify_mailbox_name(client, mailbox, FALSE, TRUE)) return TRUE; - if (!storage->create_mailbox(storage, mailbox, directory)) { + if (mail_storage_mailbox_create(storage, mailbox, directory) < 0) client_send_storage_error(client, storage); - return TRUE; - } - - client_send_tagline(client, "OK Create completed."); + else + client_send_tagline(client, "OK Create completed."); return TRUE; }
--- a/src/imap/cmd-delete.c Tue Apr 27 23:14:15 2004 +0300 +++ b/src/imap/cmd-delete.c Tue Apr 27 23:25:52 2004 +0300 @@ -20,12 +20,12 @@ } mailbox = client->mailbox; - if (mailbox != NULL && strcmp(mailbox->name, name) == 0) { + if (mailbox != NULL && strcmp(mailbox_get_name(mailbox), name) == 0) { /* deleting selected mailbox. close it first */ - storage = mailbox->storage; + storage = mailbox_get_storage(mailbox); client->mailbox = NULL; - if (!mailbox->close(mailbox)) + if (mailbox_close(mailbox) < 0) client_send_untagged_storage_error(client, storage); } else { storage = client_find_storage(client, name); @@ -33,9 +33,9 @@ return TRUE; } - if (storage->delete_mailbox(storage, name)) + if (mail_storage_mailbox_delete(storage, name) < 0) + client_send_storage_error(client, storage); + else client_send_tagline(client, "OK Delete completed."); - else - client_send_storage_error(client, storage); return TRUE; }
--- a/src/imap/cmd-expunge.c Tue Apr 27 23:14:15 2004 +0300 +++ b/src/imap/cmd-expunge.c Tue Apr 27 23:25:52 2004 +0300 @@ -9,10 +9,13 @@ if (!client_verify_open_mailbox(client)) return TRUE; - if (imap_expunge(client->mailbox, TRUE)) + if (imap_expunge(client->mailbox)) { + client_sync_full(client); client_send_tagline(client, "OK Expunge completed."); - else - client_send_storage_error(client, client->mailbox->storage); + } else { + client_send_storage_error(client, + mailbox_get_storage(client->mailbox)); + } return TRUE; }
--- a/src/imap/cmd-fetch.c Tue Apr 27 23:14:15 2004 +0300 +++ b/src/imap/cmd-fetch.c Tue Apr 27 23:25:52 2004 +0300 @@ -3,6 +3,8 @@ #include "common.h" #include "commands.h" #include "imap-fetch.h" +#include "imap-search.h" +#include "mail-search.h" /* Parse next digits in string into integer. Returns FALSE if the integer becomes too big and wraps. */ @@ -315,6 +317,7 @@ enum mail_fetch_field fetch_data; enum imap_fetch_field imap_data; struct imap_fetch_body_data *bodies, **bodies_p; + struct mail_search_arg *search_arg; const char *messageset; int ret; @@ -351,9 +354,12 @@ if (client->cmd_uid) imap_data |= IMAP_FETCH_UID; - ret = imap_fetch(client, fetch_data, imap_data, - bodies, messageset, client->cmd_uid); - if (ret >= 0) { + search_arg = imap_search_get_arg(client, messageset, client->cmd_uid); + if (search_arg == NULL) + return TRUE; + + ret = imap_fetch(client, fetch_data, imap_data, bodies, search_arg); + if (ret == 0) { if ((client_workarounds & WORKAROUND_OE6_FETCH_NO_NEWMAIL) == 0) { if (client->cmd_uid) @@ -362,10 +368,26 @@ client_sync_without_expunges(client); } - client_send_tagline(client, ret > 0 ? "OK Fetch completed." : - "NO Some of the requested messages no longer exist."); + client_send_tagline(client, "OK Fetch completed."); } else { - client_send_storage_error(client, client->mailbox->storage); + struct mail_storage *storage; + const char *error; + int syntax; + + storage = mailbox_get_storage(client->mailbox); + error = mail_storage_get_last_error(storage, &syntax); + if (!syntax) { + /* We never want to reply NO to FETCH requests, + BYE is preferrable (see imap-ml for reasons). */ + if (error == NULL) { + error = "Out of sync: " + "Trying to fetch expunged message"; + } + client_disconnect_with_error(client, error); + } else { + /* user error, we'll reply with BAD */ + client_send_storage_error(client, storage); + } } return TRUE;
--- a/src/imap/cmd-idle.c Tue Apr 27 23:14:15 2004 +0300 +++ b/src/imap/cmd-idle.c Tue Apr 27 23:25:52 2004 +0300 @@ -29,11 +29,10 @@ IO_READ, _client_input, client); if (client->mailbox != NULL) { - client->mailbox->auto_sync(client->mailbox, - mailbox_check_interval != 0 ? - MAILBOX_SYNC_FLAG_NO_EXPUNGES : - MAILBOX_SYNC_NONE, - mailbox_check_interval); + mailbox_auto_sync(client->mailbox, mailbox_check_interval != 0 ? + MAILBOX_SYNC_FLAG_NO_EXPUNGES : + MAILBOX_SYNC_AUTO_STOP, + mailbox_check_interval); } client_sync_full(client); @@ -84,18 +83,16 @@ timeout_remove(client->idle_to); client->idle_to = NULL; - if (!client->mailbox->get_status(client->mailbox, STATUS_MESSAGES, - &status)) { + if (mailbox_get_status(client->mailbox, STATUS_MESSAGES, &status) < 0) { client_send_untagged_storage_error(client, - client->mailbox->storage); + mailbox_get_storage(client->mailbox)); idle_finish(client, TRUE); } else { client->idle_expunge = status.messages+1; client_send_line(client, t_strdup_printf("* %u EXISTS", client->idle_expunge)); - client->mailbox->auto_sync(client->mailbox, - MAILBOX_SYNC_NONE, 0); + mailbox_auto_sync(client->mailbox, MAILBOX_SYNC_AUTO_STOP, 0); } } @@ -116,10 +113,8 @@ if (interval == 0) interval = DEFAULT_IDLE_CHECK_INTERVAL; - if (client->mailbox != NULL) { - client->mailbox->auto_sync(client->mailbox, - MAILBOX_SYNC_FULL, interval); - } + if (client->mailbox != NULL) + mailbox_auto_sync(client->mailbox, 0, interval); client_send_line(client, "+ idling");
--- a/src/imap/cmd-list.c Tue Apr 27 23:14:15 2004 +0300 +++ b/src/imap/cmd-list.c Tue Apr 27 23:25:52 2004 +0300 @@ -56,12 +56,12 @@ struct mailbox_list *list; string_t *str; - ctx = storage->list_mailbox_init(storage, mask, list_flags); + ctx = mail_storage_mailbox_list_init(storage, mask, list_flags); if (ctx == NULL) return FALSE; str = t_str_new(256); - while ((list = storage->list_mailbox_next(ctx)) != NULL) { + while ((list = mail_storage_mailbox_list_next(ctx)) != NULL) { str_truncate(str, 0); str_printfa(str, "* %s (%s) \"%s\" ", reply, mailbox_flags2str(list->flags, list_flags), @@ -73,7 +73,7 @@ client_send_line(client, str_c(str)); } - return storage->list_mailbox_deinit(ctx); + return mail_storage_mailbox_list_deinit(ctx); } static int parse_list_flags(struct client *client, struct imap_arg *args, @@ -152,7 +152,7 @@ else storage = client->namespaces->storage; - sep_chr = storage->hierarchy_sep; + sep_chr = mail_storage_get_hierarchy_sep(storage); if (sep_chr == '"' || sep_chr == '\\') { sep[0] = '\\'; sep[1] = sep_chr; @@ -184,8 +184,8 @@ } } - failed = !mailbox_list(client, storage, mask, sep, - lsub ? "LSUB" : "LIST", list_flags); + failed = mailbox_list(client, storage, mask, sep, + lsub ? "LSUB" : "LIST", list_flags) < 0; } if (failed)
--- a/src/imap/cmd-logout.c Tue Apr 27 23:14:15 2004 +0300 +++ b/src/imap/cmd-logout.c Tue Apr 27 23:25:52 2004 +0300 @@ -11,7 +11,7 @@ /* this could be done at client_disconnect() as well, but eg. mbox rewrite takes a while so the waiting is better to happen before "OK" message. */ - client->mailbox->close(client->mailbox); + mailbox_close(client->mailbox); client->mailbox = NULL; }
--- a/src/imap/cmd-rename.c Tue Apr 27 23:14:15 2004 +0300 +++ b/src/imap/cmd-rename.c Tue Apr 27 23:25:52 2004 +0300 @@ -29,10 +29,10 @@ return TRUE; } - if (old_storage->rename_mailbox(old_storage, oldname, newname)) + if (mail_storage_mailbox_rename(old_storage, oldname, newname) < 0) + client_send_storage_error(client, old_storage); + else client_send_tagline(client, "OK Rename completed."); - else - client_send_storage_error(client, old_storage); return TRUE; }
--- a/src/imap/cmd-search.c Tue Apr 27 23:14:15 2004 +0300 +++ b/src/imap/cmd-search.c Tue Apr 27 23:25:52 2004 +0300 @@ -12,6 +12,7 @@ struct mail_search_arg *sargs) { struct mail_search_context *ctx; + struct mailbox_transaction_context *trans; const struct mail *mail; string_t *str; int ret, uid, first = TRUE; @@ -19,13 +20,16 @@ str = t_str_new(STRBUF_SIZE); uid = client->cmd_uid; - ctx = client->mailbox->search_init(client->mailbox, charset, sargs, - NULL, 0, NULL); - if (ctx == NULL) + trans = mailbox_transaction_begin(client->mailbox, FALSE); + ctx = mailbox_search_init(trans, charset, sargs, + NULL, 0, NULL); + if (ctx == NULL) { + mailbox_transaction_rollback(trans); return FALSE; + } str_append(str, "* SEARCH"); - while ((mail = client->mailbox->search_next(ctx)) != NULL) { + while ((mail = mailbox_search_next(ctx)) != NULL) { if (str_len(str) >= STRBUF_SIZE-MAX_INT_STRLEN) { /* flush */ o_stream_send(client->output, @@ -37,13 +41,16 @@ str_printfa(str, " %u", uid ? mail->uid : mail->seq); } - ret = client->mailbox->search_deinit(ctx, NULL); + ret = mailbox_search_deinit(ctx); - if (!first || ret) { + if (mailbox_transaction_commit(trans) < 0) + ret = -1; + + if (!first || ret == 0) { str_append(str, "\r\n"); o_stream_send(client->output, str_data(str), str_len(str)); } - return ret; + return ret == 0; } int cmd_search(struct client *client) @@ -86,7 +93,7 @@ pool = pool_alloconly_create("mail_search_args", 2048); - sargs = imap_search_args_build(pool, args, &error); + sargs = imap_search_args_build(pool, client->mailbox, args, &error); if (sargs == NULL) { /* error in search arguments */ client_send_tagline(client, t_strconcat("NO ", error, NULL)); @@ -97,7 +104,8 @@ client_sync_without_expunges(client); client_send_tagline(client, "OK Search completed."); } else { - client_send_storage_error(client, client->mailbox->storage); + client_send_storage_error(client, + mailbox_get_storage(client->mailbox)); } pool_unref(pool);
--- a/src/imap/cmd-select.c Tue Apr 27 23:14:15 2004 +0300 +++ b/src/imap/cmd-select.c Tue Apr 27 23:25:52 2004 +0300 @@ -18,9 +18,9 @@ if (client->mailbox != NULL) { box = client->mailbox; client->mailbox = NULL; - if (!box->close(box)) { + if (mailbox_close(box) < 0) { client_send_untagged_storage_error(client, - box->storage); + mailbox_get_storage(box)); } } @@ -31,17 +31,18 @@ flags = mailbox_open_flags; if (readonly) flags |= MAILBOX_OPEN_READONLY; - box = storage->open_mailbox(storage, mailbox, flags); + box = mailbox_open(storage, mailbox, flags); if (box == NULL) { client_send_storage_error(client, storage); return TRUE; } - if (!box->get_status(box, STATUS_MESSAGES | STATUS_RECENT | - STATUS_FIRST_UNSEEN_SEQ | STATUS_UIDVALIDITY | - STATUS_UIDNEXT | STATUS_CUSTOM_FLAGS, &status)) { + if (mailbox_get_status(box, STATUS_MESSAGES | STATUS_RECENT | + STATUS_FIRST_UNSEEN_SEQ | STATUS_UIDVALIDITY | + STATUS_UIDNEXT | STATUS_CUSTOM_FLAGS, + &status) < 0) { client_send_storage_error(client, storage); - box->close(box); + mailbox_close(box); return TRUE; } @@ -80,13 +81,13 @@ "Disk space is full, delete some messages."); } - client_send_tagline(client, box->is_readonly(box) ? + client_send_tagline(client, mailbox_is_readonly(box) ? "OK [READ-ONLY] Select completed." : "OK [READ-WRITE] Select completed."); if (mailbox_check_interval != 0) { - box->auto_sync(box, MAILBOX_SYNC_FLAG_NO_EXPUNGES, - mailbox_check_interval); + mailbox_auto_sync(box, MAILBOX_SYNC_FLAG_NO_EXPUNGES, + mailbox_check_interval); } return TRUE;
--- a/src/imap/cmd-sort.c Tue Apr 27 23:14:15 2004 +0300 +++ b/src/imap/cmd-sort.c Tue Apr 27 23:25:52 2004 +0300 @@ -116,7 +116,7 @@ pool = pool_alloconly_create("mail_search_args", 2048); - sargs = imap_search_args_build(pool, args, &error); + sargs = imap_search_args_build(pool, client->mailbox, args, &error); if (sargs == NULL) { /* error in search arguments */ client_send_tagline(client, t_strconcat("NO ", error, NULL)); @@ -128,7 +128,8 @@ client_sync_without_expunges(client); client_send_tagline(client, "OK Sort completed."); } else { - client_send_storage_error(client, client->mailbox->storage); + client_send_storage_error(client, + mailbox_get_storage(client->mailbox)); } pool_unref(pool);
--- a/src/imap/cmd-status.c Tue Apr 27 23:14:15 2004 +0300 +++ b/src/imap/cmd-status.c Tue Apr 27 23:25:52 2004 +0300 @@ -60,23 +60,21 @@ int failed; if (client->mailbox != NULL && - mailbox_name_equals(client->mailbox->name, mailbox)) { + mailbox_name_equals(mailbox_get_name(client->mailbox), mailbox)) { /* this mailbox is selected */ box = client->mailbox; } else { /* open the mailbox */ - box = storage->open_mailbox(storage, mailbox, - mailbox_open_flags | - MAILBOX_OPEN_FAST | - MAILBOX_OPEN_READONLY); + box = mailbox_open(storage, mailbox, mailbox_open_flags | + MAILBOX_OPEN_FAST | MAILBOX_OPEN_READONLY); if (box == NULL) return FALSE; } - failed = !box->get_status(box, items, status); + failed = mailbox_get_status(box, items, status) < 0; if (box != client->mailbox) - box->close(box); + mailbox_close(box); return !failed; }
--- a/src/imap/cmd-store.c Tue Apr 27 23:14:15 2004 +0300 +++ b/src/imap/cmd-store.c Tue Apr 27 23:25:52 2004 +0300 @@ -3,6 +3,7 @@ #include "common.h" #include "commands.h" #include "imap-search.h" +#include "imap-util.h" static int get_modify_type(struct client *client, const char *item, enum modify_type *modify_type, int *silent) @@ -62,9 +63,10 @@ struct mailbox *box; struct mail_search_arg *search_arg; struct mail_search_context *search_ctx; + struct mailbox_transaction_context *t; struct mail *mail; const char *messageset, *item; - int silent, all_found, failed; + int silent, modify, failed = FALSE; if (!client_read_args(client, 0, 0, &args)) return FALSE; @@ -95,35 +97,35 @@ return TRUE; } - /* and update the flags */ box = client->mailbox; + search_arg = imap_search_get_arg(client, messageset, client->cmd_uid); + if (search_arg == NULL) + return TRUE; - if (box->is_readonly(box)) { - /* read-only, don't every try to get write locking */ - failed = FALSE; + t = mailbox_transaction_begin(box, silent); + if (!mailbox_is_readonly(box)) + modify = TRUE; + else { /* flag changes will fail, notify client about them */ - silent = FALSE; - } else { - failed = !box->lock(box, MAILBOX_LOCK_FLAGS | - MAILBOX_LOCK_READ); + modify = FALSE; } - search_arg = imap_search_get_msgset_arg(messageset, client->cmd_uid); search_ctx = failed ? NULL : - box->search_init(box, NULL, search_arg, NULL, - MAIL_FETCH_FLAGS, NULL); + mailbox_search_init(t, NULL, search_arg, NULL, + MAIL_FETCH_FLAGS, NULL); if (search_ctx == NULL) failed = TRUE; else { failed = FALSE; - while ((mail = box->search_next(search_ctx)) != NULL) { - if (!mail->update_flags(mail, &flags, modify_type)) { - failed = TRUE; - break; - } - - if (!silent) { + while ((mail = mailbox_search_next(search_ctx)) != NULL) { + if (modify) { + if (mail->update_flags(mail, &flags, + modify_type) < 0) { + failed = TRUE; + break; + } + } else { if (!mail_send_flags(client, mail)) { failed = TRUE; break; @@ -132,20 +134,24 @@ } } - if (!box->search_deinit(search_ctx, &all_found)) + if (mailbox_search_deinit(search_ctx) < 0) failed = TRUE; - (void)box->lock(box, MAILBOX_LOCK_UNLOCK); + if (failed) + mailbox_transaction_rollback(t); + else { + if (mailbox_transaction_commit(t) < 0) + failed = TRUE; + } if (!failed) { if (client->cmd_uid) client_sync_full_fast(client); else client_sync_without_expunges(client); - client_send_tagline(client, all_found ? "OK Store completed." : - "NO Some of the messages no longer exist."); + client_send_tagline(client, "OK Store completed."); } else { - client_send_storage_error(client, client->mailbox->storage); + client_send_storage_error(client, mailbox_get_storage(box)); } return TRUE;
--- a/src/imap/cmd-subscribe.c Tue Apr 27 23:14:15 2004 +0300 +++ b/src/imap/cmd-subscribe.c Tue Apr 27 23:25:52 2004 +0300 @@ -19,14 +19,13 @@ if (storage == NULL) return FALSE; - if (storage->set_subscribed(storage, mailbox, subscribe)) { + if (mail_storage_set_subscribed(storage, mailbox, subscribe) == 0) + client_send_storage_error(client, storage); + else { client_send_tagline(client, subscribe ? "OK Subscribe completed." : "OK Unsubscribe completed."); - } else { - client_send_storage_error(client, storage); } - return TRUE; }
--- a/src/imap/cmd-thread.c Tue Apr 27 23:14:15 2004 +0300 +++ b/src/imap/cmd-thread.c Tue Apr 27 23:25:52 2004 +0300 @@ -58,7 +58,7 @@ pool = pool_alloconly_create("mail_search_args", 2048); - sargs = imap_search_args_build(pool, args, &error); + sargs = imap_search_args_build(pool, client->mailbox, args, &error); if (sargs == NULL) { /* error in search arguments */ client_send_tagline(client, t_strconcat("NO ", error, NULL)); @@ -70,7 +70,8 @@ client_sync_without_expunges(client); client_send_tagline(client, "OK Search completed."); } else { - client_send_storage_error(client, client->mailbox->storage); + client_send_storage_error(client, + mailbox_get_storage(client->mailbox)); } pool_unref(pool);
--- a/src/imap/cmd-unselect.c Tue Apr 27 23:14:15 2004 +0300 +++ b/src/imap/cmd-unselect.c Tue Apr 27 23:25:52 2004 +0300 @@ -12,8 +12,10 @@ client->mailbox = NULL; - if (!mailbox->close(mailbox)) - client_send_untagged_storage_error(client, mailbox->storage); + if (mailbox_close(mailbox) < 0) { + client_send_untagged_storage_error(client, + mailbox_get_storage(mailbox)); + } client_send_tagline(client, "OK Unselect completed."); return TRUE;
--- a/src/imap/commands-util.c Tue Apr 27 23:14:15 2004 +0300 +++ b/src/imap/commands-util.c Tue Apr 27 23:25:52 2004 +0300 @@ -40,7 +40,7 @@ return FALSE; /* make sure it even looks valid */ - sep = storage->hierarchy_sep; + sep = mail_storage_get_hierarchy_sep(storage); if (*mailbox == '\0' || strspn(mailbox, "\r\n*%?") != 0) { client_send_tagline(client, "NO Invalid mailbox name."); return FALSE; @@ -60,8 +60,8 @@ } /* check what our storage thinks of it */ - if (!storage->get_mailbox_name_status(storage, mailbox, - &mailbox_status)) { + if (mail_storage_get_mailbox_name_status(storage, mailbox, + &mailbox_status) < 0) { client_send_storage_error(client, storage); return FALSE; } @@ -115,9 +115,9 @@ if (client->mailbox == NULL) return; - if (!client->mailbox->sync(client->mailbox, 0)) { + if (mailbox_sync(client->mailbox, 0) < 0) { client_send_untagged_storage_error(client, - client->mailbox->storage); + mailbox_get_storage(client->mailbox)); } } @@ -126,9 +126,9 @@ if (client->mailbox == NULL) return; - if (!client->mailbox->sync(client->mailbox, MAILBOX_SYNC_FAST)) { + if (mailbox_sync(client->mailbox, MAILBOX_SYNC_FLAG_FAST) < 0) { client_send_untagged_storage_error(client, - client->mailbox->storage); + mailbox_get_storage(client->mailbox)); } } @@ -137,10 +137,10 @@ if (client->mailbox == NULL) return; - if (!client->mailbox->sync(client->mailbox, MAILBOX_SYNC_FAST | - MAILBOX_SYNC_FLAG_NO_EXPUNGES)) { + if (mailbox_sync(client->mailbox, MAILBOX_SYNC_FLAG_FAST | + MAILBOX_SYNC_FLAG_NO_EXPUNGES) < 0) { client_send_untagged_storage_error(client, - client->mailbox->storage); + mailbox_get_storage(client->mailbox)); } } @@ -151,14 +151,14 @@ int syntax; if (client->mailbox != NULL && - client->mailbox->is_inconsistency_error(client->mailbox)) { + mailbox_is_inconsistent(client->mailbox)) { /* we can't do forced CLOSE, so have to disconnect */ client_disconnect_with_error(client, "Mailbox is in inconsistent state, please relogin."); return; } - error = storage->get_last_error(storage, &syntax); + error = mail_storage_get_last_error(storage, &syntax); client_send_tagline(client, t_strconcat(syntax ? "BAD " : "NO ", error, NULL)); } @@ -170,14 +170,14 @@ int syntax; if (client->mailbox != NULL && - client->mailbox->is_inconsistency_error(client->mailbox)) { + mailbox_is_inconsistent(client->mailbox)) { /* we can't do forced CLOSE, so have to disconnect */ client_disconnect_with_error(client, "Mailbox is in inconsistent state, please relogin."); return; } - error = storage->get_last_error(storage, &syntax); + error = mail_storage_get_last_error(storage, &syntax); client_send_line(client, t_strconcat(syntax ? "* BAD " : "* NO ", error, NULL)); } @@ -262,7 +262,7 @@ return FALSE; } - if (i == flags->custom_flags_count) { + if (i == flag_pos) { if (!is_valid_custom_flag(client, old_flags, atom)) return FALSE; @@ -316,13 +316,13 @@ client_send_line(client, t_strconcat("* FLAGS ("SYSTEM_FLAGS, str, ")", NULL)); - if (box->is_readonly(box)) { + if (mailbox_is_readonly(box)) { client_send_line(client, "* OK [PERMANENTFLAGS ()] " "Read-only mailbox."); } else { client_send_line(client, t_strconcat("* OK [PERMANENTFLAGS ("SYSTEM_FLAGS, str, - box->allow_new_custom_flags(box) ? + mailbox_allow_new_custom_flags(box) ? " \\*" : "", ")] Flags permitted.", NULL)); } }
--- a/src/imap/imap-expunge.c Tue Apr 27 23:14:15 2004 +0300 +++ b/src/imap/imap-expunge.c Tue Apr 27 23:25:52 2004 +0300 @@ -2,28 +2,43 @@ #include "common.h" #include "mail-storage.h" +#include "mail-search.h" #include "imap-expunge.h" -int imap_expunge(struct mailbox *box, int notify) +int imap_expunge(struct mailbox *box) { - struct mail_expunge_context *ctx; + struct mail_search_context *ctx; + struct mailbox_transaction_context *t; struct mail *mail; + struct mail_search_arg search_arg; int failed = FALSE; - ctx = box->expunge_init(box, 0, FALSE); - if (ctx == NULL) - return FALSE; + memset(&search_arg, 0, sizeof(search_arg)); + search_arg.type = SEARCH_DELETED; - while ((mail = box->expunge_fetch_next(ctx)) != NULL) { - if (!mail->expunge(mail, ctx, NULL, notify)) { - failed = TRUE; - break; + t = mailbox_transaction_begin(box, FALSE); + ctx = mailbox_search_init(t, NULL, &search_arg, NULL, 0, NULL); + if (ctx == NULL) + failed = TRUE; + else { + while ((mail = mailbox_search_next(ctx)) != NULL) { + if (mail->expunge(mail) < 0) { + failed = TRUE; + break; + } } } - if (!box->expunge_deinit(ctx)) + if (mailbox_search_deinit(ctx) < 0) return FALSE; + if (failed) + mailbox_transaction_rollback(t); + else { + if (mailbox_transaction_commit(t) < 0) + failed = TRUE; + } + return !failed; }
--- a/src/imap/imap-expunge.h Tue Apr 27 23:14:15 2004 +0300 +++ b/src/imap/imap-expunge.h Tue Apr 27 23:25:52 2004 +0300 @@ -1,6 +1,6 @@ #ifndef __IMAP_EXPUNGE_H #define __IMAP_EXPUNGE_H -int imap_expunge(struct mailbox *box, int notify); +int imap_expunge(struct mailbox *box); #endif
--- a/src/imap/imap-fetch-body-section.c Tue Apr 27 23:14:15 2004 +0300 +++ b/src/imap/imap-fetch-body-section.c Tue Apr 27 23:25:52 2004 +0300 @@ -463,9 +463,10 @@ if (part != NULL && (part->flags & MESSAGE_PART_FLAG_MESSAGE_RFC822) && - (*path >= '0' && *path <= '9' || strncmp(path, "HEADER", 6) == 0)) { + ((*path >= '0' && *path <= '9') || + strncmp(path, "HEADER", 6) == 0)) { /* if remainder of path is a number or "HEADER", - skip the message/rfc822 part */ + skip the message/rfc822 part */ part = part->children; } }
--- a/src/imap/imap-fetch.c Tue Apr 27 23:14:15 2004 +0300 +++ b/src/imap/imap-fetch.c Tue Apr 27 23:25:52 2004 +0300 @@ -10,7 +10,7 @@ #include "imap-date.h" #include "commands.h" #include "imap-fetch.h" -#include "imap-search.h" +#include "imap-util.h" #include <unistd.h> @@ -43,12 +43,21 @@ static int fetch_flags(struct imap_fetch_context *ctx, struct mail *mail, const struct mail_full_flags *flags) { + struct mail_full_flags full_flags; + if (flags == NULL) { flags = mail->get_flags(mail); if (flags == NULL) return FALSE; } + if (ctx->update_seen) { + /* \Seen change isn't shown by get_flags() yet */ + full_flags = *flags; + full_flags.flags |= MAIL_SEEN; + flags = &full_flags; + } + str_printfa(ctx->str, "FLAGS (%s) ", imap_write_flags(flags)); return TRUE; } @@ -242,11 +251,9 @@ return FALSE; if ((flags->flags & MAIL_SEEN) == 0) { - if (!mail->update_flags(mail, &ctx->seen_flag, - MODIFY_ADD)) + if (mail->update_flags(mail, &ctx->seen_flag, + MODIFY_ADD) < 0) return FALSE; - - flags = NULL; /* \Seen won't update automatically */ seen_updated = TRUE; } } @@ -328,17 +335,16 @@ enum mail_fetch_field fetch_data, enum imap_fetch_field imap_data, struct imap_fetch_body_data *bodies, - const char *messageset, int uidset) + struct mail_search_arg *search_args) { struct mailbox *box = client->mailbox; - struct mail_search_arg *search_arg; struct imap_fetch_context ctx; + struct mailbox_transaction_context *t; struct mail *mail; struct imap_fetch_body_data *body; const char *null = NULL; const char *const *wanted_headers, *const *arr; buffer_t *buffer; - int all_found; memset(&ctx, 0, sizeof(ctx)); ctx.fetch_data = fetch_data; @@ -348,7 +354,7 @@ ctx.select_counter = client->select_counter; ctx.seen_flag.flags = MAIL_SEEN; - if (!box->is_readonly(box)) { + if (!mailbox_is_readonly(box)) { /* If we have any BODY[..] sections, \Seen flag is added for all messages. */ for (body = bodies; body != NULL; body = body->next) { @@ -383,19 +389,14 @@ wanted_headers = !ctx.body_fetch_from_cache ? NULL : buffer_get_data(buffer, NULL); - if (ctx.update_seen) { - if (!box->lock(box, MAILBOX_LOCK_FLAGS | MAILBOX_LOCK_READ)) - return -1; - } - - search_arg = imap_search_get_msgset_arg(messageset, uidset); - ctx.search_ctx = box->search_init(box, NULL, search_arg, NULL, - fetch_data, wanted_headers); + t = mailbox_transaction_begin(box, TRUE); + ctx.search_ctx = mailbox_search_init(t, NULL, search_args, NULL, + fetch_data, wanted_headers); if (ctx.search_ctx == NULL) ctx.failed = TRUE; else { ctx.str = str_new(default_pool, 8192); - while ((mail = box->search_next(ctx.search_ctx)) != NULL) { + while ((mail = mailbox_search_next(ctx.search_ctx)) != NULL) { if (!fetch_mail(&ctx, mail)) { ctx.failed = TRUE; break; @@ -403,11 +404,15 @@ } str_free(ctx.str); - if (!box->search_deinit(ctx.search_ctx, &all_found)) + if (mailbox_search_deinit(ctx.search_ctx) < 0) ctx.failed = TRUE; } - (void)box->lock(box, MAILBOX_LOCK_UNLOCK); - - return ctx.failed ? -1 : all_found; + if (ctx.failed) + mailbox_transaction_rollback(t); + else { + if (mailbox_transaction_commit(t) < 0) + ctx.failed = TRUE; + } + return ctx.failed ? -1 : 0; }
--- a/src/imap/imap-fetch.h Tue Apr 27 23:14:15 2004 +0300 +++ b/src/imap/imap-fetch.h Tue Apr 27 23:25:52 2004 +0300 @@ -40,7 +40,7 @@ enum mail_fetch_field fetch_data, enum imap_fetch_field imap_data, struct imap_fetch_body_data *bodies, - const char *messageset, int uidset); + struct mail_search_arg *search_args); int imap_fetch_body_section(struct imap_fetch_context *ctx, const struct imap_fetch_body_data *body,
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/imap/imap-messageset.c Tue Apr 27 23:25:52 2004 +0300 @@ -0,0 +1,85 @@ +/* Copyright (C) 2002-2004 Timo Sirainen */ + +#include "lib.h" +#include "mail-search.h" +#include "imap-search.h" +#include "imap-messageset.h" + +static uint32_t get_next_number(const char **str) +{ + uint32_t num; + + num = 0; + while (**str != '\0') { + if (**str < '0' || **str > '9') + break; + + num = num*10 + (**str - '0'); + (*str)++; + } + + if (num == (uint32_t)-1) { + /* FIXME: ugly hack, we're using this number to mean the + last existing message. In reality UIDs should never get + this high, so we can quite safely just drop this one down. */ + num--; + } + + return num; +} + +struct mail_search_seqset *imap_messageset_parse(const char *messageset) +{ + struct mail_search_seqset *ret, **next; + uint32_t seq1, seq2; + + ret = NULL; + next = &ret; + + while (*messageset != '\0') { + if (*messageset == '*') { + /* last message */ + seq1 = (uint32_t)-1; + messageset++; + } else { + seq1 = get_next_number(&messageset); + if (seq1 == 0) + return NULL; + } + + if (*messageset != ':') + seq2 = seq1; + else { + /* first:last range */ + messageset++; + + if (*messageset == '*') { + seq2 = (uint32_t)-1; + messageset++; + } else { + seq2 = get_next_number(&messageset); + if (seq2 == 0) + return NULL; + } + } + + if (*messageset == ',') + messageset++; + else if (*messageset != '\0') + return NULL; + + if (seq1 > seq2) { + /* swap, as specified by RFC-3501 */ + uint32_t temp = seq1; + seq1 = seq2; + seq2 = temp; + } + + *next = t_new(struct mail_search_seqset, 1); + (*next)->seq1 = seq1; + (*next)->seq2 = seq2; + next = &(*next)->next; + } + + return ret; +}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/imap/imap-messageset.h Tue Apr 27 23:25:52 2004 +0300 @@ -0,0 +1,6 @@ +#ifndef __IMAP_MESSAGESET_H +#define __IMAP_MESSAGESET_H + +struct mail_search_seqset *imap_messageset_parse(const char *messageset); + +#endif
--- a/src/imap/imap-search.c Tue Apr 27 23:14:15 2004 +0300 +++ b/src/imap/imap-search.c Tue Apr 27 23:25:52 2004 +0300 @@ -1,15 +1,62 @@ /* Copyright (C) 2002 Timo Sirainen */ #include "common.h" +#include "mail-storage.h" #include "mail-search.h" #include "imap-search.h" #include "imap-parser.h" +#include "imap-messageset.h" struct search_build_data { pool_t pool; + struct mailbox *box; const char *error; }; +static int +imap_uidset_parse(struct mailbox *box, const char *uidset, + struct mail_search_seqset **seqset_r, const char **error_r) +{ + struct mail_search_seqset *seqset, **p; + int syntax, last; + + *seqset_r = imap_messageset_parse(uidset); + if (*seqset_r == NULL) { + *error_r = "Invalid UID messageset"; + return -1; + } + + p = seqset_r; + for (seqset = *seqset_r; seqset != NULL; seqset = seqset->next) { + if (seqset->seq1 == (uint32_t)-1) { + /* last message, stays same */ + continue; + } + + last = seqset->seq2 == (uint32_t)-1; + if (mailbox_get_uids(box, seqset->seq1, seqset->seq2, + &seqset->seq1, &seqset->seq2) < 0) { + struct mail_storage *storage = mailbox_get_storage(box); + *error_r = mail_storage_get_last_error(storage, + &syntax); + return -1; + } + + if (seqset->seq1 == 0 && last) { + /* we need special case for too_high_uid:* case */ + seqset->seq1 = seqset->seq2 = (uint32_t)-1; + } + + if (seqset->seq1 != 0) + p = &seqset->next; + else + *p = seqset->next; + } + + *error_r = NULL; + return 0; +} + static struct mail_search_arg * search_arg_new(pool_t pool, enum mail_search_arg_type type) { @@ -65,6 +112,7 @@ struct imap_arg **args, struct mail_search_arg **next_sarg) { + struct mail_search_seqset *seqset; struct mail_search_arg **subargs; struct imap_arg *arg; char *str; @@ -277,7 +325,13 @@ case 'U': if (strcmp(str, "UID") == 0) { /* <message set> */ - return ARG_NEW(SEARCH_UID); + if (!ARG_NEW(SEARCH_SEQSET)) + return FALSE; + + return imap_uidset_parse(data->box, + (*next_sarg)->value.str, + &(*next_sarg)->value.seqset, + &data->error) == 0; } else if (strcmp(str, "UNANSWERED") == 0) { if (!ARG_NEW_FLAG(SEARCH_ANSWERED)) return FALSE; @@ -313,10 +367,16 @@ default: if (*str == '*' || (*str >= '0' && *str <= '9')) { /* <message-set> */ - if (!ARG_NEW_FLAG(SEARCH_SET)) + seqset = imap_messageset_parse(str); + if (seqset == NULL) { + data->error = "Invalid messageset"; + return FALSE; + } + + if (!ARG_NEW_FLAG(SEARCH_SEQSET)) return FALSE; - (*next_sarg)->value.str = str; + (*next_sarg)->value.seqset = seqset; return TRUE; } break; @@ -327,11 +387,15 @@ } struct mail_search_arg * -imap_search_args_build(pool_t pool, struct imap_arg *args, const char **error) +imap_search_args_build(pool_t pool, struct mailbox *box, struct imap_arg *args, + const char **error_r) { struct search_build_data data; struct mail_search_arg *first_sarg, **sargs; + *error_r = NULL; + + data.box = box; data.pool = pool; data.error = NULL; @@ -339,23 +403,62 @@ first_sarg = NULL; sargs = &first_sarg; while (args->type != IMAP_ARG_EOL) { if (!search_arg_build(&data, &args, sargs)) { - *error = data.error; + *error_r = data.error; return NULL; } sargs = &(*sargs)->next; } - *error = NULL; return first_sarg; } -struct mail_search_arg * -imap_search_get_msgset_arg(const char *messageset, int uidset) +int imap_search_get_msgset_arg(const char *messageset, + struct mail_search_arg **arg_r, + const char **error_r) +{ + struct mail_search_arg *arg; + + arg = t_new(struct mail_search_arg, 1); + arg->type = SEARCH_SEQSET; + arg->value.seqset = imap_messageset_parse(messageset); + if (arg->value.seqset == NULL) { + *error_r = "Invalid messageset"; + return -1; + } + *arg_r = arg; + return 0; +} + +int imap_search_get_uidset_arg(struct mailbox *box, const char *uidset, + struct mail_search_arg **arg_r, + const char **error_r) { struct mail_search_arg *arg; arg = t_new(struct mail_search_arg, 1); - arg->type = uidset ? SEARCH_UID : SEARCH_SET; - arg->value.str = t_strdup(messageset); - return arg; + arg->type = SEARCH_SEQSET; + *arg_r = arg; + return imap_uidset_parse(box, uidset, &arg->value.seqset, error_r); } + +struct mail_search_arg * +imap_search_get_arg(struct client *client, const char *set, int uid) +{ + struct mail_search_arg *search_arg; + const char *error; + int ret; + + if (!uid) { + ret = imap_search_get_msgset_arg(set, &search_arg, + &error); + } else { + ret = imap_search_get_uidset_arg(client->mailbox, set, + &search_arg, &error); + } + if (ret < 0) { + client_send_tagline(client, t_strconcat("BAD ", error, NULL)); + return NULL; + } + + return search_arg; +}
--- a/src/imap/imap-search.h Tue Apr 27 23:14:15 2004 +0300 +++ b/src/imap/imap-search.h Tue Apr 27 23:25:52 2004 +0300 @@ -1,11 +1,21 @@ #ifndef __IMAP_SEARCH_H #define __IMAP_SEARCH_H +struct imap_arg; +struct mailbox; + /* Builds search arguments based on IMAP arguments. */ struct mail_search_arg * -imap_search_args_build(pool_t pool, struct imap_arg *args, const char **error); +imap_search_args_build(pool_t pool, struct mailbox *box, struct imap_arg *args, + const char **error_r); +int imap_search_get_msgset_arg(const char *messageset, + struct mail_search_arg **arg_r, + const char **error_r); +int imap_search_get_uidset_arg(struct mailbox *box, const char *uidset, + struct mail_search_arg **arg_r, + const char **error_r); struct mail_search_arg * -imap_search_get_msgset_arg(const char *messageset, int uidset); +imap_search_get_arg(struct client *client, const char *set, int uid); #endif
--- a/src/imap/imap-sort.c Tue Apr 27 23:14:15 2004 +0300 +++ b/src/imap/imap-sort.c Tue Apr 27 23:25:52 2004 +0300 @@ -29,6 +29,7 @@ struct sort_context { struct mail_search_context *search_ctx; + struct mailbox_transaction_context *t; enum mail_sort_type sort_program[MAX_SORT_PROGRAM_SIZE]; enum mail_sort_type common_mask, cache_mask; @@ -209,7 +210,7 @@ /* remove the common part from sort program, we already know input is sorted that much so we don't have to worry about it. */ - if (!client->mailbox->search_get_sorting(client->mailbox, norm_prog)) + if (mailbox_search_get_sorting(client->mailbox, norm_prog) < 0) return FALSE; ctx->common_mask = mail_sort_get_common_mask(ctx->sort_program, norm_prog, &count); @@ -223,11 +224,14 @@ wanted_fields = init_sort_elements(ctx, wanted_headers); /* initialize searching */ - ctx->search_ctx = client->mailbox-> - search_init(client->mailbox, charset, args, norm_prog, - wanted_fields, wanted_headers); - if (ctx->search_ctx == NULL) + ctx->t = mailbox_transaction_begin(client->mailbox, FALSE); + ctx->search_ctx = + mailbox_search_init(ctx->t, charset, args, norm_prog, + wanted_fields, wanted_headers); + if (ctx->search_ctx == NULL) { + mailbox_transaction_rollback(ctx->t); return FALSE; + } ctx->box = client->mailbox; ctx->output = client->output; @@ -240,11 +244,13 @@ ctx->id_is_uid = client->cmd_uid; - while ((mail = client->mailbox->search_next(ctx->search_ctx)) != NULL) + while ((mail = mailbox_search_next(ctx->search_ctx)) != NULL) mail_sort_input(ctx, mail); mail_sort_flush(ctx); - ret = client->mailbox->search_deinit(ctx->search_ctx, NULL); + ret = mailbox_search_deinit(ctx->search_ctx); + + mailbox_transaction_rollback(ctx->t); if (ctx->written || ret) { str_append(ctx->str, "\r\n"); @@ -487,11 +493,15 @@ static struct mail *get_mail(struct sort_context *ctx, const unsigned char *buf) { unsigned int id = *((unsigned int *) buf); + uint32_t seq; - if (ctx->id_is_uid) - return ctx->box->fetch_uid(ctx->box, id, 0); - else - return ctx->box->fetch_seq(ctx->box, id, 0); + if (!ctx->id_is_uid) + seq = id; + else { + if (mailbox_get_uids(ctx->box, id, id, &seq, &seq) < 0) + return NULL; + } + return mailbox_fetch(ctx->t, seq, 0); }
--- a/src/imap/imap-thread.c Tue Apr 27 23:14:15 2004 +0300 +++ b/src/imap/imap-thread.c Tue Apr 27 23:25:52 2004 +0300 @@ -70,6 +70,7 @@ struct thread_context { struct mail_search_context *search_ctx; + struct mailbox_transaction_context *t; struct mailbox *box; struct ostream *output; @@ -116,11 +117,14 @@ ctx = t_new(struct thread_context, 1); /* initialize searching */ - ctx->search_ctx = client->mailbox-> - search_init(client->mailbox, charset, args, NULL, - MAIL_FETCH_DATE, wanted_headers); - if (ctx->search_ctx == NULL) + ctx->t = mailbox_transaction_begin(client->mailbox, FALSE); + ctx->search_ctx = + mailbox_search_init(ctx->t, charset, args, NULL, + MAIL_FETCH_DATE, wanted_headers); + if (ctx->search_ctx == NULL) { + mailbox_transaction_rollback(ctx->t); return FALSE; + } ctx->box = client->mailbox; ctx->output = client->output; @@ -135,16 +139,17 @@ (hash_cmp_callback_t *)strcmp); ctx->id_is_uid = client->cmd_uid; - while ((mail = client->mailbox->search_next(ctx->search_ctx)) != NULL) + while ((mail = mailbox_search_next(ctx->search_ctx)) != NULL) mail_thread_input(ctx, mail); o_stream_send_str(client->output, "* THREAD"); mail_thread_finish(ctx); o_stream_send_str(client->output, "\r\n"); - ret = client->mailbox->search_deinit(ctx->search_ctx, NULL); + ret = mailbox_search_deinit(ctx->search_ctx); + mailbox_transaction_rollback(ctx->t); mail_thread_deinit(ctx); - return ret; + return ret == 0; } static void add_root(struct thread_context *ctx, struct node *node) @@ -651,6 +656,7 @@ struct mail *mail; struct node *node; unsigned int id; + uint32_t seq; ctx->subject_hash = hash_create(default_pool, ctx->temp_pool, ctx->root_count * 2, @@ -668,10 +674,14 @@ node->u.info->sorted = TRUE; } - if (ctx->id_is_uid) - mail = ctx->box->fetch_uid(ctx->box, id, 0); - else - mail = ctx->box->fetch_seq(ctx->box, id, 0); + if (!ctx->id_is_uid) + seq = id; + else { + if (mailbox_get_uids(ctx->box, id, id, &seq, &seq) < 0) + seq = 0; + } + + mail = seq == 0 ? NULL : mailbox_fetch(ctx->t, seq, 0); if (mail != NULL) { t_push();
--- a/src/imap/mail-storage-callbacks.c Tue Apr 27 23:14:15 2004 +0300 +++ b/src/imap/mail-storage-callbacks.c Tue Apr 27 23:25:52 2004 +0300 @@ -45,8 +45,7 @@ client_send_line(client, str); } -static void update_flags(struct mailbox *mailbox, - unsigned int seq, unsigned int uid __attr_unused__, +static void update_flags(struct mailbox *mailbox, unsigned int seq, const struct mail_full_flags *flags, void *context) { struct client *client = context;
--- a/src/imap/namespace.c Tue Apr 27 23:14:15 2004 +0300 +++ b/src/imap/namespace.c Tue Apr 27 23:25:52 2004 +0300 @@ -47,7 +47,7 @@ if (hook_mail_storage_created != NULL) hook_mail_storage_created(&ns->storage); - ns->hierarchy_sep = ns->storage->hierarchy_sep; + ns->hierarchy_sep = mail_storage_get_hierarchy_sep(ns->storage); return ns; } @@ -106,7 +106,7 @@ ns->type = NAMESPACE_PRIVATE; ns->inbox = TRUE; ns->prefix = p_strdup(pool, ""); - ns->hierarchy_sep = ns->storage->hierarchy_sep; + ns->hierarchy_sep = mail_storage_get_hierarchy_sep(ns->storage); if (hook_mail_storage_created != NULL) hook_mail_storage_created(&ns->storage);
--- a/src/lib-imap/imap-util.c Tue Apr 27 23:14:15 2004 +0300 +++ b/src/lib-imap/imap-util.c Tue Apr 27 23:25:52 2004 +0300 @@ -2,12 +2,13 @@ #include "lib.h" #include "str.h" +#include "mail-types.h" #include "imap-util.h" const char *imap_write_flags(const struct mail_full_flags *flags) { string_t *str; - const char *sysflags, *name; + const char *sysflags; unsigned int i; i_assert(flags->custom_flags_count <= MAIL_CUSTOM_FLAGS_COUNT); @@ -27,13 +28,23 @@ if (*sysflags != '\0') sysflags++; - if ((flags->flags & MAIL_CUSTOM_FLAGS_MASK) == 0) + if (flags->custom_flags_count == 0) return sysflags; /* we have custom flags too */ str = t_str_new(256); str_append(str, sysflags); +#if 1 + for (i = 0; i < flags->custom_flags_count; i++) { + if (str_len(str) > 0) + str_append_c(str, ' '); + str_append(str, flags->custom_flags[i]); + } +#else // FIXME + if ((flags->flags & MAIL_CUSTOM_FLAGS_MASK) == 0) + return sysflags; + for (i = 0; i < flags->custom_flags_count; i++) { if (flags->flags & (1 << (i + MAIL_CUSTOM_FLAG_1_BIT))) { name = flags->custom_flags[i]; @@ -44,6 +55,6 @@ } } } - +#endif return str_c(str); }
--- a/src/lib-imap/imap-util.h Tue Apr 27 23:14:15 2004 +0300 +++ b/src/lib-imap/imap-util.h Tue Apr 27 23:25:52 2004 +0300 @@ -1,33 +1,7 @@ #ifndef __IMAP_UTIL_H #define __IMAP_UTIL_H -enum modify_type { - MODIFY_ADD, - MODIFY_REMOVE, - MODIFY_REPLACE -}; - -enum mail_flags { - MAIL_ANSWERED = 0x0000001, - MAIL_FLAGGED = 0x0000002, - MAIL_DELETED = 0x0000004, - MAIL_SEEN = 0x0000008, - MAIL_DRAFT = 0x0000010, - MAIL_RECENT = 0x0000020, - - /* rest of the bits are custom flags */ - MAIL_CUSTOM_FLAG_1 = 0x0000040, - - MAIL_SYSTEM_FLAGS_MASK = 0x000003f, - MAIL_CUSTOM_FLAGS_MASK = 0xfffffc0 -}; - -struct mail_full_flags { - enum mail_flags flags; - - const char **custom_flags; - unsigned int custom_flags_count; -}; +struct mail_full_flags; /* growing number of flags isn't very easy. biggest problem is that they're stored into unsigned int, which is 32bit almost everywhere. another thing
--- a/src/lib-index/Makefile.am Tue Apr 27 23:14:15 2004 +0300 +++ b/src/lib-index/Makefile.am Tue Apr 27 23:25:52 2004 +0300 @@ -1,26 +1,34 @@ -SUBDIRS = maildir mbox - noinst_LIBRARIES = libindex.a INCLUDES = \ -I$(top_srcdir)/src/lib \ - -I$(top_srcdir)/src/lib-mail \ - -I$(top_srcdir)/src/lib-imap + -I$(top_srcdir)/src/lib-mail libindex_a_SOURCES = \ - mail-cache.c \ - mail-custom-flags.c \ + mail-cache.c \ + mail-cache-lookup.c \ + mail-cache-transaction.c \ mail-index.c \ - mail-index-file.c \ mail-index-fsck.c \ - mail-index-open.c \ - mail-index-rebuild.c \ - mail-index-util.c \ - mail-modifylog.c + mail-index-lock.c \ + mail-index-transaction.c \ + mail-index-reset.c \ + mail-index-sync.c \ + mail-index-sync-update.c \ + mail-index-view.c \ + mail-index-view-sync.c \ + mail-transaction-log.c \ + mail-transaction-log-view.c \ + mail-transaction-util.c noinst_HEADERS = \ - mail-cache.h \ - mail-custom-flags.h \ + mail-cache.h \ + mail-cache-private.h \ mail-index.h \ - mail-index-util.h \ - mail-modifylog.h + mail-index-private.h \ + mail-index-sync-private.h \ + mail-index-transaction-private.h \ + mail-index-view-private.h \ + mail-transaction-log.h \ + mail-transaction-log-private.h \ + mail-transaction-util.h
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-index/mail-cache-compress.c Tue Apr 27 23:25:52 2004 +0300 @@ -0,0 +1,248 @@ +static const struct mail_cache_record * +mail_cache_compress_record(struct mail_cache *cache, + struct mail_index_record *rec, int header_idx, + uint32_t *size_r) +{ + enum mail_cache_field orig_cached_fields, cached_fields, field; + struct mail_cache_record cache_rec; + buffer_t *buffer; + const void *data; + size_t size, pos; + uint32_t nb_size; + int i; + + memset(&cache_rec, 0, sizeof(cache_rec)); + buffer = buffer_create_dynamic(pool_datastack_create(), + 4096, (size_t)-1); + + orig_cached_fields = mail_cache_get_fields(cache, rec); + cached_fields = orig_cached_fields & ~MAIL_CACHE_HEADERS_MASK; + buffer_append(buffer, &cache_rec, sizeof(cache_rec)); + for (i = 0, field = 1; i < 31; i++, field <<= 1) { + if ((cached_fields & field) == 0) + continue; + + if (!mail_cache_lookup_field(cache, rec, field, &data, &size)) { + cached_fields &= ~field; + continue; + } + + nb_size = uint32_to_nbo((uint32_t)size); + + if ((field & MAIL_CACHE_FIXED_MASK) == 0) + buffer_append(buffer, &nb_size, sizeof(nb_size)); + buffer_append(buffer, data, size); + if ((size & 3) != 0) + buffer_append(buffer, null4, 4 - (size & 3)); + } + + /* now merge all the headers if we have them all */ + if ((orig_cached_fields & mail_cache_header_fields[header_idx]) != 0) { + nb_size = 0; + pos = buffer_get_used_size(buffer); + buffer_append(buffer, &nb_size, sizeof(nb_size)); + + for (i = 0; i <= header_idx; i++) { + field = mail_cache_header_fields[i]; + if (mail_cache_lookup_field(cache, rec, field, + &data, &size) && size > 1) { + size--; /* terminating \0 */ + buffer_append(buffer, data, size); + nb_size += size; + } + } + buffer_append(buffer, "", 1); + nb_size++; + if ((nb_size & 3) != 0) + buffer_append(buffer, null4, 4 - (nb_size & 3)); + + nb_size = uint32_to_nbo(nb_size); + buffer_write(buffer, pos, &nb_size, sizeof(nb_size)); + + cached_fields |= MAIL_CACHE_HEADERS1; + } + + cache_rec.fields = cached_fields; + cache_rec.size = uint32_to_nbo(buffer_get_used_size(buffer)); + buffer_write(buffer, 0, &cache_rec, sizeof(cache_rec)); + + data = buffer_get_data(buffer, &size); + *size_r = size; + return data; +} + +static int mail_cache_copy(struct mail_cache *cache, int fd) +{ +#if 0 + struct mail_cache_header *hdr; + const struct mail_cache_record *cache_rec; + struct mail_index_record *rec; + enum mail_cache_field used_fields; + unsigned char *mmap_base; + const char *str; + uint32_t new_file_size, offset, size, nb_size; + int i, header_idx; + + /* pick some reasonably good file size */ + new_file_size = cache->used_file_size - + nbo_to_uint32(cache->hdr->deleted_space); + new_file_size = (new_file_size + 1023) & ~1023; + if (new_file_size < MAIL_CACHE_INITIAL_SIZE) + new_file_size = MAIL_CACHE_INITIAL_SIZE; + + if (file_set_size(fd, new_file_size) < 0) + return mail_cache_set_syscall_error(cache, "file_set_size()"); + + mmap_base = mmap(NULL, new_file_size, PROT_READ | PROT_WRITE, + MAP_SHARED, fd, 0); + if (mmap_base == MAP_FAILED) + return mail_cache_set_syscall_error(cache, "mmap()"); + + /* skip file's header */ + hdr = (struct mail_cache_header *) mmap_base; + offset = sizeof(*hdr); + + /* merge all the header pieces into one. if some message doesn't have + all the required pieces, we'll just have to drop them all. */ + for (i = MAIL_CACHE_HEADERS_COUNT-1; i >= 0; i--) { + str = mail_cache_get_header_fields_str(cache, i); + if (str != NULL) + break; + } + + if (str == NULL) + header_idx = -1; + else { + hdr->header_offsets[0] = uint32_to_offset(offset); + header_idx = i; + + size = strlen(str) + 1; + nb_size = uint32_to_nbo(size); + + memcpy(mmap_base + offset, &nb_size, sizeof(nb_size)); + offset += sizeof(nb_size); + memcpy(mmap_base + offset, str, size); + offset += (size + 3) & ~3; + } + + // FIXME: recreate index file with new cache_offsets + + used_fields = 0; + rec = cache->index->lookup(cache->index, 1); + while (rec != NULL) { + cache_rec = mail_cache_lookup(cache, rec, 0); + if (cache_rec == NULL) + rec->cache_offset = 0; + else if (offset_to_uint32(cache_rec->next_offset) == 0) { + /* just one unmodified block, copy it */ + size = nbo_to_uint32(cache_rec->size); + i_assert(offset + size <= new_file_size); + + memcpy(mmap_base + offset, cache_rec, size); + rec->cache_offset = uint32_to_offset(offset); + + size = (size + 3) & ~3; + offset += size; + } else { + /* multiple blocks, sort them into buffer */ + t_push(); + cache_rec = mail_cache_compress_record(cache, rec, + header_idx, + &size); + i_assert(offset + size <= new_file_size); + memcpy(mmap_base + offset, cache_rec, size); + used_fields |= cache_rec->fields; + t_pop(); + + rec->cache_offset = uint32_to_offset(offset); + offset += size; + } + + rec = cache->index->next(cache->index, rec); + } + + /* update header */ + hdr->indexid = cache->index->indexid; + hdr->file_seq = cache->index->hdr->cache_sync_id+1; + hdr->used_file_size = uint32_to_nbo(offset); + hdr->used_fields = used_fields; + hdr->field_usage_start = uint32_to_nbo(ioloop_time); + + /* write everything to disk */ + if (msync(mmap_base, offset, MS_SYNC) < 0) + return mail_cache_set_syscall_error(cache, "msync()"); + + if (munmap(mmap_base, new_file_size) < 0) + return mail_cache_set_syscall_error(cache, "munmap()"); + + if (fdatasync(fd) < 0) + return mail_cache_set_syscall_error(cache, "fdatasync()"); + return TRUE; +#endif +} + +int mail_cache_compress(struct mail_cache *cache) +{ + int fd, ret = TRUE; + + i_assert(cache->trans_ctx == NULL); + + if (cache->anon_mmap) + return TRUE; + + if (!cache->index->set_lock(cache->index, MAIL_LOCK_EXCLUSIVE)) + return FALSE; + + if (mail_cache_lock(cache, TRUE) <= 0) + return FALSE; + +#ifdef DEBUG + i_warning("Compressing cache file %s", cache->filepath); +#endif + + fd = file_dotlock_open(cache->filepath, NULL, MAIL_CACHE_LOCK_TIMEOUT, + MAIL_CACHE_LOCK_CHANGE_TIMEOUT, + MAIL_CACHE_LOCK_IMMEDIATE_TIMEOUT, NULL, NULL); + if (fd == -1) { + mail_cache_set_syscall_error(cache, "file_dotlock_open()"); + return FALSE; + } + + /* now we'll begin the actual moving. keep rebuild-flag on + while doing it. */ + cache->index->hdr->flags |= MAIL_INDEX_HDR_FLAG_REBUILD; + if (!mail_index_fmdatasync(cache->index, cache->index->hdr_size)) + return FALSE; + + if (!mail_cache_copy(cache, fd)) { + (void)file_dotlock_delete(cache->filepath, fd); + ret = FALSE; + } else { + mail_cache_file_close(cache); + cache->fd = dup(fd); + + if (file_dotlock_replace(cache->filepath, fd, FALSE) < 0) { + mail_cache_set_syscall_error(cache, + "file_dotlock_replace()"); + ret = FALSE; + } + + if (!mmap_update(cache, 0, 0)) + ret = FALSE; + } + + /* headers could have changed, reread them */ + memset(cache->split_offsets, 0, sizeof(cache->split_offsets)); + memset(cache->split_headers, 0, sizeof(cache->split_headers)); + + if (ret) { + cache->index->hdr->flags &= + ~(MAIL_INDEX_HDR_FLAG_REBUILD | + MAIL_INDEX_HDR_FLAG_COMPRESS_CACHE); + } + + if (!mail_cache_unlock(cache)) + ret = FALSE; + + return ret; +}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-index/mail-cache-lookup.c Tue Apr 27 23:25:52 2004 +0300 @@ -0,0 +1,322 @@ +/* Copyright (C) 2003-2004 Timo Sirainen */ + +#include "lib.h" +#include "buffer.h" +#include "byteorder.h" +#include "mail-cache-private.h" + +#if 0 +const char * +mail_cache_get_header_fields_str(struct mail_cache *cache, unsigned int idx) +{ + uint32_t offset, data_size; + unsigned char *buf; + + offset = mail_cache_offset_to_uint32(cache->hdr->header_offsets[idx]); + + if (offset == 0) + return NULL; + + if (!mmap_update(cache, offset, 1024)) + return NULL; + + if (offset + sizeof(data_size) > cache->mmap_length) { + mail_cache_set_corrupted(cache, "Header %u points outside file", + idx); + return NULL; + } + + buf = cache->mmap_base; + memcpy(&data_size, buf + offset, sizeof(data_size)); + data_size = nbo_to_uint32(data_size); + offset += sizeof(data_size); + + if (data_size == 0) { + mail_cache_set_corrupted(cache, + "Header %u points to empty string", idx); + return NULL; + } + + if (!mmap_update(cache, offset, data_size)) + return NULL; + + if (offset + data_size > cache->mmap_length) { + mail_cache_set_corrupted(cache, "Header %u points outside file", + idx); + return NULL; + } + + buf = cache->mmap_base; + if (buf[offset + data_size - 1] != '\0') { + mail_cache_set_corrupted(cache, + "Header %u points to invalid string", idx); + return NULL; + } + + return buf + offset; +} + +const char *const * +mail_cache_split_header(struct mail_cache *cache, const char *header) +{ + const char *const *arr, *const *tmp; + const char *null = NULL; + char *str; + buffer_t *buf; + + if (header == NULL) + return NULL; + + arr = t_strsplit(header, "\n"); + buf = buffer_create_dynamic(cache->split_header_pool, 32, (size_t)-1); + for (tmp = arr; *tmp != NULL; tmp++) { + str = p_strdup(cache->split_header_pool, *tmp); + buffer_append(buf, &str, sizeof(str)); + } + buffer_append(buf, &null, sizeof(null)); + + return buffer_get_data(buf, NULL); +} + +const char *const *mail_cache_get_header_fields(struct mail_cache_view *view, + unsigned int idx) +{ + struct mail_cache *cache = view->cache; + const char *str; + int i; + + i_assert(idx < MAIL_CACHE_HEADERS_COUNT); + + /* t_strsplit() is a bit slow, so we cache it */ + if (cache->hdr->header_offsets[idx] != cache->split_offsets[idx]) { + p_clear(cache->split_header_pool); + + t_push(); + for (i = 0; i < MAIL_CACHE_HEADERS_COUNT; i++) { + cache->split_offsets[i] = + cache->hdr->header_offsets[i]; + + str = mail_cache_get_header_fields_str(cache, i); + cache->split_headers[i] = + mail_cache_split_header(cache, str); + } + t_pop(); + } + + return cache->split_headers[idx]; +} + +struct mail_cache_record * +mail_cache_get_record(struct mail_cache *cache, uint32_t offset) +{ +#define CACHE_PREFETCH 1024 + struct mail_cache_record *cache_rec; + size_t size; + + offset = mail_cache_offset_to_uint32(offset); + if (offset == 0) + return NULL; + + if (!mmap_update(cache, offset, sizeof(*cache_rec) + CACHE_PREFETCH)) + return NULL; + + if (offset + sizeof(*cache_rec) > cache->mmap_length) { + mail_cache_set_corrupted(cache, "record points outside file"); + return NULL; + } + cache_rec = CACHE_RECORD(cache, offset); + + size = nbo_to_uint32(cache_rec->size); + if (size < sizeof(*cache_rec)) { + mail_cache_set_corrupted(cache, "invalid record size"); + return NULL; + } + if (size > CACHE_PREFETCH) { + if (!mmap_update(cache, offset, size)) + return NULL; + } + + if (offset + size > cache->mmap_length) { + mail_cache_set_corrupted(cache, "record points outside file"); + return NULL; + } + return cache_rec; +} + +struct mail_cache_record * +mail_cache_get_next_record(struct mail_cache *cache, + struct mail_cache_record *rec) +{ + struct mail_cache_record *next; + + next = mail_cache_get_record(cache, rec->next_offset); + if (next != NULL && next <= rec) { + mail_cache_set_corrupted(cache, "next_offset points backwards"); + return NULL; + } + return next; +} + +struct mail_cache_record * +mail_cache_lookup(struct mail_cache_view *view, uint32_t seq, + enum mail_cache_field fields) +{ + const struct mail_index_record *rec; + + if (mail_cache_transaction_autocommit(view, seq, fields) < 0) + return NULL; + // FIXME: check cache_offset in transaction + if (mail_index_lookup_latest(view->view, seq, &rec) < 0) + return NULL; + + return mail_cache_get_record(view->cache, rec->cache_offset); +} + +enum mail_cache_field +mail_cache_get_fields(struct mail_cache_view *view, uint32_t seq) +{ + struct mail_cache_record *cache_rec; + enum mail_cache_field fields = 0; + + cache_rec = mail_cache_lookup(view, seq, 0); + while (cache_rec != NULL) { + fields |= cache_rec->fields; + cache_rec = mail_cache_get_next_record(view->cache, cache_rec); + } + + return fields; +} + +static int cache_get_field(struct mail_cache *cache, + struct mail_cache_record *cache_rec, + enum mail_cache_field field, + void **data_r, size_t *size_r) +{ + unsigned char *buf; + unsigned int mask; + uint32_t rec_size, data_size; + size_t offset, next_offset; + int i; + + rec_size = nbo_to_uint32(cache_rec->size); + buf = (unsigned char *) cache_rec; + offset = sizeof(*cache_rec); + + for (i = 0, mask = 1; i < 31; i++, mask <<= 1) { + if ((cache_rec->fields & mask) == 0) + continue; + + /* all records are at least 32bit. we have to check this + before getting data_size. */ + if (offset + sizeof(uint32_t) > rec_size) { + mail_cache_set_corrupted(cache, + "Record continues outside it's allocated size"); + return FALSE; + } + + if ((mask & MAIL_CACHE_FIXED_MASK) != 0) + data_size = mail_cache_field_sizes[i]; + else { + memcpy(&data_size, buf + offset, sizeof(data_size)); + data_size = nbo_to_uint32(data_size); + offset += sizeof(data_size); + } + + next_offset = offset + ((data_size + 3) & ~3); + if (next_offset > rec_size) { + mail_cache_set_corrupted(cache, + "Record continues outside it's allocated size"); + return FALSE; + } + + if (field == mask) { + if (data_size == 0) { + mail_cache_set_corrupted(cache, + "Field size is 0"); + return FALSE; + } + *data_r = buf + offset; + *size_r = data_size; + return TRUE; + } + offset = next_offset; + } + + i_unreached(); + return FALSE; +} + +static int cache_lookup_field(struct mail_cache_view *view, uint32_t seq, + enum mail_cache_field field, + void **data_r, size_t *size_r) +{ + struct mail_cache_record *cache_rec; + + cache_rec = mail_cache_lookup(view, seq, field); + while (cache_rec != NULL) { + if ((cache_rec->fields & field) != 0) { + return cache_get_field(view->cache, cache_rec, field, + data_r, size_r); + } + cache_rec = mail_cache_get_next_record(view->cache, cache_rec); + } + + return FALSE; +} + +int mail_cache_lookup_field(struct mail_cache_view *view, uint32_t seq, + enum mail_cache_field field, + const void **data_r, size_t *size_r) +{ + void *data; + + if (!cache_lookup_field(view, seq, field, &data, size_r)) + return FALSE; + + *data_r = data; + return TRUE; +} + +const char * +mail_cache_lookup_string_field(struct mail_cache_view *view, uint32_t seq, + enum mail_cache_field field) +{ + const void *data; + size_t size; + + i_assert((field & MAIL_CACHE_STRING_MASK) != 0); + + if (!mail_cache_lookup_field(view, seq, field, &data, &size)) + return NULL; + + if (((const char *) data)[size-1] != '\0') { + mail_cache_set_corrupted(view->cache, + "String field %x doesn't end with NUL", field); + return NULL; + } + return data; +} + +int mail_cache_copy_fixed_field(struct mail_cache_view *view, uint32_t seq, + enum mail_cache_field field, + void *buffer, size_t buffer_size) +{ + const void *data; + size_t size; + + i_assert((field & MAIL_CACHE_FIXED_MASK) != 0); + + if (!mail_cache_lookup_field(view, seq, field, &data, &size)) + return FALSE; + + if (buffer_size != size) { + i_panic("cache: fixed field %x wrong size " + "(%"PRIuSIZE_T" vs %"PRIuSIZE_T")", + field, size, buffer_size); + } + + memcpy(buffer, data, buffer_size); + return TRUE; +} +#else +#endif
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-index/mail-cache-old.c Tue Apr 27 23:25:52 2004 +0300 @@ -0,0 +1,935 @@ +static const char * +mail_cache_get_header_fields_str(struct mail_cache *cache, unsigned int idx); +static int mail_cache_write(struct mail_cache_transaction_ctx *ctx); +static struct mail_cache_record * +mail_cache_lookup(struct mail_cache *cache, + const struct mail_index_record *rec, + enum mail_cache_field fields); + +static void mail_cache_file_close(struct mail_cache *cache) +{ + if (cache->mmap_base != NULL) { + if (munmap(cache->mmap_base, cache->mmap_length) < 0) + mail_cache_set_syscall_error(cache, "munmap()"); + } + + cache->mmap_base = NULL; + cache->hdr = NULL; + cache->mmap_length = 0; + + if (cache->fd != -1) { + if (close(cache->fd) < 0) + mail_cache_set_syscall_error(cache, "close()"); + cache->fd = -1; + } +} + +static int mail_cache_file_reopen(struct mail_cache *cache) +{ + int fd; + + fd = open(cache->filepath, O_RDWR); + if (fd == -1) { + mail_cache_set_syscall_error(cache, "open()"); + return -1; + } + + mail_cache_file_close(cache); + + cache->fd = fd; + return 0; +} + +static int mmap_verify_header(struct mail_cache *cache) +{ + struct mail_cache_header *hdr; + + /* check that the header is still ok */ + if (cache->mmap_length < sizeof(struct mail_cache_header)) { + mail_cache_set_corrupted(cache, "File too small"); + return 0; + } + cache->hdr = hdr = cache->mmap_base; + + if (cache->hdr->indexid != cache->index->indexid) { + /* index id changed */ + if (cache->hdr->indexid != 0) + mail_cache_set_corrupted(cache, "indexid changed"); + return 0; + } + + if (cache->trans_ctx != NULL) { + /* we've updated used_file_size, do nothing */ + return 1; + } + + cache->used_file_size = nbo_to_uint32(hdr->used_file_size); + + /* only check the header if we're locked */ + if (cache->locks == 0) + return 1; + + if (cache->used_file_size < sizeof(struct mail_cache_header)) { + mail_cache_set_corrupted(cache, "used_file_size too small"); + return 0; + } + if ((cache->used_file_size % sizeof(uint32_t)) != 0) { + mail_cache_set_corrupted(cache, "used_file_size not aligned"); + return 0; + } + + if (cache->used_file_size > cache->mmap_length) { + /* maybe a crash truncated the file - just fix it */ + hdr->used_file_size = uint32_to_nbo(cache->mmap_length & ~3); + if (msync(cache->mmap_base, sizeof(*hdr), MS_SYNC) < 0) { + mail_cache_set_syscall_error(cache, "msync()"); + return -1; + } + } + return 1; +} + +static int mmap_update_nocheck(struct mail_cache *cache, + size_t offset, size_t size) +{ + struct stat st; + + /* if sequence has changed, the file has to be reopened. + note that if main index isn't locked, it may change again */ + if (cache->hdr->file_seq != cache->index->hdr->cache_file_seq && + cache->mmap_base != NULL) { + if (!mail_cache_file_reopen(cache)) + return -1; + } + + if (offset < cache->mmap_length && + size <= cache->mmap_length - offset && + !cache->mmap_refresh) { + /* already mapped */ + if (size != 0 || cache->anon_mmap) + return 1; + + /* requesting the whole file - see if we need to + re-mmap */ + if (fstat(cache->fd, &st) < 0) { + mail_cache_set_syscall_error(cache, "fstat()"); + return -1; + } + if ((uoff_t)st.st_size == cache->mmap_length) + return 1; + } + cache->mmap_refresh = FALSE; + + if (cache->anon_mmap) + return 1; + + if (cache->mmap_base != NULL) { + if (cache->locks != 0) { + /* in the middle of transaction - write the changes */ + if (msync(cache->mmap_base, cache->mmap_length, + MS_SYNC) < 0) { + mail_cache_set_syscall_error(cache, "msync()"); + return -1; + } + } + + if (munmap(cache->mmap_base, cache->mmap_length) < 0) + mail_cache_set_syscall_error(cache, "munmap()"); + } + + i_assert(cache->fd != -1); + + /* map the whole file */ + cache->hdr = NULL; + cache->mmap_length = 0; + + cache->mmap_base = mmap_rw_file(cache->fd, &cache->mmap_length); + if (cache->mmap_base == MAP_FAILED) { + cache->mmap_base = NULL; + mail_cache_set_syscall_error(cache, "mmap()"); + return -1; + } + + /* re-mmaped, check header */ + return 0; +} + +static int mmap_update(struct mail_cache *cache, size_t offset, size_t size) +{ + int synced, ret; + + for (synced = FALSE;; synced = TRUE) { + ret = mmap_update_nocheck(cache, offset, size); + if (ret > 0) + return TRUE; + if (ret < 0) + return FALSE; + + if (!mmap_verify_header(cache)) + return FALSE; + + /* see if cache file was rebuilt - do it only once to avoid + infinite looping */ + if (cache->hdr->sync_id == cache->index->cache_sync_id || + synced) + break; + + if (!mail_cache_file_reopen(cache)) + return FALSE; + } + return TRUE; +} + +static int mail_cache_open_and_verify(struct mail_cache *cache, int silent) +{ + struct stat st; + + mail_cache_file_close(cache); + + cache->fd = open(cache->filepath, O_RDWR); + if (cache->fd == -1) { + if (errno == ENOENT) + return 0; + + mail_cache_set_syscall_error(cache, "open()"); + return -1; + } + + if (fstat(cache->fd, &st) < 0) { + mail_cache_set_syscall_error(cache, "fstat()"); + return -1; + } + + if (st.st_size < sizeof(struct mail_cache_header)) + return 0; + + cache->mmap_refresh = TRUE; + if (mmap_update_nocheck(cache, 0, sizeof(struct mail_cache_header)) < 0) + return -1; + + /* verify that this really is the cache for wanted index */ + cache->silent = silent; + if (!mmap_verify_header(cache)) { + cache->silent = FALSE; + return 0; + } + + cache->silent = FALSE; + return 1; +} + +static int mail_cache_open_or_create_file(struct mail_cache *cache, + struct mail_cache_header *hdr) +{ + int ret, fd; + + cache->filepath = i_strconcat(cache->index->filepath, + MAIL_CACHE_FILE_PREFIX, NULL); + + ret = mail_cache_open_and_verify(cache, FALSE); + if (ret != 0) + return ret > 0; + + /* we'll have to clear cache_offsets which requires exclusive lock */ + if (!mail_index_set_lock(cache->index, MAIL_LOCK_EXCLUSIVE)) + return FALSE; + + /* maybe a rebuild.. */ + fd = file_dotlock_open(cache->filepath, NULL, MAIL_CACHE_LOCK_TIMEOUT, + MAIL_CACHE_LOCK_CHANGE_TIMEOUT, + MAIL_CACHE_LOCK_IMMEDIATE_TIMEOUT, NULL, NULL); + if (fd == -1) { + mail_cache_set_syscall_error(cache, "file_dotlock_open()"); + return FALSE; + } + + /* see if someone else just created the cache file */ + ret = mail_cache_open_and_verify(cache, TRUE); + if (ret != 0) { + (void)file_dotlock_delete(cache->filepath, fd); + return ret > 0; + } + + /* rebuild then */ + if (write_full(fd, hdr, sizeof(*hdr)) < 0) { + mail_cache_set_syscall_error(cache, "write_full()"); + (void)file_dotlock_delete(cache->filepath, fd); + return FALSE; + } + if (file_set_size(fd, MAIL_CACHE_INITIAL_SIZE) < 0) { + mail_cache_set_syscall_error(cache, "file_set_size()"); + (void)file_dotlock_delete(cache->filepath, fd); + return FALSE; + } + + if (cache->index->hdr.cache_file_seq != 0) { + // FIXME: recreate index file with cache_offsets cleared + } + + mail_cache_file_close(cache); + cache->fd = dup(fd); + + if (file_dotlock_replace(cache->filepath, fd, FALSE) < 0) { + mail_cache_set_syscall_error(cache, "file_dotlock_replace()"); + return FALSE; + } + + if (!mmap_update(cache, 0, sizeof(struct mail_cache_header))) + return FALSE; + + return TRUE; +} + +int mail_cache_open_or_create(struct mail_index *index) +{ + struct mail_cache_header hdr; + struct mail_cache *cache; + + memset(&hdr, 0, sizeof(hdr)); + hdr.indexid = index->indexid; + hdr.sync_id = index->hdr->cache_file_seq; // FIXME + hdr.used_file_size = uint32_to_nbo(sizeof(hdr)); + + cache = i_new(struct mail_cache, 1); + cache->index = index; + cache->fd = -1; + cache->split_header_pool = pool_alloconly_create("Headers", 512); + + index->cache = cache; + + /* we'll do anon-mmaping only if initially requested. if we fail + because of out of disk space, we'll just let the main index code + know it and fail. */ + if (!mail_cache_open_or_create_file(cache, &hdr)) { + mail_cache_free(cache); + return FALSE; + } + + return TRUE; +} + +void mail_cache_free(struct mail_cache *cache) +{ + i_assert(cache->trans_ctx == NULL); + + cache->index->cache = NULL; + + mail_cache_file_close(cache); + + pool_unref(cache->split_header_pool); + i_free(cache->filepath); + i_free(cache); +} + +void mail_cache_set_defaults(struct mail_cache *cache, + enum mail_cache_field default_cache_fields, + enum mail_cache_field never_cache_fields) +{ + cache->default_cache_fields = default_cache_fields; + cache->never_cache_fields = never_cache_fields; +} + +int mail_cache_reset(struct mail_cache *cache) +{ + struct mail_cache_header hdr; + int ret, fd; + + i_assert(cache->index->lock_type == MAIL_LOCK_EXCLUSIVE); + + memset(&hdr, 0, sizeof(hdr)); + hdr.indexid = cache->index->indexid; + hdr.sync_id = cache->sync_id = cache->index->cache_sync_id = + ++cache->index->hdr->cache_sync_id; + hdr.used_file_size = uint32_to_nbo(sizeof(hdr)); + cache->used_file_size = sizeof(hdr); + + fd = file_dotlock_open(cache->filepath, NULL, MAIL_CACHE_LOCK_TIMEOUT, + MAIL_CACHE_LOCK_CHANGE_TIMEOUT, + MAIL_CACHE_LOCK_IMMEDIATE_TIMEOUT, NULL, NULL); + if (fd == -1) { + mail_cache_set_syscall_error(cache, "file_dotlock_open()"); + return -1; + } + + if (write_full(fd, &hdr, sizeof(hdr)) < 0) { + mail_cache_set_syscall_error(cache, "write_full()"); + (void)file_dotlock_delete(cache->filepath, fd); + return -1; + } + if (file_set_size(fd, MAIL_CACHE_INITIAL_SIZE) < 0) { + mail_cache_set_syscall_error(cache, "file_set_size()"); + (void)file_dotlock_delete(cache->filepath, fd); + return -1; + } + + mail_cache_file_close(cache); + cache->fd = dup(fd); + + if (file_dotlock_replace(cache->filepath, fd, FALSE) < 0) { + mail_cache_set_syscall_error(cache, "file_dotlock_replace()"); + return -1; + } + + cache->mmap_refresh = TRUE; + if (!mmap_update(cache, 0, sizeof(struct mail_cache_header))) + return -1; + + return 0; +} + +int mail_cache_lock(struct mail_cache *cache, int nonblock) +{ + int ret; + + if (cache->locks++ != 0) + return TRUE; + + if (cache->anon_mmap) + return TRUE; + + if (nonblock) { + ret = file_try_lock(cache->fd, F_WRLCK); + if (ret < 0) + mail_cache_set_syscall_error(cache, "file_try_lock()"); + } else { + ret = file_wait_lock(cache->fd, F_WRLCK); + if (ret <= 0) + mail_cache_set_syscall_error(cache, "file_wait_lock()"); + } + + if (ret > 0) { + if (!mmap_update(cache, 0, 0)) { + (void)mail_cache_unlock(cache); + return -1; + } + if (cache->sync_id != cache->index->cache_sync_id) { + /* we have the cache file locked and sync_id still + doesn't match. it means we crashed between updating + cache file and updating sync_id in index header. + just update the sync_ids so they match. */ + i_warning("Updating broken sync_id in cache file %s", + cache->filepath); + cache->sync_id = cache->hdr->sync_id = + cache->index->cache_sync_id; + } + } + return ret; +} + +int mail_cache_unlock(struct mail_cache *cache) +{ + if (--cache->locks > 0) + return TRUE; + + if (cache->anon_mmap) + return TRUE; + + if (file_wait_lock(cache->fd, F_UNLCK) <= 0) { + mail_cache_set_syscall_error(cache, "file_wait_lock(F_UNLCK)"); + return FALSE; + } + + return TRUE; +} + +int mail_cache_is_locked(struct mail_cache *cache) +{ + return cache->locks > 0; +} + +struct mail_cache_view * +mail_cache_view_open(struct mail_cache *cache, struct mail_index_view *view) +{ + struct mail_cache_view *view; + + view = i_new(struct mail_cache_view, 1); + view->cache = cache; + view->view = view; + return view; +} + +void mail_cache_view_close(struct mail_cache_view *view) +{ + i_free(view); +} + +static const char * +mail_cache_get_header_fields_str(struct mail_cache *cache, unsigned int idx) +{ + uint32_t offset, data_size; + unsigned char *buf; + + offset = offset_to_uint32(cache->hdr->header_offsets[idx]); + + if (offset == 0) + return NULL; + + if (!mmap_update(cache, offset, 1024)) + return NULL; + + if (offset + sizeof(data_size) > cache->mmap_length) { + mail_cache_set_corrupted(cache, "Header %u points outside file", + idx); + return NULL; + } + + buf = cache->mmap_base; + memcpy(&data_size, buf + offset, sizeof(data_size)); + data_size = nbo_to_uint32(data_size); + offset += sizeof(data_size); + + if (data_size == 0) { + mail_cache_set_corrupted(cache, + "Header %u points to empty string", idx); + return NULL; + } + + if (!mmap_update(cache, offset, data_size)) + return NULL; + + if (offset + data_size > cache->mmap_length) { + mail_cache_set_corrupted(cache, "Header %u points outside file", + idx); + return NULL; + } + + buf = cache->mmap_base; + if (buf[offset + data_size - 1] != '\0') { + mail_cache_set_corrupted(cache, + "Header %u points to invalid string", idx); + return NULL; + } + + return buf + offset; +} + +static const char *const * +split_header(struct mail_cache *cache, const char *header) +{ + const char *const *arr, *const *tmp; + const char *null = NULL; + char *str; + buffer_t *buf; + + if (header == NULL) + return NULL; + + arr = t_strsplit(header, "\n"); + buf = buffer_create_dynamic(cache->split_header_pool, 32, (size_t)-1); + for (tmp = arr; *tmp != NULL; tmp++) { + str = p_strdup(cache->split_header_pool, *tmp); + buffer_append(buf, &str, sizeof(str)); + } + buffer_append(buf, &null, sizeof(null)); + + return buffer_get_data(buf, NULL); +} + +const char *const *mail_cache_get_header_fields(struct mail_cache *cache, + unsigned int idx) +{ + const char *str; + int i; + + i_assert(idx < MAIL_CACHE_HEADERS_COUNT); + + /* t_strsplit() is a bit slow, so we cache it */ + if (cache->hdr->header_offsets[idx] != cache->split_offsets[idx]) { + p_clear(cache->split_header_pool); + + t_push(); + for (i = 0; i < MAIL_CACHE_HEADERS_COUNT; i++) { + cache->split_offsets[i] = + cache->hdr->header_offsets[i]; + + str = mail_cache_get_header_fields_str(cache, i); + cache->split_headers[i] = split_header(cache, str); + } + t_pop(); + } + + return cache->split_headers[idx]; +} + +static const char *write_header_string(const char *const headers[], + uint32_t *size_r) +{ + buffer_t *buffer; + size_t size; + + buffer = buffer_create_dynamic(pool_datastack_create(), + 512, (size_t)-1); + + while (*headers != NULL) { + if (buffer_get_used_size(buffer) != 0) + buffer_append(buffer, "\n", 1); + buffer_append(buffer, *headers, strlen(*headers)); + headers++; + } + buffer_append(buffer, null4, 1); + + size = buffer_get_used_size(buffer); + if ((size & 3) != 0) { + buffer_append(buffer, null4, 4 - (size & 3)); + size += 4 - (size & 3); + } + *size_r = size; + return buffer_get_data(buffer, NULL); +} + +int mail_cache_set_header_fields(struct mail_cache_transaction_ctx *ctx, + unsigned int idx, const char *const headers[]) +{ + struct mail_cache *cache = ctx->cache; + uint32_t offset, update_offset, size; + const char *header_str, *prev_str; + + i_assert(*headers != NULL); + i_assert(idx < MAIL_CACHE_HEADERS_COUNT); + i_assert(idx >= ctx->next_unused_header_lowwater); + i_assert(offset_to_uint32(cache->hdr->header_offsets[idx]) == 0); + + t_push(); + + header_str = write_header_string(headers, &size); + if (idx != 0) { + prev_str = mail_cache_get_header_fields_str(cache, idx-1); + if (prev_str == NULL) { + t_pop(); + return FALSE; + } + + i_assert(strcmp(header_str, prev_str) != 0); + } + + offset = mail_cache_append_space(ctx, size + sizeof(uint32_t)); + if (offset != 0) { + memcpy((char *) cache->mmap_base + offset + sizeof(uint32_t), + header_str, size); + + size = uint32_to_nbo(size); + memcpy((char *) cache->mmap_base + offset, + &size, sizeof(uint32_t)); + + /* update cached headers */ + cache->split_offsets[idx] = cache->hdr->header_offsets[idx]; + cache->split_headers[idx] = split_header(cache, header_str); + + /* mark used-bit to be updated later. not really needed for + read-safety, but if transaction get rolled back we can't let + this point to invalid location. */ + update_offset = (char *) &cache->hdr->header_offsets[idx] - + (char *) cache->mmap_base; + mark_update(&ctx->cache_marks, update_offset, + uint32_to_offset(offset)); + + /* make sure get_header_fields() still works for this header + while the transaction isn't yet committed. */ + ctx->next_unused_header_lowwater = idx + 1; + } + + t_pop(); + return offset > 0; +} + +static struct mail_cache_record * +cache_get_record(struct mail_cache *cache, uint32_t offset) +{ +#define CACHE_PREFETCH 1024 + struct mail_cache_record *cache_rec; + size_t size; + + offset = offset_to_uint32(offset); + if (offset == 0) + return NULL; + + if (!mmap_update(cache, offset, sizeof(*cache_rec) + CACHE_PREFETCH)) + return NULL; + + if (offset + sizeof(*cache_rec) > cache->mmap_length) { + mail_cache_set_corrupted(cache, "record points outside file"); + return NULL; + } + cache_rec = CACHE_RECORD(cache, offset); + + size = nbo_to_uint32(cache_rec->size); + if (size < sizeof(*cache_rec)) { + mail_cache_set_corrupted(cache, "invalid record size"); + return NULL; + } + if (size > CACHE_PREFETCH) { + if (!mmap_update(cache, offset, size)) + return NULL; + } + + if (offset + size > cache->mmap_length) { + mail_cache_set_corrupted(cache, "record points outside file"); + return NULL; + } + return cache_rec; +} + +static struct mail_cache_record * +cache_get_next_record(struct mail_cache *cache, struct mail_cache_record *rec) +{ + struct mail_cache_record *next; + + next = cache_get_record(cache, rec->next_offset); + if (next != NULL && next <= rec) { + mail_cache_set_corrupted(cache, "next_offset points backwards"); + return NULL; + } + return next; +} + +static struct mail_cache_record * +mail_cache_lookup(struct mail_cache *cache, const struct mail_index_record *rec, + enum mail_cache_field fields) +{ + struct mail_cache_record *cache_rec; + unsigned int idx; + + if (cache->trans_ctx != NULL && + cache->trans_ctx->first_uid <= rec->uid && + cache->trans_ctx->last_uid >= rec->uid && + (cache->trans_ctx->prev_uid != rec->uid || fields == 0 || + (cache->trans_ctx->prev_fields & fields) != 0)) { + /* we have to auto-commit since we're not capable of looking + into uncommitted records. it would be possible by checking + index_marks and cache_marks, but it's just more trouble + than worth. */ + idx = INDEX_RECORD_INDEX(cache->index, rec); + if (cache->trans_ctx->last_idx == idx) { + if (!mail_cache_write(cache->trans_ctx)) + return NULL; + } + + if (!mail_cache_transaction_commit(cache->trans_ctx)) + return NULL; + } + + cache_rec = cache_get_record(cache, rec->cache_offset); + if (cache_rec == NULL) + return NULL; + + return cache_rec; +} + +enum mail_cache_field +mail_cache_get_fields(struct mail_cache *cache, + const struct mail_index_record *rec) +{ + struct mail_cache_record *cache_rec; + enum mail_cache_field fields = 0; + + cache_rec = mail_cache_lookup(cache, rec, 0); + while (cache_rec != NULL) { + fields |= cache_rec->fields; + cache_rec = cache_get_next_record(cache, cache_rec); + } + + return fields; +} + +static int cache_get_field(struct mail_cache *cache, + struct mail_cache_record *cache_rec, + enum mail_cache_field field, + void **data_r, size_t *size_r) +{ + unsigned char *buf; + unsigned int mask; + uint32_t rec_size, data_size; + size_t offset, next_offset; + int i; + + rec_size = nbo_to_uint32(cache_rec->size); + buf = (unsigned char *) cache_rec; + offset = sizeof(*cache_rec); + + for (i = 0, mask = 1; i < 31; i++, mask <<= 1) { + if ((cache_rec->fields & mask) == 0) + continue; + + /* all records are at least 32bit. we have to check this + before getting data_size. */ + if (offset + sizeof(uint32_t) > rec_size) { + mail_cache_set_corrupted(cache, + "Record continues outside it's allocated size"); + return FALSE; + } + + if ((mask & MAIL_CACHE_FIXED_MASK) != 0) + data_size = mail_cache_field_sizes[i]; + else { + memcpy(&data_size, buf + offset, sizeof(data_size)); + data_size = nbo_to_uint32(data_size); + offset += sizeof(data_size); + } + + next_offset = offset + ((data_size + 3) & ~3); + if (next_offset > rec_size) { + mail_cache_set_corrupted(cache, + "Record continues outside it's allocated size"); + return FALSE; + } + + if (field == mask) { + if (data_size == 0) { + mail_cache_set_corrupted(cache, + "Field size is 0"); + return FALSE; + } + *data_r = buf + offset; + *size_r = data_size; + return TRUE; + } + offset = next_offset; + } + + i_unreached(); + return FALSE; +} + +static int cache_lookup_field(struct mail_cache *cache, + const struct mail_index_record *rec, + enum mail_cache_field field, + void **data_r, size_t *size_r) +{ + struct mail_cache_record *cache_rec; + + cache_rec = mail_cache_lookup(cache, rec, field); + while (cache_rec != NULL) { + if ((cache_rec->fields & field) != 0) { + return cache_get_field(cache, cache_rec, field, + data_r, size_r); + } + cache_rec = cache_get_next_record(cache, cache_rec); + } + + return FALSE; +} + +int mail_cache_lookup_field(struct mail_cache *cache, + const struct mail_index_record *rec, + enum mail_cache_field field, + const void **data_r, size_t *size_r) +{ + void *data; + + if (!cache_lookup_field(cache, rec, field, &data, size_r)) + return FALSE; + + *data_r = data; + return TRUE; +} + +const char *mail_cache_lookup_string_field(struct mail_cache *cache, + const struct mail_index_record *rec, + enum mail_cache_field field) +{ + const void *data; + size_t size; + + i_assert((field & MAIL_CACHE_STRING_MASK) != 0); + + if (!mail_cache_lookup_field(cache, rec, field, &data, &size)) + return NULL; + + if (((const char *) data)[size-1] != '\0') { + mail_cache_set_corrupted(cache, + "String field %x doesn't end with NUL", field); + return NULL; + } + return data; +} + +int mail_cache_copy_fixed_field(struct mail_cache *cache, + const struct mail_index_record *rec, + enum mail_cache_field field, + void *buffer, size_t buffer_size) +{ + const void *data; + size_t size; + + i_assert((field & MAIL_CACHE_FIXED_MASK) != 0); + + if (!mail_cache_lookup_field(cache, rec, field, &data, &size)) + return FALSE; + + if (buffer_size != size) { + i_panic("cache: fixed field %x wrong size " + "(%"PRIuSIZE_T" vs %"PRIuSIZE_T")", + field, size, buffer_size); + } + + memcpy(buffer, data, buffer_size); + return TRUE; +} + +void mail_cache_mark_missing(struct mail_cache *cache, + enum mail_cache_field fields) +{ + // FIXME: count these +} + +enum mail_index_record_flag +mail_cache_get_index_flags(struct mail_cache *cache, + const struct mail_index_record *rec) +{ + enum mail_index_record_flag flags; + + if (!mail_cache_copy_fixed_field(cache, rec, MAIL_CACHE_INDEX_FLAGS, + &flags, sizeof(flags))) + return 0; + + return flags; +} + +int mail_cache_update_index_flags(struct mail_cache *cache, + const struct mail_index_record *rec, + enum mail_index_record_flag flags) +{ + void *data; + size_t size; + + i_assert(cache->locks > 0); + + if (!cache_lookup_field(cache, rec, MAIL_CACHE_INDEX_FLAGS, + &data, &size)) { + mail_cache_set_corrupted(cache, + "Missing index flags for record %u", rec->uid); + return FALSE; + } + + memcpy(data, &flags, sizeof(flags)); + return TRUE; +} + +int mail_cache_update_location_offset(struct mail_cache *cache, + const struct mail_index_record *rec, + uoff_t offset) +{ + void *data; + size_t size; + + i_assert(cache->locks > 0); + + if (!cache_lookup_field(cache, rec, MAIL_CACHE_LOCATION_OFFSET, + &data, &size)) { + mail_cache_set_corrupted(cache, + "Missing location offset for record %u", rec->uid); + return FALSE; + } + + memcpy(data, &offset, sizeof(offset)); + return TRUE; +} + +void *mail_cache_get_mmaped(struct mail_cache *cache, size_t *size) +{ + if (!mmap_update(cache, 0, 0)) + return NULL; + + *size = cache->mmap_length; + return cache->mmap_base; +}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-index/mail-cache-private.h Tue Apr 27 23:25:52 2004 +0300 @@ -0,0 +1,114 @@ +#ifndef __MAIL_CACHE_PRIVATE_H +#define __MAIL_CACHE_PRIVATE_H + +#include "mail-index-private.h" +#include "mail-cache.h" + +/* Never compress the file if it's smaller than this */ +#define COMPRESS_MIN_SIZE (1024*50) + +/* Compress the file when deleted space reaches n% of total size */ +#define COMPRESS_PERCENTAGE 20 + +/* Compress the file when n% of rows contain continued rows. + 200% means that there's 2 continued rows per record. */ +#define COMPRESS_CONTINUED_PERCENTAGE 200 + +/* Initial size for the file */ +#define MAIL_CACHE_INITIAL_SIZE (sizeof(struct mail_cache_header) + 10240) + +/* When more space is needed, grow the file n% larger than the previous size */ +#define MAIL_CACHE_GROW_PERCENTAGE 10 + +#define MAIL_CACHE_LOCK_TIMEOUT 120 +#define MAIL_CACHE_LOCK_CHANGE_TIMEOUT 60 +#define MAIL_CACHE_LOCK_IMMEDIATE_TIMEOUT (5*60) + +#define CACHE_RECORD(cache, offset) \ + ((struct mail_cache_record *) ((char *) (cache)->mmap_base + offset)) + +struct mail_cache_header { + uint32_t indexid; + uint32_t file_seq; + + uint32_t continued_record_count; + + uint32_t used_file_size; + uint32_t deleted_space; + + uint32_t used_fields; /* enum mail_cache_field */ + + uint32_t field_usage_start; /* time_t */ + uint32_t field_usage_counts[32]; + + uint32_t header_offsets[MAIL_CACHE_HEADERS_COUNT]; +}; + +struct mail_cache_record { + uint32_t fields; /* enum mail_cache_field */ + uint32_t next_offset; + uint32_t size; /* full record size, including this header */ +}; + +struct mail_cache { + struct mail_index *index; + + char *filepath; + int fd; + + void *mmap_base; + size_t mmap_length; + uint32_t used_file_size; + + struct mail_cache_header *hdr; + + pool_t split_header_pool; + uint32_t split_offsets[MAIL_CACHE_HEADERS_COUNT]; + const char *const *split_headers[MAIL_CACHE_HEADERS_COUNT]; + + enum mail_cache_field default_cache_fields; + enum mail_cache_field never_cache_fields; + + struct mail_cache_transaction_ctx *trans_ctx; + unsigned int locks; + + unsigned int mmap_refresh:1; + unsigned int silent:1; +}; + +struct mail_cache_view { + struct mail_cache *cache; + struct mail_index_view *view; + + unsigned int broken:1; +}; + +extern unsigned int mail_cache_field_sizes[32]; +extern enum mail_cache_field mail_cache_header_fields[MAIL_CACHE_HEADERS_COUNT]; + +uint32_t mail_cache_uint32_to_offset(uint32_t offset); +uint32_t mail_cache_offset_to_uint32(uint32_t offset); + +const char * +mail_cache_get_header_fields_str(struct mail_cache *cache, unsigned int idx); +const char *const * +mail_cache_split_header(struct mail_cache *cache, const char *header); + +struct mail_cache_record * +mail_cache_get_record(struct mail_cache *cache, uint32_t offset); +struct mail_cache_record * +mail_cache_get_next_record(struct mail_cache *cache, + struct mail_cache_record *rec); + +struct mail_cache_record * +mail_cache_lookup(struct mail_cache_view *view, uint32_t seq, + enum mail_cache_field fields); + +int +mail_cache_transaction_autocommit(struct mail_cache_view *view, + uint32_t seq, enum mail_cache_field fields); + +void mail_cache_set_syscall_error(struct mail_cache *cache, + const char *function); + +#endif
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-index/mail-cache-transaction.c Tue Apr 27 23:25:52 2004 +0300 @@ -0,0 +1,562 @@ +/* Copyright (C) 2003-2004 Timo Sirainen */ + +#include "lib.h" +#include "buffer.h" +#include "byteorder.h" +#include "file-set-size.h" +#include "mmap-util.h" +#include "mail-cache-private.h" + +#include <sys/stat.h> + +#if 0 +struct mail_cache_transaction_ctx { + struct mail_cache *cache; + struct mail_cache_view *view; + struct mail_index_transaction *trans; + + unsigned int next_unused_header_lowwater; + + struct mail_cache_record cache_rec; + buffer_t *cache_data; + + uint32_t first_seq, last_seq, prev_seq; + enum mail_cache_field prev_fields; + buffer_t *cache_marks; +}; + +static const unsigned char *null4[] = { 0, 0, 0, 0 }; + +int mail_cache_transaction_begin(struct mail_cache_view *view, int nonblock, + struct mail_index_transaction *t, + struct mail_cache_transaction_ctx **ctx_r) +{ + struct mail_cache_transaction_ctx *ctx; + int ret; + + i_assert(view->cache->trans_ctx == NULL); + + ret = mail_cache_lock(view->cache, nonblock); + if (ret <= 0) + return ret; + + ctx = i_new(struct mail_cache_transaction_ctx, 1); + ctx->cache = view->cache; + ctx->view = view; + ctx->trans = t; + ctx->cache_data = buffer_create_dynamic(system_pool, 8192, (size_t)-1); + + view->cache->trans_ctx = ctx; + *ctx_r = ctx; + return 1; +} + +int mail_cache_transaction_end(struct mail_cache_transaction_ctx *ctx) +{ + int ret = 0; + + i_assert(ctx->cache->trans_ctx != NULL); + + (void)mail_cache_transaction_rollback(ctx); + + if (mail_cache_unlock(ctx->cache) < 0) + ret = -1; + + ctx->cache->trans_ctx = NULL; + + if (ctx->cache_marks != NULL) + buffer_free(ctx->cache_marks); + buffer_free(ctx->cache_data); + i_free(ctx); + return ret; +} + +static void mail_cache_transaction_flush(struct mail_cache_transaction_ctx *ctx) +{ + memset(&ctx->cache_rec, 0, sizeof(ctx->cache_rec)); + + ctx->next_unused_header_lowwater = 0; + ctx->first_seq = ctx->last_seq = ctx->prev_seq = 0; + ctx->prev_fields = 0; + + if (ctx->cache_marks != NULL) + buffer_set_used_size(ctx->cache_marks, 0); + buffer_set_used_size(ctx->cache_data, 0); +} + +static void mark_update(buffer_t **buf, uint32_t offset, uint32_t data) +{ + if (*buf == NULL) + *buf = buffer_create_dynamic(system_pool, 1024, (size_t)-1); + + buffer_append(*buf, &offset, sizeof(offset)); + buffer_append(*buf, &data, sizeof(data)); +} + +static int write_mark_updates(struct mail_cache *cache) +{ + const uint32_t *data, *end; + size_t size; + + data = buffer_get_data(cache->trans_ctx->cache_marks, &size); + end = data + size/sizeof(uint32_t); + + while (data < end) { + if (pwrite(cache->fd, data+1, sizeof(*data), data[0]) < 0) { + mail_cache_set_syscall_error(cache, "pwrite()"); + return -1; + } + data += 2; + } + return 0; +} + +static int commit_all_changes(struct mail_cache_transaction_ctx *ctx) +{ + struct mail_cache *cache = ctx->cache; + uint32_t cont; + + /* write everything to disk */ + if (msync(cache->mmap_base, cache->mmap_length, MS_SYNC) < 0) { + mail_cache_set_syscall_error(cache, "msync()"); + return -1; + } + + if (fdatasync(cache->fd) < 0) { + mail_cache_set_syscall_error(cache, "fdatasync()"); + return -1; + } + + if (ctx->cache_marks == NULL || + buffer_get_used_size(ctx->cache_marks) == 0) + return 0; + + /* now that we're sure it's written, set on all the used-bits */ + if (write_mark_updates(cache) < 0) + return -1; + + /* update continued records count */ + cont = nbo_to_uint32(cache->hdr->continued_record_count); + cont += buffer_get_used_size(ctx->cache_marks) / + (sizeof(uint32_t) * 2); + + if (cont * 100 / cache->index->hdr->messages_count >= + COMPRESS_CONTINUED_PERCENTAGE && + cache->used_file_size >= COMPRESS_MIN_SIZE) { + /* too many continued rows, compress */ + //FIXME:cache->index->set_flags |= MAIL_INDEX_HDR_FLAG_COMPRESS_CACHE; + } + + cache->hdr->continued_record_count = uint32_to_nbo(cont); + return 0; +} + +static int mail_cache_grow(struct mail_cache *cache, uint32_t size) +{ + struct stat st; + uoff_t grow_size, new_fsize; + + new_fsize = cache->used_file_size + size; + grow_size = new_fsize / 100 * MAIL_CACHE_GROW_PERCENTAGE; + if (grow_size < 16384) + grow_size = 16384; + + new_fsize += grow_size; + new_fsize &= ~1023; + + if (fstat(cache->fd, &st) < 0) { + mail_cache_set_syscall_error(cache, "fstat()"); + return -1; + } + + if (cache->used_file_size + size <= (uoff_t)st.st_size) { + /* no need to grow, just update mmap */ + if (mmap_update(cache, 0, 0) < 0) + return -1; + + i_assert(cache->mmap_length >= (uoff_t)st.st_size); + return 0; + } + + if (file_set_size(cache->fd, (off_t)new_fsize) < 0) { + mail_cache_set_syscall_error(cache, "file_set_size()"); + return -1; + } + + return mmap_update(cache, 0, 0); +} + +static uint32_t mail_cache_append_space(struct mail_cache_transaction_ctx *ctx, + uint32_t size) +{ + /* NOTE: must be done within transaction or rollback would break it */ + uint32_t offset; + + i_assert((size & 3) == 0); + + offset = ctx->cache->used_file_size; + if (offset >= 0x40000000) { + mail_index_set_error(ctx->cache->index, + "Cache file too large: %s", + ctx->cache->filepath); + return 0; + } + + if (offset + size > ctx->cache->mmap_length) { + if (mail_cache_grow(ctx->cache, size) < 0) + return 0; + } + + ctx->cache->used_file_size += size; + return offset; +} + +static int mail_cache_write(struct mail_cache_transaction_ctx *ctx) +{ + struct mail_cache *cache = ctx->cache; + struct mail_cache_record *cache_rec, *next; + const struct mail_index_record *rec; + uint32_t write_offset, update_offset; + const void *buf; + size_t size, buf_size; + + buf = buffer_get_data(ctx->cache_data, &buf_size); + + size = sizeof(*cache_rec) + buf_size; + ctx->cache_rec.size = uint32_to_nbo(size); + + write_offset = mail_cache_append_space(ctx, size); + if (write_offset == 0) + return -1; + + // FIXME: check cache_offset in transaction + if (mail_index_lookup_latest(ctx->view->view, ctx->prev_seq, &rec) < 0) + return -1; + + cache_rec = mail_cache_get_record(cache, rec->cache_offset); + if (cache_rec == NULL) { + /* first cache record - update offset in index file */ + mail_index_update_cache(ctx->trans, ctx->prev_seq, + write_offset); + } else { + /* find the last cache record */ + while ((next = mail_cache_get_next_record(cache, + cache_rec)) != NULL) + cache_rec = next; + + /* mark next_offset to be updated later */ + update_offset = (char *) &cache_rec->next_offset - + (char *) cache->mmap_base; + mark_update(&ctx->cache_marks, update_offset, + mail_cache_uint32_to_offset(write_offset)); + } + ctx->prev_seq = 0; + ctx->prev_fields = 0; + + memcpy((char *) cache->mmap_base + write_offset, + &ctx->cache_rec, sizeof(ctx->cache_rec)); + memcpy((char *) cache->mmap_base + write_offset + + sizeof(ctx->cache_rec), buf, buf_size); + + /* reset the write context */ + memset(&ctx->cache_rec, 0, sizeof(ctx->cache_rec)); + buffer_set_used_size(ctx->cache_data, 0); + return 0; +} + +int mail_cache_transaction_commit(struct mail_cache_transaction_ctx *ctx) +{ + int ret = 0; + + if (ctx->prev_seq != 0) { + if (mail_cache_write(ctx) < 0) + return -1; + } + + ctx->cache->hdr->used_file_size = + uint32_to_nbo(ctx->cache->used_file_size); + + if (commit_all_changes(ctx) < 0) + ret = -1; + + if (ctx->next_unused_header_lowwater == MAIL_CACHE_HEADERS_COUNT) { + /* they're all used - compress the cache to get more */ + /* FIXME: ctx->cache->index->set_flags |= + MAIL_INDEX_HDR_FLAG_COMPRESS_CACHE;*/ + } + + mail_cache_transaction_flush(ctx); + return ret; +} + +void mail_cache_transaction_rollback(struct mail_cache_transaction_ctx *ctx) +{ + struct mail_cache *cache = ctx->cache; + unsigned int i; + + /* no need to actually modify the file - we just didn't update + used_file_size */ + cache->used_file_size = nbo_to_uint32(cache->hdr->used_file_size); + + /* make sure we don't cache the headers */ + for (i = 0; i < ctx->next_unused_header_lowwater; i++) { + uint32_t offset = cache->hdr->header_offsets[i]; + if (mail_cache_offset_to_uint32(offset) == 0) + cache->split_offsets[i] = 1; + } + + mail_cache_transaction_flush(ctx); +} + +static const char *write_header_string(const char *const headers[], + uint32_t *size_r) +{ + buffer_t *buffer; + size_t size; + + buffer = buffer_create_dynamic(pool_datastack_create(), + 512, (size_t)-1); + + while (*headers != NULL) { + if (buffer_get_used_size(buffer) != 0) + buffer_append(buffer, "\n", 1); + buffer_append(buffer, *headers, strlen(*headers)); + headers++; + } + buffer_append(buffer, null4, 1); + + size = buffer_get_used_size(buffer); + if ((size & 3) != 0) { + buffer_append(buffer, null4, 4 - (size & 3)); + size += 4 - (size & 3); + } + *size_r = size; + return buffer_get_data(buffer, NULL); +} + +int mail_cache_set_header_fields(struct mail_cache_transaction_ctx *ctx, + unsigned int idx, const char *const headers[]) +{ + struct mail_cache *cache = ctx->cache; + uint32_t offset, update_offset, size; + const char *header_str, *prev_str; + + i_assert(*headers != NULL); + i_assert(idx < MAIL_CACHE_HEADERS_COUNT); + i_assert(idx >= ctx->next_unused_header_lowwater); + i_assert(mail_cache_offset_to_uint32(cache->hdr-> + header_offsets[idx]) == 0); + + t_push(); + + header_str = write_header_string(headers, &size); + if (idx != 0) { + prev_str = mail_cache_get_header_fields_str(cache, idx-1); + if (prev_str == NULL) { + t_pop(); + return FALSE; + } + + i_assert(strcmp(header_str, prev_str) != 0); + } + + offset = mail_cache_append_space(ctx, size + sizeof(uint32_t)); + if (offset != 0) { + memcpy((char *) cache->mmap_base + offset + sizeof(uint32_t), + header_str, size); + + size = uint32_to_nbo(size); + memcpy((char *) cache->mmap_base + offset, + &size, sizeof(uint32_t)); + + /* update cached headers */ + cache->split_offsets[idx] = cache->hdr->header_offsets[idx]; + cache->split_headers[idx] = + mail_cache_split_header(cache, header_str); + + /* mark used-bit to be updated later. not really needed for + read-safety, but if transaction get rolled back we can't let + this point to invalid location. */ + update_offset = (char *) &cache->hdr->header_offsets[idx] - + (char *) cache->mmap_base; + mark_update(&ctx->cache_marks, update_offset, + mail_cache_uint32_to_offset(offset)); + + /* make sure get_header_fields() still works for this header + while the transaction isn't yet committed. */ + ctx->next_unused_header_lowwater = idx + 1; + } + + t_pop(); + return offset > 0; +} + +static size_t get_insert_offset(struct mail_cache_transaction_ctx *ctx, + enum mail_cache_field field) +{ + const unsigned char *buf; + unsigned int mask; + uint32_t data_size; + size_t offset = 0; + int i; + + buf = buffer_get_data(ctx->cache_data, NULL); + + for (i = 0, mask = 1; i < 31; i++, mask <<= 1) { + if ((field & mask) != 0) + return offset; + + if ((ctx->cache_rec.fields & mask) != 0) { + if ((mask & MAIL_CACHE_FIXED_MASK) != 0) + data_size = mail_cache_field_sizes[i]; + else { + memcpy(&data_size, buf + offset, + sizeof(data_size)); + data_size = nbo_to_uint32(data_size); + offset += sizeof(data_size); + } + offset += (data_size + 3) & ~3; + } + } + + i_unreached(); + return offset; +} + +static int get_field_num(enum mail_cache_field field) +{ + unsigned int mask; + int i; + + for (i = 0, mask = 1; i < 31; i++, mask <<= 1) { + if ((field & mask) != 0) + return i; + } + + return -1; +} + +int mail_cache_add(struct mail_cache_transaction_ctx *ctx, uint32_t seq, + enum mail_cache_field field, + const void *data, size_t data_size) +{ + uint32_t nb_data_size; + size_t full_size, offset; + unsigned char *buf; + int field_num; + + i_assert(data_size > 0); + i_assert(data_size < (uint32_t)-1); + + nb_data_size = uint32_to_nbo((uint32_t)data_size); + + if ((field & MAIL_CACHE_FIXED_MASK) != 0) { + field_num = get_field_num(field); + i_assert(field_num != -1); + i_assert(mail_cache_field_sizes[field_num] == data_size); + } else if ((field & MAIL_CACHE_STRING_MASK) != 0) { + i_assert(((char *) data)[data_size-1] == '\0'); + } + + if (ctx->prev_seq != seq && ctx->prev_seq != 0) { + if (mail_cache_write(ctx) < 0) + return -1; + } + ctx->prev_seq = seq; + + i_assert((ctx->cache_rec.fields & field) == 0); + + full_size = (data_size + 3) & ~3; + if ((field & MAIL_CACHE_FIXED_MASK) == 0) + full_size += sizeof(nb_data_size); + + /* fields must be ordered. find where to insert it. */ + if (field > ctx->cache_rec.fields) + buf = buffer_append_space_unsafe(ctx->cache_data, full_size); + else { + offset = get_insert_offset(ctx, field); + buffer_copy(ctx->cache_data, offset + full_size, + ctx->cache_data, offset, (size_t)-1); + buf = buffer_get_space_unsafe(ctx->cache_data, + offset, full_size); + } + ctx->cache_rec.fields |= field; + + /* @UNSAFE */ + if ((field & MAIL_CACHE_FIXED_MASK) == 0) { + memcpy(buf, &nb_data_size, sizeof(nb_data_size)); + buf += sizeof(nb_data_size); + } + memcpy(buf, data, data_size); buf += data_size; + if ((data_size & 3) != 0) + memset(buf, 0, 4 - (data_size & 3)); + + /* remember the transaction uid range */ + if (seq < ctx->first_seq || ctx->first_seq == 0) + ctx->first_seq = seq; + if (seq > ctx->last_seq) + ctx->last_seq = seq; + ctx->prev_fields |= field; + + return 0; +} + +int mail_cache_delete(struct mail_cache_transaction_ctx *ctx, uint32_t seq) +{ + struct mail_cache *cache = ctx->cache; + struct mail_cache_record *cache_rec; + uint32_t deleted_space; + uoff_t max_del_space; + + cache_rec = mail_cache_lookup(ctx->view, seq, 0); + if (cache_rec == NULL) + return 0; + + /* we'll only update the deleted_space in header. we can't really + do any actual deleting as other processes might still be using + the data. also it's actually useful as some index views are still + able to ask cached data from messages that have already been + expunged. */ + deleted_space = nbo_to_uint32(cache->hdr->deleted_space); + + do { + deleted_space -= nbo_to_uint32(cache_rec->size); + cache_rec = mail_cache_get_next_record(cache, cache_rec); + } while (cache_rec != NULL); + + /* see if we've reached the max. deleted space in file */ + max_del_space = cache->used_file_size / 100 * COMPRESS_PERCENTAGE; + if (deleted_space >= max_del_space && + cache->used_file_size >= COMPRESS_MIN_SIZE) { + //FIXME:cache->index->set_flags |= MAIL_INDEX_HDR_FLAG_COMPRESS_CACHE; + } + + cache->hdr->deleted_space = uint32_to_nbo(deleted_space); + return 0; +} + +int +mail_cache_transaction_autocommit(struct mail_cache_view *view, + uint32_t seq, enum mail_cache_field fields) +{ + struct mail_cache *cache = view->cache; + + if (cache->trans_ctx != NULL && + cache->trans_ctx->first_seq <= seq && + cache->trans_ctx->last_seq >= seq && + (cache->trans_ctx->prev_seq != seq || fields == 0 || + (cache->trans_ctx->prev_fields & fields) != 0)) { + /* write non-index changes */ + if (cache->trans_ctx->prev_seq == seq) { + if (mail_cache_write(cache->trans_ctx) < 0) + return -1; + } + + if (mail_cache_transaction_commit(cache->trans_ctx) < 0) + return -1; + } + + return 0; +} +#else +#endif
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-index/mail-cache.c Tue Apr 27 23:25:52 2004 +0300 @@ -0,0 +1,663 @@ +/* Copyright (C) 2003-2004 Timo Sirainen */ + +#include "lib.h" +#include "buffer.h" +#include "byteorder.h" +#include "file-lock.h" +#include "mmap-util.h" +#include "write-full.h" +#include "mail-cache-private.h" + +#include <stddef.h> +#include <unistd.h> +#include <sys/stat.h> + +unsigned int mail_cache_field_sizes[32] = { + sizeof(enum mail_index_record_flag), + sizeof(uoff_t), + 16, + sizeof(struct mail_sent_date), + sizeof(time_t), + sizeof(uoff_t), + sizeof(uoff_t), + + 0, 0, 0, 0, 0, + + /* variable sized */ + (unsigned int)-1, (unsigned int)-1, (unsigned int)-1, (unsigned int)-1, + (unsigned int)-1, (unsigned int)-1, (unsigned int)-1, (unsigned int)-1, + (unsigned int)-1, (unsigned int)-1, (unsigned int)-1, (unsigned int)-1, + (unsigned int)-1, (unsigned int)-1, (unsigned int)-1, (unsigned int)-1, + (unsigned int)-1, (unsigned int)-1, (unsigned int)-1, (unsigned int)-1 +}; + +enum mail_cache_field mail_cache_header_fields[MAIL_CACHE_HEADERS_COUNT] = { + MAIL_CACHE_HEADERS1, + MAIL_CACHE_HEADERS2, + MAIL_CACHE_HEADERS3, + MAIL_CACHE_HEADERS4 +}; + +#if 0 +uint32_t mail_cache_uint32_to_offset(uint32_t offset) +{ + unsigned char buf[4]; + + i_assert(offset < 0x40000000); + i_assert((offset & 3) == 0); + + offset >>= 2; + buf[0] = 0x80 | ((offset & 0x0fe00000) >> 21); + buf[1] = 0x80 | ((offset & 0x001fc000) >> 14); + buf[2] = 0x80 | ((offset & 0x00003f80) >> 7); + buf[3] = 0x80 | (offset & 0x0000007f); + return *((uint32_t *) buf); +} + +uint32_t mail_cache_offset_to_uint32(uint32_t offset) +{ + const unsigned char *buf = (const unsigned char *) &offset; + + if ((offset & 0x80808080) != 0x80808080) + return 0; + + return (((uint32_t)buf[3] & 0x7f) << 2) | + (((uint32_t)buf[2] & 0x7f) << 9) | + (((uint32_t)buf[1] & 0x7f) << 16) | + (((uint32_t)buf[0] & 0x7f) << 23); +} + +void mail_cache_set_syscall_error(struct mail_cache *cache, + const char *function) +{ + i_assert(function != NULL); + + if (ENOSPACE(errno)) { + cache->index->nodiskspace = TRUE; + return; + } + + mail_index_set_error(cache->index, + "%s failed with index cache file %s: %m", + function, cache->filepath); +} + +void mail_cache_set_corrupted(struct mail_cache *cache, const char *fmt, ...) +{ + va_list va; + + (void)mail_cache_reset(cache); + + if (cache->silent) + return; + + va_start(va, fmt); + t_push(); + mail_index_set_error(cache->index, "Corrupted index cache file %s: %s", + cache->filepath, t_strdup_vprintf(fmt, va)); + t_pop(); + va_end(va); +} + +static void mail_cache_file_close(struct mail_cache *cache) +{ + if (cache->mmap_base != NULL) { + if (munmap(cache->mmap_base, cache->mmap_length) < 0) + mail_cache_set_syscall_error(cache, "munmap()"); + } + + cache->mmap_base = NULL; + cache->hdr = NULL; + cache->mmap_length = 0; + + if (cache->fd != -1) { + if (close(cache->fd) < 0) + mail_cache_set_syscall_error(cache, "close()"); + cache->fd = -1; + } +} + +static int mail_cache_file_reopen(struct mail_cache *cache) +{ + int fd; + + fd = open(cache->filepath, O_RDWR); + if (fd == -1) { + mail_cache_set_syscall_error(cache, "open()"); + return -1; + } + + mail_cache_file_close(cache); + + cache->fd = fd; + return 0; +} + +static int mmap_verify_header(struct mail_cache *cache) +{ + struct mail_cache_header *hdr; + + /* check that the header is still ok */ + if (cache->mmap_length < sizeof(struct mail_cache_header)) { + mail_cache_set_corrupted(cache, "File too small"); + return 0; + } + cache->hdr = hdr = cache->mmap_base; + + if (cache->hdr->indexid != cache->index->indexid) { + /* index id changed */ + if (cache->hdr->indexid != 0) + mail_cache_set_corrupted(cache, "indexid changed"); + return 0; + } + + if (cache->trans_ctx != NULL) { + /* we've updated used_file_size, do nothing */ + return 1; + } + + cache->used_file_size = nbo_to_uint32(hdr->used_file_size); + + /* only check the header if we're locked */ + if (cache->locks == 0) + return 1; + + if (cache->used_file_size < sizeof(struct mail_cache_header)) { + mail_cache_set_corrupted(cache, "used_file_size too small"); + return 0; + } + if ((cache->used_file_size % sizeof(uint32_t)) != 0) { + mail_cache_set_corrupted(cache, "used_file_size not aligned"); + return 0; + } + + if (cache->used_file_size > cache->mmap_length) { + /* maybe a crash truncated the file - just fix it */ + hdr->used_file_size = uint32_to_nbo(cache->mmap_length & ~3); + if (msync(cache->mmap_base, sizeof(*hdr), MS_SYNC) < 0) { + mail_cache_set_syscall_error(cache, "msync()"); + return -1; + } + } + return 1; +} + +static int mmap_update_nocheck(struct mail_cache *cache, + size_t offset, size_t size) +{ + struct stat st; + + /* if sequence has changed, the file has to be reopened. + note that if main index isn't locked, it may change again */ + if (cache->hdr->file_seq != cache->index->hdr->cache_file_seq && + cache->mmap_base != NULL) { + if (!mail_cache_file_reopen(cache)) + return -1; + } + + if (offset < cache->mmap_length && + size <= cache->mmap_length - offset && + !cache->mmap_refresh) { + /* already mapped */ + if (size != 0) + return 1; + + /* requesting the whole file - see if we need to + re-mmap */ + if (fstat(cache->fd, &st) < 0) { + mail_cache_set_syscall_error(cache, "fstat()"); + return -1; + } + if ((uoff_t)st.st_size == cache->mmap_length) + return 1; + } + cache->mmap_refresh = FALSE; + + if (cache->mmap_base != NULL) { + if (cache->locks != 0) { + /* in the middle of transaction - write the changes */ + if (msync(cache->mmap_base, cache->mmap_length, + MS_SYNC) < 0) { + mail_cache_set_syscall_error(cache, "msync()"); + return -1; + } + } + + if (munmap(cache->mmap_base, cache->mmap_length) < 0) + mail_cache_set_syscall_error(cache, "munmap()"); + } + + i_assert(cache->fd != -1); + + /* map the whole file */ + cache->hdr = NULL; + cache->mmap_length = 0; + + cache->mmap_base = mmap_rw_file(cache->fd, &cache->mmap_length); + if (cache->mmap_base == MAP_FAILED) { + cache->mmap_base = NULL; + mail_cache_set_syscall_error(cache, "mmap()"); + return -1; + } + + /* re-mmaped, check header */ + return 0; +} + +static int mmap_update(struct mail_cache *cache, size_t offset, size_t size) +{ + int synced, ret; + + for (synced = FALSE;; synced = TRUE) { + ret = mmap_update_nocheck(cache, offset, size); + if (ret > 0) + return TRUE; + if (ret < 0) + return FALSE; + + if (!mmap_verify_header(cache)) + return FALSE; + + /* see if cache file was rebuilt - do it only once to avoid + infinite looping */ + if (cache->hdr->sync_id == cache->index->cache_sync_id || + synced) + break; + + if (!mail_cache_file_reopen(cache)) + return FALSE; + } + return TRUE; +} + +static int mail_cache_open_and_verify(struct mail_cache *cache, int silent) +{ + struct stat st; + + mail_cache_file_close(cache); + + cache->fd = open(cache->filepath, O_RDWR); + if (cache->fd == -1) { + if (errno == ENOENT) + return 0; + + mail_cache_set_syscall_error(cache, "open()"); + return -1; + } + + if (fstat(cache->fd, &st) < 0) { + mail_cache_set_syscall_error(cache, "fstat()"); + return -1; + } + + if (st.st_size < sizeof(struct mail_cache_header)) + return 0; + + cache->mmap_refresh = TRUE; + if (mmap_update_nocheck(cache, 0, sizeof(struct mail_cache_header)) < 0) + return -1; + + /* verify that this really is the cache for wanted index */ + cache->silent = silent; + if (!mmap_verify_header(cache)) { + cache->silent = FALSE; + return 0; + } + + cache->silent = FALSE; + return 1; +} + +static int mail_cache_open_or_create_file(struct mail_cache *cache, + struct mail_cache_header *hdr) +{ + int ret, fd; + + cache->filepath = i_strconcat(cache->index->filepath, + MAIL_CACHE_FILE_PREFIX, NULL); + + ret = mail_cache_open_and_verify(cache, FALSE); + if (ret != 0) + return ret > 0; + + /* we'll have to clear cache_offsets which requires exclusive lock */ + if (!mail_index_set_lock(cache->index, MAIL_LOCK_EXCLUSIVE)) + return FALSE; + + /* maybe a rebuild.. */ + fd = file_dotlock_open(cache->filepath, NULL, MAIL_CACHE_LOCK_TIMEOUT, + MAIL_CACHE_LOCK_CHANGE_TIMEOUT, + MAIL_CACHE_LOCK_IMMEDIATE_TIMEOUT, NULL, NULL); + if (fd == -1) { + mail_cache_set_syscall_error(cache, "file_dotlock_open()"); + return FALSE; + } + + /* see if someone else just created the cache file */ + ret = mail_cache_open_and_verify(cache, TRUE); + if (ret != 0) { + (void)file_dotlock_delete(cache->filepath, fd); + return ret > 0; + } + + /* rebuild then */ + if (write_full(fd, hdr, sizeof(*hdr)) < 0) { + mail_cache_set_syscall_error(cache, "write_full()"); + (void)file_dotlock_delete(cache->filepath, fd); + return FALSE; + } + if (file_set_size(fd, MAIL_CACHE_INITIAL_SIZE) < 0) { + mail_cache_set_syscall_error(cache, "file_set_size()"); + (void)file_dotlock_delete(cache->filepath, fd); + return FALSE; + } + + if (cache->index->hdr.cache_file_seq != 0) { + // FIXME: recreate index file with cache_offsets cleared + } + + mail_cache_file_close(cache); + cache->fd = dup(fd); + + if (file_dotlock_replace(cache->filepath, fd, FALSE) < 0) { + mail_cache_set_syscall_error(cache, "file_dotlock_replace()"); + return FALSE; + } + + if (!mmap_update(cache, 0, sizeof(struct mail_cache_header))) + return FALSE; + + return TRUE; +} + +int mail_cache_open_or_create(struct mail_index *index) +{ + struct mail_cache_header hdr; + struct mail_cache *cache; + + memset(&hdr, 0, sizeof(hdr)); + hdr.indexid = index->indexid; + hdr.sync_id = index->hdr->cache_file_seq; // FIXME + hdr.used_file_size = uint32_to_nbo(sizeof(hdr)); + + cache = i_new(struct mail_cache, 1); + cache->index = index; + cache->fd = -1; + cache->split_header_pool = pool_alloconly_create("Headers", 512); + + index->cache = cache; + + /* we'll do anon-mmaping only if initially requested. if we fail + because of out of disk space, we'll just let the main index code + know it and fail. */ + if (!mail_cache_open_or_create_file(cache, &hdr)) { + mail_cache_free(cache); + return FALSE; + } + + return TRUE; +} + +void mail_cache_free(struct mail_cache *cache) +{ + i_assert(cache->trans_ctx == NULL); + + cache->index->cache = NULL; + + mail_cache_file_close(cache); + + pool_unref(cache->split_header_pool); + i_free(cache->filepath); + i_free(cache); +} + +void mail_cache_set_defaults(struct mail_cache *cache, + enum mail_cache_field default_cache_fields, + enum mail_cache_field never_cache_fields) +{ + cache->default_cache_fields = default_cache_fields; + cache->never_cache_fields = never_cache_fields; +} + +int mail_cache_reset(struct mail_cache *cache) +{ + struct mail_cache_header hdr; + int ret, fd; + + i_assert(cache->index->lock_type == MAIL_LOCK_EXCLUSIVE); + + memset(&hdr, 0, sizeof(hdr)); + hdr.indexid = cache->index->indexid; + hdr.sync_id = cache->sync_id = cache->index->cache_sync_id = + ++cache->index->hdr->cache_sync_id; + hdr.used_file_size = uint32_to_nbo(sizeof(hdr)); + cache->used_file_size = sizeof(hdr); + + fd = file_dotlock_open(cache->filepath, NULL, MAIL_CACHE_LOCK_TIMEOUT, + MAIL_CACHE_LOCK_CHANGE_TIMEOUT, + MAIL_CACHE_LOCK_IMMEDIATE_TIMEOUT, NULL, NULL); + if (fd == -1) { + mail_cache_set_syscall_error(cache, "file_dotlock_open()"); + return -1; + } + + if (write_full(fd, &hdr, sizeof(hdr)) < 0) { + mail_cache_set_syscall_error(cache, "write_full()"); + (void)file_dotlock_delete(cache->filepath, fd); + return -1; + } + if (file_set_size(fd, MAIL_CACHE_INITIAL_SIZE) < 0) { + mail_cache_set_syscall_error(cache, "file_set_size()"); + (void)file_dotlock_delete(cache->filepath, fd); + return -1; + } + + mail_cache_file_close(cache); + cache->fd = dup(fd); + + if (file_dotlock_replace(cache->filepath, fd, FALSE) < 0) { + mail_cache_set_syscall_error(cache, "file_dotlock_replace()"); + return -1; + } + + cache->mmap_refresh = TRUE; + if (!mmap_update(cache, 0, sizeof(struct mail_cache_header))) + return -1; + + return 0; +} + +int mail_cache_lock(struct mail_cache *cache, int nonblock) +{ + int ret; + + if (cache->locks++ != 0) + return TRUE; + + if (nonblock) { + ret = file_try_lock(cache->fd, F_WRLCK); + if (ret < 0) + mail_cache_set_syscall_error(cache, "file_try_lock()"); + } else { + ret = file_wait_lock(cache->fd, F_WRLCK); + if (ret <= 0) + mail_cache_set_syscall_error(cache, "file_wait_lock()"); + } + + if (ret > 0) { + if (!mmap_update(cache, 0, 0)) { + (void)mail_cache_unlock(cache); + return -1; + } + if (cache->sync_id != cache->index->cache_sync_id) { + /* we have the cache file locked and sync_id still + doesn't match. it means we crashed between updating + cache file and updating sync_id in index header. + just update the sync_ids so they match. */ + i_warning("Updating broken sync_id in cache file %s", + cache->filepath); + cache->sync_id = cache->hdr->sync_id = + cache->index->cache_sync_id; + } + } + return ret; +} + +int mail_cache_unlock(struct mail_cache *cache) +{ + if (--cache->locks > 0) + return TRUE; + + if (file_wait_lock(cache->fd, F_UNLCK) <= 0) { + mail_cache_set_syscall_error(cache, "file_wait_lock(F_UNLCK)"); + return FALSE; + } + + return TRUE; +} + +int mail_cache_is_locked(struct mail_cache *cache) +{ + return cache->locks > 0; +} + +struct mail_cache_view * +mail_cache_view_open(struct mail_cache *cache, struct mail_index_view *iview) +{ + struct mail_cache_view *view; + + view = i_new(struct mail_cache_view, 1); + view->cache = cache; + view->view = iview; + return view; +} + +void mail_cache_view_close(struct mail_cache_view *view) +{ + i_free(view); +} +#else + +int mail_cache_open_or_create(struct mail_index *index) +{ + return 0; +} + +void mail_cache_free(struct mail_cache *cache) +{ +} + +void mail_cache_set_defaults(struct mail_cache *cache, + enum mail_cache_field default_cache_fields, + enum mail_cache_field never_cache_fields) {} + +/* Compress cache file. */ +int mail_cache_compress(struct mail_cache *cache) {return 0;} + +/* Reset the cache file, clearing all data. */ +int mail_cache_reset(struct mail_cache *cache) {return 0;} + +/* Explicitly lock the cache file. Returns 1 if ok, 0 if nonblock is TRUE and + we couldn't immediately get a lock, or -1 if error. */ +int mail_cache_lock(struct mail_cache *cache, int nonblock) {return 0;} +int mail_cache_unlock(struct mail_cache *cache) {return 0;} + +/* Returns TRUE if cache file is locked. */ +int mail_cache_is_locked(struct mail_cache *cache) {return TRUE;} + +struct mail_cache_view * +mail_cache_view_open(struct mail_cache *cache, struct mail_index_view *iview) +{return i_new(struct mail_cache_view, 1);} +void mail_cache_view_close(struct mail_cache_view *view) {i_free(view);} + +/* Begin transaction. Cache transaction may be committed or rollbacked multiple + times. It will finish when index transaction is committed or rollbacked. + The transaction might also be partially committed automatically, so this + is kind of fake transaction, it's only purpose being optimizing writes. + Returns same as mail_cache_lock(). */ +int mail_cache_transaction_begin(struct mail_cache_view *view, int nonblock, + struct mail_index_transaction *t, + struct mail_cache_transaction_ctx **ctx_r) +{ + *ctx_r = NULL; + return 1; +} +int mail_cache_transaction_commit(struct mail_cache_transaction_ctx *ctx) +{return 0;} +void mail_cache_transaction_rollback(struct mail_cache_transaction_ctx *ctx) {} + +/* Should be called only by mail_transaction_commit/rollback: */ +int mail_cache_transaction_end(struct mail_cache_transaction_ctx *ctx) +{return 0;} + +/* Return NULL-terminated list of headers for given index, or NULL if + header index isn't used. */ +const char *const *mail_cache_get_header_fields(struct mail_cache_view *view, + unsigned int idx) +{return NULL;} +/* Set list of headers for given index. */ +int mail_cache_set_header_fields(struct mail_cache_transaction_ctx *ctx, + unsigned int idx, const char *const headers[]) +{return 0;} + +/* Add new field to given record. Updates are not allowed. Fixed size fields + must be exactly the expected size and they're converted to network byte + order in disk. */ +int mail_cache_add(struct mail_cache_transaction_ctx *ctx, uint32_t seq, + enum mail_cache_field field, + const void *data, size_t data_size) +{return 0;} + +/* Mark the given record deleted. */ +int mail_cache_delete(struct mail_cache_transaction_ctx *ctx, uint32_t seq) +{return 0;} + +/* Return all fields that are currently cached for record. */ +enum mail_cache_field +mail_cache_get_fields(struct mail_cache_view *view, uint32_t seq) {return 0;} + +/* Set data_r and size_r to point to wanted field in cache file. + Returns TRUE if field was found. If field contains multiple fields, + first one found is returned. This is mostly useful for finding headers. */ +int mail_cache_lookup_field(struct mail_cache_view *view, uint32_t seq, + enum mail_cache_field field, + const void **data_r, size_t *size_r) {return 0;} + +/* Return string field. */ +const char * +mail_cache_lookup_string_field(struct mail_cache_view *view, uint32_t seq, + enum mail_cache_field field) {return 0;} + +/* Copy fixed size field to given buffer. buffer_size must be exactly the + expected size. The result will be converted to host byte order. + Returns TRUE if field was found. */ +int mail_cache_copy_fixed_field(struct mail_cache_view *view, uint32_t seq, + enum mail_cache_field field, + void *buffer, size_t buffer_size) {return 0;} + +/* Mark given fields as missing, ie. they should be cached when possible. */ +void mail_cache_mark_missing(struct mail_cache_view *view, + enum mail_cache_field fields) {} + +/* Return index flags. */ +enum mail_index_record_flag +mail_cache_get_index_flags(struct mail_cache_view *view, uint32_t seq) +{return 0;} + +/* Update index flags. The cache file must be locked and the flags must be + already inserted to the record. */ +int mail_cache_update_index_flags(struct mail_cache_view *view, uint32_t seq, + enum mail_index_record_flag flags) +{return 0;} + +/* Update location offset. External locking is assumed to take care of locking + readers out to prevent race conditions. */ +int mail_cache_update_location_offset(struct mail_cache_view *view, + uint32_t seq, uoff_t offset) +{return 0;} + +/* "Error in index cache file %s: ...". */ +void mail_cache_set_corrupted(struct mail_cache *cache, const char *fmt, ...) +{} + +#endif
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-index/mail-cache.h Tue Apr 27 23:25:52 2004 +0300 @@ -0,0 +1,165 @@ +#ifndef __MAIL_CACHE_H +#define __MAIL_CACHE_H + +#include "mail-index.h" + +#define MAIL_CACHE_FILE_PREFIX ".cache" + +#define MAIL_CACHE_HEADERS_COUNT 4 + +struct mail_cache; +struct mail_cache_view; +struct mail_cache_transaction_ctx; + +enum mail_cache_field { + /* fixed size fields */ + MAIL_CACHE_INDEX_FLAGS = 0x00000001, + MAIL_CACHE_LOCATION_OFFSET = 0x00000002, + MAIL_CACHE_MD5 = 0x00000004, + MAIL_CACHE_SENT_DATE = 0x00000008, + MAIL_CACHE_RECEIVED_DATE = 0x00000010, + MAIL_CACHE_VIRTUAL_FULL_SIZE = 0x00000020, + MAIL_CACHE_PHYSICAL_BODY_SIZE = 0x00000040, + + /* variable sized field */ + MAIL_CACHE_HEADERS1 = 0x40000000, + MAIL_CACHE_HEADERS2 = 0x20000000, + MAIL_CACHE_HEADERS3 = 0x10000000, + MAIL_CACHE_HEADERS4 = 0x08000000, + MAIL_CACHE_LOCATION = 0x04000000, + MAIL_CACHE_BODY = 0x02000000, + MAIL_CACHE_BODYSTRUCTURE = 0x01000000, + MAIL_CACHE_ENVELOPE = 0x00800000, + MAIL_CACHE_MESSAGEPART = 0x00400000, + + MAIL_CACHE_FIXED_MASK = MAIL_CACHE_INDEX_FLAGS | + MAIL_CACHE_LOCATION_OFFSET | + MAIL_CACHE_MD5 | + MAIL_CACHE_SENT_DATE | + MAIL_CACHE_RECEIVED_DATE | + MAIL_CACHE_VIRTUAL_FULL_SIZE | + MAIL_CACHE_PHYSICAL_BODY_SIZE, + MAIL_CACHE_HEADERS_MASK = MAIL_CACHE_HEADERS1 | + MAIL_CACHE_HEADERS2 | + MAIL_CACHE_HEADERS3 | + MAIL_CACHE_HEADERS4, + MAIL_CACHE_STRING_MASK = MAIL_CACHE_HEADERS_MASK | + MAIL_CACHE_LOCATION | + MAIL_CACHE_BODY | + MAIL_CACHE_BODYSTRUCTURE | + MAIL_CACHE_ENVELOPE, + MAIL_CACHE_BODYSTRUCTURE_MASK = MAIL_CACHE_BODY | + MAIL_CACHE_BODYSTRUCTURE | + MAIL_CACHE_MESSAGEPART +}; + +struct mail_sent_date { + time_t time; + int32_t timezone; +}; + +extern enum mail_cache_field mail_cache_header_fields[MAIL_CACHE_HEADERS_COUNT]; + +int mail_cache_open_or_create(struct mail_index *index); +void mail_cache_free(struct mail_cache *cache); + +void mail_cache_set_defaults(struct mail_cache *cache, + enum mail_cache_field default_cache_fields, + enum mail_cache_field never_cache_fields); + +/* Compress cache file. */ +int mail_cache_compress(struct mail_cache *cache); + +/* Reset the cache file, clearing all data. */ +int mail_cache_reset(struct mail_cache *cache); + +/* Explicitly lock the cache file. Returns 1 if ok, 0 if nonblock is TRUE and + we couldn't immediately get a lock, or -1 if error. */ +int mail_cache_lock(struct mail_cache *cache, int nonblock); +int mail_cache_unlock(struct mail_cache *cache); + +/* Returns TRUE if cache file is locked. */ +int mail_cache_is_locked(struct mail_cache *cache); + +struct mail_cache_view * +mail_cache_view_open(struct mail_cache *cache, struct mail_index_view *iview); +void mail_cache_view_close(struct mail_cache_view *view); + +/* Begin transaction. Cache transaction may be committed or rollbacked multiple + times. It will finish when index transaction is committed or rollbacked. + The transaction might also be partially committed automatically, so this + is kind of fake transaction, it's only purpose being optimizing writes. + Returns same as mail_cache_lock(). */ +int mail_cache_transaction_begin(struct mail_cache_view *view, int nonblock, + struct mail_index_transaction *t, + struct mail_cache_transaction_ctx **ctx_r); +int mail_cache_transaction_commit(struct mail_cache_transaction_ctx *ctx); +void mail_cache_transaction_rollback(struct mail_cache_transaction_ctx *ctx); + +/* Should be called only by mail_transaction_commit/rollback: */ +int mail_cache_transaction_end(struct mail_cache_transaction_ctx *ctx); + +/* Return NULL-terminated list of headers for given index, or NULL if + header index isn't used. */ +const char *const *mail_cache_get_header_fields(struct mail_cache_view *view, + unsigned int idx); +/* Set list of headers for given index. */ +int mail_cache_set_header_fields(struct mail_cache_transaction_ctx *ctx, + unsigned int idx, const char *const headers[]); + +/* Add new field to given record. Updates are not allowed. Fixed size fields + must be exactly the expected size and they're converted to network byte + order in disk. */ +int mail_cache_add(struct mail_cache_transaction_ctx *ctx, uint32_t seq, + enum mail_cache_field field, + const void *data, size_t data_size); + +/* Mark the given record deleted. */ +int mail_cache_delete(struct mail_cache_transaction_ctx *ctx, uint32_t seq); + +/* Return all fields that are currently cached for record. */ +enum mail_cache_field +mail_cache_get_fields(struct mail_cache_view *view, uint32_t seq); + +/* Set data_r and size_r to point to wanted field in cache file. + Returns TRUE if field was found. If field contains multiple fields, + first one found is returned. This is mostly useful for finding headers. */ +int mail_cache_lookup_field(struct mail_cache_view *view, uint32_t seq, + enum mail_cache_field field, + const void **data_r, size_t *size_r); + +/* Return string field. */ +const char * +mail_cache_lookup_string_field(struct mail_cache_view *view, uint32_t seq, + enum mail_cache_field field); + +/* Copy fixed size field to given buffer. buffer_size must be exactly the + expected size. The result will be converted to host byte order. + Returns TRUE if field was found. */ +int mail_cache_copy_fixed_field(struct mail_cache_view *view, uint32_t seq, + enum mail_cache_field field, + void *buffer, size_t buffer_size); + +/* Mark given fields as missing, ie. they should be cached when possible. */ +void mail_cache_mark_missing(struct mail_cache_view *view, + enum mail_cache_field fields); + +/* Return index flags. */ +enum mail_index_record_flag +mail_cache_get_index_flags(struct mail_cache_view *view, uint32_t seq); + +/* Update index flags. The cache file must be locked and the flags must be + already inserted to the record. */ +int mail_cache_update_index_flags(struct mail_cache_view *view, uint32_t seq, + enum mail_index_record_flag flags); + +/* Update location offset. External locking is assumed to take care of locking + readers out to prevent race conditions. */ +int mail_cache_update_location_offset(struct mail_cache_view *view, + uint32_t seq, uoff_t offset); + +/* "Error in index cache file %s: ...". */ +void mail_cache_set_corrupted(struct mail_cache *cache, const char *fmt, ...) + __attr_format__(2, 3); + +#endif
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-index/mail-index-fsck.c Tue Apr 27 23:25:52 2004 +0300 @@ -0,0 +1,135 @@ +/* Copyright (C) 2004 Timo Sirainen */ + +#include "lib.h" +#include "mail-index-private.h" +#include "mail-transaction-log.h" + +static void mail_index_fsck_error(struct mail_index *index, + const char *fmt, ...) __attr_format__(2, 3); +static void mail_index_fsck_error(struct mail_index *index, + const char *fmt, ...) +{ + va_list va; + + va_start(va, fmt); + mail_index_set_error(index, "Fixed index file %s: %s", + index->filepath, t_strdup_vprintf(fmt, va)); + va_end(va); +} + +#define CHECK(field, oper) \ + if (hdr.field oper index->hdr->field) { \ + mail_index_fsck_error(index, #field" %u -> %u", \ + index->hdr->field, hdr.field); \ + } + +static int mail_index_fsck_locked(struct mail_index *index, + const char **error_r) +{ + struct mail_index_header hdr; + const struct mail_index_record *rec, *end; + uint32_t last_uid; + + *error_r = NULL; + + /* locking already does the most important sanity checks for header */ + hdr = *index->hdr; + + if (hdr.uid_validity == 0) { + *error_r = "uid_validity = 0"; + return 0; + } + + hdr.messages_count = 0; + hdr.seen_messages_count = 0; + hdr.deleted_messages_count = 0; + + hdr.first_recent_uid_lowwater = 0; + hdr.first_unseen_uid_lowwater = 0; + hdr.first_deleted_uid_lowwater = 0; + + end = index->map->records + index->map->records_count; last_uid = 0; + for (rec = index->map->records; rec != end; rec++) { + if (rec->uid <= last_uid) { + *error_r = "Record UIDs are not ordered"; + return 0; + } + + hdr.messages_count++; + if ((rec->flags & MAIL_SEEN) != 0) + hdr.seen_messages_count++; + if ((rec->flags & MAIL_DELETED) != 0) + hdr.deleted_messages_count++; + + if ((rec->flags & MAIL_INDEX_MAIL_FLAG_NONRECENT) == 0 && + hdr.first_recent_uid_lowwater == 0) + hdr.first_recent_uid_lowwater = rec->uid; + if ((rec->flags & MAIL_SEEN) == 0 && + hdr.first_unseen_uid_lowwater == 0) + hdr.first_unseen_uid_lowwater = rec->uid; + if ((rec->flags & MAIL_DELETED) != 0 && + hdr.first_deleted_uid_lowwater == 0) + hdr.first_deleted_uid_lowwater = rec->uid; + + last_uid = rec->uid; + } + + if (hdr.next_uid <= last_uid) { + mail_index_fsck_error(index, "next_uid %u -> %u", + hdr.next_uid, last_uid+1); + hdr.next_uid = last_uid+1; + } + + if (hdr.first_recent_uid_lowwater == 0) + hdr.first_recent_uid_lowwater = hdr.next_uid; + if (hdr.first_unseen_uid_lowwater == 0) + hdr.first_unseen_uid_lowwater = hdr.next_uid; + if (hdr.first_deleted_uid_lowwater == 0) + hdr.first_deleted_uid_lowwater = hdr.next_uid; + + CHECK(messages_count, !=); + CHECK(seen_messages_count, !=); + CHECK(deleted_messages_count, !=); + + CHECK(first_recent_uid_lowwater, <); + CHECK(first_unseen_uid_lowwater, <); + CHECK(first_deleted_uid_lowwater, <); + + if (mail_index_write_header(index, &hdr) < 0) + return -1; + + return 1; +} + +int mail_index_fsck(struct mail_index *index) +{ + const char *error; + unsigned int lock_id; + uint32_t file_seq; + uoff_t file_offset; + int ret; + + if (mail_transaction_log_sync_lock(index->log, &file_seq, + &file_offset) < 0) + return -1; + if (mail_index_lock_exclusive(index, 0, 0, &lock_id) < 0) { + mail_transaction_log_sync_unlock(index->log); + return -1; + } + + error = NULL; + ret = mail_index_map(index, FALSE); + if (ret >= 0) + ret = mail_index_fsck_locked(index, &error); + + mail_index_unlock(index, lock_id); + mail_transaction_log_sync_unlock(index->log); + + if (error != NULL) { + mail_index_set_error(index, "Corrupted index file %s: %s", + index->filepath, error); + } + if (ret == 0) + mail_index_reset(index); + return ret; +}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-index/mail-index-lock.c Tue Apr 27 23:25:52 2004 +0300 @@ -0,0 +1,337 @@ +/* Copyright (C) 2003-2004 Timo Sirainen */ + +/* + Locking is meant to be as transparent as possible. Anything that locks + the index must either keep it only a short time, or be prepared that the + lock is lost. + + Lock is lost in only one situation: when we try to get an exclusive lock + but we already have a shared lock. Then we'll drop all shared locks and + get the exclusive lock. + + Locking should never fail or timeout. Exclusive locks must be kept as short + time as possible. Shared locks can be long living, so if can't get exclusive + lock directly within 2 seconds, we'll replace the index file with a copy of + it. That means the shared lock holders can keep using the old file while + we're modifying the new file. + + lock_id is used to figure out if acquired lock is still valid. Shared + locks have even numbers, exclusive locks have odd numbers. The number is + increased by two every time the lock is dropped. + + mail_index_lock_shared() -> lock_id=2 + mail_index_lock_shared() -> lock_id=2 + mail_index_lock_exclusive() -> lock_id=5 (had to drop shared locks) + mail_index_lock_shared() -> lock_id=4 + + Only 4 and 5 locks are valid at this time. +*/ + +#include "lib.h" +#include "file-lock.h" +#include "write-full.h" +#include "mail-index-private.h" + +#include <stdio.h> +#include <sys/stat.h> + +static int mail_index_reopen(struct mail_index *index, int fd) +{ + int ret; + + mail_index_unmap(index, index->map); + index->map = NULL; + + if (close(index->fd) < 0) + mail_index_set_syscall_error(index, "close()"); + index->fd = fd; + + ret = fd < 0 ? mail_index_try_open(index) : + mail_index_map(index, FALSE); + if (ret <= 0) { + // FIXME: serious problem, we'll just crash later.. + return -1; + } + + return 0; +} + +static int mail_index_has_changed(struct mail_index *index) +{ + struct stat st1, st2; + + if (fstat(index->fd, &st1) < 0) + return mail_index_set_syscall_error(index, "fstat()"); + if (stat(index->filepath, &st2) < 0) + return mail_index_set_syscall_error(index, "stat()"); + + if (st1.st_ino != st2.st_ino || + !CMP_DEV_T(st1.st_dev, st2.st_dev)) { + if (mail_index_reopen(index, -1) < 0) + return -1; + return 1; + } else { + return 0; + } +} + +static int mail_index_lock(struct mail_index *index, int lock_type, + unsigned int timeout_secs, int update_index, + unsigned int *lock_id_r) +{ + // FIXME: mprotect() the index to make sure we don't access it unlocked! + int ret; + + i_assert(lock_type == F_RDLCK || lock_type == F_WRLCK); + + if (lock_type == F_WRLCK && index->lock_type == F_RDLCK) { + /* drop shared locks */ + i_assert(index->excl_lock_count == 0); + + if (file_wait_lock(index->fd, F_UNLCK) < 0) + mail_index_set_syscall_error(index, "file_wait_lock()"); + + index->shared_lock_count = 0; + index->lock_type = F_UNLCK; + index->lock_id += 2; /* make sure failures below work right */ + } + + if (index->excl_lock_count > 0 || index->shared_lock_count > 0) { + i_assert(lock_type == F_RDLCK || index->excl_lock_count > 0); + if (lock_type == F_RDLCK) { + index->shared_lock_count++; + *lock_id_r = index->lock_id; + } else { + index->excl_lock_count++; + *lock_id_r = index->lock_id + 1; + } + return 1; + } + + i_assert(index->lock_type == F_UNLCK); + + if (update_index && lock_type != F_WRLCK) { + if (mail_index_has_changed(index) < 0) + return -1; + } + + do { + ret = file_wait_lock_full(index->fd, lock_type, timeout_secs, + NULL, NULL); + if (ret <= 0) { + if (ret == 0) + return 0; + mail_index_set_syscall_error(index, "file_wait_lock()"); + return -1; + } + + if (lock_type == F_WRLCK) { + /* we need to have the latest index file locked - + check if it's been updated. */ + if ((ret = mail_index_has_changed(index)) < 0) { + (void)file_wait_lock(index->fd, F_UNLCK); + return -1; + } + if (ret > 0) + continue; + } + } while (0); + + index->lock_type = lock_type; + index->lock_id += 2; + + if (lock_type == F_RDLCK) { + index->shared_lock_count++; + *lock_id_r = index->lock_id; + } else { + index->excl_lock_count++; + *lock_id_r = index->lock_id + 1; + } + return 1; +} + +int mail_index_lock_shared(struct mail_index *index, int update_index, + unsigned int *lock_id_r) +{ + int ret; + + ret = mail_index_lock(index, F_RDLCK, DEFAULT_LOCK_TIMEOUT, + update_index, lock_id_r); + if (ret > 0) + return 0; + if (ret < 0) + return -1; + + mail_index_set_error(index, "Timeout while waiting for release of " + "shared fcntl() lock for index file %s", + index->filepath); + index->index_lock_timeout = TRUE; + return -1; +} + +static int mail_index_copy(struct mail_index *index) +{ + const char *path; + int ret, fd; + + fd = mail_index_create_tmp_file(index, &path); + if (fd == -1) + return -1; + + ret = 0; + if (write_full(fd, index->map->mmap_base, + index->map->mmap_used_size) < 0) { + mail_index_file_set_syscall_error(index, path, "write_full()"); + (void)close(fd); + (void)unlink(path); + return -1; + } + + i_assert(index->copy_lock_path == NULL); + index->copy_lock_path = i_strdup(path); + return fd; +} + +static int mail_index_need_lock(struct mail_index *index, + uint32_t log_file_seq, uoff_t log_file_offset) +{ + if (mail_index_map(index, FALSE) <= 0) + return 1; + + if (log_file_seq != 0 && + (index->hdr->log_file_seq > log_file_seq || + (index->hdr->log_file_seq == log_file_seq && + index->hdr->log_file_offset >= log_file_offset))) { + /* already synced */ + return 0; + } + + return 1; +} + +int mail_index_lock_exclusive(struct mail_index *index, + uint32_t log_file_seq, uoff_t log_file_offset, + unsigned int *lock_id_r) +{ + unsigned int lock_id; + int ret; + + /* exclusive transaction log lock protects exclusive locking + for the main index file */ + i_assert(index->log_locked); + + /* wait two seconds for exclusive lock */ + ret = mail_index_lock(index, F_WRLCK, 2, TRUE, lock_id_r); + if (ret > 0) { + if (mail_index_need_lock(index, log_file_seq, log_file_offset)) + return 1; + + mail_index_unlock(index, *lock_id_r); + return 0; + } + if (ret < 0) + return -1; + + /* Grab shared lock to make sure it's not already being + exclusively locked */ + if (mail_index_lock_shared(index, TRUE, &lock_id) < 0) + return -1; + + if (log_file_seq != 0) { + /* check first if we really need to recreate it */ + ret = mail_index_need_lock(index, log_file_seq, + log_file_offset); + if (ret == 0) { + mail_index_unlock(index, lock_id); + return 0; + } + } + + mail_index_unlock(index, lock_id); + + *lock_id_r = 0; + return mail_index_lock_exclusive_copy(index); +} + +int mail_index_lock_exclusive_copy(struct mail_index *index) +{ + int fd; + + if (index->copy_lock_path != NULL) { + index->excl_lock_count++; + return 1; + } + + /* copy the index to index.tmp and use it. when */ + fd = mail_index_copy(index); + if (fd == -1) + return -1; + + if (mail_index_reopen(index, fd) < 0) { + (void)mail_index_reopen(index, -1); + i_free(index->copy_lock_path); + index->copy_lock_path = NULL; + return -1; + } + + index->lock_type = F_WRLCK; + index->excl_lock_count++; + return 1; +} + +static void mail_index_copy_lock_finish(struct mail_index *index) +{ + if (fsync(index->fd) < 0) { + mail_index_file_set_syscall_error(index, index->copy_lock_path, + "fsync()"); + } + + if (rename(index->copy_lock_path, index->filepath) < 0) { + mail_index_set_error(index, "rename(%s, %s) failed: %m", + index->copy_lock_path, index->filepath); + // FIXME: this isn't good + } + + i_free(index->copy_lock_path); + index->copy_lock_path = NULL; + + index->shared_lock_count = 0; + index->lock_id += 2; + index->lock_type = F_UNLCK; +} + +void mail_index_unlock(struct mail_index *index, unsigned int lock_id) +{ + if (index->copy_lock_path != NULL) { + i_assert(index->log_locked); + i_assert(index->excl_lock_count > 0); + if (--index->excl_lock_count == 0) + mail_index_copy_lock_finish(index); + return; + } + + if ((lock_id & 1) == 0) { + /* shared lock */ + if (mail_index_is_locked(index, lock_id)) { + i_assert(index->shared_lock_count > 0); + index->shared_lock_count--; + } + } else { + /* exclusive lock */ + i_assert(lock_id == index->lock_id+1); + i_assert(index->excl_lock_count > 0); + index->excl_lock_count--; + } + + if (index->shared_lock_count == 0 && index->excl_lock_count == 0) { + index->lock_id += 2; + index->lock_type = F_UNLCK; + if (file_wait_lock(index->fd, F_UNLCK) < 0) + mail_index_set_syscall_error(index, "file_wait_lock()"); + } +} + +int mail_index_is_locked(struct mail_index *index, unsigned int lock_id) +{ + return (index->lock_id ^ lock_id) <= 1; +}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-index/mail-index-private.h Tue Apr 27 23:25:52 2004 +0300 @@ -0,0 +1,134 @@ +#ifndef __MAIL_INDEX_PRIVATE_H +#define __MAIL_INDEX_PRIVATE_H + +#include "mail-index.h" + +struct mail_transaction_header; + +/* number of records to always keep allocated in index file, + either used or unused */ +#define INDEX_MIN_RECORDS_COUNT 64 +/* when empty space in index file gets full, grow the file n% larger */ +#define INDEX_GROW_PERCENTAGE 10 +/* ftruncate() the index file when only n% of it is in use */ +#define INDEX_TRUNCATE_PERCENTAGE 30 +/* don't truncate whole file anyway, keep n% of the empty space */ +#define INDEX_TRUNCATE_KEEP_PERCENTAGE 10 +/* Compress the file when deleted space reaches n% of total size */ +#define INDEX_COMPRESS_PERCENTAGE 50 +/* Compress the file when searching deleted records tree has to go this deep */ +#define INDEX_COMPRESS_DEPTH 10 + +enum mail_index_mail_flags { + MAIL_INDEX_MAIL_FLAG_DIRTY = 0x80, + MAIL_INDEX_MAIL_FLAG_EXPUNGED = 0x40, + MAIL_INDEX_MAIL_FLAG_NONRECENT = MAIL_RECENT +}; + +#define MAIL_INDEX_MAP_IS_IN_MEMORY(map) \ + ((map)->buffer != NULL) + +struct mail_index_map { + int refcount; + + const struct mail_index_header *hdr; + struct mail_index_record *records; + unsigned int records_count; + + void *mmap_base; + size_t mmap_size, mmap_used_size; + + buffer_t *buffer; + + struct mail_index_header hdr_copy; +}; + +struct mail_index { + char *dir, *prefix; + + struct mail_cache *cache; + struct mail_transaction_log *log; + + mode_t mode; + gid_t gid; + + char *filepath; + int fd; + + struct mail_index_map *map; + const struct mail_index_header *hdr; + uint32_t indexid; + + int lock_type, shared_lock_count, excl_lock_count; + unsigned int lock_id, copy_lock_id; + char *copy_lock_path; + + char *error; + unsigned int nodiskspace:1; + unsigned int index_lock_timeout:1; + + unsigned int opened:1; + unsigned int log_locked:1; + unsigned int use_mmap:1; + unsigned int readonly:1; + unsigned int fsck:1; +}; + +void mail_index_header_init(struct mail_index_header *hdr); +int mail_index_write_header(struct mail_index *index, + const struct mail_index_header *hdr); + +int mail_index_create(struct mail_index *index, struct mail_index_header *hdr); +int mail_index_try_open(struct mail_index *index); +int mail_index_create_tmp_file(struct mail_index *index, const char **path_r); + +/* Returns 0 = ok, -1 = error. If update_index is TRUE, reopens the index + file if needed to get later version of it (not necessarily latest due to + races, unless transaction log is exclusively locked). */ +int mail_index_lock_shared(struct mail_index *index, int update_index, + unsigned int *lock_id_r); +/* Returns 1 = ok, 0 = already synced up to given log_file_offset, -1 = error */ +int mail_index_lock_exclusive(struct mail_index *index, + uint32_t log_file_seq, uoff_t log_file_offset, + unsigned int *lock_id_r); +int mail_index_lock_exclusive_copy(struct mail_index *index); +void mail_index_unlock(struct mail_index *index, unsigned int lock_id); +/* Returns 1 if given lock_id is valid, 0 if not. */ +int mail_index_is_locked(struct mail_index *index, unsigned int lock_id); + +/* Map index file to memory, replacing the previous mapping for index. + Returns 1 = ok, 0 = corrupted, -1 = error. If index needs fscking, it + returns 1 but sets index->fsck = TRUE. */ +int mail_index_map(struct mail_index *index, int force); +/* Unreference given mapping and unmap it if it's dropped to zero. */ +void mail_index_unmap(struct mail_index *index, struct mail_index_map *map); +struct mail_index_map *mail_index_map_to_memory(struct mail_index_map *map); + +void mail_index_update_cache(struct mail_index_transaction *t, + uint32_t seq, uint32_t offset); + +int mail_index_fix_header(struct mail_index *index, struct mail_index_map *map, + struct mail_index_header *hdr, const char **error_r); + +void mail_index_view_transaction_ref(struct mail_index_view *view); +void mail_index_view_transaction_unref(struct mail_index_view *view); + +int mail_index_sync_get_rec(struct mail_index_view *view, + struct mail_index_sync_rec *rec, + const struct mail_transaction_header *hdr, + const void *data, size_t *data_offset); + +int mail_index_mark_corrupted(struct mail_index *index); + +int mail_index_set_error(struct mail_index *index, const char *fmt, ...) + __attr_format__(2, 3); +/* "%s failed with index file %s: %m" */ +int mail_index_set_syscall_error(struct mail_index *index, + const char *function); +/* "%s failed with file %s: %m" */ +int mail_index_file_set_syscall_error(struct mail_index *index, + const char *filepath, + const char *function); +void mail_index_reset_error(struct mail_index *index); + +#endif
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-index/mail-index-reset.c Tue Apr 27 23:25:52 2004 +0300 @@ -0,0 +1,37 @@ +/* Copyright (C) 2004 Timo Sirainen */ + +#include "lib.h" +#include "mmap-util.h" +#include "write-full.h" +#include "mail-index-private.h" +#include "mail-transaction-log.h" + +int mail_index_reset(struct mail_index *index) +{ + struct mail_index_header hdr; + + /* this invalidates all views even if we fail later */ + index->indexid = 0; + + if (mail_index_mark_corrupted(index) < 0) + return -1; + + mail_index_header_init(&hdr); + if (hdr.indexid == index->indexid) + hdr.indexid++; + + // FIXME: close it? .. + if (mail_index_create(index, &hdr) < 0) + return -1; + + /* reopen transaction log - FIXME: doesn't work, we have log views + open.. */ + mail_transaction_log_close(index->log); + index->log = mail_transaction_log_open_or_create(index); + if (index->log == NULL) { + /* FIXME: creates potential crashes.. */ + return -1; + } + + return 0; +}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-index/mail-index-sync-private.h Tue Apr 27 23:25:52 2004 +0300 @@ -0,0 +1,32 @@ +#ifndef __MAIL_INDEX_SYNC_PRIVATE_H +#define __MAIL_INDEX_SYNC_PRIVATE_H + +struct mail_index_sync_ctx { + struct mail_index *index; + struct mail_index_view *view; + + buffer_t *expunges_buf, *updates_buf, *appends_buf; + + const struct mail_transaction_expunge *expunges; + const struct mail_transaction_flag_update *updates; + size_t expunges_count, updates_count; + + const struct mail_transaction_header *hdr; + const void *data; + + size_t expunge_idx, update_idx; + uint32_t next_seq; + + unsigned int lock_id; + + unsigned int sync_appends:1; +}; + +int mail_index_sync_update_index(struct mail_index_sync_ctx *sync_ctx); + +void mail_index_header_update_counts(struct mail_index_header *hdr, + uint8_t old_flags, uint8_t new_flags); +void mail_index_header_update_lowwaters(struct mail_index_header *hdr, + const struct mail_index_record *rec); + +#endif
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-index/mail-index-sync-update.c Tue Apr 27 23:25:52 2004 +0300 @@ -0,0 +1,273 @@ +/* Copyright (C) 2004 Timo Sirainen */ + +#include "lib.h" +#include "buffer.h" +#include "file-set-size.h" +#include "mmap-util.h" +#include "mail-index-view-private.h" +#include "mail-index-sync-private.h" +#include "mail-transaction-log.h" + +struct mail_index_update_ctx { + struct mail_index *index; + struct mail_index_header hdr; + struct mail_transaction_log_view *log_view; +}; + +void mail_index_header_update_counts(struct mail_index_header *hdr, + uint8_t old_flags, uint8_t new_flags) +{ + if (((old_flags ^ new_flags) & MAIL_SEEN) != 0) { + /* different seen-flag */ + if ((old_flags & MAIL_SEEN) == 0) + hdr->seen_messages_count++; + else + hdr->seen_messages_count--; + } + + if (((old_flags ^ new_flags) & MAIL_DELETED) != 0) { + /* different deleted-flag */ + if ((old_flags & MAIL_DELETED) == 0) + hdr->deleted_messages_count++; + else + hdr->deleted_messages_count--; + } +} + +void mail_index_header_update_lowwaters(struct mail_index_header *hdr, + const struct mail_index_record *rec) +{ + if ((rec->flags & MAIL_RECENT) != 0 && + rec->uid < hdr->first_recent_uid_lowwater) + hdr->first_recent_uid_lowwater = rec->uid; + if ((rec->flags & MAIL_SEEN) == 0 && + rec->uid < hdr->first_unseen_uid_lowwater) + hdr->first_unseen_uid_lowwater = rec->uid; + if ((rec->flags & MAIL_DELETED) != 0 && + rec->uid < hdr->first_deleted_uid_lowwater) + hdr->first_deleted_uid_lowwater = rec->uid; +} + +static void mail_index_sync_update_expunges(struct mail_index_update_ctx *ctx, + uint32_t seq1, uint32_t seq2) +{ + struct mail_index_record *rec; + + rec = &ctx->index->map->records[seq1-1]; + for (; seq1 <= seq2; seq1++, rec++) + mail_index_header_update_counts(&ctx->hdr, rec->flags, 0); +} + +static void mail_index_sync_update_flags(struct mail_index_update_ctx *ctx, + struct mail_index_sync_rec *syncrec) +{ + struct mail_index_record *rec, *end; + uint8_t flag_mask, old_flags; + custom_flags_mask_t custom_mask; + int i, update_custom; + + update_custom = FALSE; + for (i = 0; i < INDEX_CUSTOM_FLAGS_BYTE_COUNT; i++) { + if (syncrec->add_custom_flags[i] != 0) + update_custom = TRUE; + if (syncrec->remove_custom_flags[i] != 0) + update_custom = TRUE; + custom_mask[i] = ~syncrec->remove_custom_flags[i]; + } + + flag_mask = ~syncrec->remove_flags; + rec = &ctx->index->map->records[syncrec->seq1-1]; + end = rec + (syncrec->seq2 - syncrec->seq1) + 1; + for (; rec != end; rec++) { + old_flags = rec->flags; + rec->flags = (rec->flags & flag_mask) | syncrec->add_flags; + if (update_custom) { + for (i = 0; i < INDEX_CUSTOM_FLAGS_BYTE_COUNT; i++) { + rec->custom_flags[i] = + (rec->custom_flags[i]&custom_mask[i]) | + syncrec->add_custom_flags[i]; + } + } + + mail_index_header_update_counts(&ctx->hdr, + old_flags, rec->flags); + mail_index_header_update_lowwaters(&ctx->hdr, rec); + } +} + +static int mail_index_grow(struct mail_index *index, unsigned int count) +{ + size_t size, mmap_used_size; + unsigned int records_count; + + // FIXME: grow exponentially + size = index->map->mmap_used_size + + count * sizeof(struct mail_index_record); + if (file_set_size(index->fd, (off_t)size) < 0) + return mail_index_set_syscall_error(index, "file_set_size()"); + + records_count = index->map->records_count; + mmap_used_size = index->map->mmap_used_size; + + if (mail_index_map(index, TRUE) <= 0) + return -1; + + i_assert(index->map->mmap_size >= size); + index->map->records_count = records_count; + index->map->mmap_used_size = mmap_used_size; + return 0; +} + +static int mail_index_sync_appends(struct mail_index_update_ctx *ctx, + const struct mail_index_record *appends, + unsigned int count) +{ + struct mail_index_map *map = ctx->index->map; + unsigned int i; + size_t space; + uint32_t next_uid; + + if (!ctx->index->use_mmap) { + // FIXME + } + + space = (map->mmap_size - map->mmap_used_size) / sizeof(*appends); + if (space < count) { + if (mail_index_grow(ctx->index, count) < 0) + return -1; + + if (mprotect(map->mmap_base, map->mmap_size, + PROT_READ|PROT_WRITE) < 0) { + mail_index_set_syscall_error(ctx->index, "mprotect()"); + return -1; + } + } + + next_uid = ctx->hdr.next_uid; + for (i = 0; i < count; i++) { + mail_index_header_update_counts(&ctx->hdr, 0, appends[i].flags); + mail_index_header_update_lowwaters(&ctx->hdr, &appends[i]); + + if (appends[i].uid < next_uid) { + /* FIXME: should we rather just update the record? + this can actually happen if append was written to + transaction log but index wasn't updated, then + another sync wrote it again.. */ + mail_transaction_log_view_set_corrupted(ctx->log_view, + "Append with UID %u, but next_uid = %u", + appends[i].uid, next_uid); + return -1; + } + next_uid = appends[i].uid+1; + } + ctx->hdr.next_uid = next_uid; + + memcpy(map->records + map->records_count, appends, + count * sizeof(*appends)); + map->records_count += count; + map->mmap_used_size += count * sizeof(struct mail_index_record); + return 0; +} + +int mail_index_sync_update_index(struct mail_index_sync_ctx *sync_ctx) +{ + struct mail_index *index = sync_ctx->index; + struct mail_index_map *map = index->map; + struct mail_index_update_ctx ctx; + struct mail_index_sync_rec rec; + const struct mail_index_record *appends; + unsigned int append_count; + uint32_t count, file_seq, src_idx, dest_idx; + uoff_t file_offset; + int ret, locked = FALSE; + + if (mprotect(map->mmap_base, map->mmap_size, PROT_READ|PROT_WRITE) < 0) + return mail_index_set_syscall_error(index, "mprotect()"); + + /* rewind */ + sync_ctx->update_idx = sync_ctx->expunge_idx = 0; + sync_ctx->sync_appends = + buffer_get_used_size(sync_ctx->appends_buf) != 0; + + memset(&ctx, 0, sizeof(ctx)); + ctx.index = index; + ctx.hdr = *index->hdr; + ctx.log_view = sync_ctx->view->log_view; + + src_idx = dest_idx = 0; + append_count = 0; appends = NULL; + while (mail_index_sync_next(sync_ctx, &rec) > 0) { + switch (rec.type) { + case MAIL_INDEX_SYNC_TYPE_APPEND: + i_assert(appends == NULL); + append_count = rec.seq2 - rec.seq1 + 1; + appends = rec.appends; + break; + case MAIL_INDEX_SYNC_TYPE_EXPUNGE: + if (src_idx != 0) { + count = (rec.seq1-1) - src_idx; + memmove(map->records + dest_idx, + map->records + src_idx, + count * sizeof(*map->records)); + dest_idx += count; + } else { + dest_idx = rec.seq1-1; + if (mail_index_lock_exclusive_copy(index) <= 0) + return -1; + map = index->map; + if (mprotect(map->mmap_base, map->mmap_size, + PROT_READ|PROT_WRITE) < 0) { + mail_index_set_syscall_error(index, + "mprotect()"); + return -1; + } + locked = TRUE; + } + + mail_index_sync_update_expunges(&ctx, rec.seq1, + rec.seq2); + src_idx = rec.seq2; + break; + case MAIL_INDEX_SYNC_TYPE_FLAGS: + mail_index_sync_update_flags(&ctx, &rec); + break; + } + } + + if (src_idx != 0) { + count = map->records_count - src_idx; + memmove(map->records + dest_idx, + map->records + src_idx, + count * sizeof(*map->records)); + dest_idx += count; + + map->records_count = dest_idx; + map->mmap_used_size = index->hdr->header_size + + map->records_count * sizeof(struct mail_index_record); + } + + ret = 0; + if (append_count > 0) + ret = mail_index_sync_appends(&ctx, appends, append_count); + + mail_transaction_log_get_head(index->log, &file_seq, &file_offset); + + ctx.hdr.messages_count = map->records_count; + ctx.hdr.log_file_seq = file_seq; + ctx.hdr.log_file_offset = file_offset; + + if (index->use_mmap) { + memcpy(map->mmap_base, &ctx.hdr, sizeof(ctx.hdr)); + if (msync(map->mmap_base, map->mmap_used_size, MS_SYNC) < 0) + return mail_index_set_syscall_error(index, "msync()"); + } else { + // FIXME + } + + if (mprotect(map->mmap_base, map->mmap_size, PROT_READ) < 0) + mail_index_set_syscall_error(index, "mprotect()"); + + if (locked) + mail_index_unlock(index, 0); + return ret; +}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-index/mail-index-sync.c Tue Apr 27 23:25:52 2004 +0300 @@ -0,0 +1,421 @@ +/* Copyright (C) 2003-2004 Timo Sirainen */ + +#include "lib.h" +#include "buffer.h" +#include "mail-index-view-private.h" +#include "mail-index-sync-private.h" +#include "mail-transaction-log.h" +#include "mail-transaction-util.h" + +#include <stdlib.h> + +static void mail_index_sync_sort_flags(struct mail_index_sync_ctx *ctx) +{ + const struct mail_transaction_flag_update *src, *src_end; + const struct mail_transaction_flag_update *dest; + struct mail_transaction_flag_update new_update; + struct mail_transaction_expunge_traverse_ctx *exp_ctx; + uint32_t last; + size_t i, dest_count; + + src = ctx->data; + src_end = PTR_OFFSET(src, ctx->hdr->size); + + dest = buffer_get_data(ctx->updates_buf, &dest_count); + dest_count /= sizeof(*dest); + + exp_ctx = mail_transaction_expunge_traverse_init(ctx->expunges_buf); + + for (i = 0; src != src_end; src++) { + new_update = *src; + + /* find seq1 */ + new_update.seq1 += + mail_transaction_expunge_traverse_to(exp_ctx, + src->seq1); + + /* find seq2 */ + new_update.seq2 += + mail_transaction_expunge_traverse_to(exp_ctx, + src->seq2); + + /* insert it into buffer, split it in multiple parts if needed + to make sure the ordering stays the same */ + for (; i < dest_count; i++) { + if (dest[i].seq1 <= new_update.seq1) + continue; + + if (dest[i].seq1 > new_update.seq2) + break; + + /* partial */ + last = new_update.seq2; + new_update.seq2 = dest[i].seq1-1; + + buffer_insert(ctx->updates_buf, i * sizeof(new_update), + &new_update, sizeof(new_update)); + dest = buffer_get_data(ctx->updates_buf, NULL); + dest_count++; + + new_update.seq1 = new_update.seq2+1; + new_update.seq2 = last; + } + + buffer_insert(ctx->updates_buf, i * sizeof(new_update), + &new_update, sizeof(new_update)); + dest = buffer_get_data(ctx->updates_buf, NULL); + dest_count++; + } + mail_transaction_expunge_traverse_deinit(exp_ctx); +} + +static void mail_index_sync_sort_transaction(struct mail_index_sync_ctx *ctx) +{ + switch (ctx->hdr->type & MAIL_TRANSACTION_TYPE_MASK) { + case MAIL_TRANSACTION_EXPUNGE: + if (buffer_get_used_size(ctx->expunges_buf) == 0) { + buffer_append(ctx->expunges_buf, ctx->data, + ctx->hdr->size); + } else { + mail_transaction_log_sort_expunges(ctx->expunges_buf, + ctx->data, + ctx->hdr->size); + } + break; + case MAIL_TRANSACTION_FLAG_UPDATE: + if (buffer_get_used_size(ctx->expunges_buf) == 0 && + buffer_get_used_size(ctx->updates_buf) == 0) { + buffer_append(ctx->updates_buf, ctx->data, + ctx->hdr->size); + } else { + mail_index_sync_sort_flags(ctx); + } + break; + case MAIL_TRANSACTION_APPEND: + buffer_append(ctx->appends_buf, ctx->data, ctx->hdr->size); + ctx->sync_appends = TRUE; + break; + } +} + +static int mail_index_sync_read_and_sort(struct mail_index_sync_ctx *ctx, + int external) +{ + enum mail_transaction_type flag; + int ret; + + flag = external ? MAIL_TRANSACTION_EXTERNAL : 0; + while ((ret = mail_transaction_log_view_next(ctx->view->log_view, + &ctx->hdr, + &ctx->data, NULL)) > 0) { + if ((ctx->hdr->type & MAIL_TRANSACTION_EXTERNAL) == flag) + mail_index_sync_sort_transaction(ctx); + } + + return ret; +} + +int mail_index_sync_begin(struct mail_index *index, + struct mail_index_sync_ctx **ctx_r, + struct mail_index_view **view_r, + uint32_t log_file_seq, uoff_t log_file_offset) +{ + struct mail_index_sync_ctx *ctx; + uint32_t seq; + uoff_t offset; + size_t size; + unsigned int lock_id; + int ret; + + if (mail_transaction_log_sync_lock(index->log, &seq, &offset) < 0) + return -1; + + /* FIXME: really needed yet? If there are readers, the index file + is copied even if there are no changes.. */ + ret = mail_index_lock_exclusive(index, log_file_seq, + log_file_offset, &lock_id); + if (ret <= 0) { + mail_transaction_log_sync_unlock(index->log); + return ret; + } + + if (mail_index_map(index, FALSE) <= 0) { + mail_transaction_log_sync_unlock(index->log); + return -1; + } + + ctx = i_new(struct mail_index_sync_ctx, 1); + ctx->index = index; + ctx->lock_id = lock_id; + + ctx->view = mail_index_view_open(index); + ctx->view->external = TRUE; + + if (mail_transaction_log_view_set(ctx->view->log_view, + index->hdr->log_file_seq, + index->hdr->log_file_offset, + seq, offset, + MAIL_TRANSACTION_TYPE_MASK) < 0) { + mail_index_sync_end(ctx); + return -1; + } + + /* we need to have all the transactions sorted to optimize + caller's mailbox access patterns */ + ctx->expunges_buf = buffer_create_dynamic(default_pool, + 1024, (size_t)-1); + ctx->updates_buf = buffer_create_dynamic(default_pool, + 1024, (size_t)-1); + ctx->appends_buf = buffer_create_dynamic(default_pool, + 1024, (size_t)-1); + if (mail_index_sync_read_and_sort(ctx, FALSE) < 0) { + mail_index_sync_end(ctx); + return -1; + } + + ctx->expunges = buffer_get_data(ctx->expunges_buf, &size); + ctx->expunges_count = size / sizeof(*ctx->expunges); + ctx->updates = buffer_get_data(ctx->updates_buf, &size); + ctx->updates_count = size / sizeof(*ctx->updates); + + *ctx_r = ctx; + *view_r = ctx->view; + return 1; +} + +static void +mail_index_sync_get_expunge(struct mail_index_sync_rec *rec, + const struct mail_transaction_expunge *exp) +{ + rec->type = MAIL_INDEX_SYNC_TYPE_EXPUNGE; + rec->seq1 = exp->seq1; + rec->seq2 = exp->seq2; +} + +static void +mail_index_sync_get_update(struct mail_index_sync_rec *rec, + const struct mail_transaction_flag_update *update) +{ + rec->type = MAIL_INDEX_SYNC_TYPE_FLAGS; + rec->seq1 = update->seq1; + rec->seq2 = update->seq2; + + rec->add_flags = update->add_flags; + memcpy(rec->add_custom_flags, update->add_custom_flags, + sizeof(rec->add_custom_flags)); + rec->remove_flags = update->remove_flags; + memcpy(rec->remove_custom_flags, update->remove_custom_flags, + sizeof(rec->remove_custom_flags)); +} + +static int mail_index_sync_rec_check(struct mail_index_view *view, + struct mail_index_sync_rec *rec) +{ + uint32_t message_count; + + switch (rec->type) { + case MAIL_INDEX_SYNC_TYPE_EXPUNGE: + case MAIL_INDEX_SYNC_TYPE_FLAGS: + if (rec->seq1 > rec->seq2 || rec->seq1 == 0) { + mail_transaction_log_view_set_corrupted(view->log_view, + "Broken sequence: %u..%u", + rec->seq1, rec->seq2); + return FALSE; + } + + message_count = mail_index_view_get_message_count(view); + if (rec->seq2 > message_count) { + mail_transaction_log_view_set_corrupted(view->log_view, + "Sequence out of range: %u > %u", + rec->seq2, message_count); + return FALSE; + } + break; + case MAIL_INDEX_SYNC_TYPE_APPEND: + break; + } + return TRUE; +} + +int mail_index_sync_get_rec(struct mail_index_view *view, + struct mail_index_sync_rec *rec, + const struct mail_transaction_header *hdr, + const void *data, size_t *data_offset) +{ + switch (hdr->type & MAIL_TRANSACTION_TYPE_MASK) { + case MAIL_TRANSACTION_APPEND: { + rec->type = MAIL_INDEX_SYNC_TYPE_APPEND; + rec->seq1 = view->index->map->records_count + 1; + rec->seq2 = rec->seq1 + hdr->size / + sizeof(struct mail_index_record) - 1; + rec->appends = NULL; + + *data_offset += hdr->size; + break; + } + case MAIL_TRANSACTION_EXPUNGE: { + const struct mail_transaction_expunge *exp = + CONST_PTR_OFFSET(data, *data_offset); + + *data_offset += sizeof(*exp); + mail_index_sync_get_expunge(rec, exp); + break; + } + case MAIL_TRANSACTION_FLAG_UPDATE: { + const struct mail_transaction_flag_update *update = + CONST_PTR_OFFSET(data, *data_offset); + + *data_offset += sizeof(*update); + mail_index_sync_get_update(rec, update); + break; + } + default: + i_unreached(); + } + + return mail_index_sync_rec_check(view, rec); +} + +int mail_index_sync_next(struct mail_index_sync_ctx *ctx, + struct mail_index_sync_rec *sync_rec) +{ + const struct mail_transaction_expunge *next_exp; + const struct mail_transaction_flag_update *next_update; + + next_exp = ctx->expunge_idx == ctx->expunges_count ? NULL : + &ctx->expunges[ctx->expunge_idx]; + next_update = ctx->update_idx == ctx->updates_count ? NULL : + &ctx->updates[ctx->update_idx]; + + /* the ugliness here is to avoid returning overlapping expunge + and update areas. For example: + + updates[] = A { 1, 7 }, B { 1, 3 } + expunges[] = { 5, 6 } + + will make us return + + update A: 1, 4 + update B: 1, 3 + expunge : 5, 6 + update A: 7, 7 + */ + while (next_update != NULL && + (next_exp == NULL || next_update->seq1 < next_exp->seq1)) { + if (next_update->seq2 >= ctx->next_seq) { + mail_index_sync_get_update(sync_rec, next_update); + if (next_exp != NULL && + next_exp->seq1 <= next_update->seq2) { + /* it's overlapping.. */ + sync_rec->seq2 = next_exp->seq1-1; + } + + if (sync_rec->seq1 < ctx->next_seq) + sync_rec->seq1 = ctx->next_seq; + + i_assert(sync_rec->seq1 <= sync_rec->seq2); + ctx->update_idx++; + return mail_index_sync_rec_check(ctx->view, sync_rec); + } + + if (++ctx->update_idx == ctx->updates_count) + break; + next_update++; + } + + if (next_exp != NULL) { + /* a few sanity checks here, we really don't ever want to + accidentally expunge a message. If sequence and UID matches, + it's quite unlikely this expunge was caused by some bug. */ + uint32_t uid1, uid2; + + if (mail_index_lookup_uid(ctx->view, next_exp->seq1, &uid1) < 0) + return -1; + if (mail_index_lookup_uid(ctx->view, next_exp->seq2, &uid2) < 0) + return -1; + if (next_exp->uid1 != uid1 || next_exp->uid2 != uid2) { + mail_transaction_log_view_set_corrupted( + ctx->view->log_view, "Expunge range %u..%u: " + "UIDs %u..%u doesn't match real UIDs %u..%u", + next_exp->seq1, next_exp->seq2, + next_exp->uid1, next_exp->uid2, uid1, uid2); + return -1; + } + + mail_index_sync_get_expunge(sync_rec, next_exp); + ctx->expunge_idx++; + + /* scan updates again from the beginning */ + ctx->update_idx = 0; + ctx->next_seq = next_exp->seq2; + return mail_index_sync_rec_check(ctx->view, sync_rec); + } + + if (ctx->sync_appends) { + ctx->sync_appends = FALSE; + sync_rec->type = MAIL_INDEX_SYNC_TYPE_APPEND; + sync_rec->seq1 = ctx->index->map->records_count+1; + sync_rec->seq2 = sync_rec->seq1-1 + + buffer_get_used_size(ctx->appends_buf) / + sizeof(struct mail_index_record); + sync_rec->appends = buffer_get_data(ctx->appends_buf, NULL); + return 1; + } + + return 0; +} + +int mail_index_sync_end(struct mail_index_sync_ctx *ctx) +{ + uint32_t seq; + uoff_t offset; + int ret = 0; + + if (mail_transaction_log_view_is_corrupted(ctx->view->log_view)) + ret = -1; + + mail_transaction_log_get_head(ctx->index->log, &seq, &offset); + + if (mail_transaction_log_view_set(ctx->view->log_view, + ctx->index->hdr->log_file_seq, + ctx->index->hdr->log_file_offset, + seq, offset, + MAIL_TRANSACTION_TYPE_MASK) < 0) + ret = -1; + + if (ret == 0) { + mail_index_sync_read_and_sort(ctx, TRUE); + if (mail_index_sync_update_index(ctx) < 0) + ret = -1; + } + + mail_index_unlock(ctx->index, ctx->lock_id); + mail_transaction_log_sync_unlock(ctx->index->log); + + if (ctx->view != NULL) + mail_index_view_close(ctx->view); + if (ctx->expunges_buf != NULL) + buffer_free(ctx->expunges_buf); + if (ctx->updates_buf != NULL) + buffer_free(ctx->updates_buf); + if (ctx->appends_buf != NULL) + buffer_free(ctx->appends_buf); + i_free(ctx); + return ret; +} + +void mail_index_sync_flags_apply(const struct mail_index_sync_rec *sync_rec, + uint8_t *flags, + custom_flags_mask_t custom_flags) +{ + int i; + + i_assert(sync_rec->type == MAIL_INDEX_SYNC_TYPE_FLAGS); + + *flags = (*flags & ~sync_rec->remove_flags) | sync_rec->add_flags; + for (i = 0; i < INDEX_CUSTOM_FLAGS_BYTE_COUNT; i++) { + custom_flags[i] = + (custom_flags[i] & ~sync_rec->remove_custom_flags[i]) | + sync_rec->add_custom_flags[i]; + } +}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-index/mail-index-transaction-private.h Tue Apr 27 23:25:52 2004 +0300 @@ -0,0 +1,20 @@ +#ifndef __MAIL_INDEX_TRANSACTION_PRIVATE_H +#define __MAIL_INDEX_TRANSACTION_PRIVATE_H + +struct mail_index_transaction { + struct mail_index_view *view; + + buffer_t *appends; + uint32_t first_new_seq, last_new_seq, next_uid; + + buffer_t *expunges; + + buffer_t *updates; + struct mail_transaction_flag_update last_update; + enum modify_type last_update_modify_type; + + buffer_t *cache_updates; + unsigned int hide_transaction:1; +}; + +#endif
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-index/mail-index-transaction.c Tue Apr 27 23:25:52 2004 +0300 @@ -0,0 +1,461 @@ +/* Copyright (C) 2003-2004 Timo Sirainen */ + +#include "lib.h" +#include "buffer.h" +#include "mail-index-view-private.h" +#include "mail-transaction-log.h" +#include "mail-index-transaction-private.h" + +static void mail_index_transaction_add_last(struct mail_index_transaction *t); + +struct mail_index_transaction * +mail_index_transaction_begin(struct mail_index_view *view, int hide) +{ + struct mail_index_transaction *t; + + /* don't allow syncing view while there's ongoing transactions */ + mail_index_view_transaction_ref(view); + + t = i_new(struct mail_index_transaction, 1); + t->view = view; + t->hide_transaction = hide; + t->next_uid = view->index->hdr->next_uid; + return t; +} + +static void mail_index_transaction_free(struct mail_index_transaction *t) +{ + mail_index_view_transaction_unref(t->view); + if (t->appends != NULL) + buffer_free(t->appends); + if (t->expunges != NULL) + buffer_free(t->expunges); + if (t->updates != NULL) + buffer_free(t->updates); + i_free(t); +} + +static void +mail_index_transaction_expunge_updates(struct mail_index_transaction *t) +{ + /* FIXME: is this useful? do we even want this? */ + const struct mail_transaction_expunge *expunges, *last_expunge; + struct mail_transaction_flag_update *updates; + size_t expunge_size, update_count, i, dest; + uint32_t seq1, seq2; + int cut; + + expunges = buffer_get_data(t->expunges, &expunge_size); + last_expunge = CONST_PTR_OFFSET(expunges, expunge_size); + + if (expunge_size == 0) + return; + + updates = buffer_get_modifyable_data(t->updates, &update_count); + update_count /= sizeof(*updates); + + /* Cut off the updates that contain expunged messages. However if + the cutting would require creating another flag update entry + (eg. updates=1..3, expunge=2), don't do it. */ + for (i = 0, dest = 0; i < update_count; i++) { + while (expunges->seq2 < updates[i].seq1) { + if (++expunges == last_expunge) + break; + } + + cut = FALSE; + if (expunges->seq1 <= updates[i].seq2) { + /* they're overlapping at least partially */ + seq1 = I_MIN(expunges->seq1, updates[i].seq1); + seq2 = I_MAX(expunges->seq2, updates[i].seq2); + + if (seq1 == expunges->seq1 && seq2 == expunges->seq2) { + /* cut it off completely */ + cut = TRUE; + } else if (seq1 == expunges->seq1) { + /* cut the beginning */ + updates[i].seq1 = expunges->seq2+1; + } else if (seq2 == expunges->seq2) { + /* cut the end */ + updates[i].seq2 = expunges->seq1-1; + } else { + /* expunge range is in the middle - + don't bother cutting it */ + } + } + + if (!cut) { + if (i != dest) + updates[dest] = updates[i]; + dest++; + } + } + + if (i != dest) + buffer_set_used_size(t->updates, dest * sizeof(*updates)); +} + +int mail_index_transaction_commit(struct mail_index_transaction *t, + uint32_t *log_file_seq_r, + uoff_t *log_file_offset_r) +{ + int ret; + + if (mail_index_view_is_inconsistent(t->view)) { + mail_index_transaction_free(t); + return -1; + } + + if (t->last_update.seq1 != 0) + mail_index_transaction_add_last(t); + if (t->updates != NULL && t->expunges != NULL) + mail_index_transaction_expunge_updates(t); + + ret = mail_transaction_log_append(t, log_file_seq_r, log_file_offset_r); + + mail_index_transaction_free(t); + return ret; +} + +void mail_index_transaction_rollback(struct mail_index_transaction *t) +{ + mail_index_transaction_free(t); +} + +void mail_index_append(struct mail_index_transaction *t, uint32_t uid, + uint32_t *seq_r) +{ + struct mail_index_record *rec; + + i_assert(uid >= t->next_uid); + + if (t->appends == NULL) { + t->appends = buffer_create_dynamic(default_pool, + 4096, (size_t)-1); + } + + /* sequence number is visible only inside given view, + so let it generate it */ + if (t->last_new_seq != 0) + *seq_r = ++t->last_new_seq; + else { + *seq_r = t->first_new_seq = t->last_new_seq = + mail_index_view_get_message_count(t->view)+1; + } + + rec = buffer_append_space_unsafe(t->appends, sizeof(*rec)); + memset(rec, 0, sizeof(*rec)); + rec->uid = uid; + + t->next_uid = uid+1; +} + +void mail_index_expunge(struct mail_index_transaction *t, uint32_t seq) +{ + struct mail_transaction_expunge exp, *data; + unsigned int idx, left_idx, right_idx; + uint32_t uid; + size_t size; + + i_assert(seq > 0 && seq <= mail_index_view_get_message_count(t->view)); + + uid = t->view->map->records[seq-1].uid; + exp.seq1 = exp.seq2 = seq; + exp.uid1 = exp.uid2 = uid; + + /* expunges is a sorted array of {seq1, seq2, ..}, .. */ + + if (t->expunges == NULL) { + t->expunges = buffer_create_dynamic(default_pool, + 1024, (size_t)-1); + buffer_append(t->expunges, &exp, sizeof(exp)); + return; + } + + data = buffer_get_modifyable_data(t->expunges, &size); + size /= sizeof(*data); + i_assert(size > 0); + + /* quick checks */ + if (data[size-1].seq2 == seq-1) { + /* grow last range */ + data[size-1].seq2 = seq; + data[size-1].uid2 = uid; + return; + } + if (data[size-1].seq2 < seq) { + buffer_append(t->expunges, &exp, sizeof(exp)); + return; + } + if (data[0].seq1 == seq+1) { + /* grow down first range */ + data[0].seq1 = seq; + data[0].uid1 = uid; + return; + } + if (data[0].seq1 > seq) { + buffer_insert(t->expunges, 0, &exp, sizeof(exp)); + return; + } + + /* somewhere in the middle, array is sorted so find it with + binary search */ + idx = 0; left_idx = 0; right_idx = size; + while (left_idx < right_idx) { + idx = (left_idx + right_idx) / 2; + + if (data[idx].seq1 < seq) + left_idx = idx+1; + else if (data[idx].seq1 > seq) + right_idx = idx; + else + break; + } + + if (data[idx].seq2 < seq) + idx++; + + /* idx == size couldn't happen because we already handle it above */ + i_assert(idx < size && data[idx].seq1 >= seq); + + if (data[idx].seq1 <= seq && data[idx].seq2 >= seq) { + /* already expunged */ + return; + } + + if (data[idx].seq1 == seq+1) { + data[idx].seq1 = seq; + data[idx].uid1 = uid; + if (idx > 0 && data[idx-1].seq2 == seq-1) { + /* merge */ + data[idx-1].seq2 = data[idx].seq2; + data[idx-1].uid2 = data[idx].uid2; + buffer_delete(t->expunges, idx * sizeof(*data), + sizeof(*data)); + } + } else if (data[idx].seq2 == seq-1) { + i_assert(idx+1 < size); /* already handled above */ + data[idx].seq2 = seq; + data[idx].uid2 = uid; + if (data[idx+1].seq1 == seq+1) { + /* merge */ + data[idx+1].seq1 = data[idx].seq1; + data[idx+1].uid1 = data[idx].uid1; + buffer_delete(t->expunges, idx * sizeof(*data), + sizeof(*data)); + } + } else { + buffer_insert(t->expunges, idx * sizeof(*data), + &exp, sizeof(exp)); + } +} + +static void mail_index_record_modify_flags(struct mail_index_record *rec, + enum modify_type modify_type, + enum mail_flags flags, + custom_flags_mask_t custom_flags) +{ + int i; + + switch (modify_type) { + case MODIFY_REPLACE: + rec->flags = flags; + memcpy(rec->custom_flags, custom_flags, + INDEX_CUSTOM_FLAGS_BYTE_COUNT); + break; + case MODIFY_ADD: + rec->flags |= flags; + for (i = 0; i < INDEX_CUSTOM_FLAGS_BYTE_COUNT; i++) + rec->custom_flags[i] |= custom_flags[i]; + break; + case MODIFY_REMOVE: + rec->flags &= ~flags; + for (i = 0; i < INDEX_CUSTOM_FLAGS_BYTE_COUNT; i++) + rec->custom_flags[i] &= ~custom_flags[i]; + break; + } +} + +#define IS_COMPATIBLE_UPDATE(t, modify_type, flags, custom_flags) \ + ((t)->last_update_modify_type == (modify_type) && \ + (t)->last_update.add_flags == (flags) && \ + memcmp((t)->last_update.add_custom_flags, custom_flags, \ + INDEX_CUSTOM_FLAGS_BYTE_COUNT) == 0) + +void mail_index_update_flags(struct mail_index_transaction *t, uint32_t seq, + enum modify_type modify_type, + enum mail_flags flags, + custom_flags_mask_t custom_flags) +{ + struct mail_index_record *rec; + size_t pos; + + if (t->first_new_seq != 0 && seq >= t->first_new_seq) { + /* just appended message, modify it directly */ + i_assert(seq > 0 && seq <= t->last_new_seq); + + pos = (seq - t->first_new_seq) * sizeof(*rec); + rec = buffer_get_space_unsafe(t->appends, pos, sizeof(*rec)); + mail_index_record_modify_flags(rec, modify_type, + flags, custom_flags); + return; + } + + i_assert(seq > 0 && seq <= mail_index_view_get_message_count(t->view)); + + /* first get group updates into same structure. this allows faster + updates if same mails have multiple flag updates during same + transaction (eg. 1:10 +seen, 1:10 +deleted) */ + if (t->last_update.seq2 == seq-1) { + if (t->last_update.seq1 != 0 && + IS_COMPATIBLE_UPDATE(t, modify_type, flags, custom_flags)) { + t->last_update.seq2 = seq; + return; + } + } else if (t->last_update.seq1 == seq+1) { + if (t->last_update.seq1 != 0 && + IS_COMPATIBLE_UPDATE(t, modify_type, flags, custom_flags)) { + t->last_update.seq1 = seq; + return; + } + } + + if (t->last_update.seq1 != 0) + mail_index_transaction_add_last(t); + + t->last_update_modify_type = modify_type; + t->last_update.seq1 = t->last_update.seq2 = seq; + t->last_update.add_flags = flags; + memcpy(t->last_update.add_custom_flags, custom_flags, + INDEX_CUSTOM_FLAGS_BYTE_COUNT); +} + +static void +mail_index_transaction_get_last(struct mail_index_transaction *t, + struct mail_transaction_flag_update *update) +{ + int i; + + *update = t->last_update; + switch (t->last_update_modify_type) { + case MODIFY_REPLACE: + /* remove_flags = ~add_flags */ + update->remove_flags = + ~update->add_flags & MAIL_INDEX_FLAGS_MASK; + for (i = 0; i < INDEX_CUSTOM_FLAGS_BYTE_COUNT; i++) { + update->remove_custom_flags[i] = + ~update->add_custom_flags[i]; + } + break; + case MODIFY_ADD: + /* already in add_flags */ + break; + case MODIFY_REMOVE: + /* add_flags -> remove_flags */ + update->remove_flags = update->add_flags; + memcpy(&update->remove_custom_flags, &update->add_custom_flags, + INDEX_CUSTOM_FLAGS_BYTE_COUNT); + update->add_flags = 0; + memset(&update->add_custom_flags, 0, + INDEX_CUSTOM_FLAGS_BYTE_COUNT); + break; + } +} + +static void mail_index_transaction_add_last(struct mail_index_transaction *t) +{ + struct mail_transaction_flag_update update, *data; + unsigned int idx, left_idx, right_idx; + uint32_t last; + size_t size; + + mail_index_transaction_get_last(t, &update); + + if (t->updates == NULL) { + t->updates = buffer_create_dynamic(default_pool, + 4096, (size_t)-1); + } + + data = buffer_get_modifyable_data(t->updates, &size); + size /= sizeof(*data); + + /* find the nearest sequence from existing updates */ + idx = 0; left_idx = 0; right_idx = size; + while (left_idx < right_idx) { + idx = (left_idx + right_idx) / 2; + + if (data[idx].seq1 < update.seq1) + left_idx = idx+1; + else if (data[idx].seq1 > update.seq1) + right_idx = idx; + else + break; + } + if (idx < size && data[idx].seq2 < update.seq1) + idx++; + + i_assert(idx == size || data[idx].seq1 < update.seq1); + + /* insert it into buffer, split it in multiple parts if needed + to make sure the ordering stays the same */ + for (; idx < size; idx++) { + if (data[idx].seq1 > update.seq2) + break; + + /* partial */ + last = update.seq2; + update.seq2 = data[idx].seq1-1; + + buffer_insert(t->updates, idx * sizeof(update), + &update, sizeof(update)); + data = buffer_get_modifyable_data(t->updates, NULL); + size++; + + update.seq1 = update.seq2+1; + update.seq2 = last; + } + + buffer_insert(t->updates, idx * sizeof(update), + &update, sizeof(update)); +} + +void mail_index_update_cache(struct mail_index_transaction *t, + uint32_t seq, uint32_t offset) +{ + struct mail_transaction_cache_update *data, update; + unsigned int idx, left_idx, right_idx; + size_t size; + + if (t->cache_updates == NULL) { + t->cache_updates = buffer_create_dynamic(default_pool, + 1024, (size_t)-1); + } + + data = buffer_get_modifyable_data(t->cache_updates, &size); + size /= sizeof(*data); + + /* we're probably appending it, check */ + if (size == 0 || data[size-1].seq < seq) + idx = size; + else { + idx = 0; left_idx = 0; right_idx = size; + while (left_idx < right_idx) { + idx = (left_idx + right_idx) / 2; + + if (data[idx].seq < seq) + left_idx = idx+1; + else if (data[idx].seq > seq) + right_idx = idx; + else { + /* already there, update */ + data[idx].cache_offset = offset; + return; + } + } + } + + update.seq = seq; + update.cache_offset = offset; + buffer_insert(t->updates, idx * sizeof(update), + &update, sizeof(update)); +}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-index/mail-index-view-private.h Tue Apr 27 23:25:52 2004 +0300 @@ -0,0 +1,29 @@ +#ifndef __MAIL_INDEX_VIEW_PRIVATE_H +#define __MAIL_INDEX_VIEW_PRIVATE_H + +#include "mail-index-private.h" + +struct mail_index_view { + struct mail_index *index; + struct mail_transaction_log_view *log_view; + + struct mail_index_map *map; + + uint32_t log_file_seq; + uoff_t log_file_offset; + buffer_t *log_syncs; + + int transactions; + unsigned int lock_id; + + unsigned int inconsistent:1; + unsigned int syncing:1; + unsigned int external:1; +}; + +int mail_index_view_lock(struct mail_index_view *view, int update_index); +void mail_index_view_add_synced_transaction(struct mail_index_view *view, + uint32_t log_file_seq, + uoff_t log_file_offset); + +#endif
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-index/mail-index-view-sync.c Tue Apr 27 23:25:52 2004 +0300 @@ -0,0 +1,335 @@ +/* Copyright (C) 2003-2004 Timo Sirainen */ + +#include "lib.h" +#include "buffer.h" +#include "mail-index-view-private.h" +#include "mail-index-sync-private.h" +#include "mail-transaction-log.h" +#include "mail-transaction-util.h" + +struct mail_index_view_sync_ctx { + struct mail_index_view *view; + enum mail_index_sync_type sync_mask; + struct mail_index_map *sync_map; + buffer_t *expunges; + + const struct mail_transaction_header *hdr; + const void *data; + + size_t data_offset; + unsigned int skipped:1; + unsigned int last_read:1; +}; + +static int +view_sync_get_expunges(struct mail_index_view *view, buffer_t **expunges_r) +{ + const struct mail_transaction_expunge *exp, *end; + buffer_t *expunges; + size_t size; + + /* with mask 0 we don't get anything, we'll just read the expunges + while seeking to end */ + if (mail_transaction_log_view_set(view->log_view, + view->log_file_seq, + view->log_file_offset, + view->index->hdr->log_file_seq, + view->index->hdr->log_file_offset, + 0) < 0) + return -1; + if (mail_transaction_log_view_next(view->log_view, + NULL, NULL, NULL) < 0) + return -1; + + expunges = mail_transaction_log_view_get_expunges(view->log_view); + exp = buffer_get_data(expunges, &size); + end = CONST_PTR_OFFSET(exp, size); + + *expunges_r = buffer_create_dynamic(default_pool, size, (size_t)-1); + for (; exp != end; exp++) { + buffer_append(*expunges_r, &exp->seq1, sizeof(exp->seq1)); + buffer_append(*expunges_r, &exp->seq2, sizeof(exp->seq2)); + } + return 0; +} + +int mail_index_view_sync_begin(struct mail_index_view *view, + enum mail_index_sync_type sync_mask, + struct mail_index_view_sync_ctx **ctx_r) +{ + const struct mail_index_header *hdr; + struct mail_index_view_sync_ctx *ctx; + struct mail_index_map *map; + enum mail_transaction_type mask; + buffer_t *expunges = NULL; + + /* We must sync flags as long as view is mmap()ed, as the flags may + have already changed under us. */ + i_assert((sync_mask & MAIL_INDEX_SYNC_TYPE_FLAGS) != 0); + i_assert(view->transactions == 0); + i_assert(!view->syncing); + + if (mail_index_view_lock(view, TRUE) < 0) + return -1; + + hdr = view->index->hdr; + if ((sync_mask & MAIL_INDEX_SYNC_TYPE_EXPUNGE) != 0) { + /* get list of all expunges first */ + if (view_sync_get_expunges(view, &expunges) < 0) + return -1; + } + + mask = mail_transaction_type_mask_get(sync_mask); + if (mail_transaction_log_view_set(view->log_view, + view->log_file_seq, + view->log_file_offset, + hdr->log_file_seq, + hdr->log_file_offset, mask) < 0) { + if (expunges != NULL) + buffer_free(expunges); + return -1; + } + + if (sync_mask == MAIL_INDEX_SYNC_MASK_ALL) { + map = view->index->map; + map->refcount++; + } else { + map = mail_index_map_to_memory(view->map); + } + view->syncing = TRUE; + + ctx = i_new(struct mail_index_view_sync_ctx, 1); + ctx->view = view; + ctx->sync_mask = sync_mask; + ctx->sync_map = map; + ctx->expunges = expunges; + + *ctx_r = ctx; + return 0; +} + +static int view_is_transaction_synced(struct mail_index_view *view, + uint32_t seq, uoff_t offset) +{ + const unsigned char *data, *end; + size_t size; + + if (view->log_syncs == NULL) + return 0; + + data = buffer_get_data(view->log_syncs, &size); + end = data + size; + + for (; data < end; ) { + if (*((const uoff_t *)data) == offset && + *((const uint32_t *)(data + sizeof(uoff_t))) == seq) + return 1; + data += sizeof(uoff_t) + sizeof(uint32_t); + } + + return 0; +} + +static int sync_expunge(const struct mail_transaction_expunge *e, void *context) +{ + struct mail_index_map *map = context; + unsigned int idx, count; + + for (idx = e->seq1-1; idx < e->seq2; idx++) { + mail_index_header_update_counts(&map->hdr_copy, + map->records[idx].flags, 0); + } + + count = e->seq2 - e->seq1 + 1; + buffer_delete(map->buffer, + (e->seq1-1) * sizeof(struct mail_index_record), + count * sizeof(struct mail_index_record)); + map->records = buffer_get_modifyable_data(map->buffer, NULL); + + map->records_count -= count; + map->hdr_copy.messages_count -= count; + return 1; +} + +static int sync_append(const struct mail_index_record *rec, void *context) +{ + struct mail_index_map *map = context; + + buffer_append(map->buffer, rec, sizeof(*rec)); + map->records = buffer_get_modifyable_data(map->buffer, NULL); + + map->records_count++; + map->hdr_copy.messages_count++; + + mail_index_header_update_counts(&map->hdr_copy, 0, rec->flags); + mail_index_header_update_lowwaters(&map->hdr_copy, rec); + return 1; +} + +static int sync_flag_update(const struct mail_transaction_flag_update *u, + void *context) +{ + struct mail_index_map *map = context; + struct mail_index_record *rec; + unsigned int i, idx; + uint8_t old_flags; + + for (idx = u->seq1-1; idx < u->seq2; idx++) { + rec = &map->records[idx]; + + old_flags = rec->flags; + rec->flags = (rec->flags & ~u->remove_flags) | u->add_flags; + for (i = 0; i < INDEX_CUSTOM_FLAGS_BYTE_COUNT; i++) { + rec->custom_flags[i] = + (rec->custom_flags[i] & + ~u->remove_custom_flags[i]) | + u->add_custom_flags[i]; + } + + mail_index_header_update_counts(&map->hdr_copy, old_flags, + rec->flags); + mail_index_header_update_lowwaters(&map->hdr_copy, rec); + } + return 1; +} + +static int sync_cache_update(const struct mail_transaction_cache_update *u, + void *context) +{ + struct mail_index_map *map = context; + + map->records[u->seq-1].cache_offset = u->cache_offset; + return 1; +} + +static int mail_index_view_sync_map(struct mail_index_view_sync_ctx *ctx) +{ + static struct mail_transaction_map_functions map_funcs = { + sync_expunge, sync_append, sync_flag_update, sync_cache_update + }; + + return mail_transaction_map(ctx->hdr, ctx->data, + &map_funcs, ctx->sync_map); +} + +static int mail_index_view_sync_next_trans(struct mail_index_view_sync_ctx *ctx, + uint32_t *seq_r, uoff_t *offset_r) +{ + struct mail_transaction_log_view *log_view = ctx->view->log_view; + struct mail_index_view *view = ctx->view; + int ret, skipped; + + ret = mail_transaction_log_view_next(log_view, &ctx->hdr, &ctx->data, + &skipped); + if (ret <= 0) { + if (ret < 0) + return -1; + + ctx->last_read = TRUE; + return 1; + } + + if (skipped) + ctx->skipped = TRUE; + + mail_transaction_log_view_get_prev_pos(log_view, seq_r, offset_r); + + /* skip flag changes that we committed ourself or have already synced */ + if (view_is_transaction_synced(view, *seq_r, *offset_r)) + return 0; + + if (ctx->sync_mask != MAIL_INDEX_SYNC_MASK_ALL) { + if (mail_index_view_sync_map(ctx) < 0) + return -1; + } + + return 1; +} + +int mail_index_view_sync_next(struct mail_index_view_sync_ctx *ctx, + struct mail_index_sync_rec *sync_rec) +{ + struct mail_index_view *view = ctx->view; + uint32_t seq; + uoff_t offset; + int ret; + + if (ctx->hdr == NULL || ctx->data_offset == ctx->hdr->size) { + ctx->data_offset = 0; + do { + ret = mail_index_view_sync_next_trans(ctx, &seq, + &offset); + if (ret < 0) + return -1; + + if (ctx->last_read) + return 0; + + if (!ctx->skipped) { + view->log_file_seq = seq; + view->log_file_offset = offset + + sizeof(*ctx->hdr) + ctx->hdr->size; + } + } while (ret == 0); + + if (ctx->skipped) { + mail_index_view_add_synced_transaction(view, seq, + offset); + } + } + + if (!mail_index_sync_get_rec(view, sync_rec, ctx->hdr, ctx->data, + &ctx->data_offset)) + return -1; + return 1; +} + +const uint32_t * +mail_index_view_sync_get_expunges(struct mail_index_view_sync_ctx *ctx, + size_t *count_r) +{ + const uint32_t *data; + size_t size; + + data = buffer_get_data(ctx->expunges, &size); + *count_r = size / (sizeof(uint32_t)*2); + return data; +} + +void mail_index_view_sync_end(struct mail_index_view_sync_ctx *ctx) +{ + struct mail_index_view *view = ctx->view; + + i_assert(view->syncing); + + if (view->log_syncs != NULL && !ctx->skipped) + buffer_set_used_size(view->log_syncs, 0); + + if (!ctx->last_read && ctx->hdr != NULL && + ctx->data_offset != ctx->hdr->size) { + /* we didn't sync everything */ + view->inconsistent = TRUE; + } + + mail_index_unmap(view->index, view->map); + view->map = ctx->sync_map; + + if (ctx->expunges != NULL) + buffer_free(ctx->expunges); + + view->syncing = FALSE; + i_free(ctx); +} + +void mail_index_view_add_synced_transaction(struct mail_index_view *view, + uint32_t log_file_seq, + uoff_t log_file_offset) +{ + if (view->log_syncs == NULL) { + view->log_syncs = buffer_create_dynamic(default_pool, + 128, (size_t)-1); + } + buffer_append(view->log_syncs, &log_file_offset, + sizeof(log_file_offset)); + buffer_append(view->log_syncs, &log_file_seq, sizeof(log_file_seq)); +}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-index/mail-index-view.c Tue Apr 27 23:25:52 2004 +0300 @@ -0,0 +1,275 @@ +/* Copyright (C) 2003-2004 Timo Sirainen */ + +#include "lib.h" +#include "buffer.h" +#include "file-lock.h" +#include "mail-index-view-private.h" +#include "mail-transaction-log.h" + +struct mail_index_view *mail_index_view_open(struct mail_index *index) +{ + struct mail_index_view *view; + + view = i_new(struct mail_index_view, 1); + view->index = index; + view->log_view = mail_transaction_log_view_open(index->log); + + view->map = index->map; + view->map->refcount++; + + view->log_file_seq = view->index->hdr->log_file_seq; + view->log_file_offset = view->index->hdr->log_file_offset; + return view; +} + +void mail_index_view_close(struct mail_index_view *view) +{ + mail_index_view_unlock(view); + mail_transaction_log_view_close(view->log_view); + + if (view->log_syncs != NULL) + buffer_free(view->log_syncs); + mail_index_unmap(view->index, view->map); + i_free(view); +} + +static int +mail_index_view_lock_head(struct mail_index_view *view, int update_index) +{ + if (!mail_index_is_locked(view->index, view->lock_id)) { + if (view->index->indexid != view->map->hdr->indexid) { + /* index was rebuilt */ + view->inconsistent = TRUE; + return -1; + } + + if (mail_index_lock_shared(view->index, update_index, + &view->lock_id) < 0) + return -1; + + if (mail_index_map(view->index, FALSE) <= 0) { + view->inconsistent = TRUE; + return -1; + } + } + + return 0; +} + +int mail_index_view_lock(struct mail_index_view *view, int update_index) +{ + if (view->inconsistent) + return -1; + + if (view->map != view->index->map) { + /* not head mapping, no need to lock */ + return 0; + } + + return mail_index_view_lock_head(view, update_index); +} + +void mail_index_view_unlock(struct mail_index_view *view) +{ + if (view->lock_id != 0) { + mail_index_unlock(view->index, view->lock_id); + view->lock_id = 0; + } +} + +uint32_t mail_index_view_get_message_count(struct mail_index_view *view) +{ + return view->map->records_count; +} + +int mail_index_view_is_inconsistent(struct mail_index_view *view) +{ + return view->inconsistent; +} + +struct mail_index *mail_index_view_get_index(struct mail_index_view *view) +{ + return view->index; +} + +void mail_index_view_transaction_ref(struct mail_index_view *view) +{ + view->transactions++; +} + +void mail_index_view_transaction_unref(struct mail_index_view *view) +{ + i_assert(view->transactions > 0); + + view->transactions--; +} + +const struct mail_index_header * +mail_index_get_header(struct mail_index_view *view) +{ + return view->map->hdr; +} + +int mail_index_lookup(struct mail_index_view *view, uint32_t seq, + const struct mail_index_record **rec_r) +{ + struct mail_index_map *map; + const struct mail_index_record *rec; + uint32_t uid; + + i_assert(seq > 0); + i_assert(seq <= view->map->records_count); + + if (mail_index_view_lock(view, FALSE) < 0) + return -1; + + rec = &view->map->records[seq-1]; + if (view->map == view->index->map) { + *rec_r = rec; + return 0; + } + + if (mail_index_view_lock_head(view, FALSE) < 0) + return -1; + + /* look for it in the head mapping */ + uid = rec->uid; + if (seq > view->index->hdr->messages_count) + seq = view->index->hdr->messages_count; + + map = view->index->map; + while (seq > 0) { + // FIXME: we could be skipping more by uid diff + if (map->records[--seq].uid <= uid) + break; + } + + *rec_r = map->records[seq].uid == uid ? + &map->records[seq] : rec; + return 0; +} + +int mail_index_lookup_uid(struct mail_index_view *view, uint32_t seq, + uint32_t *uid_r) +{ + i_assert(seq > 0); + i_assert(seq <= view->map->records_count); + + if (mail_index_view_lock(view, FALSE) < 0) + return -1; + + *uid_r = view->map->records[seq-1].uid; + return 0; +} + +static uint32_t mail_index_bsearch_uid(struct mail_index_view *view, + uint32_t uid, uint32_t *left_idx_p, + int nearest_side) +{ + const struct mail_index_record *rec; + uint32_t idx, left_idx, right_idx; + + rec = view->map->records; + + idx = 0; + left_idx = *left_idx_p; + right_idx = view->map->records_count; + + while (left_idx < right_idx) { + idx = (left_idx + right_idx) / 2; + + if (rec[idx].uid < uid) + left_idx = idx+1; + else if (rec[idx].uid > uid) + right_idx = idx; + else + break; + } + + *left_idx_p = left_idx; + if (rec[idx].uid != uid) { + if (nearest_side > 0) { + /* we want uid or larger */ + return rec[idx].uid > uid ? idx+1 : + idx == view->map->records_count-1 ? 0 : idx+2; + } else { + /* we want uid or smaller */ + return rec[idx].uid < uid ? idx + 1 : idx; + } + } + + return idx+1; +} + +int mail_index_lookup_uid_range(struct mail_index_view *view, + uint32_t first_uid, uint32_t last_uid, + uint32_t *first_seq_r, uint32_t *last_seq_r) +{ + uint32_t left_idx; + + i_assert(first_uid > 0); + i_assert(first_uid <= last_uid); + + if (mail_index_view_lock(view, FALSE) < 0) + return -1; + + left_idx = 0; + *first_seq_r = mail_index_bsearch_uid(view, first_uid, &left_idx, 1); + if (*first_seq_r == 0 || + view->map->records[*first_seq_r-1].uid > last_uid) { + *first_seq_r = 0; + *last_seq_r = 0; + return 0; + } + if (first_uid == last_uid) { + *last_seq_r = *first_seq_r; + return 0; + } + + /* optimization - binary lookup only from right side: */ + *last_seq_r = mail_index_bsearch_uid(view, last_uid, &left_idx, -1); + i_assert(*last_seq_r >= *first_seq_r); + return 0; +} + +int mail_index_lookup_first(struct mail_index_view *view, enum mail_flags flags, + uint8_t flags_mask, uint32_t *seq_r) +{ +#define LOW_UPDATE(x) \ + STMT_START { if ((x) > low_uid) low_uid = x; } STMT_END + const struct mail_index_record *rec; + uint32_t seq, low_uid = 1; + + *seq_r = 0; + + if (mail_index_view_lock(view, FALSE) < 0) + return -1; + + if ((flags_mask & MAIL_RECENT) != 0 && (flags & MAIL_RECENT) != 0) + LOW_UPDATE(view->map->hdr->first_recent_uid_lowwater); + if ((flags_mask & MAIL_SEEN) != 0 && (flags & MAIL_SEEN) == 0) + LOW_UPDATE(view->map->hdr->first_unseen_uid_lowwater); + if ((flags_mask & MAIL_DELETED) != 0 && (flags & MAIL_DELETED) != 0) + LOW_UPDATE(view->map->hdr->first_deleted_uid_lowwater); + + if (low_uid == 1) + seq = 1; + else { + if (mail_index_lookup_uid_range(view, low_uid, low_uid, + &seq, &seq) < 0) + return -1; + + if (seq == 0) + return 0; + } + + rec = &view->map->records[seq-1]; + for (; seq <= view->map->records_count; seq++, rec++) { + if ((rec->flags & flags_mask) == (uint8_t)flags) { + *seq_r = seq; + break; + } + } + + return 0; +}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-index/mail-index.c Tue Apr 27 23:25:52 2004 +0300 @@ -0,0 +1,588 @@ +/* Copyright (C) 2003-2004 Timo Sirainen */ + +#include "lib.h" +#include "buffer.h" +#include "file-lock.h" +#include "mmap-util.h" +#include "write-full.h" +#include "mail-index-private.h" +#include "mail-transaction-log.h" + +#include <stdio.h> +#include <stddef.h> +#include <time.h> + +struct mail_index *mail_index_alloc(const char *dir, const char *prefix) +{ + struct mail_index *index; + + index = i_new(struct mail_index, 1); + index->dir = i_strdup(dir); + index->prefix = i_strdup(prefix); + index->fd = -1; + + index->mode = 0600; + index->gid = (gid_t)-1; + return index; +} + +void mail_index_free(struct mail_index *index) +{ + i_free(index->error); + i_free(index->dir); + i_free(index->prefix); + i_free(index); +} + +static int mail_index_check_quick_header(struct mail_index *index, + struct mail_index_map *map, + const struct mail_index_header *hdr) +{ + if ((hdr->flags & MAIL_INDEX_HDR_FLAG_CORRUPTED) != 0) { + /* either a crash or we've already complained about it */ + return -1; + } + + if (map->mmap_used_size > map->mmap_size) { + map->records_count = + (map->mmap_size - hdr->header_size) / + sizeof(struct mail_index_record); + map->mmap_used_size = map->mmap_size; + + mail_index_set_error(index, "Corrupted index file %s: " + "messages_count too large (%u > %u)", + index->filepath, hdr->messages_count, + map->records_count); + return 0; + } + + return 1; +} + +static int mail_index_check_header(struct mail_index *index, + struct mail_index_map *map, + const struct mail_index_header *hdr) +{ + unsigned char compat_data[3]; + int ret; + +#ifndef WORDS_BIGENDIAN + compat_data[0] = MAIL_INDEX_COMPAT_LITTLE_ENDIAN; +#else + compat_data[0] = 0; +#endif + compat_data[1] = sizeof(uoff_t); + compat_data[2] = sizeof(time_t); + + if (hdr->major_version != MAIL_INDEX_MAJOR_VERSION) { + /* major version change - handle silently(?) */ + return -1; + } + if (memcmp(hdr->compat_data, compat_data, sizeof(compat_data)) != 0) { + /* architecture change - handle silently(?) */ + return -1; + } + + if ((ret = mail_index_check_quick_header(index, map, hdr)) <= 0) + return ret; + + /* following some extra checks that only take a bit of CPU */ + if (hdr->uid_validity == 0) { + mail_index_set_error(index, "Corrupted index file %s: " + "uid_validity = 0", index->filepath); + return -1; + } + + if (hdr->next_uid == 0) + return 0; + + if (hdr->seen_messages_count > hdr->messages_count || + hdr->deleted_messages_count > hdr->messages_count) + return 0; + if (hdr->first_recent_uid_lowwater > hdr->next_uid || + hdr->first_unseen_uid_lowwater > hdr->next_uid || + hdr->first_deleted_uid_lowwater > hdr->next_uid) + return 0; + + return 1; +} + +void mail_index_unmap(struct mail_index *index, struct mail_index_map *map) +{ + if (--map->refcount > 0) + return; + + if (map->buffer != NULL) { + i_assert(map->mmap_base == NULL); + buffer_free(map->buffer); + } else { + i_assert(map->buffer == NULL); + if (munmap(map->mmap_base, map->mmap_size) < 0) + mail_index_set_syscall_error(index, "munmap()"); + } + i_free(map); +} + +int mail_index_map(struct mail_index *index, int force) +{ + const struct mail_index_header *hdr; + struct mail_index_map *map; + size_t used_size; + int ret; + + if (!index->use_mmap) { + // FIXME + return -1; + } + + if (index->map != NULL) { + map = index->map; + + /* see if re-mmaping is needed (file has grown) */ + hdr = map->mmap_base; + used_size = hdr->header_size + + hdr->messages_count * sizeof(struct mail_index_record); + if (map->mmap_size >= used_size && !force) + return 1; + + if (munmap(map->mmap_base, map->mmap_size) < 0) + mail_index_set_syscall_error(index, "munmap()"); + map->mmap_base = NULL; + } else { + map = i_new(struct mail_index_map, 1); + map->refcount = 1; + } + + index->hdr = NULL; + index->map = NULL; + + map->mmap_base = mmap_ro_file(index->fd, &map->mmap_size); + if (map->mmap_base == MAP_FAILED) { + map->mmap_base = NULL; + mail_index_set_syscall_error(index, "mmap()"); + mail_index_unmap(index, map); + return -1; + } + + if (map->mmap_size < MAIL_INDEX_HEADER_MIN_SIZE) { + mail_index_set_error(index, "Corrupted index file %s: " + "File too small (%"PRIuSIZE_T")", + index->filepath, map->mmap_size); + mail_index_unmap(index, map); + return 0; + } + + hdr = map->mmap_base; + if (hdr->header_size < sizeof(*hdr)) { + /* header smaller than ours, make a copy so our newer headers + won't have garbage in them */ + memcpy(&map->hdr_copy, hdr, hdr->header_size); + hdr = &map->hdr_copy; + } + + map->hdr = map->mmap_base; + map->records = PTR_OFFSET(map->mmap_base, hdr->header_size); + map->records_count = hdr->messages_count; + map->mmap_used_size = hdr->header_size + + map->records_count * sizeof(struct mail_index_record); + + ret = mail_index_check_header(index, map, hdr); + if (ret < 0) { + mail_index_unmap(index, map); + return 0; + } + if (ret == 0) + index->fsck = TRUE; + + index->hdr = map->mmap_base; + index->map = map; + return 1; +} + +struct mail_index_map *mail_index_map_to_memory(struct mail_index_map *map) +{ + const struct mail_index_header *hdr; + struct mail_index_map *mem_map; + size_t size; + + if (MAIL_INDEX_MAP_IS_IN_MEMORY(map)) { + map->refcount++; + return map; + } + + size = map->records_count * sizeof(struct mail_index_record); + + mem_map = i_new(struct mail_index_map, 1); + mem_map->refcount = 1; + mem_map->buffer = buffer_create_dynamic(default_pool, size, (size_t)-1); + buffer_append(mem_map->buffer, map->records, size); + + mem_map->records = buffer_get_modifyable_data(mem_map->buffer, NULL); + mem_map->records_count = map->records_count; + + hdr = map->mmap_base; + memcpy(&mem_map->hdr_copy, map->mmap_base, + I_MIN(hdr->header_size, sizeof(mem_map->hdr_copy))); + mem_map->hdr = &mem_map->hdr_copy; + return mem_map; +} + +void mail_index_header_init(struct mail_index_header *hdr) +{ + time_t now = time(NULL); + + memset(hdr, 0, sizeof(*hdr)); + + hdr->major_version = MAIL_INDEX_MAJOR_VERSION; + hdr->minor_version = MAIL_INDEX_MINOR_VERSION; + hdr->header_size = sizeof(*hdr); + +#ifndef WORDS_BIGENDIAN + hdr->compat_data[0] = MAIL_INDEX_COMPAT_LITTLE_ENDIAN; +#endif + hdr->compat_data[1] = sizeof(uoff_t); + hdr->compat_data[2] = sizeof(time_t); + + hdr->indexid = now; + + hdr->uid_validity = now; + hdr->next_uid = 1; +} + +int mail_index_write_header(struct mail_index *index, + const struct mail_index_header *hdr) +{ + if (index->use_mmap) { + if (mprotect(index->map->mmap_base, sizeof(*hdr), + PROT_READ | PROT_WRITE) < 0) { + mail_index_set_syscall_error(index, "mprotect()"); + return -1; + } + + memcpy(index->map->mmap_base, hdr, sizeof(*hdr)); + if (msync(index->map->mmap_base, sizeof(*hdr), MS_SYNC) < 0) + return mail_index_set_syscall_error(index, "msync()"); + + if (mprotect(index->map->mmap_base, sizeof(*hdr), + PROT_READ) < 0) { + mail_index_set_syscall_error(index, "mprotect()"); + return -1; + } + } else { + if (pwrite_full(index->fd, hdr, sizeof(*hdr), 0) < 0) { + mail_index_set_syscall_error(index, "pwrite_full()"); + return -1; + } + + index->map->hdr_copy = *hdr; + index->hdr = &index->map->hdr_copy; + } + + return 0; +} + +int mail_index_create_tmp_file(struct mail_index *index, const char **path_r) +{ + const char *path; + int fd; + + path = *path_r = t_strconcat(index->filepath, ".tmp", NULL); + fd = open(path, O_RDWR|O_CREAT|O_TRUNC, index->mode); + if (fd == -1) + return mail_index_file_set_syscall_error(index, path, "open()"); + + if (index->gid != (gid_t)-1 && + fchown(index->fd, (uid_t)-1, index->gid) < 0) { + mail_index_file_set_syscall_error(index, path, "fchown()"); + return -1; + } + + return fd; +} + +int mail_index_create(struct mail_index *index, struct mail_index_header *hdr) +{ + const char *path; + uint32_t seq; + uoff_t offset; + int ret; + + /* log file lock protects index creation */ + if (mail_transaction_log_sync_lock(index->log, &seq, &offset) < 0) + return -1; + + hdr->log_file_seq = seq; + hdr->log_file_offset = offset; + + ret = mail_index_try_open(index); + if (ret != 0) { + mail_transaction_log_sync_unlock(index->log); + return ret; + } + + /* create it fully in index.tmp first */ + index->fd = mail_index_create_tmp_file(index, &path); + if (index->fd == -1) + ret = -1; + else if (write_full(index->fd, hdr, sizeof(*hdr)) < 0) { + mail_index_file_set_syscall_error(index, path, "write_full()"); + ret = -1; + } else { + ret = mail_index_map(index, FALSE); + } + + if (ret == 0) { + /* it's corrupted even while we just created it, + should never happen unless someone pokes the file directly */ + mail_index_set_error(index, + "Newly created index file is corrupted: %s", path); + ret = -1; + } + + if (ret < 0) { + if (unlink(path) < 0 && errno != ENOENT) { + mail_index_file_set_syscall_error(index, path, + "unlink()"); + } + return -1; + } + + /* make it visible to others */ + if (rename(path, index->filepath) < 0) { + mail_index_set_error(index, "rename(%s, %s) failed: %m", + path, index->filepath); + return -1; + } + + mail_transaction_log_sync_unlock(index->log); + return 1; +} + +int mail_index_try_open(struct mail_index *index) +{ + unsigned int lock_id; + int ret; + + index->fd = open(index->filepath, O_RDWR); + if (index->fd == -1 && errno == EACCES) { + index->fd = open(index->filepath, O_RDONLY); + index->readonly = TRUE; + } + if (index->fd == -1) { + if (errno != ENOENT) + return mail_index_set_syscall_error(index, "open()"); + + /* have to create it */ + return 0; + } else { + if (mail_index_lock_shared(index, FALSE, &lock_id) < 0) + return -1; + ret = mail_index_map(index, FALSE); + mail_index_unlock(index, lock_id); + + if (ret == 0) { + /* it's corrupted - recreate it */ + (void)close(index->fd); + index->fd = -1; + } + return ret; + } +} + +static int +mail_index_open2(struct mail_index *index, enum mail_index_open_flags flags) +{ + struct mail_index_header hdr; + int ret; + + ret = mail_index_try_open(index); + if (ret == 1) + hdr = *index->hdr; + else if (ret == 0) { + /* doesn't exist, or corrupted */ + if ((flags & MAIL_INDEX_OPEN_FLAG_CREATE) == 0) + return 0; + mail_index_header_init(&hdr); + index->hdr = &hdr; + } else if (ret < 0) + return -1; + + index->indexid = hdr.indexid; + + index->log = mail_transaction_log_open_or_create(index); + if (index->log == NULL) + return -1; + return index->fd != -1 ? 1 : mail_index_create(index, &hdr); +} + +int mail_index_open(struct mail_index *index, enum mail_index_open_flags flags) +{ + int i = 0, ret; + + if (index->opened) + return 0; + + do { + index->shared_lock_count = 0; + index->excl_lock_count = 0; + index->lock_type = F_UNLCK; + + index->nodiskspace = FALSE; + index->index_lock_timeout = FALSE; + index->log_locked = FALSE; + index->use_mmap = (flags & MAIL_INDEX_OPEN_FLAG_NO_MMAP) == 0; + index->readonly = FALSE; + + index->filepath = i_strconcat(index->dir, "/", + index->prefix, NULL); + ret = mail_index_open2(index, flags); + if (ret <= 0) + break; + + index->opened = TRUE; + if (index->fsck) { + index->fsck = FALSE; + ret = mail_index_fsck(index); + if (ret == 0) { + /* completely broken, reopen */ + if (i++ < 3) + continue; + /* too many tries */ + ret = -1; + } + } + break; + } while (1); + + if (ret <= 0) + mail_index_close(index); + + return ret; +} + +void mail_index_close(struct mail_index *index) +{ + if (index->log != NULL) { + mail_transaction_log_close(index->log); + index->log = NULL; + } + + mail_index_unmap(index, index->map); + index->map = NULL; + + if (index->fd != -1) { + if (close(index->fd) < 0) + mail_index_set_syscall_error(index, "close()"); + index->fd = -1; + } + + i_free(index->copy_lock_path); + i_free(index->filepath); + index->filepath = NULL; + + index->indexid = 0; + index->opened = FALSE; +} + +struct mail_cache *mail_index_get_cache(struct mail_index *index) +{ + return index->cache; +} + +int mail_index_set_error(struct mail_index *index, const char *fmt, ...) +{ + va_list va; + + i_free(index->error); + + if (fmt == NULL) + index->error = NULL; + else { + va_start(va, fmt); + index->error = i_strdup_vprintf(fmt, va); + va_end(va); + + i_error("%s", index->error); + } + + return -1; +} + +int mail_index_mark_corrupted(struct mail_index *index) +{ + struct mail_index_header hdr; + + if (index->readonly || index->hdr == NULL || + (index->hdr->flags & MAIL_INDEX_HDR_FLAG_CORRUPTED) != 0) + return 0; + + hdr = *index->hdr; + hdr.flags |= MAIL_INDEX_HDR_FLAG_CORRUPTED; + if (mail_index_write_header(index, &hdr) < 0) + return -1; + + if (fsync(index->fd) < 0) + return mail_index_set_syscall_error(index, "fsync()"); + return 0; +} + +int mail_index_set_syscall_error(struct mail_index *index, + const char *function) +{ + i_assert(function != NULL); + + if (ENOSPACE(errno)) { + index->nodiskspace = TRUE; + return -1; + } + + return mail_index_set_error(index, "%s failed with index file %s: %m", + function, index->filepath); +} + +int mail_index_file_set_syscall_error(struct mail_index *index, + const char *filepath, + const char *function) +{ + i_assert(filepath != NULL); + i_assert(function != NULL); + + if (ENOSPACE(errno)) { + index->nodiskspace = TRUE; + return -1; + } + + return mail_index_set_error(index, "%s failed with file %s: %m", + function, filepath); +} + +enum mail_index_error mail_index_get_last_error(struct mail_index *index) +{ + if (index->nodiskspace) + return MAIL_INDEX_ERROR_DISKSPACE; + if (index->error != NULL) + return MAIL_INDEX_ERROR_INTERNAL; + + return MAIL_INDEX_ERROR_NONE; +} + +const char *mail_index_get_error_message(struct mail_index *index) +{ + return index->error; +} + +void mail_index_reset_error(struct mail_index *index) +{ + if (index->error != NULL) { + i_free(index->error); + index->error = NULL; + } + + index->nodiskspace = FALSE; + index->index_lock_timeout = FALSE; +} + +int mail_index_is_in_memory(struct mail_index *index) +{ + return FALSE; // FIXME +}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-index/mail-index.h Tue Apr 27 23:25:52 2004 +0300 @@ -0,0 +1,281 @@ +#ifndef __MAIL_INDEX_H +#define __MAIL_INDEX_H + +#include "mail-types.h" + +#define MAIL_INDEX_MAJOR_VERSION 4 +#define MAIL_INDEX_MINOR_VERSION 0 + +#define MAIL_INDEX_HEADER_MIN_SIZE 68 + +/* Number of custom flags in mail_index_record. */ +#define INDEX_CUSTOM_FLAGS_COUNT (3*8) +#define INDEX_CUSTOM_FLAGS_BYTE_COUNT ((INDEX_CUSTOM_FLAGS_COUNT*7)/8) + +enum mail_index_open_flags { + /* Create index if it doesn't exist */ + MAIL_INDEX_OPEN_FLAG_CREATE = 0x01, + /* Open the index as fast as possible - do only minimal checks and + delay opening cache/log files unless they're needed. */ + MAIL_INDEX_OPEN_FLAG_FAST = 0x02, + /* Don't try to mmap() index files */ + MAIL_INDEX_OPEN_FLAG_NO_MMAP = 0x04, + /* Use only dotlocking, no fcntl() */ + MAIL_INDEX_OPEN_FLAG_USE_DOTLOCKS = 0x08 +}; + +enum mail_index_header_compat_flags { + MAIL_INDEX_COMPAT_LITTLE_ENDIAN = 0x01 +}; + +enum mail_index_header_flag { + /* Corrupted-flag should be set while anything dangerous is done to + index file, such as when expunging messages. Once the operation + is finished, the corrupted-flag is removed. If this flag is noticed + unexpectedly, the index must be assumed to be corrupted and must + not be used. */ + MAIL_INDEX_HDR_FLAG_CORRUPTED = 0x0001, + MAIL_INDEX_HDR_FLAG_COMPRESS_CACHE = 0x0002 +}; + +enum mail_index_record_flag { + /* If binary flags are set, it's not checked whether mail is + missing CRs. So this flag may be set as an optimization for + regular non-binary mails as well if it's known that it contains + valid CR+LF line breaks. */ + MAIL_INDEX_FLAG_BINARY_HEADER = 0x0001, + MAIL_INDEX_FLAG_BINARY_BODY = 0x0002, + + /* Mail header or body is known to contain NUL characters. */ + MAIL_INDEX_FLAG_HAS_NULS = 0x0004, + /* Mail header or body is known to not contain NUL characters. */ + MAIL_INDEX_FLAG_HAS_NO_NULS = 0x0008 +}; + +enum mail_index_error { + /* No errors */ + MAIL_INDEX_ERROR_NONE, + /* Internal error, see get_error_text() for more information. */ + MAIL_INDEX_ERROR_INTERNAL, + /* We ran out of available disk space. */ + MAIL_INDEX_ERROR_DISKSPACE +}; + +#define MAIL_INDEX_FLAGS_MASK \ + (MAIL_ANSWERED | MAIL_FLAGGED | MAIL_DELETED | MAIL_SEEN | MAIL_DRAFT) + +typedef unsigned char custom_flags_mask_t[INDEX_CUSTOM_FLAGS_BYTE_COUNT]; + +struct mail_index_header { + /* major version is increased only when you can't have backwards + compatibility. minor version is increased when header size is + increased to contain new non-critical fields. */ + uint8_t major_version; + uint8_t minor_version; + uint16_t header_size; + + /* 0 = flags + 1 = sizeof(uoff_t) + 2 = sizeof(time_t) + 3 = reserved, 0 for now */ + uint8_t compat_data[4]; + + uint32_t indexid; + uint32_t flags; + + uint32_t uid_validity; + uint32_t next_uid; + + uint32_t messages_count; + uint32_t seen_messages_count; + uint32_t deleted_messages_count; + + /* these UIDs may not exist and may not even be unseen */ + uint32_t first_recent_uid_lowwater; + uint32_t first_unseen_uid_lowwater; + uint32_t first_deleted_uid_lowwater; + + uint32_t log_file_seq; + uint32_t log_file_offset; + + uint64_t sync_size; + uint32_t sync_stamp; + + uint32_t cache_file_seq; +}; + +struct mail_index_record { + uint32_t uid; + uint8_t flags; /* mail_flags | mail_index_mail_flags */ + custom_flags_mask_t custom_flags; + uint32_t cache_offset; +}; + +enum mail_index_sync_type { + MAIL_INDEX_SYNC_TYPE_APPEND = 0x01, + MAIL_INDEX_SYNC_TYPE_EXPUNGE = 0x02, + MAIL_INDEX_SYNC_TYPE_FLAGS = 0x04 +}; +#define MAIL_INDEX_SYNC_MASK_ALL 0xff + +struct mail_index_sync_rec { + uint32_t seq1, seq2; + enum mail_index_sync_type type; + + /* MAIL_INDEX_SYNC_TYPE_FLAGS: */ + uint8_t add_flags; + custom_flags_mask_t add_custom_flags; + uint8_t remove_flags; + custom_flags_mask_t remove_custom_flags; + + /* MAIL_INDEX_SYNC_TYPE_APPEND: */ + const struct mail_index_record *appends; +}; + +struct mail_index; +struct mail_index_view; +struct mail_index_transaction; +struct mail_index_sync_ctx; +struct mail_index_view_sync_ctx; + +struct mail_index *mail_index_alloc(const char *dir, const char *prefix); +void mail_index_free(struct mail_index *index); + +int mail_index_open(struct mail_index *index, enum mail_index_open_flags flags); +void mail_index_close(struct mail_index *index); + +struct mail_cache *mail_index_get_cache(struct mail_index *index); + +/* View can be used to look into index. Sequence numbers inside view change + only when you synchronize it. The view acquires required locks + automatically, but you'll have to drop them manually. Opening view + acquires a lock immediately. */ +struct mail_index_view *mail_index_view_open(struct mail_index *index); +void mail_index_view_close(struct mail_index_view *view); + +/* Returns the index for given view. */ +struct mail_index *mail_index_view_get_index(struct mail_index_view *view); +/* Call whenever you've done with requesting messages from view for a while. */ +void mail_index_view_unlock(struct mail_index_view *view); +/* Returns number of mails in view. */ +uint32_t mail_index_view_get_message_count(struct mail_index_view *view); +/* Returns TRUE if we lost track of changes for some reason. */ +int mail_index_view_is_inconsistent(struct mail_index_view *view); + +/* Transaction has to be opened to be able to modify index. You can have + multiple transactions open simultaneously. Note that committed transactions + won't show up until you've synchronized mailbox (mail_index_sync_begin). */ +struct mail_index_transaction * +mail_index_transaction_begin(struct mail_index_view *view, int hide); +int mail_index_transaction_commit(struct mail_index_transaction *t, + uint32_t *log_file_seq_r, + uoff_t *log_file_offset_r); +void mail_index_transaction_rollback(struct mail_index_transaction *t); + +/* Begin synchronizing mailbox with index file. This call locks the index + exclusively against other modifications. Returns 1 if ok, -1 if error. + + If log_file_seq is not (uint32_t)-1 and index is already synchronized up + to given log_file_offset, the synchronization isn't started and this + function returns 0. This should be done when you wish to sync your previous + transaction instead of doing a full mailbox synchronization. + + mail_index_sync_next() returns all changes from previously committed + transactions which haven't yet been committed to the actual mailbox. + They're returned in ascending order. You must go through all of them and + update the mailbox accordingly. + + None of the changes actually show up in index until at + mail_index_sync_end(). + + Note that there may be multiple overlapping flag changes. They're returned + sorted by their beginning sequence. They never overlap expunges however. + Returned sequence numbers describe the mailbox state at the beginning of + synchronization, ie. expunges don't affect them. + + You may create a new transaction for the returned view. That transaction + acts as "external mailbox changes" transaction. Any changes done there are + expected to describe mailbox's current state. */ +int mail_index_sync_begin(struct mail_index *index, + struct mail_index_sync_ctx **ctx_r, + struct mail_index_view **view_r, + uint32_t log_file_seq, uoff_t log_file_offset); +/* Returns -1 if error, 0 if sync is finished, 1 if record was filled. */ +int mail_index_sync_next(struct mail_index_sync_ctx *ctx, + struct mail_index_sync_rec *sync_rec); +/* End synchronization by unlocking the index and closing the view. */ +int mail_index_sync_end(struct mail_index_sync_ctx *ctx); + +/* Reset index by erasing everything in it. Invalidates all views. */ +int mail_index_reset(struct mail_index *index); +/* Check and fix any found problems. If index is broken beyond repair, calls + mail_index_reset() and returns 0. Otherwise returns -1 if there was some + I/O error or 1 if everything went ok. */ +int mail_index_fsck(struct mail_index *index); + +/* Synchronize changes in view. You have to go through all records, or view + will be marked inconsistent. Only sync_mask type records are + synchronized. */ +int mail_index_view_sync_begin(struct mail_index_view *view, + enum mail_index_sync_type sync_mask, + struct mail_index_view_sync_ctx **ctx_r); +/* Returns -1 if error, 0 if sync is finished, 1 if record was filled. */ +int mail_index_view_sync_next(struct mail_index_view_sync_ctx *ctx, + struct mail_index_sync_rec *sync_rec); +const uint32_t * +mail_index_view_sync_get_expunges(struct mail_index_view_sync_ctx *ctx, + size_t *count_r); +void mail_index_view_sync_end(struct mail_index_view_sync_ctx *ctx); + +/* Returns the index header. */ +const struct mail_index_header * +mail_index_get_header(struct mail_index_view *view); + +/* Returns the given message. */ +int mail_index_lookup(struct mail_index_view *view, uint32_t seq, + const struct mail_index_record **rec_r); +/* Returns the UID for given message. May be slightly faster than + mail_index_lookup()->uid */ +int mail_index_lookup_uid(struct mail_index_view *view, uint32_t seq, + uint32_t *uid_r); +/* Convert UID range to sequence range. If no UIDs are found, sequences are + set to 0. Note that any of the returned sequences may have been expunged + already. */ +int mail_index_lookup_uid_range(struct mail_index_view *view, + uint32_t first_uid, uint32_t last_uid, + uint32_t *first_seq_r, uint32_t *last_seq_r); +/* Find first mail with (mail->flags & flags_mask) == flags. Useful mostly for + taking advantage of lowwater-fields in headers. */ +int mail_index_lookup_first(struct mail_index_view *view, enum mail_flags flags, + uint8_t flags_mask, uint32_t *seq_r); + +/* Append a new record to index. */ +void mail_index_append(struct mail_index_transaction *t, uint32_t uid, + uint32_t *seq_r); +/* Expunge record from index. Note that this doesn't affect sequence numbers + until transaction is committed and mailbox is synced. */ +void mail_index_expunge(struct mail_index_transaction *t, uint32_t seq); +/* Update flags in index. */ +void mail_index_update_flags(struct mail_index_transaction *t, uint32_t seq, + enum modify_type modify_type, + enum mail_flags flags, + custom_flags_mask_t custom_flags); + +/* Returns the last error code. */ +enum mail_index_error mail_index_get_last_error(struct mail_index *index); +/* Returns the full error message for last error. This message may + contain paths etc. so it shouldn't be shown to users. */ +const char *mail_index_get_error_message(struct mail_index *index); +/* Reset the error message. */ +void mail_index_reset_error(struct mail_index *index); + +/* Returns TRUE if index is currently only in memory. */ +int mail_index_is_in_memory(struct mail_index *index); + +/* Apply changes in MAIL_INDEX_SYNC_TYPE_FLAGS typed sync records to given + flags variables. */ +void mail_index_sync_flags_apply(const struct mail_index_sync_rec *sync_rec, + uint8_t *flags, + custom_flags_mask_t custom_flags); + +#endif
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-index/mail-transaction-log-private.h Tue Apr 27 23:25:52 2004 +0300 @@ -0,0 +1,47 @@ +#ifndef __MAIL_TRANSACTION_LOG_VIEW_H +#define __MAIL_TRANSACTION_LOG_VIEW_H + +#include "mail-transaction-log.h" + +struct mail_transaction_log_file { + struct mail_transaction_log *log; + struct mail_transaction_log_file *next; + + int refcount; + + char *filepath; + int fd; + int lock_type; + + ino_t st_ino; + dev_t st_dev; + + buffer_t *buffer; + uoff_t buffer_offset; + size_t buffer_size; + void *mmap_base; + size_t mmap_size; + + struct mail_transaction_log_header hdr; +}; + +struct mail_transaction_log { + struct mail_index *index; + struct mail_transaction_log_view *views; + struct mail_transaction_log_file *head, *tail; +}; + +void +mail_transaction_log_file_set_corrupted(struct mail_transaction_log_file *file, + const char *fmt, ...); + +int mail_transaction_log_file_find(struct mail_transaction_log *log, + uint32_t file_seq, + struct mail_transaction_log_file **file_r); + +int mail_transaction_log_file_map(struct mail_transaction_log_file *file, + uoff_t start_offset, uoff_t end_offset); + +void mail_transaction_logs_clean(struct mail_transaction_log *log); + +#endif
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-index/mail-transaction-log-view.c Tue Apr 27 23:25:52 2004 +0300 @@ -0,0 +1,398 @@ +/* Copyright (C) 2003-2004 Timo Sirainen */ + +#include "lib.h" +#include "buffer.h" +#include "mail-index-private.h" +#include "mail-transaction-log-private.h" +#include "mail-transaction-util.h" + +struct mail_transaction_log_view { + struct mail_transaction_log *log; + struct mail_transaction_log_view *next; + + uint32_t min_file_seq, max_file_seq; + uoff_t min_file_offset, max_file_offset; + + enum mail_transaction_type type_mask; + buffer_t *expunges_buf, *data_buf; + struct mail_transaction_expunge_traverse_ctx *exp_ctx; + struct mail_transaction_header tmp_hdr; + + struct mail_transaction_log_file *file; + uoff_t file_offset; + + uint32_t prev_file_seq; + uoff_t prev_file_offset; + + unsigned int broken:1; +}; + +struct mail_transaction_log_view * +mail_transaction_log_view_open(struct mail_transaction_log *log) +{ + struct mail_transaction_log_view *view; + + view = i_new(struct mail_transaction_log_view, 1); + view->log = log; + view->expunges_buf = + buffer_create_dynamic(default_pool, 512, (size_t)-1); + + view->next = log->views; + log->views = view; + return view; +} + +static void +mail_transaction_log_view_close_files(struct mail_transaction_log_view *view) +{ + struct mail_transaction_log_file *file; + + for (file = view->log->tail; file != NULL; file = file->next) { + if (file->hdr.file_seq > view->max_file_seq) + break; + if (file->hdr.file_seq >= view->min_file_seq) + file->refcount--; + } + + mail_transaction_logs_clean(view->log); +} + +void mail_transaction_log_view_close(struct mail_transaction_log_view *view) +{ + mail_transaction_log_view_close_files(view); + if (view->data_buf != NULL) + buffer_free(view->data_buf); + buffer_free(view->expunges_buf); + i_free(view); +} + +int +mail_transaction_log_view_set(struct mail_transaction_log_view *view, + uint32_t min_file_seq, uoff_t min_file_offset, + uint32_t max_file_seq, uoff_t max_file_offset, + enum mail_transaction_type type_mask) +{ + /* FIXME: error handling for "not found" case is bad.. should the + caller after all check it and handle as it sees best..? */ + struct mail_transaction_log_file *file, *first; + uint32_t seq; + uoff_t end_offset; + int ret; + + i_assert(min_file_seq <= max_file_seq); + i_assert(min_file_offset >= sizeof(struct mail_transaction_log_header)); + i_assert(max_file_offset >= sizeof(struct mail_transaction_log_header)); + + view->broken = TRUE; + + mail_transaction_log_view_close_files(view); + + ret = mail_transaction_log_file_find(view->log, min_file_seq, &file); + if (ret <= 0) + return -1; + end_offset = min_file_seq == max_file_seq ? + max_file_offset : (uoff_t)-1; + ret = mail_transaction_log_file_map(file, min_file_offset, end_offset); + if (ret <= 0) + return -1; + first = file; + + for (seq = min_file_seq+1; seq <= max_file_seq; seq++) { + file = file->next; + if (file == NULL || file->hdr.file_seq != seq) + return -1; + + end_offset = file->hdr.file_seq == max_file_seq ? + max_file_offset : (uoff_t)-1; + ret = mail_transaction_log_file_map(file, + sizeof(struct mail_transaction_log_header), + end_offset); + if (ret <= 0) + return -1; + } + + i_assert(max_file_offset <= file->hdr.used_size); + + /* we have it all, refcount the files */ + for (file = first, seq = min_file_seq; seq <= max_file_seq; seq++) { + file->refcount++; + file = file->next; + } + + buffer_set_used_size(view->expunges_buf, 0); + + view->prev_file_seq = 0; + view->prev_file_offset = 0; + + view->file = first; + view->file_offset = min_file_offset; + + view->min_file_seq = min_file_seq; + view->min_file_offset = min_file_offset; + view->max_file_seq = max_file_seq; + view->max_file_offset = max_file_offset; + view->type_mask = type_mask; + view->broken = FALSE; + return 0; +} + +void +mail_transaction_log_view_get_prev_pos(struct mail_transaction_log_view *view, + uint32_t *file_seq_r, + uoff_t *file_offset_r) +{ + *file_seq_r = view->prev_file_seq; + *file_offset_r = view->prev_file_offset; +} + +void +mail_transaction_log_view_set_corrupted(struct mail_transaction_log_view *view, + const char *fmt, ...) +{ + va_list va; + + i_assert(view->file != NULL); + + view->broken = TRUE; + + va_start(va, fmt); + t_push(); + mail_transaction_log_file_set_corrupted(view->file, "%s", + t_strdup_vprintf(fmt, va)); + t_pop(); + va_end(va); +} + +int +mail_transaction_log_view_is_corrupted(struct mail_transaction_log_view *view) +{ + return view->broken; +} + +static int log_view_get_next(struct mail_transaction_log_view *view, + const struct mail_transaction_header **hdr_r, + const void **data_r) +{ + const struct mail_transaction_header *hdr; + struct mail_transaction_log_file *file = view->file; + const struct mail_transaction_type_map *type_rec; + const void *data; + unsigned int record_size; + size_t size; + + view->prev_file_seq = file->hdr.file_seq; + view->prev_file_offset = view->file_offset; + + if (view->file_offset == file->hdr.used_size) { + view->file = file->next; + view->file_offset = sizeof(struct mail_transaction_log_header); + return 0; + } + + data = buffer_get_data(file->buffer, &size); + if (view->file_offset + sizeof(*hdr) > file->hdr.used_size) { + mail_transaction_log_file_set_corrupted(file, + "offset points outside file (%u + %"PRIuSIZE_T" > %u)", + view->file_offset, sizeof(*hdr), size); + return -1; + } + + hdr = CONST_PTR_OFFSET(data, view->file_offset - file->buffer_offset); + view->file_offset += sizeof(*hdr); + + if (file->hdr.used_size - view->file_offset < hdr->size) { + mail_transaction_log_file_set_corrupted(file, + "record size too large " + "(type=0x%x, offset=%u, size=%u, end=%u)", + hdr->type & MAIL_TRANSACTION_TYPE_MASK, + view->file_offset, hdr->size, file->hdr.used_size); + view->file_offset = file->hdr.used_size; + return -1; + } + + type_rec = mail_transaction_type_lookup(hdr->type); + if (type_rec != NULL) + record_size = type_rec->record_size; + else { + mail_transaction_log_file_set_corrupted(file, + "unknown record type 0x%x", + hdr->type & MAIL_TRANSACTION_TYPE_MASK); + view->file_offset = file->hdr.used_size; + return -1; + } + + if ((hdr->type & MAIL_TRANSACTION_EXPUNGE) != 0) { + if ((hdr->type & MAIL_TRANSACTION_TYPE_MASK) != + (MAIL_TRANSACTION_EXPUNGE|MAIL_TRANSACTION_EXPUNGE_PROT)) { + mail_transaction_log_file_set_corrupted(file, + "found expunge without protection mask"); + return -1; + } + } else if ((hdr->type & MAIL_TRANSACTION_TYPE_MASK) != type_rec->type) { + mail_transaction_log_file_set_corrupted(file, + "extra bits in header type: 0x%x", + hdr->type & MAIL_TRANSACTION_TYPE_MASK); + return -1; + } + + if (hdr->size % record_size != 0) { + mail_transaction_log_file_set_corrupted(file, + "record size wrong (type 0x%x, %u %% %u != 0)", + hdr->type & MAIL_TRANSACTION_TYPE_MASK, + hdr->size, record_size); + view->file_offset = file->hdr.used_size; + return -1; + } + + *hdr_r = hdr; + *data_r = CONST_PTR_OFFSET(data, view->file_offset - + file->buffer_offset); + view->file_offset += hdr->size; + return 1; +} + +static int seqfix_expunge(const struct mail_transaction_expunge *e, + void *context) +{ + struct mail_transaction_log_view *view = context; + struct mail_transaction_expunge new_e; + uint32_t expunges_before; + + expunges_before = mail_transaction_expunge_traverse_to(view->exp_ctx, + e->seq2); + if (expunges_before == 0) { + buffer_append(view->data_buf, e, sizeof(*e)); + return 1; + } + + /* FIXME: if there's expunges in the middle of the + range, we'd have to split this to multiple records */ + + new_e = *e; + new_e.seq2 += expunges_before; + new_e.seq1 += mail_transaction_expunge_traverse_to(view->exp_ctx, + new_e.seq1); + buffer_append(view->data_buf, &new_e, sizeof(new_e)); + return 1; +} + +static int seqfix_flag_update(const struct mail_transaction_flag_update *u, + void *context) +{ + struct mail_transaction_log_view *view = context; + struct mail_transaction_flag_update new_u; + uint32_t expunges_before; + + expunges_before = mail_transaction_expunge_traverse_to(view->exp_ctx, + u->seq2); + if (expunges_before == 0) { + buffer_append(view->data_buf, u, sizeof(*u)); + return 1; + } + + /* FIXME: if there's expunges in the middle of the + range, we'd have to split this to multiple records */ + + new_u = *u; + new_u.seq2 += expunges_before; + new_u.seq1 += mail_transaction_expunge_traverse_to(view->exp_ctx, + new_u.seq1); + buffer_append(view->data_buf, &new_u, sizeof(new_u)); + return 1; +} + +static int seqfix_cache_update(const struct mail_transaction_cache_update *u, + void *context) +{ + struct mail_transaction_log_view *view = context; + struct mail_transaction_cache_update new_u; + uint32_t expunges_before; + + expunges_before = mail_transaction_expunge_traverse_to(view->exp_ctx, + u->seq); + if (expunges_before != 0) { + new_u = *u; + new_u.seq += expunges_before; + u = &new_u; + } + + buffer_append(view->data_buf, u, sizeof(*u)); + return 1; +} + +int mail_transaction_log_view_next(struct mail_transaction_log_view *view, + const struct mail_transaction_header **hdr_r, + const void **data_r, int *skipped_r) +{ + struct mail_transaction_map_functions seqfix_funcs = { + seqfix_expunge, NULL, seqfix_flag_update, seqfix_cache_update + }; + const struct mail_transaction_header *hdr; + const void *data; + int ret = 0; + + if (skipped_r != NULL) + *skipped_r = FALSE; + if (view->broken) + return -1; + + while ((ret = log_view_get_next(view, &hdr, &data)) > 0) { + if ((view->type_mask & hdr->type) != 0) + break; + + /* we don't want this record */ + if (skipped_r != NULL) + *skipped_r = TRUE; + + if ((hdr->type & MAIL_TRANSACTION_EXPUNGE) != 0) { + mail_transaction_log_sort_expunges(view->expunges_buf, + data, hdr->size); + } + + /* FIXME: hide flag/cache updates for appends if + append isn't in mask */ + } + + if (ret <= 0) + return ret; + + *hdr_r = hdr; + *data_r = data; + + if (buffer_get_used_size(view->expunges_buf) > 0) { + /* we have to fix sequences in the data */ + if (view->data_buf == NULL) { + view->data_buf = + buffer_create_dynamic(default_pool, + hdr->size, (size_t)-1); + } else { + buffer_set_used_size(view->data_buf, 0); + } + + view->exp_ctx = mail_transaction_expunge_traverse_init( + view->expunges_buf); + ret = mail_transaction_map(hdr, data, &seqfix_funcs, view); + mail_transaction_expunge_traverse_deinit(view->exp_ctx); + i_assert(buffer_get_used_size(view->data_buf) == hdr->size); + + *data_r = buffer_get_data(view->data_buf, NULL); + } + + if ((hdr->type & MAIL_TRANSACTION_EXPUNGE) != 0) { + mail_transaction_log_sort_expunges(view->expunges_buf, + data, hdr->size); + + /* hide expunge protection */ + view->tmp_hdr = *hdr; + view->tmp_hdr.type &= ~MAIL_TRANSACTION_EXPUNGE_PROT; + *hdr_r = &view->tmp_hdr; + } + + return 1; +} + +buffer_t * +mail_transaction_log_view_get_expunges(struct mail_transaction_log_view *view) +{ + return view->expunges_buf; +}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-index/mail-transaction-log.c Tue Apr 27 23:25:52 2004 +0300 @@ -0,0 +1,985 @@ +/* Copyright (C) 2003-2004 Timo Sirainen */ + +#include "lib.h" +#include "buffer.h" +#include "file-lock.h" +#include "file-dotlock.h" +#include "read-full.h" +#include "write-full.h" +#include "mmap-util.h" +#include "mail-index-private.h" +#include "mail-index-view-private.h" +#include "mail-transaction-log-private.h" +#include "mail-transaction-util.h" +#include "mail-index-transaction-private.h" + +#include <stddef.h> +#include <sys/stat.h> + +struct mail_transaction_add_ctx { + struct mail_transaction_log *log; + struct mail_index_view *view; + + buffer_t *appends, *expunges; + buffer_t *flag_updates, *cache_updates; +}; + +static struct mail_transaction_log_file * +mail_transaction_log_file_open_or_create(struct mail_transaction_log *log, + const char *path); +static int mail_transaction_log_rotate(struct mail_transaction_log *log); + +static int +mail_transaction_log_file_lock(struct mail_transaction_log_file *file, + int lock_type); +static int mail_transaction_log_lock_head(struct mail_transaction_log *log); + +void +mail_transaction_log_file_set_corrupted(struct mail_transaction_log_file *file, + const char *fmt, ...) +{ + va_list va; + + file->hdr.indexid = 0; + if (pwrite_full(file->fd, &file->hdr.indexid, + sizeof(file->hdr.indexid), 0) < 0) { + mail_index_file_set_syscall_error(file->log->index, + file->filepath, "pwrite()"); + } + + va_start(va, fmt); + t_push(); + mail_index_set_error(file->log->index, + "Corrupted transaction log file %s: %s", + file->filepath, t_strdup_vprintf(fmt, va)); + t_pop(); + va_end(va); +} + +static int mail_transaction_log_check_file_seq(struct mail_transaction_log *log) +{ + struct mail_index *index = log->index; + struct mail_transaction_log_file *file; + unsigned int lock_id; + int ret; + + if (mail_transaction_log_lock_head(log) < 0) + return -1; + + file = log->head; + ret = mail_index_lock_shared(index, TRUE, &lock_id); + if (ret == 0) { + ret = mail_index_map(index, FALSE); + if (ret <= 0) + ret = -1; + else if (file->hdr.file_seq != index->hdr->log_file_seq) { + /* broken - fix it by creating a new log file */ + ret = mail_transaction_log_rotate(log); + } + } + (void)mail_transaction_log_file_lock(file, F_UNLCK); + return ret; +} + +struct mail_transaction_log * +mail_transaction_log_open_or_create(struct mail_index *index) +{ + struct mail_transaction_log *log; + const char *path; + + log = i_new(struct mail_transaction_log, 1); + log->index = index; + + path = t_strconcat(log->index->filepath, + MAIL_TRANSACTION_LOG_PREFIX, NULL); + log->head = mail_transaction_log_file_open_or_create(log, path); + if (log->head == NULL) { + i_free(log); + return NULL; + } + + if (index->fd != -1 && + log->head->hdr.file_seq != index->hdr->log_file_seq) { + /* head log file isn't same as head index file - + shouldn't happen except in race conditions. lock them and + check again - FIXME: missing error handling */ + (void)mail_transaction_log_check_file_seq(log); + } + return log; +} + +void mail_transaction_log_close(struct mail_transaction_log *log) +{ + i_assert(log->views == NULL); + + i_free(log); +} + +static int +mail_transaction_log_file_lock(struct mail_transaction_log_file *file, + int lock_type) +{ + int ret; + + if (lock_type == F_UNLCK) { + i_assert(file->lock_type != F_UNLCK); + } else { + i_assert(file->lock_type == F_UNLCK); + } + + ret = file_wait_lock_full(file->fd, lock_type, DEFAULT_LOCK_TIMEOUT, + NULL, NULL); + if (ret > 0) { + file->lock_type = lock_type; + return 0; + } + if (ret < 0) { + mail_index_file_set_syscall_error(file->log->index, + file->filepath, + "file_wait_lock()"); + return -1; + } + + mail_index_set_error(file->log->index, + "Timeout while waiting for release of " + "%s fcntl() lock for transaction log file %s", + lock_type == F_WRLCK ? "exclusive" : "shared", + file->filepath); + file->log->index->index_lock_timeout = TRUE; + return -1; +} + +static void +mail_transaction_log_file_close(struct mail_transaction_log_file *file) +{ + if (close(file->fd) < 0) { + mail_index_file_set_syscall_error(file->log->index, + file->filepath, "close()"); + } + + i_free(file->filepath); + i_free(file); +} + +static int +mail_transaction_log_file_read_hdr(struct mail_transaction_log_file *file, + struct stat *st) +{ + int ret; + uint32_t old_size = file->hdr.used_size; + + if (file->lock_type != F_UNLCK) + ret = pread_full(file->fd, &file->hdr, sizeof(file->hdr), 0); + else { + if (mail_transaction_log_file_lock(file, F_RDLCK) < 0) + return -1; + ret = pread_full(file->fd, &file->hdr, sizeof(file->hdr), 0); + (void)mail_transaction_log_file_lock(file, F_UNLCK); + } + + if (ret < 0) { + mail_index_file_set_syscall_error(file->log->index, + file->filepath, "pread()"); + return -1; + } + if (ret == 0) { + mail_transaction_log_file_set_corrupted(file, + "unexpected end of file while reading header"); + return 0; + } + if (file->hdr.indexid == 0) { + /* corrupted */ + mail_index_set_error(file->log->index, + "Transaction log file %s: marked corrupted", + file->filepath); + return 0; + } + if (file->hdr.indexid != file->log->index->indexid && + file->log->index->indexid != 0) { + /* either index was just recreated, or transaction has wrong + indexid. we don't know here which one is the case, so we'll + just fail. If index->indexid == 0, we're rebuilding it and + we just want to lock the transaction log. */ + mail_index_set_error(file->log->index, + "Transaction log file %s: invalid indexid", + file->filepath); + return 0; + } + if (file->hdr.used_size > st->st_size) { + mail_transaction_log_file_set_corrupted(file, + "used_size (%u) > file size (%"PRIuUOFF_T")", + file->hdr.used_size, (uoff_t)st->st_size); + return 0; + } + if (file->hdr.used_size < old_size) { + mail_transaction_log_file_set_corrupted(file, + "used_size (%u) < old_size (%u)", + file->hdr.used_size, old_size); + return 0; + } + + return 1; +} + +static int mail_transaction_log_file_create(struct mail_transaction_log *log, + const char *path, + dev_t dev, ino_t ino) +{ + struct mail_index *index = log->index; + struct mail_transaction_log_header hdr; + struct stat st; + unsigned int lock_id; + int fd, fd2, ret; + + /* this lock should never exist for a long time.. */ + fd = file_dotlock_open(path, NULL, 30, 0, 120, NULL, NULL); + if (fd == -1) { + mail_index_file_set_syscall_error(index, path, + "file_dotlock_open()"); + return -1; + } + + /* log creation is locked now - see if someone already created it */ + fd2 = open(path, O_RDWR); + if (fd2 != -1) { + if ((ret = fstat(fd2, &st)) < 0) { + mail_index_file_set_syscall_error(index, path, + "fstat()"); + } else if (st.st_dev == dev && st.st_ino == ino) { + /* same file, still broken */ + } else { + (void)file_dotlock_delete(path, fd2); + return fd2; + } + + (void)close(fd2); + fd2 = -1; + + if (ret < 0) + return -1; + } else if (errno != ENOENT) { + mail_index_file_set_syscall_error(index, path, "open()"); + return -1; + } + + memset(&hdr, 0, sizeof(hdr)); + hdr.indexid = index->indexid; + hdr.used_size = sizeof(hdr); + + if (index->fd != -1) { + index->log_locked = TRUE; /* kludging around assert.. */ + if (mail_index_lock_exclusive(index, 0, 0, &lock_id) < 0) { + (void)file_dotlock_delete(path, fd); + index->log_locked = FALSE; + return -1; + } + index->log_locked = FALSE; + + ret = mail_index_map(index, FALSE); + if (ret > 0) { + /* update log_file_* fields in header */ + struct mail_index_header idx_hdr; + + idx_hdr = *index->hdr; + idx_hdr.log_file_seq++; + idx_hdr.log_file_offset = sizeof(hdr); + if (mail_index_write_header(index, &idx_hdr) < 0) + ret = -1; + } + mail_index_unlock(index, lock_id); + + if (ret <= 0) { + (void)file_dotlock_delete(path, fd); + return -1; + } + hdr.file_seq = index->hdr->log_file_seq; + } else { + /* creating new index file */ + hdr.file_seq = index->hdr->log_file_seq+1; + } + + if (write_full(fd, &hdr, sizeof(hdr)) < 0) { + mail_index_file_set_syscall_error(index, path, "write_full()"); + (void)file_dotlock_delete(path, fd); + return -1; + } + + fd2 = dup(fd); + if (fd2 < 0) { + mail_index_file_set_syscall_error(index, path, "dup()"); + (void)file_dotlock_delete(path, fd); + return -1; + } + + if (file_dotlock_replace(path, fd, FALSE) <= 0) + return -1; + + /* success */ + return fd2; +} + +static struct mail_transaction_log_file * +mail_transaction_log_file_fd_open(struct mail_transaction_log *log, + const char *path, int fd) +{ + struct mail_transaction_log_file **p; + struct mail_transaction_log_file *file; + struct stat st; + int ret; + + if (fstat(fd, &st) < 0) { + mail_index_file_set_syscall_error(log->index, path, "stat()"); + return NULL; + } + + file = i_new(struct mail_transaction_log_file, 1); + file->refcount = 1; + file->log = log; + file->filepath = i_strdup(path); + file->fd = fd; + file->lock_type = F_UNLCK; + file->st_dev = st.st_dev; + file->st_ino = st.st_ino; + + ret = mail_transaction_log_file_read_hdr(file, &st); + if (ret == 0) { + /* corrupted header */ + fd = mail_transaction_log_file_create(log, path, + st.st_dev, st.st_ino); + if (fstat(fd, &st) < 0) { + mail_index_file_set_syscall_error(log->index, path, + "stat()"); + (void)close(fd); + fd = -1; + ret = -1; + } + + if (fd != -1) { + (void)close(file->fd); + file->fd = fd; + + file->st_dev = st.st_dev; + file->st_ino = st.st_ino; + + memset(&file->hdr, 0, sizeof(file->hdr)); + ret = mail_transaction_log_file_read_hdr(file, &st); + } + } + if (ret <= 0) { + mail_transaction_log_file_close(file); + return NULL; + } + + for (p = &log->tail; *p != NULL; p = &(*p)->next) + ; + *p = file; + + return file; +} + +static struct mail_transaction_log_file * +mail_transaction_log_file_open_or_create(struct mail_transaction_log *log, + const char *path) +{ + int fd; + + fd = open(path, O_RDWR); + if (fd == -1) { + if (errno != ENOENT) { + mail_index_file_set_syscall_error(log->index, path, + "open()"); + return NULL; + } + + fd = mail_transaction_log_file_create(log, path, 0, 0); + if (fd == -1) + return NULL; + } + + return mail_transaction_log_file_fd_open(log, path, fd); +} + +void mail_transaction_logs_clean(struct mail_transaction_log *log) +{ + struct mail_transaction_log_file **p; + + for (p = &log->tail; *p != NULL; ) { + if ((*p)->refcount != 0) + p = &(*p)->next; + else { + mail_transaction_log_file_close(*p); + *p = (*p)->next; + } + } +} + +static int mail_transaction_log_rotate(struct mail_transaction_log *log) +{ + struct mail_transaction_log_file *file; + struct stat st; + int fd; + + if (fstat(log->head->fd, &st) < 0) { + mail_index_file_set_syscall_error(log->index, + log->head->filepath, + "fstat()"); + return -1; + } + + fd = mail_transaction_log_file_create(log, log->head->filepath, + st.st_dev, st.st_ino); + if (fd == -1) + return 0; + + file = mail_transaction_log_file_fd_open(log, log->head->filepath, fd); + if (file == NULL) + return -1; + + if (log->head != NULL) { + if (--log->head->refcount == 0) + mail_transaction_logs_clean(log); + } + + log->head = file; + return 0; +} + +static int mail_transaction_log_refresh(struct mail_transaction_log *log) +{ + struct mail_transaction_log_file *file; + struct stat st; + const char *path; + int ret; + + path = t_strconcat(log->index->filepath, + MAIL_TRANSACTION_LOG_PREFIX, NULL); + if (stat(path, &st) < 0) { + mail_index_file_set_syscall_error(log->index, path, "stat()"); + return -1; + } + + if (log->head != NULL && + log->head->st_ino == st.st_ino && + log->head->st_dev == st.st_dev) { + /* same file */ + ret = mail_transaction_log_file_read_hdr(log->head, &st); + return ret <= 0 ? -1 : 0; + } + + file = mail_transaction_log_file_open_or_create(log, path); + if (file == NULL) + return -1; + + if (log->head != NULL) { + if (--log->head->refcount == 0) + mail_transaction_logs_clean(log); + } + + log->head = file; + return 0; +} + +int mail_transaction_log_file_find(struct mail_transaction_log *log, + uint32_t file_seq, + struct mail_transaction_log_file **file_r) +{ + struct mail_transaction_log_file *file; + + if (file_seq > log->head->hdr.file_seq) { + if (mail_transaction_log_refresh(log) < 0) + return -1; + } + + for (file = log->tail; file != NULL; file = file->next) { + if (file->hdr.file_seq == file_seq) { + *file_r = file; + return 1; + } + } + + return 0; +} + +static int +mail_transaction_log_file_read(struct mail_transaction_log_file *file, + uoff_t offset) +{ + void *data; + size_t size; + int ret; + + i_assert(file->mmap_base == NULL); + i_assert(offset <= file->hdr.used_size); + + if (file->buffer != NULL && file->buffer_offset > offset) { + /* we have to insert missing data to beginning of buffer */ + size = file->buffer_offset - offset; + buffer_copy(file->buffer, size, file->buffer, 0, (size_t)-1); + file->buffer_offset = offset; + + data = buffer_get_modifyable_data(file->buffer, NULL); + ret = pread(file->fd, data, size, offset); + if (ret < 0 && errno == ESTALE) { + /* log file was deleted in NFS server, fail silently */ + ret = 0; + } + if (ret <= 0) + return ret; + } + + if (file->buffer == NULL) { + size = file->hdr.used_size - offset; + file->buffer = buffer_create_dynamic(default_pool, + size, (size_t)-1); + file->buffer_offset = offset; + size = 0; + } else { + size = buffer_get_used_size(file->buffer); + if (file->buffer_offset + size >= file->hdr.used_size) { + /* caller should have checked this.. */ + return 1; + } + } + + size = file->hdr.used_size - file->buffer_offset - size; + data = buffer_append_space_unsafe(file->buffer, size); + + ret = pread(file->fd, data, size, offset); + if (ret < 0 && errno == ESTALE) { + /* log file was deleted in NFS server, fail silently */ + ret = 0; + } + return ret; +} + +int mail_transaction_log_file_map(struct mail_transaction_log_file *file, + uoff_t start_offset, uoff_t end_offset) +{ + size_t size; + struct stat st; + int ret; + + i_assert(start_offset <= end_offset); + + if (file->hdr.indexid == 0) { + /* corrupted */ + return 0; + } + + if (file->buffer != NULL && file->buffer_offset <= start_offset) { + /* see if we already have it */ + size = buffer_get_used_size(file->buffer); + if (file->buffer_offset + size >= end_offset) + return 1; + } + + if (fstat(file->fd, &st) < 0) { + mail_index_file_set_syscall_error(file->log->index, + file->filepath, "fstat()"); + return -1; + } + + if (st.st_size == file->hdr.used_size && end_offset == (uoff_t)-1) { + /* we've seen the whole file.. do we have all of it mapped? */ + size = buffer_get_used_size(file->buffer); + if (file->buffer_offset + size == file->hdr.used_size) + return 1; + } + + if (file->buffer != NULL && + (file->mmap_base != NULL || file->log->index->use_mmap)) { + buffer_free(file->buffer); + file->buffer = NULL; + } + if (file->mmap_base != NULL) { + if (munmap(file->mmap_base, file->mmap_size) < 0) { + mail_index_file_set_syscall_error(file->log->index, + file->filepath, + "munmap()"); + } + file->mmap_base = NULL; + } + + if (mail_transaction_log_file_read_hdr(file, &st) <= 0) + return -1; + + if (end_offset == (uoff_t)-1) + end_offset = file->hdr.used_size; + + if (start_offset < sizeof(file->hdr)) { + mail_transaction_log_file_set_corrupted(file, + "offset (%"PRIuUOFF_T"u) < header size (%"PRIuSIZE_T")", + start_offset, sizeof(file->hdr)); + return -1; + } + if (end_offset > file->hdr.used_size) { + mail_transaction_log_file_set_corrupted(file, + "offset (%"PRIuUOFF_T"u) > used_size (%u)", + end_offset, file->hdr.used_size); + return -1; + } + + if (!file->log->index->use_mmap) { + ret = mail_transaction_log_file_read(file, start_offset); + if (ret <= 0) { + /* make sure we don't leave ourself in + inconsistent state */ + if (file->buffer != NULL) { + buffer_free(file->buffer); + file->buffer = NULL; + } + file->buffer_size = 0; + } else { + file->buffer_size = buffer_get_used_size(file->buffer); + } + return ret; + } + + file->mmap_size = file->hdr.used_size; + file->mmap_base = mmap(NULL, file->mmap_size, PROT_READ, + MAP_SHARED, file->fd, 0); + if (file->mmap_base == MAP_FAILED) { + file->mmap_base = NULL; + mail_index_file_set_syscall_error(file->log->index, + file->filepath, "mmap()"); + return -1; + } + file->buffer = buffer_create_const_data(default_pool, file->mmap_base, + file->mmap_size); + file->buffer_size = buffer_get_used_size(file->buffer); + return 1; +} + +static int mail_transaction_log_lock_head(struct mail_transaction_log *log) +{ + struct mail_transaction_log_file *file; + int ret = 0; + + /* we want to get the head file locked. this is a bit racy, + since by the time we have it locked a new log file may have been + created. + + creating new log file requires locking the head file, so if we + can lock it and don't see another file, we can be sure no-one is + creating a new log at the moment */ + + for (;;) { + file = log->head; + if (mail_transaction_log_file_lock(file, F_WRLCK) < 0) + return -1; + + file->refcount++; + ret = mail_transaction_log_refresh(log); + if (--file->refcount == 0) { + mail_transaction_logs_clean(log); + file = NULL; + } + + if (ret == 0 && log->head == file) { + /* success */ + break; + } + + if (file != NULL) { + if (mail_transaction_log_file_lock(file, F_UNLCK) < 0) + return -1; + } + + if (ret < 0) + break; + + /* try again */ + } + + return ret; +} + +static int get_expunge_buf(struct mail_transaction_log *log, + struct mail_index_view *view, buffer_t *expunges) +{ + struct mail_transaction_log_view *sync_view; + const struct mail_transaction_header *hdr; + const void *data; + int ret; + + sync_view = mail_transaction_log_view_open(log); + ret = mail_transaction_log_view_set(sync_view, view->log_file_seq, + view->log_file_offset, + log->head->hdr.file_seq, + log->head->hdr.used_size, + MAIL_TRANSACTION_TYPE_MASK); + while ((ret = mail_transaction_log_view_next(sync_view, + &hdr, &data, NULL)) == 1) { + if ((hdr->type & MAIL_TRANSACTION_TYPE_MASK) == + MAIL_TRANSACTION_EXPUNGE) { + mail_transaction_log_sort_expunges(expunges, + data, hdr->size); + } + } + mail_transaction_log_view_close(sync_view); + return ret; +} + +static void +log_view_fix_sequences(struct mail_index_view *view, buffer_t *view_expunges, + buffer_t *buf, size_t record_size, int two, int uids) +{ + // FIXME: make sure this function works correctly + const struct mail_transaction_expunge *exp, *exp_end, *exp2; + unsigned char *data; + uint32_t *seq, expunges_before, count; + size_t src_idx, dest_idx, size; + + if (buf == NULL) + return; + + exp = buffer_get_data(view_expunges, &size); + exp_end = exp + (size / sizeof(*exp)); + if (exp == exp_end) + return; + + data = buffer_get_modifyable_data(buf, &size); + + expunges_before = 0; + for (src_idx = dest_idx = 0; src_idx < size; src_idx += record_size) { + seq = (uint32_t *)&data[src_idx]; + + while (exp != exp_end && exp->seq1 < seq[0]) { + expunges_before += exp->seq2 - exp->seq1 + 1; + exp++; + } + if (exp != exp_end && exp->seq1 == seq[0]) { + /* this sequence was expunged */ + if (!two) + continue; + + /* we point to next non-expunged message */ + } + if (expunges_before != 0) { + if (uids) { + (void)mail_index_lookup_uid(view, seq[0], + &seq[2]); + } + seq[0] -= expunges_before; + } + + if (two) { + exp2 = exp; + count = expunges_before; + while (exp2 != exp_end && exp2->seq1 <= seq[1]) { + count += exp->seq2 - exp->seq1 + 1; + exp2++; + } + if (seq[1] < count || seq[1]-count < seq[0]) { + /* whole range is expunged */ + continue; + } + if (count != 0) { + if (uids) { + (void)mail_index_lookup_uid(view, + seq[1], + &seq[3]); + } + seq[1] -= count; + } + } + + if (src_idx != dest_idx) + memcpy(&data[dest_idx], &data[src_idx], record_size); + dest_idx += record_size; + } + buffer_set_used_size(buf, dest_idx); +} + +static int +mail_transaction_log_fix_sequences(struct mail_transaction_log *log, + struct mail_index_transaction *t) +{ + buffer_t *view_expunges; + + if (t->updates == NULL && t->cache_updates == NULL && + t->expunges == NULL) + return 0; + + /* all sequences are currently relative to given view. we have to + find out all the expunges since then, even the ones that aren't + yet synchronized to index file. */ + view_expunges = buffer_create_dynamic(default_pool, 1024, (size_t)-1); + if (get_expunge_buf(log, t->view, view_expunges) < 0) { + buffer_free(view_expunges); + return -1; + } + + log_view_fix_sequences(t->view, view_expunges, t->updates, + sizeof(struct mail_transaction_flag_update), + TRUE, FALSE); + log_view_fix_sequences(t->view, view_expunges, t->cache_updates, + sizeof(struct mail_transaction_cache_update), + FALSE, FALSE); + log_view_fix_sequences(t->view, view_expunges, t->expunges, + sizeof(struct mail_transaction_expunge), + TRUE, TRUE); + + buffer_free(view_expunges); + return 0; +} + +static int +log_append_buffer(struct mail_transaction_log_file *file, const buffer_t *buf, + enum mail_transaction_type type, int external) +{ + struct mail_transaction_header hdr; + const void *data; + size_t size; + + i_assert((type & MAIL_TRANSACTION_TYPE_MASK) != 0); + + if (buf != NULL) { + data = buffer_get_data(buf, &size); + if (size == 0) + return 0; + } else { + /* write only the header */ + data = NULL; + size = 0; + } + + hdr.type = type; + if (type == MAIL_TRANSACTION_EXPUNGE) + hdr.type |= MAIL_TRANSACTION_EXPUNGE_PROT; + if (external) + hdr.type |= MAIL_TRANSACTION_EXTERNAL; + hdr.size = size; + + if (pwrite_full(file->fd, &hdr, sizeof(hdr), file->hdr.used_size) < 0) + return -1; + file->hdr.used_size += sizeof(hdr); + + if (size != 0) { + if (pwrite_full(file->fd, data, size, file->hdr.used_size) < 0) + return -1; + file->hdr.used_size += size; + } + return 0; +} + +int mail_transaction_log_append(struct mail_index_transaction *t, + uint32_t *log_file_seq_r, + uoff_t *log_file_offset_r) +{ + struct mail_index_view *view = t->view; + struct mail_transaction_log *log; + struct mail_transaction_log_file *file; + size_t offset; + uoff_t append_offset; + int ret; + + if (t->updates == NULL && t->cache_updates == NULL && + t->expunges == NULL && t->appends == NULL) { + /* nothing to append */ + return 0; + } + + log = mail_index_view_get_index(view)->log; + + if (log->index->log_locked) { + i_assert(view->external); + } else { + if (mail_transaction_log_lock_head(log) < 0) + return -1; + } + file = log->head; + append_offset = file->hdr.used_size; + + if (mail_transaction_log_fix_sequences(log, t) < 0) { + if (!log->index->log_locked) + (void)mail_transaction_log_file_lock(file, F_UNLCK); + return -1; + } + + ret = 0; + if (t->appends != NULL) { + ret = log_append_buffer(file, t->appends, + MAIL_TRANSACTION_APPEND, + view->external); + } + if (t->updates != NULL && ret == 0) { + ret = log_append_buffer(file, t->updates, + MAIL_TRANSACTION_FLAG_UPDATE, + view->external); + } + if (t->cache_updates != NULL && ret == 0) { + ret = log_append_buffer(file, t->cache_updates, + MAIL_TRANSACTION_CACHE_UPDATE, + view->external); + } + if (t->expunges != NULL && ret == 0) { + ret = log_append_buffer(file, t->expunges, + MAIL_TRANSACTION_EXPUNGE, + view->external); + } + + if (ret == 0) { + /* rewrite used_size */ + offset = offsetof(struct mail_transaction_log_header, + used_size); + ret = pwrite_full(file->fd, &file->hdr.used_size, + sizeof(file->hdr.used_size), offset); + } + + if (ret == 0 && (t->updates != NULL || t->appends != NULL) && + t->hide_transaction) { + mail_index_view_add_synced_transaction(view, file->hdr.file_seq, + append_offset); + } + + if (ret < 0) { + file->hdr.used_size = append_offset; + mail_index_file_set_syscall_error(log->index, file->filepath, + "pwrite()"); + } else if (fsync(file->fd) < 0) { + /* we don't know how much of it got written, + it may be corrupted now.. */ + mail_index_file_set_syscall_error(log->index, file->filepath, + "fsync()"); + ret = -1; + } + + *log_file_seq_r = file->hdr.file_seq; + *log_file_offset_r = file->hdr.used_size; + + if (!log->index->log_locked) + (void)mail_transaction_log_file_lock(file, F_UNLCK); + return ret; +} + +int mail_transaction_log_sync_lock(struct mail_transaction_log *log, + uint32_t *file_seq_r, uoff_t *file_offset_r) +{ + i_assert(!log->index->log_locked); + + if (mail_transaction_log_lock_head(log) < 0) + return -1; + + log->index->log_locked = TRUE; + *file_seq_r = log->head->hdr.file_seq; + *file_offset_r = log->head->hdr.used_size; + return 0; +} + +void mail_transaction_log_sync_unlock(struct mail_transaction_log *log) +{ + i_assert(log->index->log_locked); + + log->index->log_locked = FALSE; + (void)mail_transaction_log_file_lock(log->head, F_UNLCK); +} + +void mail_transaction_log_get_head(struct mail_transaction_log *log, + uint32_t *file_seq_r, uoff_t *file_offset_r) +{ + i_assert(log->index->log_locked); + + *file_seq_r = log->head->hdr.file_seq; + *file_offset_r = log->head->hdr.used_size; +}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-index/mail-transaction-log.h Tue Apr 27 23:25:52 2004 +0300 @@ -0,0 +1,108 @@ +#ifndef __MAIL_TRANSACTION_LOG_H +#define __MAIL_TRANSACTION_LOG_H + +#define MAIL_TRANSACTION_LOG_PREFIX ".log" + +struct mail_transaction_log_header { + uint32_t indexid; + uint32_t file_seq; + uint32_t used_size; +}; + +enum mail_transaction_type { + MAIL_TRANSACTION_EXPUNGE = 0x00000001, + MAIL_TRANSACTION_APPEND = 0x00000002, + MAIL_TRANSACTION_FLAG_UPDATE = 0x00000004, + MAIL_TRANSACTION_CACHE_UPDATE = 0x00000008, + + MAIL_TRANSACTION_TYPE_MASK = 0x0000ffff, + + /* since we'll expunge mails based on data read from transaction log, + try to avoid the possibility of corrupted transaction log expunging + messages. this value is ORed to the actual MAIL_TRANSACTION_EXPUNGE + flag. if it's not present, assume corrupted log. */ + MAIL_TRANSACTION_EXPUNGE_PROT = 0x0000cd90, + + /* Mailbox synchronization noticed this change. */ + MAIL_TRANSACTION_EXTERNAL = 0x10000000 +}; + +struct mail_transaction_header { + uint32_t size; + uint32_t type; /* enum mail_transaction_type */ +}; + +struct mail_transaction_expunge { + uint32_t seq1, seq2; + uint32_t uid1, uid2; /* only to avoid accidental expunges due to bugs */ +}; + +struct mail_transaction_cache_update { + uint32_t seq; + uint32_t cache_offset; +}; + +struct mail_transaction_flag_update { + uint32_t seq1, seq2; + uint8_t add_flags; + custom_flags_mask_t add_custom_flags; + uint8_t remove_flags; + custom_flags_mask_t remove_custom_flags; +}; + +struct mail_transaction_log * +mail_transaction_log_open_or_create(struct mail_index *index); +void mail_transaction_log_close(struct mail_transaction_log *log); + +struct mail_transaction_log_view * +mail_transaction_log_view_open(struct mail_transaction_log *log); +void mail_transaction_log_view_close(struct mail_transaction_log_view *view); + +/* Set view boundaries. Returns -1 if error, 0 if ok. */ +int +mail_transaction_log_view_set(struct mail_transaction_log_view *view, + uint32_t min_file_seq, uoff_t min_file_offset, + uint32_t max_file_seq, uoff_t max_file_offset, + enum mail_transaction_type type_mask); + +/* Read next transaction record from current position. The position is updated. + Returns -1 if error, 0 if we're at end of the view, 1 if ok. */ +int mail_transaction_log_view_next(struct mail_transaction_log_view *view, + const struct mail_transaction_header **hdr_r, + const void **data_r, int *skipped_r); + +/* Returns the position of the record returned previously with + mail_transaction_log_view_next() */ +void +mail_transaction_log_view_get_prev_pos(struct mail_transaction_log_view *view, + uint32_t *file_seq_r, + uoff_t *file_offset_r); + +buffer_t * +mail_transaction_log_view_get_expunges(struct mail_transaction_log_view *view); + +/* Marks the log file in current position to be corrupted. */ +void +mail_transaction_log_view_set_corrupted(struct mail_transaction_log_view *view, + const char *fmt, ...) + __attr_format__(2, 3); +int +mail_transaction_log_view_is_corrupted(struct mail_transaction_log_view *view); + +/* Write data to transaction log. This is atomic operation. Sequences in + updates[] and expunges[] are relative to given view, they're modified + to real ones. */ +int mail_transaction_log_append(struct mail_index_transaction *t, + uint32_t *log_file_seq_r, + uoff_t *log_file_offset_r); + +/* Lock transaction log for index synchronization. Log cannot be read or + written to while it's locked. Returns end offset. */ +int mail_transaction_log_sync_lock(struct mail_transaction_log *log, + uint32_t *file_seq_r, uoff_t *file_offset_r); +void mail_transaction_log_sync_unlock(struct mail_transaction_log *log); +/* Returns the current head. Works only when log is locked. */ +void mail_transaction_log_get_head(struct mail_transaction_log *log, + uint32_t *file_seq_r, uoff_t *file_offset_r); + +#endif
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-index/mail-transaction-util.c Tue Apr 27 23:25:52 2004 +0300 @@ -0,0 +1,252 @@ +/* Copyright (C) 2004 Timo Sirainen */ + +#include "lib.h" +#include "buffer.h" +#include "mail-index-private.h" +#include "mail-transaction-log.h" +#include "mail-transaction-util.h" + +struct mail_transaction_expunge_traverse_ctx { + const struct mail_transaction_expunge *expunges; + size_t expunges_count, cur_idx, old_idx; + uint32_t cur_seq, expunges_before; + uint32_t old_seq, old_expunges_before; +}; + +const struct mail_transaction_type_map mail_transaction_type_map[] = { + { MAIL_TRANSACTION_APPEND, MAIL_INDEX_SYNC_TYPE_APPEND, + sizeof(struct mail_index_record) }, + { MAIL_TRANSACTION_EXPUNGE, MAIL_INDEX_SYNC_TYPE_EXPUNGE, + sizeof(struct mail_transaction_expunge) }, + { MAIL_TRANSACTION_FLAG_UPDATE, MAIL_INDEX_SYNC_TYPE_FLAGS, + sizeof(struct mail_transaction_flag_update) }, + { MAIL_TRANSACTION_CACHE_UPDATE, 0, + sizeof(struct mail_transaction_cache_update) }, + { 0, 0, 0 } +}; + +const struct mail_transaction_type_map * +mail_transaction_type_lookup(enum mail_transaction_type type) +{ + int i; + + for (i = 0; mail_transaction_type_map[i].type != 0; i++) { + if ((mail_transaction_type_map[i].type & type) != 0) + return &mail_transaction_type_map[i]; + } + return NULL; +} + +enum mail_transaction_type +mail_transaction_type_mask_get(enum mail_index_sync_type sync_type) +{ + enum mail_transaction_type type = 0; + int i; + + for (i = 0; mail_transaction_type_map[i].type != 0; i++) { + if ((mail_transaction_type_map[i].sync_type & sync_type) != 0) + type |= mail_transaction_type_map[i].type; + } + return type; +} + +int mail_transaction_map(const struct mail_transaction_header *hdr, + const void *data, + struct mail_transaction_map_functions *map, + void *context) +{ + int ret = 0; + + switch (hdr->type & MAIL_TRANSACTION_TYPE_MASK) { + case MAIL_TRANSACTION_APPEND: { + const struct mail_index_record *rec, *end; + + if (map->append == NULL) + break; + + end = CONST_PTR_OFFSET(data, hdr->size); + for (rec = data; rec != end; rec++) { + ret = map->append(rec, context); + if (ret <= 0) + break; + } + break; + } + case MAIL_TRANSACTION_EXPUNGE: { + const struct mail_transaction_expunge *rec, *end; + + if (map->expunge == NULL) + break; + + end = CONST_PTR_OFFSET(data, hdr->size); + for (rec = data; rec != end; rec++) { + ret = map->expunge(rec, context); + if (ret <= 0) + break; + } + break; + } + case MAIL_TRANSACTION_FLAG_UPDATE: { + const struct mail_transaction_flag_update *rec, *end; + + if (map->flag_update == NULL) + break; + + end = CONST_PTR_OFFSET(data, hdr->size); + for (rec = data; rec != end; rec++) { + ret = map->flag_update(rec, context); + if (ret <= 0) + break; + } + break; + } + case MAIL_TRANSACTION_CACHE_UPDATE: { + const struct mail_transaction_cache_update *rec, *end; + + if (map->cache_update == NULL) + break; + + end = CONST_PTR_OFFSET(data, hdr->size); + for (rec = data; rec != end; rec++) { + ret = map->cache_update(rec, context); + if (ret <= 0) + break; + } + break; + } + default: + i_unreached(); + } + + return ret; +} + +void +mail_transaction_log_sort_expunges(buffer_t *expunges_buf, + const struct mail_transaction_expunge *src, + size_t src_buf_size) +{ + const struct mail_transaction_expunge *src_end; + struct mail_transaction_expunge *dest; + struct mail_transaction_expunge new_exp; + uint32_t cur_seq, prev_seq, expunges_before, count; + size_t first, i, dest_count; + + i_assert(src_buf_size % sizeof(*src) == 0); + src_end = CONST_PTR_OFFSET(src, src_buf_size); + + /* @UNSAFE */ + dest = buffer_get_modifyable_data(expunges_buf, &dest_count); + dest_count /= sizeof(*dest); + + cur_seq = prev_seq = 1; expunges_before = 0; + for (i = 0; src != src_end; src++) { + for (; i < dest_count; i++) { + count = dest[i].seq1 - prev_seq; + if (cur_seq + count > src->seq1) + break; + cur_seq += count; + + expunges_before += dest[i].seq2 - dest[i].seq1 + 1; + prev_seq = dest[i].seq2+1; + } + + new_exp.seq1 = src->seq1 + expunges_before; + new_exp.seq2 = src->seq2 + expunges_before; + + /* if src[] is in format {1,2}{1,2} rather than {1,2}{3,4}: + expunges_before += new_exp.seq2 - new_exp.seq1 + 1;*/ + + first = i; + while (i < dest_count && new_exp.seq2 >= dest[i].seq1-1) { + /* we can/must merge with next record */ + count = dest[i].seq2 - dest[i].seq1 + 1; + expunges_before += count; + new_exp.seq2 += count; + i++; + } + + if (first > 0 && new_exp.seq1 == dest[first-1].seq2+1) { + /* continue previous record */ + dest[first-1].seq2 = new_exp.seq2; + } else if (i == first) { + buffer_insert(expunges_buf, i * sizeof(new_exp), + &new_exp, sizeof(new_exp)); + i++; first++; + + dest = buffer_get_modifyable_data(expunges_buf, NULL); + dest_count++; + } else { + /* use next record */ + dest[first].seq1 = new_exp.seq1; + dest[first].seq2 = new_exp.seq2; + first++; + } + + if (i > first) { + buffer_delete(expunges_buf, first * sizeof(new_exp), + (i - first) * sizeof(new_exp)); + + dest = buffer_get_modifyable_data(expunges_buf, NULL); + dest_count -= i - first; + i = first + 1; + } + } +} + +struct mail_transaction_expunge_traverse_ctx * +mail_transaction_expunge_traverse_init(const buffer_t *expunges_buf) +{ + struct mail_transaction_expunge_traverse_ctx *ctx; + + ctx = i_new(struct mail_transaction_expunge_traverse_ctx, 1); + ctx->cur_seq = 1; + ctx->old_seq = 1; + + if (expunges_buf != NULL) { + ctx->expunges = + buffer_get_data(expunges_buf, &ctx->expunges_count); + ctx->expunges_count /= sizeof(*ctx->expunges); + } + return ctx; +} + +void mail_transaction_expunge_traverse_deinit( + struct mail_transaction_expunge_traverse_ctx *ctx) +{ + i_free(ctx); +} + +uint32_t mail_transaction_expunge_traverse_to( + struct mail_transaction_expunge_traverse_ctx *ctx, uint32_t seq) +{ + uint32_t idx, count, last_seq; + + if (seq < ctx->cur_seq) { + /* allow seeking one back */ + ctx->cur_idx = ctx->old_idx; + ctx->cur_seq = ctx->old_seq; + ctx->expunges_before = ctx->old_expunges_before; + } else { + ctx->old_idx = ctx->cur_idx; + ctx->old_seq = ctx->cur_seq; + ctx->old_expunges_before = ctx->expunges_before; + } + i_assert(seq >= ctx->cur_seq); + + idx = ctx->cur_idx; + last_seq = idx == 0 ? 1 : ctx->expunges[idx-1].seq2 + 1; + for (; idx < ctx->expunges_count; idx++) { + count = ctx->expunges[idx].seq1 - last_seq; + if (ctx->cur_seq + count > seq) + break; + ctx->cur_seq += count; + + ctx->expunges_before += ctx->expunges[idx].seq2 - + ctx->expunges[idx].seq1 + 1; + last_seq = ctx->expunges[idx].seq2+1; + } + + ctx->cur_idx = idx; + return ctx->expunges_before; +}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-index/mail-transaction-util.h Tue Apr 27 23:25:52 2004 +0300 @@ -0,0 +1,42 @@ +#ifndef __MAIL_TRANSACTION_UTIL_H +#define __MAIL_TRANSACTION_UTIL_H + +struct mail_transaction_type_map { + enum mail_transaction_type type; + enum mail_index_sync_type sync_type; + size_t record_size; +}; +extern const struct mail_transaction_type_map mail_transaction_type_map[]; + +struct mail_transaction_map_functions { + int (*expunge)(const struct mail_transaction_expunge *e, void *context); + int (*append)(const struct mail_index_record *rec, void *context); + int (*flag_update)(const struct mail_transaction_flag_update *u, + void *context); + int (*cache_update)(const struct mail_transaction_cache_update *u, + void *context); +}; + +const struct mail_transaction_type_map * +mail_transaction_type_lookup(enum mail_transaction_type type); +enum mail_transaction_type +mail_transaction_type_mask_get(enum mail_index_sync_type sync_type); + +int mail_transaction_map(const struct mail_transaction_header *hdr, + const void *data, + struct mail_transaction_map_functions *map, + void *context); + +void +mail_transaction_log_sort_expunges(buffer_t *expunges_buf, + const struct mail_transaction_expunge *src, + size_t src_buf_size);; + +struct mail_transaction_expunge_traverse_ctx * +mail_transaction_expunge_traverse_init(const buffer_t *expunges_buf); +void mail_transaction_expunge_traverse_deinit( + struct mail_transaction_expunge_traverse_ctx *ctx); +uint32_t mail_transaction_expunge_traverse_to( + struct mail_transaction_expunge_traverse_ctx *ctx, uint32_t seq); + +#endif
--- a/src/lib-index/maildir/Makefile.am Tue Apr 27 23:14:15 2004 +0300 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,21 +0,0 @@ -noinst_LIBRARIES = libindex_maildir.a - -INCLUDES = \ - -I$(top_srcdir)/src/lib \ - -I$(top_srcdir)/src/lib-mail \ - -I$(top_srcdir)/src/lib-imap \ - -I$(top_srcdir)/src/lib-index - -libindex_maildir_a_SOURCES = \ - maildir-index.c \ - maildir-build.c \ - maildir-clean.c \ - maildir-expunge.c \ - maildir-open.c \ - maildir-sync.c \ - maildir-uidlist.c \ - maildir-update-flags.c - -noinst_HEADERS = \ - maildir-index.h \ - maildir-uidlist.h
--- a/src/lib-index/mbox/Makefile.am Tue Apr 27 23:14:15 2004 +0300 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,22 +0,0 @@ -noinst_LIBRARIES = libindex_mbox.a - -INCLUDES = \ - -I$(top_srcdir)/src/lib \ - -I$(top_srcdir)/src/lib-mail \ - -I$(top_srcdir)/src/lib-imap \ - -I$(top_srcdir)/src/lib-index - -libindex_mbox_a_SOURCES = \ - istream-mbox.c \ - mbox-append.c \ - mbox-from.c \ - mbox-index.c \ - mbox-lock.c \ - mbox-open.c \ - mbox-rewrite.c \ - mbox-sync.c \ - mbox-sync-full.c - -noinst_HEADERS = \ - mbox-index.h \ - mbox-lock.h
--- a/src/lib-mail/Makefile.am Tue Apr 27 23:14:15 2004 +0300 +++ b/src/lib-mail/Makefile.am Tue Apr 27 23:25:52 2004 +0300 @@ -19,6 +19,7 @@ quoted-printable.c noinst_HEADERS = \ + mail-types.h \ message-address.h \ message-body-search.h \ message-content-parser.h \
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-mail/mail-types.h Tue Apr 27 23:25:52 2004 +0300 @@ -0,0 +1,28 @@ +#ifndef __MAIL_TYPES_H +#define __MAIL_TYPES_H + +enum mail_flags { + MAIL_ANSWERED = 0x01, + MAIL_FLAGGED = 0x02, + MAIL_DELETED = 0x04, + MAIL_SEEN = 0x08, + MAIL_DRAFT = 0x10, + MAIL_RECENT = 0x20, + + MAIL_FLAGS_MASK = 0x3f +}; + +struct mail_full_flags { + enum mail_flags flags; + + const char **custom_flags; + unsigned int custom_flags_count; +}; + +enum modify_type { + MODIFY_ADD, + MODIFY_REMOVE, + MODIFY_REPLACE +}; + +#endif
--- a/src/lib-mail/message-parser.c Tue Apr 27 23:14:15 2004 +0300 +++ b/src/lib-mail/message-parser.c Tue Apr 27 23:25:52 2004 +0300 @@ -707,6 +707,7 @@ } else { /* new header line */ line->continued = FALSE; + line->name_offset = ctx->input->v_offset; } for (;;) { @@ -773,6 +774,10 @@ if (msg[i] <= ':') { if (msg[i] == ':') { colon_pos = i; + // FIXME: correct? + line->full_value_offset = + ctx->input->v_offset + + i + 1; break; } if (msg[i] == '\n') {
--- a/src/lib-mail/message-parser.h Tue Apr 27 23:14:15 2004 +0300 +++ b/src/lib-mail/message-parser.h Tue Apr 27 23:25:52 2004 +0300 @@ -47,9 +47,11 @@ const unsigned char *value; size_t value_len; - const unsigned char *full_value; + const unsigned char *full_value; // FIXME: should contain \n too size_t full_value_len; + uoff_t name_offset, full_value_offset; + unsigned int continues:1; /* multiline header, continues in next line */ unsigned int continued:1; /* multiline header, continues */ unsigned int eoh:1; /* "end of headers" line */
--- a/src/lib-storage/Makefile.am Tue Apr 27 23:14:15 2004 +0300 +++ b/src/lib-storage/Makefile.am Tue Apr 27 23:25:52 2004 +0300 @@ -20,6 +20,7 @@ mail-save.h \ mail-search.h \ mail-storage.h \ + mail-storage-private.h \ mailbox-tree.h \ proxy-mail.h \ proxy-mail-storage.h \
--- a/src/lib-storage/index/Makefile.am Tue Apr 27 23:14:15 2004 +0300 +++ b/src/lib-storage/index/Makefile.am Tue Apr 27 23:25:52 2004 +0300 @@ -10,21 +10,16 @@ -I$(top_srcdir)/src/lib-storage libstorage_index_a_SOURCES = \ - index-copy.c \ - index-expunge.c \ index-fetch.c \ index-mail.c \ index-mail-headers.c \ index-mailbox-check.c \ - index-messageset.c \ index-search.c \ index-status.c \ index-storage.c \ index-sync.c \ - index-update-flags.c + index-transaction.c noinst_HEADERS = \ - index-expunge.h \ index-mail.h \ - index-messageset.h \ index-storage.h
--- a/src/lib-storage/index/index-copy.c Tue Apr 27 23:14:15 2004 +0300 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,59 +0,0 @@ -/* Copyright (C) 2002 Timo Sirainen */ - -#include "lib.h" -#include "istream.h" -#include "mail-custom-flags.h" -#include "index-storage.h" -#include "index-messageset.h" - -#include <unistd.h> - -struct mail_copy_context { - struct mailbox *box; - struct mail_save_context *save_ctx; -}; - -struct mail_copy_context *index_storage_copy_init(struct mailbox *box) -{ - struct mail_copy_context *ctx; - struct mail_save_context *save_ctx; - - save_ctx = box->save_init(box, TRUE); - if (save_ctx == NULL) - return NULL; - - ctx = i_new(struct mail_copy_context, 1); - ctx->box = box; - ctx->save_ctx = save_ctx; - - return ctx; -} - -int index_storage_copy_deinit(struct mail_copy_context *ctx, int rollback) -{ - int ret; - - ret = ctx->box->save_deinit(ctx->save_ctx, rollback); - i_free(ctx); - return ret; -} - -int index_storage_copy(struct mail *mail, struct mail_copy_context *ctx) -{ - struct index_mail *imail = (struct index_mail *) mail; - struct istream *input; - time_t received_date; - int ret, deleted; - - input = imail->ibox->index->open_mail(imail->ibox->index, - imail->data.rec, - &received_date, &deleted); - if (input == NULL) - return FALSE; - - ret = ctx->box->save_next(ctx->save_ctx, mail->get_flags(mail), - received_date, 0, input); - i_stream_unref(input); - - return ret; -}
--- a/src/lib-storage/index/index-expunge.c Tue Apr 27 23:14:15 2004 +0300 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,200 +0,0 @@ -/* Copyright (C) 2002-2003 Timo Sirainen */ - -#include "lib.h" -#include "index-storage.h" -#include "index-expunge.h" - -static int index_expunge_seek_first(struct index_mailbox *ibox, - unsigned int *seq, - struct mail_index_record **rec) -{ - struct mail_index_header *hdr; - - i_assert(ibox->index->lock_type == MAIL_LOCK_EXCLUSIVE); - - hdr = ibox->index->get_header(ibox->index); - if (hdr->deleted_messages_count == 0) { - /* no deleted messages */ - *seq = 0; - *rec = NULL; - return TRUE; - } - - /* find mails with DELETED flag and expunge them */ - if (hdr->first_deleted_uid_lowwater > 1) { - *rec = hdr->first_deleted_uid_lowwater >= hdr->next_uid ? NULL : - ibox->index->lookup_uid_range(ibox->index, - hdr->first_deleted_uid_lowwater, - hdr->next_uid-1, seq); - if (*rec == NULL) { - mail_storage_set_critical(ibox->box.storage, - "index header's deleted_messages_count (%u) " - "or first_deleted_uid_lowwater (%u) " - "is invalid.", hdr->deleted_messages_count, - hdr->first_deleted_uid_lowwater); - - /* fsck should be enough to fix it */ - ibox->index->set_flags |= MAIL_INDEX_HDR_FLAG_FSCK; - return FALSE; - } - } else { - *rec = ibox->index->lookup(ibox->index, 1); - *seq = 1; - } - - return TRUE; -} - -struct mail_expunge_context * -index_storage_expunge_init(struct mailbox *box, - enum mail_fetch_field wanted_fields, - int expunge_all) -{ - struct index_mailbox *ibox = (struct index_mailbox *) box; - struct mail_expunge_context *ctx; - - if (box->is_readonly(box)) { - box->storage->callbacks-> - notify_no(box, "Mailbox is read-only, ignoring expunge", - box->storage->callback_context); - return i_new(struct mail_expunge_context, 1); - } - - if (!index_storage_lock(ibox, MAIL_LOCK_EXCLUSIVE)) - return NULL; - - ctx = i_new(struct mail_expunge_context, 1); - ctx->ibox = ibox; - ctx->expunge_all = expunge_all; - index_mail_init(ibox, &ctx->mail, wanted_fields, NULL); - - do { - if (!index_storage_sync_and_lock(ibox, FALSE, TRUE, - MAIL_LOCK_EXCLUSIVE)) - break; - - /* modifylog must be marked synced before expunging - anything new */ - if (!index_storage_sync_modifylog(ibox, TRUE)) - break; - - if (expunge_all) { - ctx->seq = 1; - ctx->rec = ibox->index->lookup(ibox->index, 1); - } else { - if (!index_expunge_seek_first(ibox, &ctx->seq, - &ctx->rec)) - break; - - ctx->fetch_next = ctx->rec != NULL && - (ctx->rec->msg_flags & MAIL_DELETED) == 0; - } - - return ctx; - } while (0); - - (void)index_storage_lock(ctx->ibox, MAIL_LOCK_UNLOCK); - i_free(ctx); - return NULL; -} - -int index_storage_expunge_deinit(struct mail_expunge_context *ctx) -{ - int ret = !ctx->failed; - - if (ctx->first_seq != 0) { - if (!ctx->ibox->index->expunge(ctx->ibox->index, - ctx->first_rec, ctx->last_rec, - ctx->first_seq, ctx->last_seq, - FALSE)) - ret = FALSE; - } - - if (ctx->ibox != NULL) { - ctx->ibox->fetch_mail.data.rec = NULL; - - if (!index_storage_lock(ctx->ibox, MAIL_LOCK_UNLOCK)) - ret = FALSE; - } - - i_free(ctx); - return ret; -} - -struct mail *index_storage_expunge_fetch_next(struct mail_expunge_context *ctx) -{ - struct mail_index *index = ctx->ibox->index; - - if (ctx->rec == NULL) - return NULL; - - if (ctx->fetch_next) { - do { - ctx->seq++; - ctx->rec = index->next(index, ctx->rec); - if (ctx->rec == NULL) - return NULL; - } while ((ctx->rec->msg_flags & MAIL_DELETED) == 0 && - !ctx->expunge_all); - } else { - ctx->fetch_next = TRUE; - } - - ctx->mail.expunge_counter = index->expunge_counter; - ctx->mail.mail.seq = ctx->seq; - ctx->mail.mail.uid = ctx->rec->uid; - - if (index_mail_next(&ctx->mail, ctx->rec, ctx->seq, FALSE) < 0) { - ctx->failed = TRUE; - return NULL; - } - - return &ctx->mail.mail; -} - -int index_storage_expunge(struct mail *mail, struct mail_expunge_context *ctx, - unsigned int *seq_r, int notify) -{ - struct index_mail *imail = (struct index_mail *) mail; - struct index_mailbox *ibox = imail->ibox; - unsigned int seq; - - /* currently we allow expunges only from beginning to end so we can - easily update sequence numbers */ - i_assert(ctx->last_seq < ctx->seq); - - seq = mail->seq; - if (ctx->first_seq != 0) - seq -= (ctx->last_seq - ctx->first_seq) + 1; - if (seq_r != NULL) *seq_r = seq; - - if (ctx->first_seq != 0 && ctx->seq != ctx->last_seq+1) { - if (!ibox->index->expunge(ibox->index, - ctx->first_rec, ctx->last_rec, - ctx->first_seq, ctx->last_seq, FALSE)) - return FALSE; - - ctx->seq -= (ctx->last_seq - ctx->first_seq) + 1; - ctx->rec = ibox->index->lookup(ibox->index, ctx->seq); - - ctx->first_seq = 0; - } - - if (ctx->first_seq == 0) { - ctx->first_seq = ctx->seq; - ctx->first_rec = ctx->rec; - } - ctx->last_seq = ctx->seq; - ctx->last_rec = ctx->rec; - - ibox->fetch_mail.data.rec = NULL; - - ibox->synced_messages_count--; - if (notify && seq <= ibox->synced_messages_count+1) { - ibox->box.storage->callbacks-> - expunge(&ibox->box, seq, - ibox->box.storage->callback_context); - } - - return TRUE; -}
--- a/src/lib-storage/index/index-expunge.h Tue Apr 27 23:14:15 2004 +0300 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,28 +0,0 @@ -#ifndef __INDEX_EXPUNGE_H -#define __INDEX_EXPUNGE_H - -#include "mail-storage.h" -#include "index-mail.h" - -struct mail_expunge_context { - struct index_mailbox *ibox; - struct index_mail mail; - int expunge_all, fetch_next, failed; - - unsigned int seq; - struct mail_index_record *rec; - - unsigned int first_seq, last_seq; - struct mail_index_record *first_rec, *last_rec; -}; - -struct mail_expunge_context * -index_storage_expunge_init(struct mailbox *box, - enum mail_fetch_field wanted_fields, - int expunge_all); -int index_storage_expunge_deinit(struct mail_expunge_context *ctx); -struct mail *index_storage_expunge_fetch_next(struct mail_expunge_context *ctx); -int index_storage_expunge(struct mail *mail, struct mail_expunge_context *ctx, - unsigned int *seq_r, int notify); - -#endif
--- a/src/lib-storage/index/index-fetch.c Tue Apr 27 23:14:15 2004 +0300 +++ b/src/lib-storage/index/index-fetch.c Tue Apr 27 23:25:52 2004 +0300 @@ -1,62 +1,46 @@ -/* Copyright (C) 2002 Timo Sirainen */ +/* Copyright (C) 2002-2003 Timo Sirainen */ #include "lib.h" -#include "ostream.h" -#include "str.h" -#include "mail-index.h" -#include "mail-modifylog.h" -#include "mail-custom-flags.h" #include "index-storage.h" -#include "index-messageset.h" #include "index-mail.h" -static struct mail * -fetch_record(struct index_mailbox *ibox, struct mail_index_record *rec, - unsigned int idx_seq, enum mail_fetch_field wanted_fields) +struct mail * +index_storage_fetch(struct mailbox_transaction_context *_t, uint32_t seq, + enum mail_fetch_field wanted_fields) { - if (ibox->fetch_mail.pool != NULL) - index_mail_deinit(&ibox->fetch_mail); - - index_mail_init(ibox, &ibox->fetch_mail, wanted_fields, NULL); - if (index_mail_next(&ibox->fetch_mail, rec, idx_seq, FALSE) <= 0) - return NULL; + struct index_transaction_context *t = + (struct index_transaction_context *)_t; + const struct mail_index_record *rec; - return &ibox->fetch_mail.mail; -} + if (mail_index_lookup(t->ibox->view, seq, &rec) < 0) { + mail_storage_set_index_error(t->ibox); + return NULL; + } -struct mail *index_storage_fetch_uid(struct mailbox *box, unsigned int uid, - enum mail_fetch_field wanted_fields) -{ - struct index_mailbox *ibox = (struct index_mailbox *) box; - struct mail_index_record *rec; - unsigned int seq; - - i_assert(ibox->index->lock_type != MAIL_LOCK_UNLOCK); - - rec = ibox->index->lookup_uid_range(ibox->index, uid, uid, &seq); if (rec == NULL) return NULL; - return fetch_record(ibox, rec, seq, wanted_fields); + if (t->fetch_mail.pool != NULL) + index_mail_deinit(&t->fetch_mail); + + index_mail_init(t, &t->fetch_mail, wanted_fields, NULL); + if (index_mail_next(&t->fetch_mail, rec, seq, FALSE) <= 0) + return NULL; + + return &t->fetch_mail.mail; } -struct mail *index_storage_fetch_seq(struct mailbox *box, unsigned int seq, - enum mail_fetch_field wanted_fields) +int index_storage_get_uids(struct mailbox *box, + uint32_t uid1, uint32_t uid2, + uint32_t *seq1_r, uint32_t *seq2_r) { - struct index_mailbox *ibox = (struct index_mailbox *) box; - struct mail_index_record *rec; - unsigned int expunges_before; - - i_assert(ibox->index->lock_type != MAIL_LOCK_UNLOCK); + struct index_mailbox *ibox = (struct index_mailbox *)box; - if (mail_modifylog_seq_get_expunges(ibox->index->modifylog, seq, seq, - &expunges_before) == NULL) - return NULL; + if (mail_index_lookup_uid_range(ibox->view, uid1, uid2, + seq1_r, seq2_r) < 0) { + mail_storage_set_index_error(ibox); + return -1; + } - seq -= expunges_before; - rec = ibox->index->lookup(ibox->index, seq); - if (rec == NULL) - return NULL; - - return fetch_record(ibox, rec, seq, wanted_fields); + return 0; }
--- a/src/lib-storage/index/index-mail-headers.c Tue Apr 27 23:14:15 2004 +0300 +++ b/src/lib-storage/index/index-mail-headers.c Tue Apr 27 23:25:52 2004 +0300 @@ -39,6 +39,7 @@ #include "buffer.h" #include "str.h" #include "message-date.h" +#include "message-parser.h" #include "imap-envelope.h" #include "imap-bodystructure.h" #include "index-storage.h" @@ -140,7 +141,7 @@ return data; } -static int find_wanted_headers(struct mail_cache *cache, +static int find_wanted_headers(struct mail_cache_view *cache_view, const char *const wanted_headers[]) { const char *const *headers, *const *tmp; @@ -154,7 +155,7 @@ ret = -1; for (i = MAIL_CACHE_HEADERS_COUNT-1; i >= 0; i--) { - headers = mail_cache_get_header_fields(cache, i); + headers = mail_cache_get_header_fields(cache_view, i); if (headers == NULL) continue; @@ -185,7 +186,7 @@ { int idx; - idx = find_wanted_headers(mail->ibox->index->cache, wanted_headers); + idx = find_wanted_headers(mail->ibox->cache_view, wanted_headers); if (idx < 0) return -1; @@ -433,7 +434,7 @@ data->header_stream = istream; } else { str = mail_cache_lookup_string_field( - mail->ibox->index->cache, data->rec, + mail->ibox->cache_view, data->seq, mail_cache_header_fields[idx]); if (str == NULL) { /* broken - we expected the header to exist */ @@ -446,10 +447,10 @@ str, strlen(str)); } - idx_headers = mail_cache_get_header_fields(mail->ibox->index->cache, + idx_headers = mail_cache_get_header_fields(mail->ibox->cache_view, idx); if (idx_headers == NULL) { - mail_cache_set_corrupted(mail->ibox->index->cache, + mail_cache_set_corrupted(mail->ibox->cache, "Headers %d names not found", idx); t_pop(); return FALSE; @@ -485,13 +486,14 @@ int index_mail_parse_headers(struct index_mail *mail) { - struct mail_cache *cache = mail->ibox->index->cache; struct index_mail_data *data = &mail->data; const char *str, *const *headers; int idx, max; - if (!index_mail_open_stream(mail, 0)) - return FALSE; + if (data->stream == NULL) { + if (mail->mail.get_stream(&mail->mail, NULL, NULL) == NULL) + return FALSE; + } if (mail->data.header_data == NULL) mail->data.header_data = str_new(mail->pool, 4096); @@ -508,8 +510,9 @@ /* add all cached headers to beginning of header_data */ idx = data->header_data_cached; max = idx-1; for (; idx < MAIL_CACHE_HEADERS_COUNT; idx++) { - str = mail_cache_lookup_string_field(cache, data->rec, - mail_cache_header_fields[idx]); + str = mail_cache_lookup_string_field( + mail->ibox->cache_view, mail->data.seq, + mail_cache_header_fields[idx]); if (str == NULL) continue; @@ -522,7 +525,8 @@ /* make sure we cache everything */ for (idx = MAIL_CACHE_HEADERS_COUNT-1; idx >= 0; idx--) { - headers = mail_cache_get_header_fields(cache, idx); + headers = mail_cache_get_header_fields( + mail->ibox->cache_view, idx); if (headers != NULL) break; } @@ -537,8 +541,10 @@ if (max >= 0) { /* now we'll have to set value_idx for all headers that are already cached */ - if (!parse_cached_headers(mail, max)) + if (!parse_cached_headers(mail, max)) { + /* FIXME: handle better */ return FALSE; + } } /* it's possible that we're parsing headers without wanting @@ -593,8 +599,10 @@ idx = mail_find_wanted_headers(mail, arr); if (idx >= 0) { - if (!parse_cached_headers(mail, idx)) - return NULL; + if (!parse_cached_headers(mail, idx)) { + /* broken cache, parse again */ + idx = -1; + } } } @@ -647,7 +655,7 @@ } for (i = data->header_data_cached; i <= idx; i++) { str = mail_cache_lookup_string_field( - mail->ibox->index->cache, data->rec, + mail->ibox->cache_view, data->seq, mail_cache_header_fields[i]); if (str == NULL) continue; @@ -679,14 +687,14 @@ void index_mail_headers_init(struct index_mail *mail) { - struct mail_cache *cache = mail->ibox->index->cache; + struct mail_cache_view *cache_view = mail->ibox->cache_view; int idx = -2, idx2 = -2; if (mail->wanted_headers != NULL && *mail->wanted_headers != NULL) - idx = find_wanted_headers(cache, mail->wanted_headers); + idx = find_wanted_headers(cache_view, mail->wanted_headers); if (idx != -1 && (mail->wanted_fields & MAIL_FETCH_IMAP_ENVELOPE)) - idx2 = find_wanted_headers(cache, imap_envelope_headers); + idx2 = find_wanted_headers(cache_view, imap_envelope_headers); mail->wanted_headers_idx = idx == -1 || idx2 == -1 ? -1 : idx > idx2 ? idx : idx2; @@ -735,12 +743,12 @@ } } -static int find_unused_header_idx(struct mail_cache *cache) +static int find_unused_header_idx(struct mail_cache_view *cache_view) { int i; for (i = 0; i < MAIL_CACHE_HEADERS_COUNT; i++) { - if (mail_cache_get_header_fields(cache, i) == NULL) + if (mail_cache_get_header_fields(cache_view, i) == NULL) return i; } @@ -761,21 +769,21 @@ accessing headers from same message. index_mails should probably be shared.. */ headers = cached_header_get_names(mail); - idx = find_wanted_headers(mail->ibox->index->cache, headers); + idx = find_wanted_headers(mail->ibox->cache_view, headers); if (idx >= 0) { /* all headers found */ if (idx != mail->data.header_save_idx) { - mail_cache_set_corrupted(mail->ibox->index->cache, + mail_cache_set_corrupted(mail->ibox->cache, "Duplicated header names list (%d and %d)", idx, mail->data.header_save_idx); } } else { /* there's some new headers */ - idx = find_unused_header_idx(mail->ibox->index->cache); + idx = find_unused_header_idx(mail->ibox->cache_view); if (idx < 0) return; - if (!mail_cache_set_header_fields(mail->ibox->trans_ctx, + if (!mail_cache_set_header_fields(mail->trans->cache_trans, idx, headers)) return; } @@ -784,7 +792,7 @@ len = str_len(mail->data.header_data) - data->header_data_uncached_offset; - mail_cache_add(mail->ibox->trans_ctx, data->rec, + mail_cache_add(mail->trans->cache_trans, data->seq, mail_cache_header_fields[idx], str, len+1); data->header_save = FALSE; }
--- a/src/lib-storage/index/index-mail.c Tue Apr 27 23:14:15 2004 +0300 +++ b/src/lib-storage/index/index-mail.c Tue Apr 27 23:25:52 2004 +0300 @@ -6,15 +6,14 @@ #include "str.h" #include "message-date.h" #include "message-part-serialize.h" +#include "message-parser.h" #include "imap-bodystructure.h" #include "imap-envelope.h" -#include "mail-custom-flags.h" #include "mail-cache.h" #include "index-storage.h" -#include "index-expunge.h" #include "index-mail.h" -static int index_mail_parse_body(struct index_mail *mail); +static void index_mail_parse_body(struct index_mail *mail); static struct message_part *get_cached_parts(struct index_mail *mail) { @@ -24,12 +23,12 @@ size_t part_size; if ((mail->data.cached_fields & MAIL_CACHE_MESSAGEPART) == 0) { - mail_cache_mark_missing(mail->ibox->index->cache, + mail_cache_mark_missing(mail->ibox->cache_view, MAIL_CACHE_MESSAGEPART); return NULL; } - if (!mail_cache_lookup_field(mail->ibox->index->cache, mail->data.rec, + if (!mail_cache_lookup_field(mail->ibox->cache_view, mail->data.seq, MAIL_CACHE_MESSAGEPART, &part_data, &part_size)) { /* unexpected - must be an error */ @@ -39,7 +38,7 @@ part = message_part_deserialize(mail->pool, part_data, part_size, &error); if (part == NULL) { - mail_cache_set_corrupted(mail->ibox->index->cache, + mail_cache_set_corrupted(mail->ibox->cache, "Corrupted cached message_part data (%s)", error); return NULL; } @@ -56,50 +55,48 @@ return part; } -static char *get_cached_string(struct index_mail *mail, - enum mail_cache_field field) +char *index_mail_get_cached_string(struct index_mail *mail, + enum mail_cache_field field) { const char *ret; if ((mail->data.cached_fields & field) == 0) { - mail_cache_mark_missing(mail->ibox->index->cache, field); + mail_cache_mark_missing(mail->ibox->cache_view, field); return NULL; } - ret = mail_cache_lookup_string_field(mail->ibox->index->cache, - mail->data.rec, field); + ret = mail_cache_lookup_string_field(mail->ibox->cache_view, + mail->data.seq, field); return p_strdup(mail->pool, ret); } -static uoff_t get_cached_uoff_t(struct index_mail *mail, - enum mail_cache_field field) +uoff_t index_mail_get_cached_uoff_t(struct index_mail *mail, + enum mail_cache_field field) { uoff_t uoff; - if (!mail_cache_copy_fixed_field(mail->ibox->index->cache, - mail->data.rec, field, - &uoff, sizeof(uoff))) { - mail_cache_mark_missing(mail->ibox->index->cache, field); + if (!mail_cache_copy_fixed_field(mail->ibox->cache_view, mail->data.seq, + field, &uoff, sizeof(uoff))) { + mail_cache_mark_missing(mail->ibox->cache_view, field); uoff = (uoff_t)-1; } return uoff; } -static uoff_t get_cached_virtual_size(struct index_mail *mail) +uoff_t index_mail_get_cached_virtual_size(struct index_mail *mail) { - return get_cached_uoff_t(mail, MAIL_CACHE_VIRTUAL_FULL_SIZE); + return index_mail_get_cached_uoff_t(mail, MAIL_CACHE_VIRTUAL_FULL_SIZE); } -static time_t get_cached_received_date(struct index_mail *mail) +time_t index_mail_get_cached_received_date(struct index_mail *mail) { time_t t; - if (!mail_cache_copy_fixed_field(mail->ibox->index->cache, - mail->data.rec, + if (!mail_cache_copy_fixed_field(mail->ibox->cache_view, mail->data.seq, MAIL_CACHE_RECEIVED_DATE, &t, sizeof(t))) { - mail_cache_mark_missing(mail->ibox->index->cache, + mail_cache_mark_missing(mail->ibox->cache_view, MAIL_CACHE_RECEIVED_DATE); t = (time_t)-1; } @@ -110,10 +107,10 @@ static void get_cached_sent_date(struct index_mail *mail, struct mail_sent_date *sent_date) { - if (!mail_cache_copy_fixed_field(mail->ibox->index->cache, - mail->data.rec, MAIL_CACHE_SENT_DATE, + if (!mail_cache_copy_fixed_field(mail->ibox->cache_view, mail->data.seq, + MAIL_CACHE_SENT_DATE, sent_date, sizeof(*sent_date))) { - mail_cache_mark_missing(mail->ibox->index->cache, + mail_cache_mark_missing(mail->ibox->cache_view, MAIL_CACHE_SENT_DATE); sent_date->time = (time_t)-1; @@ -123,16 +120,16 @@ int index_mail_cache_transaction_begin(struct index_mail *mail) { - if (mail->ibox->trans_ctx != NULL) + if (mail->trans->cache_trans != NULL) return TRUE; - if (mail_cache_transaction_begin(mail->ibox->index->cache, TRUE, - &mail->ibox->trans_ctx) <= 0) + if (mail_cache_transaction_begin(mail->ibox->cache_view, TRUE, + mail->trans->trans, + &mail->trans->cache_trans) <= 0) return FALSE; - mail->data.cached_fields = - mail_cache_get_fields(mail->ibox->index->cache, - mail->data.rec); + mail->data.cached_fields = mail_cache_get_fields(mail->ibox->cache_view, + mail->data.seq); return TRUE; } @@ -157,60 +154,33 @@ void index_mail_cache_add(struct index_mail *mail, enum mail_cache_field field, const void *data, size_t size) { - struct index_mailbox *ibox = mail->ibox; - if (!index_mail_cache_can_add(mail, field)) return; - if (!mail_cache_add(ibox->trans_ctx, mail->data.rec, + if (!mail_cache_add(mail->trans->cache_trans, mail->data.seq, field, data, size)) - mail_cache_transaction_rollback(ibox->trans_ctx); + mail_cache_transaction_rollback(mail->trans->cache_trans); mail->data.cached_fields |= field; } -int index_mail_open_stream(struct index_mail *mail, uoff_t position) -{ - struct index_mail_data *data = &mail->data; - int deleted; - - if (data->stream == NULL) { - data->stream = mail->ibox->index-> - open_mail(mail->ibox->index, data->rec, - &data->received_date, &deleted); - data->deleted = deleted; - - if (data->stream == NULL) - return FALSE; - - if (data->received_date != (time_t)-1) { - index_mail_cache_add(mail, MAIL_CACHE_RECEIVED_DATE, - &data->received_date, - sizeof(data->received_date)); - } - } - - i_stream_seek(mail->data.stream, position); - return TRUE; -} - -static const struct mail_full_flags *get_flags(struct mail *_mail) +const struct mail_full_flags *index_mail_get_flags(struct mail *_mail) { struct index_mail *mail = (struct index_mail *) _mail; struct index_mail_data *data = &mail->data; - data->flags.flags = data->rec->msg_flags; - data->flags.custom_flags = + data->flags.flags = data->rec->flags; + /*FIXME:data->flags.custom_flags = mail_custom_flags_list_get(mail->ibox->index->custom_flags); data->flags.custom_flags_count = MAIL_CUSTOM_FLAGS_COUNT; if (data->rec->uid >= mail->ibox->index->first_recent_uid) - data->flags.flags |= MAIL_RECENT; + data->flags.flags |= MAIL_RECENT;*/ return &data->flags; } -static const struct message_part *get_parts(struct mail *_mail) +const struct message_part *index_mail_get_parts(struct mail *_mail) { struct index_mail *mail = (struct index_mail *) _mail; struct index_mail_data *data = &mail->data; @@ -228,37 +198,12 @@ if (!index_mail_parse_headers(mail)) return NULL; } - if (!index_mail_parse_body(mail)) - return NULL; + index_mail_parse_body(mail); return data->parts; } -static time_t get_received_date(struct mail *_mail) -{ - struct index_mail *mail = (struct index_mail *) _mail; - struct index_mail_data *data = &mail->data; - - if (data->received_date != (time_t)-1) - return data->received_date; - - if ((mail->wanted_fields & MAIL_FETCH_RECEIVED_DATE) == 0) { - data->received_date = get_cached_received_date(mail); - if (data->received_date != (time_t)-1) - return data->received_date; - } - - data->received_date = mail->ibox->index-> - get_received_date(mail->ibox->index, mail->data.rec); - if (data->received_date != (time_t)-1) { - index_mail_cache_add(mail, MAIL_CACHE_RECEIVED_DATE, - &data->received_date, - sizeof(data->received_date)); - } - return data->received_date; -} - -static time_t get_date(struct mail *_mail, int *timezone) +time_t index_mail_get_date(struct mail *_mail, int *timezone) { struct index_mail *mail = (struct index_mail *) _mail; struct index_mail_data *data = &mail->data; @@ -302,7 +247,7 @@ if (data->parts == NULL) { if ((mail->wanted_fields & MAIL_FETCH_MESSAGE_PARTS) != 0) - (void)get_parts(&mail->mail); + (void)index_mail_get_parts(&mail->mail); else data->parts = get_cached_parts(mail); } @@ -319,7 +264,7 @@ return data->parts != NULL; } -static uoff_t get_size(struct mail *_mail) +uoff_t index_mail_get_size(struct mail *_mail) { struct index_mail *mail = (struct index_mail *) _mail; struct index_mail_data *data = &mail->data; @@ -329,7 +274,7 @@ return data->size; if ((mail->wanted_fields & MAIL_FETCH_SIZE) == 0) { - data->size = get_cached_virtual_size(mail); + data->size = index_mail_get_cached_virtual_size(mail); if (data->size != (uoff_t)-1) return data->size; } @@ -352,7 +297,7 @@ imap_bodystructure_parse_header(pool, part, hdr); } -static int index_mail_parse_body(struct index_mail *mail) +static void index_mail_parse_body(struct index_mail *mail) { struct index_mail_data *data = &mail->data; enum mail_index_record_flag index_flags; @@ -379,7 +324,7 @@ data->body_size_set = TRUE; if (mail->mail.has_nuls || mail->mail.has_no_nuls) - return TRUE; + return; /* we know the NULs now, update them */ if ((data->parts->flags & MESSAGE_PART_FLAG_HAS_NULS) != 0) { @@ -391,19 +336,19 @@ } if (!index_mail_cache_transaction_begin(mail)) - return TRUE; + return; /* update index_flags */ - index_flags = mail_cache_get_index_flags(mail->ibox->index->cache, - mail->data.rec); + index_flags = mail_cache_get_index_flags(mail->ibox->cache_view, + mail->data.seq); if (mail->mail.has_nuls) index_flags |= MAIL_INDEX_FLAG_HAS_NULS; else index_flags |= MAIL_INDEX_FLAG_HAS_NO_NULS; - if (!mail_cache_update_index_flags(mail->ibox->index->cache, - mail->data.rec, index_flags)) - return FALSE; + if (!mail_cache_update_index_flags(mail->ibox->cache_view, + mail->data.seq, index_flags)) + return; if (index_mail_cache_can_add(mail, MAIL_CACHE_MESSAGEPART)) { t_push(); @@ -416,19 +361,15 @@ buf_data, buf_size); t_pop(); } - return TRUE; } -static struct istream *get_stream(struct mail *_mail, - struct message_size *hdr_size, - struct message_size *body_size) +struct istream *index_mail_init_stream(struct index_mail *_mail, + struct message_size *hdr_size, + struct message_size *body_size) { struct index_mail *mail = (struct index_mail *) _mail; struct index_mail_data *data = &mail->data; - if (!index_mail_open_stream(mail, 0)) - return NULL; - if (hdr_size != NULL || body_size != NULL) (void)get_msgpart_sizes(mail); @@ -442,10 +383,8 @@ } if (body_size != NULL) { - if (!data->body_size_set) { - if (!index_mail_parse_body(mail)) - return NULL; - } + if (!data->body_size_set) + index_mail_parse_body(mail); *body_size = data->body_size; } @@ -459,28 +398,30 @@ return data->stream; } -static const char *get_special(struct mail *_mail, enum mail_fetch_field field) +const char *index_mail_get_special(struct mail *_mail, + enum mail_fetch_field field) { struct index_mail *mail = (struct index_mail *) _mail; struct index_mail_data *data = &mail->data; - struct mail_cache *cache = mail->ibox->index->cache; + struct mail_cache *cache = mail->ibox->cache; enum mail_cache_field cache_field; char *str; switch (field) { case MAIL_FETCH_IMAP_BODY: if ((data->cached_fields & MAIL_CACHE_BODY) && - data->body == NULL) - data->body = get_cached_string(mail, MAIL_CACHE_BODY); + data->body == NULL) { + data->body = index_mail_get_cached_string(mail, + MAIL_CACHE_BODY); + } if (data->body != NULL) return data->body; /* fall through */ case MAIL_FETCH_IMAP_BODYSTRUCTURE: if ((data->cached_fields & MAIL_CACHE_BODYSTRUCTURE) && data->bodystructure == NULL) { - data->bodystructure = - get_cached_string(mail, - MAIL_CACHE_BODYSTRUCTURE); + data->bodystructure = index_mail_get_cached_string(mail, + MAIL_CACHE_BODYSTRUCTURE); } if (data->bodystructure != NULL) { @@ -514,8 +455,7 @@ parse_bodystructure_header, mail->pool); } else { - if (!index_mail_parse_body(mail)) - return NULL; + index_mail_parse_body(mail); } t_push(); @@ -548,39 +488,21 @@ } } -static struct mail index_mail = { - 0, 0, 0, 0, 0, - - get_flags, - get_parts, - get_received_date, - get_date, - get_size, - index_mail_get_header, - index_mail_get_headers, - get_stream, - get_special, - index_storage_update_flags, - index_storage_expunge -}; - -void index_mail_init(struct index_mailbox *ibox, struct index_mail *mail, +void index_mail_init(struct index_transaction_context *t, + struct index_mail *mail, enum mail_fetch_field wanted_fields, const char *const wanted_headers[]) { - mail->mail = index_mail; - mail->mail.box = &ibox->box; + mail->mail = *t->ibox->mail_interface; + mail->mail.box = &t->ibox->box; mail->pool = pool_alloconly_create("index_mail", 16384); - mail->ibox = ibox; + mail->ibox = t->ibox; + mail->trans = t; mail->wanted_fields = wanted_fields; mail->wanted_headers = wanted_headers; - mail->expunge_counter = ibox->index->expunge_counter; index_mail_headers_init(mail); - - if (ibox->mail_init != NULL) - ibox->mail_init(mail); } static void index_mail_close(struct index_mail *mail) @@ -591,48 +513,49 @@ index_mail_headers_close(mail); } -int index_mail_next(struct index_mail *mail, struct mail_index_record *rec, - unsigned int idx_seq, int delay_open) +int index_mail_next(struct index_mail *mail, + const struct mail_index_record *rec, + uint32_t seq, int delay_open) { - struct mail_index *index = mail->ibox->index; struct index_mail_data *data = &mail->data; enum mail_index_record_flag index_flags; int ret, open_mail; - i_assert(mail->expunge_counter == index->expunge_counter); - t_push(); index_mail_close(mail); memset(data, 0, sizeof(*data)); p_clear(mail->pool); - data->cached_fields = mail_cache_get_fields(index->cache, rec); + data->cached_fields = + mail_cache_get_fields(mail->ibox->cache_view, seq); index_flags = (data->cached_fields & MAIL_CACHE_INDEX_FLAGS) == 0 ? 0 : - mail_cache_get_index_flags(index->cache, rec); + mail_cache_get_index_flags(mail->ibox->cache_view, seq); mail->mail.has_nuls = (index_flags & MAIL_INDEX_FLAG_HAS_NULS) != 0; mail->mail.has_no_nuls = (index_flags & MAIL_INDEX_FLAG_HAS_NO_NULS) != 0; data->rec = rec; - data->idx_seq = idx_seq; + data->seq = seq; data->size = (uoff_t)-1; data->received_date = data->sent_date.time = (time_t)-1; /* if some wanted fields are cached, get them */ if (mail->wanted_fields & MAIL_FETCH_MESSAGE_PARTS) data->parts = get_cached_parts(mail); - if (mail->wanted_fields & MAIL_FETCH_IMAP_BODY) - data->body = get_cached_string(mail, MAIL_CACHE_BODY); + if (mail->wanted_fields & MAIL_FETCH_IMAP_BODY) { + data->body = + index_mail_get_cached_string(mail, MAIL_CACHE_BODY); + } if ((mail->wanted_fields & MAIL_FETCH_IMAP_BODYSTRUCTURE) || ((mail->wanted_fields & MAIL_FETCH_IMAP_BODY) && data->body == NULL)) { - data->bodystructure = - get_cached_string(mail, MAIL_CACHE_BODYSTRUCTURE); + data->bodystructure = index_mail_get_cached_string(mail, + MAIL_CACHE_BODYSTRUCTURE); } if (mail->wanted_fields & MAIL_FETCH_SIZE) - data->size = get_cached_virtual_size(mail); + data->size = index_mail_get_cached_virtual_size(mail); if (mail->wanted_fields & MAIL_FETCH_DATE) get_cached_sent_date(mail, &data->sent_date); @@ -662,14 +585,15 @@ index_mail_headers_init_next(mail); if ((open_mail || data->parse_header) && !delay_open) { - if (!index_mail_open_stream(mail, 0)) + if (mail->mail.get_stream(&mail->mail, NULL, NULL) == NULL) ret = data->deleted ? 0 : -1; else ret = 1; } else { if (mail->wanted_fields & MAIL_FETCH_RECEIVED_DATE) { /* check this only after open_mail() */ - data->received_date = get_cached_received_date(mail); + data->received_date = + index_mail_get_cached_received_date(mail); } ret = 1; } @@ -694,3 +618,40 @@ pool_unref(mail->pool); memset(mail, 0, sizeof(*mail)); } + +int index_mail_update_flags(struct mail *mail, + const struct mail_full_flags *flags, + enum modify_type modify_type) +{ + struct index_mail *imail = (struct index_mail *)mail; + enum mail_flags modify_flags; + custom_flags_mask_t custom_flags; + + /* \Recent can't be changed */ + modify_flags = flags->flags & ~MAIL_RECENT; + + /*if (!index_mailbox_fix_custom_flags(ibox, &modify_flags, + flags->custom_flags, + flags->custom_flags_count)) + return FALSE;*/ + + memset(custom_flags, 0, sizeof(custom_flags)); + mail_index_update_flags(imail->trans->trans, mail->seq, modify_type, + flags->flags, custom_flags); + + /*if (mail_custom_flags_has_changes(ibox->index->custom_flags)) { + storage->callbacks->new_custom_flags(&ibox->box, + mail_custom_flags_list_get(ibox->index->custom_flags), + MAIL_CUSTOM_FLAGS_COUNT, storage->callback_context); + }*/ + + return 0; +} + +int index_mail_expunge(struct mail *mail) +{ + struct index_mail *imail = (struct index_mail *)mail; + + mail_index_expunge(imail->trans->trans, mail->seq); + return 0; +}
--- a/src/lib-storage/index/index-mail.h Tue Apr 27 23:14:15 2004 +0300 +++ b/src/lib-storage/index/index-mail.h Tue Apr 27 23:25:52 2004 +0300 @@ -3,6 +3,7 @@ #include "message-size.h" #include "mail-cache.h" +#include "mail-storage-private.h" struct message_header_line; @@ -25,8 +26,8 @@ const char *envelope, *body, *bodystructure; struct message_part_envelope_data *envelope_data; - struct mail_index_record *rec; - unsigned int idx_seq; + uint32_t seq; + const struct mail_index_record *rec; struct istream *stream; struct message_size hdr_size, body_size; @@ -53,6 +54,7 @@ pool_t pool; struct index_mailbox *ibox; + struct index_transaction_context *trans; unsigned int expunge_counter; buffer_t *header_buf; @@ -61,11 +63,13 @@ int wanted_headers_idx; }; -void index_mail_init(struct index_mailbox *ibox, struct index_mail *mail, +void index_mail_init(struct index_transaction_context *t, + struct index_mail *mail, enum mail_fetch_field wanted_fields, const char *const wanted_headers[]); -int index_mail_next(struct index_mail *mail, struct mail_index_record *rec, - unsigned int idx_seq, int delay_open); +int index_mail_next(struct index_mail *mail, + const struct mail_index_record *rec, + uint32_t seq, int delay_open); void index_mail_deinit(struct index_mail *mail); void index_mail_parse_header_init(struct index_mail *mail, @@ -78,7 +82,6 @@ void index_mail_cache_add(struct index_mail *mail, enum mail_cache_field field, const void *data, size_t size); -int index_mail_open_stream(struct index_mail *mail, uoff_t position); int index_mail_parse_headers(struct index_mail *mail); void index_mail_headers_init(struct index_mail *mail); @@ -89,4 +92,26 @@ struct istream *index_mail_get_headers(struct mail *_mail, const char *const minimum_fields[]); +const struct mail_full_flags *index_mail_get_flags(struct mail *_mail); +const struct message_part *index_mail_get_parts(struct mail *_mail); +time_t index_mail_get_date(struct mail *_mail, int *timezone); +uoff_t index_mail_get_size(struct mail *_mail); +struct istream *index_mail_init_stream(struct index_mail *mail, + struct message_size *hdr_size, + struct message_size *body_size); +const char *index_mail_get_special(struct mail *_mail, + enum mail_fetch_field field); + +int index_mail_update_flags(struct mail *mail, + const struct mail_full_flags *flags, + enum modify_type modify_type); +int index_mail_expunge(struct mail *mail); + +char *index_mail_get_cached_string(struct index_mail *mail, + enum mail_cache_field field); +uoff_t index_mail_get_cached_uoff_t(struct index_mail *mail, + enum mail_cache_field field); +uoff_t index_mail_get_cached_virtual_size(struct index_mail *mail); +time_t index_mail_get_cached_received_date(struct index_mail *mail); + #endif
--- a/src/lib-storage/index/index-mailbox-check.c Tue Apr 27 23:14:15 2004 +0300 +++ b/src/lib-storage/index/index-mailbox-check.c Tue Apr 27 23:25:52 2004 +0300 @@ -5,6 +5,8 @@ #include "index-storage.h" #include <stdlib.h> +#include <unistd.h> +#include <fcntl.h> #include <sys/stat.h> static void check_timeout(void *context)
--- a/src/lib-storage/index/index-messageset.c Tue Apr 27 23:14:15 2004 +0300 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,384 +0,0 @@ -/* Copyright (C) 2002-2003 Timo Sirainen */ - -#include "lib.h" -#include "mail-index.h" -#include "mail-index-util.h" -#include "mail-modifylog.h" -#include "index-storage.h" -#include "index-messageset.h" - -struct messageset_context { - struct index_mailbox *ibox; - struct mail_index *index; - - const struct modify_log_expunge *expunges; - int expunges_found; - - struct messageset_mail mail; - unsigned int messages_count; - unsigned int num1, num2; - unsigned int min_uid, max_uid; - - const char *messageset, *p; - int uidset, skip_expunged; - - int first, ret; - const char *error; -}; - -static int uidset_init(struct messageset_context *ctx); -static int seqset_init(struct messageset_context *ctx); - -struct messageset_context * -index_messageset_init(struct index_mailbox *ibox, - const char *messageset, int uidset, int skip_expunged) -{ - struct messageset_context *ctx; - - i_assert(ibox->index->lock_type != MAIL_LOCK_UNLOCK); - - ctx = i_new(struct messageset_context, 1); - ctx->ibox = ibox; - ctx->index = ibox->index; - ctx->messages_count = ibox->synced_messages_count; - ctx->p = ctx->messageset = messageset; - ctx->uidset = uidset; - ctx->skip_expunged = skip_expunged; - - ctx->min_uid = 1; - ctx->max_uid = (unsigned int)-1; - - /* Reset index errors, we rely on it to check for failures */ - index_reset_error(ctx->index); - - return ctx; -} - -struct messageset_context * -index_messageset_init_range(struct index_mailbox *ibox, - unsigned int num1, unsigned int num2, int uidset) -{ - struct messageset_context *ctx; - - ctx = index_messageset_init(ibox, NULL, uidset, TRUE); - if (num1 <= num2) { - ctx->num1 = num1; - ctx->num2 = num2; - } else { - ctx->num1 = num2; - ctx->num2 = num1; - } - return ctx; -} - -void index_messageset_limit_range(struct messageset_context *ctx, - unsigned int min_uid, unsigned int max_uid) -{ - ctx->min_uid = min_uid; - ctx->max_uid = max_uid; -} - -int index_messageset_deinit(struct messageset_context *ctx) -{ - int ret = ctx->ret; - - if (ret == 0) { - /* we just didn't go through all of them */ - ret = 1; - } - - if (ret == 1 && ctx->expunges_found) { - /* some of the messages weren't found */ - ret = 0; - } - - if (ret == -1) - mail_storage_set_index_error(ctx->ibox); - else if (ret == -2) { - /* user error */ - mail_storage_set_syntax_error(ctx->ibox->box.storage, - "%s", ctx->error); - } - - i_free(ctx); - return ret; -} - -static unsigned int get_next_number(const char **str) -{ - unsigned int num; - - num = 0; - while (**str != '\0') { - if (**str < '0' || **str > '9') - break; - - num = num*10 + (**str - '0'); - (*str)++; - } - - return num; -} - -static int messageset_parse_next(struct messageset_context *ctx) -{ - unsigned int num; - - if (ctx->p == NULL) { - /* num1..num2 already set. */ - ctx->p = ""; - return TRUE; - } - - if (*ctx->p == '*') { - /* last message */ - ctx->num1 = (unsigned int)-1; - ctx->p++; - } else { - ctx->num1 = get_next_number(&ctx->p); - if (ctx->num1 == 0) { - ctx->error = t_strconcat("Invalid messageset: ", - ctx->messageset, NULL); - return FALSE; - } - } - - if (*ctx->p != ':') - ctx->num2 = ctx->num1; - else { - /* first:last range */ - ctx->p++; - - if (*ctx->p == '*') { - ctx->num2 = (unsigned int)-1; - ctx->p++; - } else { - ctx->num2 = get_next_number(&ctx->p); - if (ctx->num2 == 0) { - ctx->error = t_strconcat("Invalid messageset: ", - ctx->messageset, NULL); - return FALSE; - } - } - } - - if (*ctx->p == ',') - ctx->p++; - else if (*ctx->p != '\0') { - ctx->error = t_strdup_printf("Unexpected char '%c' " - "with messageset: %s", - *ctx->p, ctx->messageset); - return FALSE; - } - - if ((client_workarounds & WORKAROUND_OE6_FETCH_REDUNDANT_MSGSET) != 0 && - ctx->uidset && ctx->num1 == ctx->ibox->index->header->next_uid && - ctx->num2 == (unsigned int)-1) { - /* FETCH nextuid:* - it's very unlikely the client wants to - fetch the last message */ - ctx->num2 = ctx->num1; - } - - if (ctx->num1 > ctx->num2) { - /* swap, as specified by RFC-3501 */ - unsigned int temp = ctx->num1; - ctx->num1 = ctx->num2; - ctx->num2 = temp; - } - - num = ctx->num2 == (unsigned int)-1 ? ctx->num1 : ctx->num2; - if (num > ctx->messages_count && !ctx->uidset && - num != (unsigned int)-1) { - ctx->error = t_strdup_printf("Message sequence %u " - "larger than message count (%u)", - num, ctx->messages_count); - return FALSE; - } - - if (ctx->messages_count == 0 && !ctx->uidset && - num == (unsigned int)-1) { - ctx->error = "No messages in mailbox"; - return FALSE; - } - - return TRUE; -} - -static int uidset_init(struct messageset_context *ctx) -{ - unsigned int expunges_before; - - if (ctx->num1 == (unsigned int)-1) { - struct mail_index_record *rec; - - rec = ctx->index->lookup(ctx->index, ctx->messages_count); - if (rec == NULL) - return 1; - - ctx->num1 = rec->uid; - } - - if (ctx->num2 == (unsigned int)-1) { - ctx->num2 = ctx->index->header->next_uid-1; - - if (ctx->num2 == 0) - return 1; - - /* num1 might actually be larger, check */ - if (ctx->num1 > ctx->num2) { - unsigned int temp = ctx->num1; - ctx->num1 = ctx->num2; - ctx->num2 = temp; - } - } - i_assert(ctx->num1 <= ctx->num2); - - if (ctx->num1 < ctx->min_uid) - ctx->num1 = ctx->min_uid; - if (ctx->num2 > ctx->max_uid) - ctx->num2 = ctx->max_uid; - if (ctx->num1 > ctx->num2) - return 1; - - /* get list of expunged messages in our range. */ - ctx->expunges = mail_modifylog_uid_get_expunges(ctx->index->modifylog, - ctx->num1, ctx->num2, - &expunges_before); - if (ctx->expunges == NULL) - return -1; - - if (ctx->expunges->uid1 != 0) - ctx->expunges_found = TRUE; - - /* get the first message */ - ctx->mail.rec = ctx->index->lookup_uid_range(ctx->index, - ctx->num1, ctx->num2, - &ctx->mail.idx_seq); - if (ctx->mail.rec == NULL) { - return ctx->index->get_last_error(ctx->index) == - MAIL_INDEX_ERROR_NONE ? 1 : -1; - } - - ctx->mail.client_seq = ctx->mail.idx_seq + expunges_before; - return 0; -} - -static int seqset_init(struct messageset_context *ctx) -{ - unsigned int expunges_before; - - if (ctx->num1 == (unsigned int)-1) - ctx->num1 = ctx->messages_count; - - if (ctx->num2 == (unsigned int)-1) - ctx->num2 = ctx->messages_count; - - /* get list of expunged messages in our range. the expunges_before - can be used to calculate the current real sequence position */ - ctx->expunges = mail_modifylog_seq_get_expunges(ctx->index->modifylog, - ctx->num1, ctx->num2, - &expunges_before); - if (ctx->expunges == NULL) - return -1; - - i_assert(expunges_before < ctx->num1); - if (ctx->expunges->uid1 != 0) - ctx->expunges_found = TRUE; - - /* get the first non-expunged message. note that if all messages - were expunged in the range, this points outside wanted range. */ - ctx->mail.idx_seq = ctx->num1 - expunges_before; - ctx->mail.client_seq = ctx->num1; - ctx->mail.rec = ctx->index->lookup(ctx->index, ctx->mail.idx_seq); - while (ctx->mail.rec != NULL && ctx->mail.rec->uid < ctx->min_uid) { - ctx->mail.idx_seq++; - ctx->mail.client_seq++; - ctx->mail.rec = ctx->index->next(ctx->index, ctx->mail.rec); - } - - if (ctx->mail.rec != NULL && ctx->mail.rec->uid > ctx->max_uid) { - ctx->mail.rec = NULL; - return 1; - } - - if (ctx->mail.rec == NULL) { - return ctx->index->get_last_error(ctx->index) == - MAIL_INDEX_ERROR_NONE ? 1 : -1; - } - - return 0; -} - -const struct messageset_mail * -index_messageset_next(struct messageset_context *ctx) -{ - struct messageset_mail *mail = &ctx->mail; - int last; - - if (ctx->ret != 0) - return NULL; - - if (!ctx->uidset) - last = mail->rec == NULL || mail->client_seq >= ctx->num2; - else - last = mail->rec == NULL || mail->rec->uid >= ctx->num2; - - if (!last) { - mail->rec = ctx->index->next(ctx->index, mail->rec); - mail->client_seq++; - mail->idx_seq++; - - if (mail->rec == NULL) { - /* finished early (high UID larger than exists) */ - ctx->ret = 1; - return NULL; - } - } else { - do { - if (ctx->p != NULL && *ctx->p == '\0') { - /* finished */ - ctx->ret = 1; - return NULL; - } - - if (!messageset_parse_next(ctx)) { - ctx->ret = -2; - return NULL; - } - - if (ctx->uidset) - ctx->ret = uidset_init(ctx); - else - ctx->ret = seqset_init(ctx); - - if (ctx->expunges_found && !ctx->skip_expunged) { - /* we wish to abort if there's any - expunged messages */ - ctx->ret = 1; - return NULL; - } - } while (ctx->ret == 1); - - if (ctx->ret != 0) - return NULL; - } - - /* fix client_seq */ - while (ctx->expunges->uid1 != 0 && - ctx->expunges->uid1 < mail->rec->uid) { - i_assert(ctx->expunges->uid2 < mail->rec->uid); - - mail->client_seq += ctx->expunges->seq_count; - ctx->expunges++; - } - - i_assert(mail->rec->uid < ctx->expunges->uid1 || - mail->rec->uid > ctx->expunges->uid2); - - if (!ctx->uidset && mail->client_seq > ctx->num2) { - /* finished this set - see if there's more */ - return index_messageset_next(ctx); - } - - return mail; -}
--- a/src/lib-storage/index/index-messageset.h Tue Apr 27 23:14:15 2004 +0300 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,32 +0,0 @@ -#ifndef __INDEX_MESSAGESET_H -#define __INDEX_MESSAGESET_H - -struct index_mailbox; - -struct messageset_mail { - struct mail_index_record *rec; - unsigned int client_seq; - unsigned int idx_seq; -}; - -struct messageset_context; - -struct messageset_context * -index_messageset_init(struct index_mailbox *ibox, - const char *messageset, int uidset, int skip_expunged); - -struct messageset_context * -index_messageset_init_range(struct index_mailbox *ibox, - unsigned int num1, unsigned int num2, int uidset); - -void index_messageset_limit_range(struct messageset_context *ctx, - unsigned int min_uid, unsigned int max_uid); - -/* Returns 1 if all were found, 0 if some messages were expunged, - -1 if internal error occured or -2 if messageset was invalid. */ -int index_messageset_deinit(struct messageset_context *ctx); - -const struct messageset_mail * -index_messageset_next(struct messageset_context *ctx); - -#endif
--- a/src/lib-storage/index/index-search.c Tue Apr 27 23:14:15 2004 +0300 +++ b/src/lib-storage/index/index-search.c Tue Apr 27 23:25:52 2004 +0300 @@ -7,12 +7,10 @@ #include "message-date.h" #include "message-body-search.h" #include "message-header-search.h" +#include "message-parser.h" #include "imap-date.h" #include "index-storage.h" -#include "index-messageset.h" #include "index-mail.h" -#include "mail-custom-flags.h" -#include "mail-modifylog.h" #include "mail-search.h" #include <stdlib.h> @@ -21,12 +19,14 @@ #define TXT_UNKNOWN_CHARSET "[BADCHARSET] Unknown charset" #define TXT_INVALID_SEARCH_KEY "Invalid search key" -struct mail_search_context { +struct index_search_context { + struct mail_search_context mail_ctx; + struct index_transaction_context *trans; struct index_mailbox *ibox; char *charset; struct mail_search_arg *args; - struct messageset_context *msgset_ctx; + uint32_t seq1, seq2; struct index_mail imail; struct mail *mail; @@ -37,7 +37,7 @@ }; struct search_header_context { - struct mail_search_context *index_context; + struct index_search_context *index_context; struct mail_search_arg *args; struct message_header_line *hdr; @@ -47,70 +47,17 @@ }; struct search_body_context { - struct mail_search_context *index_ctx; + struct index_search_context *index_ctx; struct istream *input; const struct message_part *part; }; -static int msgset_contains(const char *set, unsigned int match_num, - unsigned int max_num) +static int seqset_contains(struct mail_search_seqset *set, uint32_t seq) { - unsigned int num, num2; - - while (*set != '\0') { - if (*set == '*') { - set++; - num = max_num; - } else { - num = 0; - while (*set >= '0' && *set <= '9') { - num = num*10 + (*set-'0'); - set++; - } - - if (num == 0) - return FALSE; - } - - if (*set == ',' || *set == '\0') { - if (num == match_num) - return TRUE; - if (*set == '\0') - return FALSE; - } else if (*set == ':') { - set++; - - if (*set == '*') { - set++; - - if (match_num >= num && num <= max_num) - return TRUE; - } else { - num2 = 0; - while (*set >= '0' && *set <= '9') { - num2 = num2*10 + (*set-'0'); - set++; - } - - if (num2 == 0) - return FALSE; - - if (num > num2) { - /* swap, as specified by RFC-3501 */ - unsigned int temp = num; - num = num2; - num2 = temp; - } - - if (match_num >= num && match_num <= num2) - return TRUE; - } - - if (*set != ',') - return FALSE; - } - - set++; + while (set != NULL) { + if (seq >= set->seq1 && seq <= set->seq2) + return TRUE; + set = set->next; } return FALSE; @@ -133,56 +80,56 @@ } static int search_keyword(struct mail_index *index, - struct mail_index_record *rec, const char *value) + const struct mail_index_record *rec, + const char *value) { const char **custom_flags; int i; - if ((rec->msg_flags & MAIL_CUSTOM_FLAGS_MASK) == 0) - return FALSE; + for (i = 0; i < INDEX_CUSTOM_FLAGS_BYTE_COUNT; i++) { + if (rec->custom_flags[i] != 0) + break; + } - custom_flags = mail_custom_flags_list_get(index->custom_flags); + if (i == INDEX_CUSTOM_FLAGS_BYTE_COUNT) + return FALSE; /* no custom flags set */ + + /*FIXME:custom_flags = mail_custom_flags_list_get(index->custom_flags); for (i = 0; i < MAIL_CUSTOM_FLAGS_COUNT; i++) { if (custom_flags[i] != NULL && strcasecmp(custom_flags[i], value) == 0) { return rec->msg_flags & (1 << (MAIL_CUSTOM_FLAG_1_BIT+i)); } - } + }*/ return FALSE; } /* Returns >0 = matched, 0 = not matched, -1 = unknown */ static int search_arg_match_index(struct index_mailbox *ibox, - struct mail_index_record *rec, - unsigned int client_seq, + const struct mail_index_record *rec, enum mail_search_arg_type type, const char *value) { switch (type) { case SEARCH_ALL: return 1; - case SEARCH_SET: - return msgset_contains(value, client_seq, - ibox->synced_messages_count); - case SEARCH_UID: - return msgset_contains(value, rec->uid, - ibox->index->header->next_uid-1); /* flags */ case SEARCH_ANSWERED: - return rec->msg_flags & MAIL_ANSWERED; + return rec->flags & MAIL_ANSWERED; case SEARCH_DELETED: - return rec->msg_flags & MAIL_DELETED; + return rec->flags & MAIL_DELETED; case SEARCH_DRAFT: - return rec->msg_flags & MAIL_DRAFT; + return rec->flags & MAIL_DRAFT; case SEARCH_FLAGGED: - return rec->msg_flags & MAIL_FLAGGED; + return rec->flags & MAIL_FLAGGED; case SEARCH_SEEN: - return rec->msg_flags & MAIL_SEEN; + return rec->flags & MAIL_SEEN; case SEARCH_RECENT: - return rec->uid >= ibox->index->first_recent_uid; + //FIXME:return rec->uid >= ibox->index->first_recent_uid; + return FALSE; case SEARCH_KEYWORD: return search_keyword(ibox->index, rec, value); @@ -193,10 +140,22 @@ static void search_index_arg(struct mail_search_arg *arg, void *context) { - struct mail_search_context *ctx = context; + struct index_search_context *ctx = context; + int found; + + if (arg->type == SEARCH_SEQSET) { + found = seqset_contains(arg->value.seqset, ctx->mail->seq); + ARG_SET_RESULT(arg, found); + return; + } + + if (ctx->imail.data.rec == NULL) { + /* expunged message */ + ARG_SET_RESULT(arg, 0); + return; + } switch (search_arg_match_index(ctx->ibox, ctx->imail.data.rec, - ctx->mail->seq, arg->type, arg->value.str)) { case -1: /* unknown */ @@ -211,7 +170,7 @@ } /* Returns >0 = matched, 0 = not matched, -1 = unknown */ -static int search_arg_match_cached(struct mail_search_context *ctx, +static int search_arg_match_cached(struct index_search_context *ctx, enum mail_search_arg_type type, const char *value) { @@ -291,7 +250,7 @@ static void search_cached_arg(struct mail_search_arg *arg, void *context) { - struct mail_search_context *ctx = context; + struct index_search_context *ctx = context; switch (search_arg_match_cached(ctx, arg->type, arg->value.str)) { @@ -340,7 +299,7 @@ } static struct header_search_context * -search_header_context(struct mail_search_context *ctx, +search_header_context(struct index_search_context *ctx, struct mail_search_arg *arg) { int unknown_charset; @@ -517,7 +476,7 @@ } static int search_arg_match_text(struct mail_search_arg *args, - struct mail_search_context *ctx) + struct index_search_context *ctx) { struct istream *input; const char *const *headers; @@ -571,249 +530,180 @@ return TRUE; } -static int seq_update(const char *set, unsigned int *first_seq, - unsigned int *last_seq, unsigned int max_value) +static int search_msgset_fix(struct index_mailbox *ibox, + const struct mail_index_header *hdr, + struct mail_search_seqset *set, + uint32_t *seq1_r, uint32_t *seq2_r) { - unsigned int seq; - int first = TRUE; + for (; set != NULL; set = set->next) { + if (set->seq1 == (uint32_t)-1) + set->seq1 = hdr->messages_count; + if (set->seq2 == (uint32_t)-1) + set->seq2 = hdr->messages_count; - while (*set != '\0') { - if (*set == '*') { - seq = max_value; - set++; - } else { - seq = 0; - while (*set >= '0' && *set <= '9') { - seq = seq*10 + (*set-'0'); - set++; - } + if (set->seq1 == 0 || set->seq2 == 0 || + set->seq1 > hdr->messages_count || + set->seq2 > hdr->messages_count) { + mail_storage_set_syntax_error(ibox->box.storage, + "Invalid messageset"); + return -1; } - if (seq == 0) - return FALSE; - - if (*first_seq == 0 || seq < *first_seq) - *first_seq = seq; - if (*last_seq == 0 || seq > *last_seq) - *last_seq = seq; - - if (*set != '\0') { - if (*set == ',') - first = TRUE; - else if (*set == ':' && first) - first = FALSE; - else - return FALSE; - set++; - } + if (*seq1_r > set->seq1 || *seq1_r == 0) + *seq1_r = set->seq1; + if (*seq2_r < set->seq2) + *seq2_r = set->seq2; } - - return TRUE; + return 0; } -struct search_msgset_context { - struct index_mailbox *ibox; - - unsigned int first_seq, last_seq; - unsigned int first_uid, last_uid; +static int search_parse_msgset_args(struct index_mailbox *ibox, + struct mail_search_arg *args, + uint32_t *seq1_r, uint32_t *seq2_r) +{ + const struct mail_index_header *hdr; - struct mail_search_arg *msgset_arg; - unsigned int msgset_arg_count; -}; + *seq1_r = *seq2_r = 0; -static int search_parse_msgset_args(struct search_msgset_context *ctx, - struct mail_search_arg *args) -{ - struct index_mailbox *ibox = ctx->ibox; - + hdr = mail_index_get_header(ibox->view); for (; args != NULL; args = args->next) { - /* FIXME: we don't check if OR condition can limit the range. - It's a bit tricky and unlikely to affect performance much. */ if (args->type == SEARCH_SUB) { - if (!search_parse_msgset_args(ctx, args->value.subargs)) - return FALSE; - } else if (args->type == SEARCH_SET) { - ctx->msgset_arg = args; - ctx->msgset_arg_count++; - if (!seq_update(args->value.str, - &ctx->first_seq, &ctx->last_seq, - ibox->synced_messages_count)) { - mail_storage_set_syntax_error(ibox->box.storage, - "Invalid messageset: %s", - args->value.str); - return FALSE; - } - } else if (args->type == SEARCH_UID) { - ctx->msgset_arg = args; - ctx->msgset_arg_count++; - if (!seq_update(args->value.str, - &ctx->first_uid, &ctx->last_uid, - ibox->index->header->next_uid-1)) { - mail_storage_set_syntax_error(ibox->box.storage, - "Invalid messageset: %s", - args->value.str); - return FALSE; - } + if (search_parse_msgset_args(ibox, args->value.subargs, + seq1_r, seq2_r) < 0) + return -1; + } else if (args->type == SEARCH_OR) { + /* FIXME: in cases like "SEEN OR 5 7" we shouldn't + limit the range, but in cases like "1 OR 5 7" we + should expand the range. A bit tricky, we'll + just go through everything now to make it work + right. */ + *seq1_r = 1; + *seq2_r = hdr->messages_count; + + /* We still have to fix potential seqsets though */ + if (search_parse_msgset_args(ibox, args->value.subargs, + seq1_r, seq2_r) < 0) + return -1; + } else if (args->type == SEARCH_SEQSET) { + if (search_msgset_fix(ibox, hdr, args->value.seqset, + seq1_r, seq2_r) < 0) + return -1; } else if (args->type == SEARCH_ALL) { - /* go through everything */ - ctx->first_seq = 1; - ctx->last_seq = ibox->synced_messages_count; - ctx->msgset_arg_count++; - return TRUE; + /* go through everything. don't stop, have to fix + seqsets. */ + *seq1_r = 1; + *seq2_r = hdr->messages_count; } } + return 0; +} - return TRUE; +static int search_limit_lowwater(struct index_mailbox *ibox, + uint32_t uid_lowwater, uint32_t *first_seq) +{ + const struct mail_index_header *hdr; + uint32_t seq1, seq2; + + if (uid_lowwater == 0) + return 0; + + hdr = mail_index_get_header(ibox->view); + if (mail_index_lookup_uid_range(ibox->view, uid_lowwater, + hdr->next_uid-1, + &seq1, &seq2) < 0) { + mail_storage_set_index_error(ibox); + return -1; + } + + if (*first_seq < seq1) + *first_seq = seq1; + return 0; } static int search_limit_by_flags(struct index_mailbox *ibox, struct mail_search_arg *args, - unsigned int *first_uid, - unsigned int *last_uid) + uint32_t *seq1, uint32_t *seq2) { - struct mail_index_header *hdr; - unsigned int uid; + const struct mail_index_header *hdr; - hdr = ibox->index->header; + hdr = mail_index_get_header(ibox->view); for (; args != NULL; args = args->next) { if (args->type == SEARCH_SEEN) { /* SEEN with 0 seen? */ if (!args->not && hdr->seen_messages_count == 0) - return FALSE; + return 0; if (hdr->seen_messages_count == hdr->messages_count) { /* UNSEEN with all seen? */ if (args->not) - return FALSE; + return 0; /* SEEN with all seen */ args->match_always = TRUE; - } else { + } else if (args->not) { /* UNSEEN with lowwater limiting */ - uid = hdr->first_unseen_uid_lowwater; - if (args->not && *first_uid < uid) - *first_uid = uid; + if (search_limit_lowwater(ibox, + hdr->first_unseen_uid_lowwater, + seq1) < 0) + return -1; } } if (args->type == SEARCH_DELETED) { /* DELETED with 0 deleted? */ if (!args->not && hdr->deleted_messages_count == 0) - return FALSE; + return 0; if (hdr->deleted_messages_count == hdr->messages_count) { /* UNDELETED with all deleted? */ if (args->not) - return FALSE; + return 0; /* DELETED with all deleted */ args->match_always = TRUE; - } else { + } else if (!args->not) { /* DELETED with lowwater limiting */ - uid = hdr->first_deleted_uid_lowwater; - if (!args->not && *first_uid < uid) - *first_uid = uid; + if (search_limit_lowwater(ibox, + hdr->first_deleted_uid_lowwater, + seq1) < 0) + return -1; } } - if (args->type == SEARCH_RECENT) { + /*FIXME:if (args->type == SEARCH_RECENT) { uid = ibox->index->first_recent_uid; if (!args->not && *first_uid < uid) *first_uid = ibox->index->first_recent_uid; else if (args->not && *last_uid >= uid) *last_uid = uid-1; - } - } - - return *first_uid <= *last_uid; -} - -static int client_seq_to_uid(struct index_mailbox *ibox, - unsigned int seq, unsigned int *uid) -{ - struct mail_index_record *rec; - unsigned int expunges_before; - - if (seq > ibox->synced_messages_count) { - mail_storage_set_syntax_error(ibox->box.storage, - "Sequence out of range: %u", seq); - return FALSE; + }*/ } - if (mail_modifylog_seq_get_expunges(ibox->index->modifylog, seq, seq, - &expunges_before) == NULL) - return FALSE; - - seq -= expunges_before; - - rec = ibox->index->lookup(ibox->index, seq); - *uid = rec == NULL ? 0 : rec->uid; - return TRUE; + return *seq1 <= *seq2; } -static int search_get_msgset(struct index_mailbox *ibox, - struct mail_search_arg *args, - struct messageset_context **msgset_r) +static int search_get_seqset(struct index_search_context *ctx, + struct mail_search_arg *args) { - struct search_msgset_context ctx; - unsigned int uid; + const struct mail_index_header *hdr; - memset(&ctx, 0, sizeof(ctx)); - ctx.ibox = ibox; - - if (!search_parse_msgset_args(&ctx, args)) + if (search_parse_msgset_args(ctx->ibox, args, + &ctx->seq1, &ctx->seq2) < 0) return -1; - /* seq_update() should make sure that these can't happen */ - i_assert(ctx.first_seq <= ctx.last_seq); - i_assert(ctx.first_uid <= ctx.last_uid); - - if (ctx.first_seq > 1) { - if (!client_seq_to_uid(ibox, ctx.first_seq, &uid)) - return -1; - if (uid == 0) - return 0; - - if (ctx.first_uid == 0 || uid < ctx.first_uid) - ctx.first_uid = uid; - } - - if (ctx.last_seq > 1 && ctx.last_seq != ibox->synced_messages_count) { - if (!client_seq_to_uid(ibox, ctx.last_seq, &uid)) - return -1; - if (uid == 0) - return 0; - - if (ctx.last_uid == 0 || uid > ctx.last_uid) - ctx.last_uid = uid; + if (ctx->seq1 == 0) { + hdr = mail_index_get_header(ctx->ibox->view); + ctx->seq1 = 1; + ctx->seq2 = hdr->messages_count; } - if (ctx.first_uid == 0) - ctx.first_uid = 1; - if (ctx.last_uid == 0 || ctx.last_seq == ibox->synced_messages_count) - ctx.last_uid = ibox->index->header->next_uid-1; + i_assert(ctx->seq1 <= ctx->seq2); /* UNSEEN and DELETED in root search level may limit the range */ - if (!search_limit_by_flags(ibox, args, &ctx.first_uid, &ctx.last_uid)) - return 0; - - i_assert(ctx.first_uid <= ctx.last_uid); - - if (ctx.msgset_arg != NULL && ctx.msgset_arg_count == 1) { - /* one messageset argument, we can use it */ - *msgset_r = index_messageset_init(ibox, - ctx.msgset_arg->value.str, - ctx.msgset_arg->type == SEARCH_UID, TRUE); - /* we might be able to limit it some more */ - index_messageset_limit_range(*msgset_r, - ctx.first_uid, ctx.last_uid); - ctx.msgset_arg->match_always = TRUE; - } else { - *msgset_r = index_messageset_init_range(ibox, ctx.first_uid, - ctx.last_uid, TRUE); - } - return 1; + if (search_limit_by_flags(ctx->ibox, args, &ctx->seq1, &ctx->seq2) < 0) + return -1; + return 0; } int index_storage_search_get_sorting(struct mailbox *box __attr_unused__, @@ -821,80 +711,58 @@ { /* currently we don't support sorting */ *sort_program = MAIL_SORT_END; - return TRUE; + return 0; } struct mail_search_context * -index_storage_search_init(struct mailbox *box, const char *charset, - struct mail_search_arg *args, +index_storage_search_init(struct mailbox_transaction_context *_t, + const char *charset, struct mail_search_arg *args, const enum mail_sort_type *sort_program, enum mail_fetch_field wanted_fields, const char *const wanted_headers[]) { - struct index_mailbox *ibox = (struct index_mailbox *) box; - struct mail_search_context *ctx; + struct index_transaction_context *t = + (struct index_transaction_context *)_t; + struct index_search_context *ctx; if (sort_program != NULL && *sort_program != MAIL_SORT_END) { - i_error("BUG: index_storage_search_init(): " - "invalid sort_program"); - return NULL; + i_fatal("BUG: index_storage_search_init(): " + "invalid sort_program"); } - if (!index_storage_sync_and_lock(ibox, TRUE, TRUE, MAIL_LOCK_SHARED)) - return NULL; + /*FIXME:if (!index_storage_sync_and_lock(ibox, TRUE, TRUE, MAIL_LOCK_SHARED)) + return NULL;*/ - ctx = i_new(struct mail_search_context, 1); - ctx->ibox = ibox; + ctx = i_new(struct index_search_context, 1); + ctx->mail_ctx.box = &t->ibox->box; + ctx->trans = t; + ctx->ibox = t->ibox; ctx->charset = i_strdup(charset); ctx->args = args; - ctx->mail = (struct mail *) &ctx->imail; - index_mail_init(ibox, &ctx->imail, wanted_fields, wanted_headers); - - if (ibox->synced_messages_count == 0) - return ctx; + ctx->mail = &ctx->imail.mail; + index_mail_init(t, &ctx->imail, wanted_fields, wanted_headers); mail_search_args_reset(ctx->args, TRUE); - /* see if we can limit the records we look at */ - switch (search_get_msgset(ibox, args, &ctx->msgset_ctx)) { - case -1: - /* error */ + if (search_get_seqset(ctx, args) < 0) { ctx->failed = TRUE; - return ctx; - case 0: - /* nothing found */ - return ctx; + ctx->seq1 = 1; + ctx->seq2 = 0; } - - return ctx; + return &ctx->mail_ctx; } -int index_storage_search_deinit(struct mail_search_context *ctx, int *all_found) +int index_storage_search_deinit(struct mail_search_context *_ctx) { - int ret, msgset_ret; - - ret = !ctx->failed && ctx->error == NULL; + struct index_search_context *ctx = (struct index_search_context *)_ctx; + int ret; - if (ctx->msgset_ctx != NULL) { - msgset_ret = index_messageset_deinit(ctx->msgset_ctx); - if (msgset_ret < 0) - ret = FALSE; - if (all_found != NULL) - *all_found = msgset_ret > 0; - } else { - if (all_found != NULL) - *all_found = !ctx->failed; - } + ret = ctx->failed || ctx->error != NULL ? -1 : 0; - if (ctx->ibox->fetch_mail.pool != NULL) - index_mail_deinit(&ctx->ibox->fetch_mail); if (ctx->imail.pool != NULL) index_mail_deinit(&ctx->imail); - if (!index_storage_lock(ctx->ibox, MAIL_LOCK_UNLOCK)) - ret = FALSE; - if (ctx->error != NULL) { mail_storage_set_error(ctx->ibox->box.storage, "%s", ctx->error); @@ -907,7 +775,7 @@ return ret; } -static int search_match_next(struct mail_search_context *ctx) +static int search_match_next(struct index_search_context *ctx) { struct mail_search_arg *arg; int ret; @@ -918,6 +786,12 @@ if (ret >= 0) return ret > 0; + if (ctx->imail.data.rec == NULL) { + /* expunged message, no way to check if the rest would have + matched */ + return FALSE; + } + /* next search only from cached arguments */ ret = mail_search_args_foreach(ctx->args, search_cached_arg, ctx); if (ret >= 0) @@ -935,33 +809,27 @@ return TRUE; } -struct mail *index_storage_search_next(struct mail_search_context *ctx) +struct mail *index_storage_search_next(struct mail_search_context *_ctx) { - const struct messageset_mail *msgset_mail; + struct index_search_context *ctx = (struct index_search_context *)_ctx; + const struct mail_index_record *rec; int ret; - if (ctx->msgset_ctx == NULL) { - /* initialization failed or didn't found any messages */ - return NULL; - } - - do { - msgset_mail = index_messageset_next(ctx->msgset_ctx); - if (msgset_mail == NULL) { - ret = -1; - break; + ret = 0; + while (ctx->seq1 <= ctx->seq2) { + if (mail_index_lookup(ctx->ibox->view, ctx->seq1, &rec) < 0) { + ctx->failed = TRUE; + mail_storage_set_index_error(ctx->ibox); + return NULL; } - ctx->mail->seq = msgset_mail->client_seq; - ctx->mail->uid = msgset_mail->rec->uid; + ctx->imail.data.rec = rec; + ctx->mail->seq = ctx->seq1++; + ctx->mail->uid = rec == NULL ? 0 : rec->uid; - ret = index_mail_next(&ctx->imail, msgset_mail->rec, - msgset_mail->idx_seq, TRUE); - if (ret <= 0) { - if (ret < 0) - break; - continue; - } + ret = index_mail_next(&ctx->imail, rec, ctx->mail->seq, TRUE); + if (ret < 0) + break; t_push(); ret = search_match_next(ctx); @@ -969,11 +837,12 @@ if (ctx->error != NULL) ret = -1; - } while (ret == 0); + if (ret != 0) + break; + } - if (ret < 0) { + if (ret <= 0) { /* error or last record */ - index_mail_deinit(&ctx->imail); return NULL; }
--- a/src/lib-storage/index/index-status.c Tue Apr 27 23:14:15 2004 +0300 +++ b/src/lib-storage/index/index-status.c Tue Apr 27 23:25:52 2004 +0300 @@ -1,69 +1,13 @@ /* Copyright (C) 2002 Timo Sirainen */ #include "lib.h" -#include "mail-custom-flags.h" -#include "mail-index-util.h" #include "index-storage.h" #define STATUS_MESSAGE_COUNTS \ (STATUS_MESSAGES | STATUS_RECENT | STATUS_UIDNEXT | \ STATUS_UIDVALIDITY | STATUS_UNSEEN | STATUS_FIRST_UNSEEN_SEQ) -static unsigned int get_first_unseen_seq(struct mail_index *index) -{ - struct mail_index_header *hdr; - struct mail_index_record *rec; - unsigned int seq, lowwater_uid; - - hdr = mail_index_get_header(index); - if (hdr->seen_messages_count == hdr->messages_count) { - /* no unseen messages */ - return 0; - } - - lowwater_uid = hdr->first_unseen_uid_lowwater; - if (lowwater_uid == hdr->next_uid) { - /* no unseen messages */ - rec = NULL; - } else if (lowwater_uid > hdr->next_uid) { - index_set_corrupted(index, "first_unseen_uid_lowwater %u >= " - "next_uid %u", lowwater_uid, hdr->next_uid); - return 0; - } else if (lowwater_uid != 0) { - /* begin scanning from the low water mark */ - rec = index->lookup_uid_range(index, lowwater_uid, - hdr->next_uid - 1, &seq); - } else { - /* begin scanning from the beginning */ - rec = index->lookup(index, 1); - seq = 1; - } - - while (rec != NULL && (rec->msg_flags & MAIL_SEEN)) { - rec = index->next(index, rec); - seq++; - } - - if (rec == NULL) { - index_set_corrupted(index, "No unseen messages found with " - "first_unseen_uid_lowwater %u, " - "seen_messages_count %u, messages_count %u", - lowwater_uid, hdr->seen_messages_count, - hdr->messages_count); - return 0; - } - - if (rec->uid != lowwater_uid) { - /* update the low water mark if we can get exclusive - lock immediately. */ - if (index->try_lock(index, MAIL_LOCK_EXCLUSIVE)) - hdr->first_unseen_uid_lowwater = rec->uid; - } - - return seq; -} - -static void +/*static void get_custom_flags(struct mail_custom_flags *mcf, struct mailbox_status *status) { const char **flags; @@ -75,55 +19,47 @@ flags = mail_custom_flags_list_get(mcf); for (i = 0; i < MAIL_CUSTOM_FLAGS_COUNT; i++) status->custom_flags[i] = t_strdup(flags[i]); -} +}*/ int index_storage_get_status(struct mailbox *box, enum mailbox_status_items items, struct mailbox_status *status) { struct index_mailbox *ibox = (struct index_mailbox *) box; - struct mail_index_header *hdr; + const struct mail_index_header *hdr; memset(status, 0, sizeof(struct mailbox_status)); if ((items & STATUS_MESSAGE_COUNTS) != 0) { - /* if we're doing STATUS for selected mailbox, we have to sync - it first or STATUS reply may give different data */ - if (!index_storage_sync_and_lock(ibox, TRUE, FALSE, - MAIL_LOCK_UNLOCK)) - return FALSE; - - if (!index_storage_sync_modifylog(ibox, FALSE)) { - (void)index_storage_lock(ibox, MAIL_LOCK_UNLOCK); - return FALSE; - } - } else { - if (!index_storage_lock(ibox, MAIL_LOCK_SHARED)) - return FALSE; + /* sync mailbox to update message counts */ + if (mailbox_sync(box, 0) < 0) + return -1; } /* we can get most of the status items without any trouble */ - hdr = mail_index_get_header(ibox->index); + hdr = mail_index_get_header(ibox->view); if ((items & STATUS_MESSAGE_COUNTS) != 0) { status->messages = hdr->messages_count; status->unseen = hdr->messages_count - hdr->seen_messages_count; status->uidvalidity = hdr->uid_validity; status->uidnext = hdr->next_uid; } - status->diskspace_full = ibox->index->nodiskspace; + //FIXME:status->diskspace_full = ibox->nodiskspace; if (items & STATUS_FIRST_UNSEEN_SEQ) { - status->first_unseen_seq = - get_first_unseen_seq(ibox->index); + if (mail_index_lookup_first(ibox->view, 0, MAIL_SEEN, + &status->first_unseen_seq) < 0) { + mail_storage_set_index_error(ibox); + return -1; + } } - if (items & STATUS_RECENT) - status->recent = index_storage_get_recent_count(ibox->index); + /*FIXME:if (items & STATUS_RECENT) + status->recent = index_storage_get_recent_count(view);*/ - if (items & STATUS_CUSTOM_FLAGS) - get_custom_flags(ibox->index->custom_flags, status); + /*FIXME:if (items & STATUS_CUSTOM_FLAGS) + get_custom_flags(ibox, status);*/ - if (!index_storage_lock(ibox, MAIL_LOCK_UNLOCK)) - return FALSE; - return TRUE; + mail_index_view_unlock(ibox->view); + return 0; }
--- a/src/lib-storage/index/index-storage.c Tue Apr 27 23:14:15 2004 +0300 +++ b/src/lib-storage/index/index-storage.c Tue Apr 27 23:25:52 2004 +0300 @@ -1,10 +1,8 @@ -/* Copyright (C) 2002 Timo Sirainen */ +/* Copyright (C) 2002-2003 Timo Sirainen */ #include "lib.h" #include "ioloop.h" #include "mail-index.h" -#include "mail-index-util.h" -#include "mail-custom-flags.h" #include "index-storage.h" #include <stdlib.h> @@ -23,8 +21,12 @@ struct index_list *next; struct mail_index *index; + char *mailbox_path; int refcount; + dev_t index_dir_dev; + ino_t index_dir_ino; + time_t destroy_time; }; @@ -32,12 +34,12 @@ static struct timeout *to_index = NULL; static int index_storage_refcount = 0; -void index_storage_init(struct mail_storage *storage __attr_unused__) +void index_storage_init(struct index_storage *storage __attr_unused__) { index_storage_refcount++; } -void index_storage_deinit(struct mail_storage *storage __attr_unused__) +void index_storage_deinit(struct index_storage *storage __attr_unused__) { if (--index_storage_refcount > 0) return; @@ -45,7 +47,8 @@ index_storage_destroy_unrefed(); } -void index_storage_add(struct mail_index *index) +static void index_storage_add(struct mail_index *index, + const char *mailbox_path, struct stat *st) { struct index_list *list; @@ -53,43 +56,56 @@ list->refcount = 1; list->index = index; + list->mailbox_path = i_strdup(mailbox_path); + list->index_dir_dev = st->st_dev; + list->index_dir_ino = st->st_ino; + list->next = indexes; indexes = list; } +static void index_list_free(struct index_list *list) +{ + mail_index_free(list->index); + i_free(list->mailbox_path); + i_free(list); +} + struct mail_index * -index_storage_lookup_ref(const char *index_dir, const char *path) +index_storage_alloc(const char *index_dir, const char *mailbox_path, + const char *prefix) { struct index_list **list, *rec; - struct mail_index *match; - struct stat st1, st2; + struct mail_index *index; + struct stat st; int destroy_count; if (index_dir != NULL) { - if (stat(index_dir, &st1) < 0) + if (stat(index_dir, &st) < 0) return NULL; + } else { + memset(&st, 0, sizeof(st)); } /* compare index_dir inodes so we don't break even with symlinks. for in-memory indexes compare just mailbox paths */ - destroy_count = 0; match = NULL; + destroy_count = 0; index = NULL; for (list = &indexes; *list != NULL;) { rec = *list; - if ((index_dir != NULL && stat(rec->index->dir, &st2) == 0 && - st1.st_ino == st2.st_ino && st1.st_dev == st2.st_dev) || - (index_dir == NULL && - strcmp(path, rec->index->mailbox_path) == 0)) { + if ((index_dir != NULL && st.st_ino == rec->index_dir_ino && + st.st_dev == rec->index_dir_dev) || + (index_dir == NULL && st.st_ino == 0 && + strcmp(mailbox_path, rec->mailbox_path) == 0)) { rec->refcount++; - match = rec->index; + index = rec->index; } if (rec->refcount == 0) { if (rec->destroy_time <= ioloop_time || destroy_count >= INDEX_CACHE_MAX) { - rec->index->free(rec->index); *list = rec->next; - i_free(rec); + index_list_free(rec); continue; } else { destroy_count++; @@ -99,7 +115,12 @@ list = &(*list)->next; } - return match; + if (index == NULL) { + index = mail_index_alloc(index_dir, prefix); + index_storage_add(index, mailbox_path, &st); + } + + return index; } static void destroy_unrefed(int all) @@ -111,9 +132,8 @@ if (rec->refcount == 0 && (all || rec->destroy_time <= ioloop_time)) { - rec->index->free(rec->index); *list = rec->next; - i_free(rec); + index_list_free(rec); } else { list = &(*list)->next; } @@ -223,11 +243,11 @@ return ret; } -static void lock_notify(enum mail_lock_notify_type notify_type, +static void lock_notify(enum mailbox_lock_notify_type notify_type, unsigned int secs_left, void *context) { struct index_mailbox *ibox = context; - struct mail_storage *storage = ibox->box.storage; + struct index_storage *storage = ibox->storage; const char *str; time_t now; @@ -241,10 +261,10 @@ /* if notify type changes, print the message immediately */ now = time(NULL); - if (ibox->last_notify_type == (enum mail_lock_notify_type)-1 || + if (ibox->last_notify_type == MAILBOX_LOCK_NOTIFY_NONE || ibox->last_notify_type == notify_type) { - if (ibox->last_notify_type == (enum mail_lock_notify_type)-1 && - notify_type == MAIL_LOCK_NOTIFY_MAILBOX_OVERRIDE) { + if (ibox->last_notify_type == MAILBOX_LOCK_NOTIFY_NONE && + notify_type == MAILBOX_LOCK_NOTIFY_MAILBOX_OVERRIDE) { /* first override notification, show it */ } else { if (now < ibox->next_lock_notify || secs_left < 15) @@ -256,73 +276,31 @@ ibox->last_notify_type = notify_type; switch (notify_type) { - case MAIL_LOCK_NOTIFY_MAILBOX_ABORT: + case MAILBOX_LOCK_NOTIFY_NONE: + break; + case MAILBOX_LOCK_NOTIFY_MAILBOX_ABORT: str = t_strdup_printf("Mailbox is locked, will abort in " "%u seconds", secs_left); storage->callbacks->notify_no(&ibox->box, str, storage->callback_context); break; - case MAIL_LOCK_NOTIFY_MAILBOX_OVERRIDE: + case MAILBOX_LOCK_NOTIFY_MAILBOX_OVERRIDE: str = t_strdup_printf("Stale mailbox lock file detected, " "will override in %u seconds", secs_left); storage->callbacks->notify_ok(&ibox->box, str, storage->callback_context); break; - case MAIL_LOCK_NOTIFY_INDEX_ABORT: - str = t_strdup_printf("Mailbox index is locked, will abort in " - "%u seconds", secs_left); - storage->callbacks->notify_no(&ibox->box, str, - storage->callback_context); - break; } } -void index_storage_init_lock_notify(struct index_mailbox *ibox) -{ - if (ibox->index->mailbox_readonly) - ibox->readonly = TRUE; - - ibox->next_lock_notify = time(NULL) + LOCK_NOTIFY_INTERVAL; - ibox->last_notify_type = (enum mail_lock_notify_type)-1; - - ibox->index->set_lock_notify_callback(ibox->index, lock_notify, ibox); -} - -int index_storage_lock(struct index_mailbox *ibox, - enum mail_lock_type lock_type) +void index_storage_reset_lock_notify(struct index_mailbox *ibox) { - int ret = TRUE; - - if (lock_type == MAIL_LOCK_UNLOCK) { - if (ibox->trans_ctx != NULL) { - if (!mail_cache_transaction_commit(ibox->trans_ctx)) - ret = FALSE; - if (!mail_cache_transaction_end(ibox->trans_ctx)) - ret = FALSE; - ibox->trans_ctx = NULL; - } - if (ibox->lock_type != MAILBOX_LOCK_UNLOCK) - return TRUE; - } else { - if (ibox->lock_type == MAIL_LOCK_EXCLUSIVE) - return TRUE; - } - - /* we have to set/reset this every time, because the same index - may be used by multiple IndexMailboxes. */ - index_storage_init_lock_notify(ibox); - if (!ibox->index->set_lock(ibox->index, lock_type)) - ret = FALSE; - ibox->index->set_lock_notify_callback(ibox->index, NULL, NULL); - - if (!ret) - return mail_storage_set_index_error(ibox); - - return TRUE; + ibox->next_lock_notify = time(NULL) + LOCK_NOTIFY_INTERVAL; + ibox->last_notify_type = MAILBOX_LOCK_NOTIFY_NONE; } struct index_mailbox * -index_storage_mailbox_init(struct mail_storage *storage, struct mailbox *box, +index_storage_mailbox_init(struct index_storage *storage, struct mailbox *box, struct mail_index *index, const char *name, enum mailbox_open_flags flags) { @@ -334,52 +312,38 @@ index_flags = MAIL_INDEX_OPEN_FLAG_CREATE; if ((flags & MAILBOX_OPEN_FAST) != 0) index_flags |= MAIL_INDEX_OPEN_FLAG_FAST; - if ((flags & MAILBOX_OPEN_READONLY) != 0) - index_flags |= MAIL_INDEX_OPEN_FLAG_UPDATE_RECENT; - if ((flags & MAILBOX_OPEN_MMAP_INVALIDATE) != 0) - index_flags |= MAIL_INDEX_OPEN_FLAG_MMAP_INVALIDATE; do { ibox = i_new(struct index_mailbox, 1); ibox->box = *box; + ibox->storage = storage; - ibox->box.storage = storage; + ibox->box.storage = &storage->storage; ibox->box.name = i_strdup(name); ibox->readonly = (flags & MAILBOX_OPEN_READONLY) != 0; ibox->index = index; ibox->next_lock_notify = time(NULL) + LOCK_NOTIFY_INTERVAL; - index->set_lock_notify_callback(index, lock_notify, ibox); + ibox->commit_log_file_seq = 0; + ibox->mail_read_mmaped = getenv("MAIL_READ_MMAPED") != NULL; - if (!index->opened) { - /* open the index first */ - if (!index->open(index, index_flags)) - break; + if (mail_index_open(index, index_flags) < 0) + break; - mail_cache_set_defaults(index->cache, - get_default_cache_fields(), - get_never_cache_fields()); + ibox->cache = mail_index_get_cache(index); + mail_cache_set_defaults(ibox->cache, + get_default_cache_fields(), + get_never_cache_fields()); - if (INDEX_IS_IN_MEMORY(index) && - storage->index_dir != NULL) { - storage->callbacks->notify_no(&ibox->box, - "Couldn't use index files", - storage->callback_context); - } + if (mail_index_is_in_memory(index) && + storage->index_dir != NULL) { + storage->callbacks->notify_no(&ibox->box, + "Couldn't use index files", + storage->callback_context); } - if (!ibox->index->set_lock(ibox->index, MAIL_LOCK_SHARED)) - break; - - ibox->synced_messages_count = - mail_index_get_header(index)->messages_count; - - if (!ibox->index->set_lock(ibox->index, MAIL_LOCK_UNLOCK)) - break; - - index->set_lock_notify_callback(index, NULL, NULL); - + ibox->view = mail_index_view_open(index); return ibox; } while (0); @@ -388,21 +352,21 @@ return NULL; } -int index_storage_mailbox_free(struct mailbox *box) +void index_storage_mailbox_free(struct mailbox *box) { struct index_mailbox *ibox = (struct index_mailbox *) box; /* make sure we're unlocked */ - (void)ibox->index->set_lock(ibox->index, MAIL_LOCK_UNLOCK); + mail_index_view_unlock(ibox->view); index_mailbox_check_remove_all(ibox); if (ibox->index != NULL) index_storage_unref(ibox->index); + i_free(ibox->path); + i_free(ibox->control_dir); i_free(box->name); i_free(box); - - return TRUE; } int index_storage_is_readonly(struct mailbox *box) @@ -416,51 +380,41 @@ { struct index_mailbox *ibox = (struct index_mailbox *) box; - return ibox->index->allow_new_custom_flags; + /* FIXME: return FALSE if we're full */ + return !ibox->readonly; } -int index_storage_is_inconsistency_error(struct mailbox *box) +int index_storage_is_inconsistent(struct mailbox *box) { struct index_mailbox *ibox = (struct index_mailbox *) box; - return ibox->inconsistent; + return mail_index_view_is_inconsistent(ibox->view); } -void index_storage_set_callbacks(struct mail_storage *storage, +void index_storage_set_callbacks(struct mail_storage *_storage, struct mail_storage_callbacks *callbacks, void *context) { - memcpy(storage->callbacks, callbacks, - sizeof(struct mail_storage_callbacks)); + struct index_storage *storage = (struct index_storage *) _storage; + + *storage->callbacks = *callbacks; storage->callback_context = context; } int mail_storage_set_index_error(struct index_mailbox *ibox) { - switch (ibox->index->get_last_error(ibox->index)) { + switch (mail_index_get_last_error(ibox->index)) { case MAIL_INDEX_ERROR_NONE: case MAIL_INDEX_ERROR_INTERNAL: mail_storage_set_internal_error(ibox->box.storage); break; - case MAIL_INDEX_ERROR_INCONSISTENT: - ibox->inconsistent = TRUE; - break; case MAIL_INDEX_ERROR_DISKSPACE: mail_storage_set_error(ibox->box.storage, "Out of disk space"); break; - case MAIL_INDEX_ERROR_INDEX_LOCK_TIMEOUT: - mail_storage_set_error(ibox->box.storage, - "Timeout while waiting for lock to index of mailbox %s", - ibox->box.name); - break; - case MAIL_INDEX_ERROR_MAILBOX_LOCK_TIMEOUT: - mail_storage_set_error(ibox->box.storage, - "Timeout while waiting for lock to mailbox %s", - ibox->box.name); - break; } - index_reset_error(ibox->index); + mail_index_view_unlock(ibox->view); + mail_index_reset_error(ibox->index); return FALSE; } @@ -469,9 +423,9 @@ const char *custom_flags[], unsigned int custom_flags_count) { - int ret; + /*FIXME:int ret; - ret = mail_custom_flags_fix_list(ibox->index->custom_flags, + ret = mail_custom_flags_fix_list(ibox->index, flags, custom_flags, custom_flags_count); switch (ret) { @@ -483,16 +437,17 @@ return FALSE; default: return mail_storage_set_index_error(ibox); - } + }*/ } -unsigned int index_storage_get_recent_count(struct mail_index *index) +unsigned int index_storage_get_recent_count(struct mail_index_view *view) { +#if 0 struct mail_index_header *hdr; struct mail_index_record *rec; unsigned int seq; - hdr = mail_index_get_header(index); + hdr = mail_index_get_header(view); if (index->first_recent_uid <= 1) { /* all are recent */ return hdr->messages_count; @@ -502,7 +457,9 @@ if (index->first_recent_uid >= hdr->next_uid) return 0; - rec = index->lookup_uid_range(index, index->first_recent_uid, - hdr->next_uid - 1, &seq); + rec = mail_index_lookup_uid_range(view, index->first_recent_uid, + hdr->next_uid - 1, &seq); return rec == NULL ? 0 : hdr->messages_count+1 - seq; +#endif + return 0; }
--- a/src/lib-storage/index/index-storage.h Tue Apr 27 23:14:15 2004 +0300 +++ b/src/lib-storage/index/index-storage.h Tue Apr 27 23:25:52 2004 +0300 @@ -1,10 +1,26 @@ #ifndef __INDEX_STORAGE_H #define __INDEX_STORAGE_H -#include "mail-storage.h" +#include "mail-storage-private.h" #include "mail-index.h" #include "index-mail.h" +/* Max. mmap()ed size for a message */ +#define MAIL_MMAP_BLOCK_SIZE (1024*256) +/* Block size when read()ing message. */ +#define MAIL_READ_BLOCK_SIZE (1024*8) + +#define MAILBOX_FULL_SYNC_INTERVAL 5 + +enum mailbox_lock_notify_type { + MAILBOX_LOCK_NOTIFY_NONE, + + /* Mailbox is locked, will abort in secs_left */ + MAILBOX_LOCK_NOTIFY_MAILBOX_ABORT, + /* Mailbox lock looks stale, will override in secs_left */ + MAILBOX_LOCK_NOTIFY_MAILBOX_OVERRIDE +}; + struct index_autosync_file { struct index_autosync_file *next; @@ -18,16 +34,30 @@ int fd; }; +struct index_storage { + struct mail_storage storage; + + char *dir; /* root directory */ + char *index_dir; + char *control_dir; + char *inbox_path; /* INBOX location */ + + char *user; /* name of user accessing the storage */ + + struct mail_storage_callbacks *callbacks; + void *callback_context; +}; + struct index_mailbox { struct mailbox box; - - /* expunge messages marked as deleted, requires index to be - exclusively locked */ - void (*mail_init)(struct index_mail *mail); + struct index_storage *storage; + char *path, *control_dir; struct mail_index *index; - enum mailbox_lock_type lock_type; - struct mail_cache_transaction_ctx *trans_ctx; + struct mail_index_view *view; + struct mail_cache *cache; + struct mail_cache_view *cache_view; + struct mail *mail_interface; struct timeout *autosync_to; struct index_autosync_file *autosync_files; @@ -36,59 +66,72 @@ time_t sync_last_check, sync_last_notify; unsigned int min_newmail_notify_interval; - struct index_mail fetch_mail; /* fetch_uid() or fetch_seq() */ - unsigned int synced_messages_count; + time_t next_lock_notify; /* temporary */ + enum mailbox_lock_notify_type last_notify_type; + + uint32_t commit_log_file_seq; + uoff_t commit_log_file_offset; - time_t next_lock_notify; /* temporary */ - enum mail_lock_notify_type last_notify_type; + /* sync: */ + struct maildir_uidlist *uidlist; + time_t last_new_mtime, last_cur_mtime, last_sync; + + mode_t mail_create_mode; + unsigned int private_flags_mask; unsigned int readonly:1; - unsigned int inconsistent:1; unsigned int sent_diskspace_warning:1; unsigned int sent_readonly_flags_warning:1; unsigned int autosync_pending:1; + unsigned int mail_read_mmaped:1; + + unsigned int maildir_keep_new:1; +}; + +struct index_transaction_context { + struct mailbox_transaction_context mailbox_ctx; + struct index_mailbox *ibox; + struct mail_index_transaction *trans; + struct mail_cache_transaction_ctx *cache_trans; + + struct index_mail fetch_mail; /* for index_storage_fetch() */ }; int mail_storage_set_index_error(struct index_mailbox *ibox); -void index_storage_init_lock_notify(struct index_mailbox *ibox); -int index_storage_lock(struct index_mailbox *ibox, - enum mail_lock_type lock_type); +void index_storage_reset_lock_notify(struct index_mailbox *ibox); -void index_storage_add(struct mail_index *index); struct mail_index * -index_storage_lookup_ref(const char *index_dir, const char *path); +index_storage_alloc(const char *index_dir, + const char *mailbox_path, const char *prefix); void index_storage_unref(struct mail_index *index); void index_storage_destroy_unrefed(void); -void index_storage_init(struct mail_storage *storage); -void index_storage_deinit(struct mail_storage *storage); +void index_storage_init(struct index_storage *storage); +void index_storage_deinit(struct index_storage *storage); struct index_mailbox * -index_storage_mailbox_init(struct mail_storage *storage, struct mailbox *box, +index_storage_mailbox_init(struct index_storage *storage, struct mailbox *box, struct mail_index *index, const char *name, enum mailbox_open_flags flags); -int index_storage_mailbox_free(struct mailbox *box); +void index_storage_mailbox_free(struct mailbox *box); int index_storage_is_readonly(struct mailbox *box); int index_storage_allow_new_custom_flags(struct mailbox *box); -int index_storage_is_inconsistency_error(struct mailbox *box); - -int index_storage_sync_and_lock(struct index_mailbox *ibox, - int sync_size, int minimal_sync, - enum mail_lock_type data_lock_type); -int index_storage_sync_modifylog(struct index_mailbox *ibox, int hide_deleted); +int index_storage_is_inconsistent(struct mailbox *box); int index_mailbox_fix_custom_flags(struct index_mailbox *ibox, enum mail_flags *flags, const char *custom_flags[], unsigned int custom_flags_count); -unsigned int index_storage_get_recent_count(struct mail_index *index); +unsigned int index_storage_get_recent_count(struct mail_index_view *view); void index_mailbox_check_add(struct index_mailbox *ibox, const char *path, int dir); void index_mailbox_check_remove_all(struct index_mailbox *ibox); +int index_storage_sync(struct mailbox *box, enum mailbox_sync_flags flags); + /* mailbox methods: */ void index_storage_set_callbacks(struct mail_storage *storage, struct mail_storage_callbacks *callbacks, @@ -96,31 +139,27 @@ int index_storage_get_status(struct mailbox *box, enum mailbox_status_items items, struct mailbox_status *status); -int index_storage_sync(struct mailbox *box, enum mailbox_sync_flags flags); -struct mail *index_storage_fetch_uid(struct mailbox *box, unsigned int uid, - enum mail_fetch_field wanted_fields); -struct mail *index_storage_fetch_seq(struct mailbox *box, unsigned int seq, - enum mail_fetch_field wanted_fields); +struct mail * +index_storage_fetch(struct mailbox_transaction_context *t, uint32_t seq, + enum mail_fetch_field wanted_fields); +int index_storage_get_uids(struct mailbox *box, uint32_t uid1, uint32_t uid2, + uint32_t *seq1_r, uint32_t *seq2_r); int index_storage_search_get_sorting(struct mailbox *box, enum mail_sort_type *sort_program); struct mail_search_context * -index_storage_search_init(struct mailbox *box, const char *charset, - struct mail_search_arg *args, +index_storage_search_init(struct mailbox_transaction_context *t, + const char *charset, struct mail_search_arg *args, const enum mail_sort_type *sort_program, enum mail_fetch_field wanted_fields, const char *const wanted_headers[]); -int index_storage_search_deinit(struct mail_search_context *ctx, - int *all_found); +int index_storage_search_deinit(struct mail_search_context *ctx); struct mail *index_storage_search_next(struct mail_search_context *ctx); -struct mail_copy_context *index_storage_copy_init(struct mailbox *box); -int index_storage_copy_deinit(struct mail_copy_context *ctx, int rollback); -int index_storage_copy(struct mail *mail, struct mail_copy_context *ctx); - -int index_storage_update_flags(struct mail *mail, - const struct mail_full_flags *flags, - enum modify_type modify_type); +struct mailbox_transaction_context * +index_transaction_begin(struct mailbox *box); +int index_transaction_commit(struct mailbox_transaction_context *t); +void index_transaction_rollback(struct mailbox_transaction_context *t); #endif
--- a/src/lib-storage/index/index-sync.c Tue Apr 27 23:14:15 2004 +0300 +++ b/src/lib-storage/index/index-sync.c Tue Apr 27 23:25:52 2004 +0300 @@ -1,250 +1,89 @@ /* Copyright (C) 2002 Timo Sirainen */ #include "lib.h" -#include "ioloop.h" #include "index-storage.h" -#include "mail-index-util.h" -#include "mail-modifylog.h" -#include "mail-custom-flags.h" - -/* How often to do full sync when fast sync flag is set. */ -#define MAILBOX_FULL_SYNC_INTERVAL 5 - -static void index_storage_sync_size(struct index_mailbox *ibox) -{ - struct mail_storage *storage = ibox->box.storage; - unsigned int messages, recent; - - if (storage->callbacks->new_messages == NULL) - return; - - messages = ibox->index->get_header(ibox->index)->messages_count; - messages += mail_modifylog_get_expunge_count(ibox->index->modifylog); - - if (messages != ibox->synced_messages_count) { - i_assert(messages > ibox->synced_messages_count); - - /* new messages in mailbox */ - recent = index_storage_get_recent_count(ibox->index); - storage->callbacks->new_messages(&ibox->box, messages, recent, - storage->callback_context); - ibox->synced_messages_count = messages; - } -} -int index_storage_sync_and_lock(struct index_mailbox *ibox, - int sync_size, int minimal_sync, - enum mail_lock_type data_lock_type) +int index_storage_sync(struct mailbox *box, enum mailbox_sync_flags flags) { - struct mail_storage *storage = ibox->box.storage; - struct mail_index *index = ibox->index; - int failed, changes, set_shared_lock; - - set_shared_lock = ibox->index->lock_type != MAIL_LOCK_EXCLUSIVE; - - index_storage_init_lock_notify(ibox); - failed = !index->sync_and_lock(index, minimal_sync, - data_lock_type, &changes); - ibox->index->set_lock_notify_callback(ibox->index, NULL, NULL); + struct index_mailbox *ibox = (struct index_mailbox *)box; + struct mail_index_view_sync_ctx *ctx; + struct mail_full_flags full_flags; + const struct mail_index_record *rec; + struct mail_index_sync_rec sync; + struct mail_storage_callbacks *sc; + const uint32_t *expunges; + size_t i, expunges_count; + void *sc_context; + enum mail_index_sync_type sync_mask; + uint32_t seq, new_count; + int ret, appends; - if (!failed) { - /* reset every time it has worked */ - ibox->sent_diskspace_warning = FALSE; - } else { - if (index->get_last_error(index) != - MAIL_INDEX_ERROR_DISKSPACE) { - (void)index_storage_lock(ibox, MAIL_LOCK_UNLOCK); - return mail_storage_set_index_error(ibox); - } + sync_mask = MAIL_INDEX_SYNC_MASK_ALL; + if ((flags & MAILBOX_SYNC_FLAG_NO_EXPUNGES) != 0) + sync_mask &= ~MAIL_INDEX_SYNC_TYPE_EXPUNGE; - /* notify client once about it */ - if (!ibox->sent_diskspace_warning && - storage->callbacks->alert_no_diskspace != NULL) { - ibox->sent_diskspace_warning = TRUE; - storage->callbacks->alert_no_diskspace( - &ibox->box, storage->callback_context); - } - - index_reset_error(index); + if (mail_index_view_sync_begin(ibox->view, sync_mask, &ctx) < 0) { + mail_storage_set_index_error(ibox); + return -1; } - if (set_shared_lock) { - /* just make sure we are locked, and that we drop our - exclusive lock if it wasn't wanted originally */ - if (!index_storage_lock(ibox, MAIL_LOCK_SHARED)) { - (void)index_storage_lock(ibox, MAIL_LOCK_UNLOCK); - return FALSE; - } - } - - /* notify about changes in mailbox size. */ - if (!changes) - return TRUE; /* no changes - must be no new mail either */ - - if (sync_size) - index_storage_sync_size(ibox); - - /* notify changes in custom flags */ - if (mail_custom_flags_has_changes(index->custom_flags) && - storage->callbacks->new_custom_flags != NULL) { - storage->callbacks->new_custom_flags(&ibox->box, - mail_custom_flags_list_get(index->custom_flags), - MAIL_CUSTOM_FLAGS_COUNT, storage->callback_context); + if ((flags & MAILBOX_SYNC_FLAG_NO_EXPUNGES) != 0) { + expunges_count = 0; + expunges = NULL; + } else { + expunges = + mail_index_view_sync_get_expunges(ctx, &expunges_count); } - return TRUE; -} - -int index_storage_sync_modifylog(struct index_mailbox *ibox, int hide_deleted) -{ - const struct modify_log_record *log1, *log2, *log, *first_flag_log; - struct mail_index_record *rec; - struct mail_full_flags flags; - struct mail_storage_callbacks *sc; - void *sc_context; - unsigned int count1, count2, total_count, seq, seq_count, i, messages; - unsigned int first_flag_change, first_flag_messages_count; - - /* show the log */ - if (!mail_modifylog_get_nonsynced(ibox->index->modifylog, - &log1, &count1, &log2, &count2)) - return mail_storage_set_index_error(ibox); - - sc = ibox->box.storage->callbacks; - sc_context = ibox->box.storage->callback_context; + sc = ibox->storage->callbacks; + sc_context = ibox->storage->callback_context; + appends = FALSE; - /* first show expunges. this makes it easier to deal with sequence - numbers. */ - total_count = count1 + count2; - messages = ibox->synced_messages_count; - first_flag_change = total_count; - first_flag_log = NULL; - first_flag_messages_count = 0; - - for (i = 0, log = log1; i < total_count; i++, log++) { - if (i == count1) - log = log2; - - if (log->seq1 > messages) { - /* client doesn't know about this message yet */ - continue; - } - - switch (log->type) { - case RECORD_TYPE_EXPUNGE: - seq_count = (log->seq2 - log->seq1) + 1; - messages -= seq_count; - - if (sc->expunge == NULL) + memset(&full_flags, 0, sizeof(full_flags)); + while ((ret = mail_index_view_sync_next(ctx, &sync)) > 0) { + switch (sync.type) { + case MAIL_INDEX_SYNC_TYPE_APPEND: + appends = TRUE; + break; + case MAIL_INDEX_SYNC_TYPE_EXPUNGE: + /* later */ + break; + case MAIL_INDEX_SYNC_TYPE_FLAGS: + if (sc->update_flags == NULL) break; - for (; seq_count > 0; seq_count--) { - sc->expunge(&ibox->box, log->seq1, - sc_context); - } - break; - case RECORD_TYPE_FLAGS_CHANGED: - if (first_flag_change == total_count) { - first_flag_change = i; - first_flag_log = log; - first_flag_messages_count = messages; + /* FIXME: hide the flag updates for expunged messages */ + for (seq = sync.seq1; seq <= sync.seq2; seq++) { + if (mail_index_lookup(ibox->view, + seq, &rec) < 0) { + ret = -1; + break; + } + full_flags.flags = rec->flags; // FIXME + sc->update_flags(&ibox->box, seq, + &full_flags, sc_context); } break; } } - /* set synced messages count before flag changes break it */ - ibox->synced_messages_count = messages; - - /* now show the flags */ - messages = first_flag_messages_count; - flags.custom_flags = - mail_custom_flags_list_get(ibox->index->custom_flags); - flags.custom_flags_count = MAIL_CUSTOM_FLAGS_COUNT; - - if (sc->update_flags == NULL) { - /* don't bother going through, we're not printing them anyway */ - total_count = 0; - } - - log = first_flag_log; - for (i = first_flag_change; i < total_count; i++, log++) { - if (i == count1) - log = log2; - - if (log->seq1 > messages) { - /* client doesn't know about this message yet */ - continue; - } - - switch (log->type) { - case RECORD_TYPE_EXPUNGE: - messages -= (log->seq2 - log->seq1) + 1; - break; - case RECORD_TYPE_FLAGS_CHANGED: - rec = ibox->index->lookup_uid_range(ibox->index, - log->uid1, - log->uid2, &seq); - while (rec != NULL && rec->uid <= log->uid2) { - flags.flags = rec->msg_flags; - if (rec->uid >= ibox->index->first_recent_uid) - flags.flags |= MAIL_RECENT; - - /* \Deleted-hiding is useful when syncing just - before doing EXPUNGE. */ - if ((flags.flags & MAIL_DELETED) == 0 || - !hide_deleted) { - sc->update_flags(&ibox->box, seq, - rec->uid, &flags, - sc_context); - } - - seq++; - rec = ibox->index->next(ibox->index, rec); - } - break; + if (sc->expunge != NULL) { + for (i = expunges_count*2; i > 0; i -= 2) { + for (seq = expunges[i-1]; seq >= expunges[i-2]; seq--) + sc->expunge(&ibox->box, seq, sc_context); } } - /* mark synced */ - if (!mail_modifylog_mark_synced(ibox->index->modifylog)) - return mail_storage_set_index_error(ibox); - - return TRUE; -} - -int index_storage_sync(struct mailbox *box, enum mailbox_sync_flags flags) -{ - struct index_mailbox *ibox = (struct index_mailbox *) box; - int ret; + mail_index_view_sync_end(ctx); - if ((flags & MAILBOX_SYNC_FAST) == 0 || - ibox->sync_last_check + MAILBOX_FULL_SYNC_INTERVAL <= ioloop_time) { - ibox->sync_last_check = ioloop_time; - - if (!index_storage_sync_and_lock(ibox, FALSE, FALSE, - MAIL_LOCK_UNLOCK)) - return FALSE; - } else { - /* check only modify log */ - if (!index_storage_lock(ibox, MAIL_LOCK_SHARED)) { - (void)index_storage_lock(ibox, MAIL_LOCK_UNLOCK); - return FALSE; - } + if (appends) { + new_count = mail_index_view_get_message_count(ibox->view); + sc->new_messages(&ibox->box, new_count, 0, sc_context); } - /* FIXME: we could sync flags always, but expunges in the middle - could make it a bit more difficult and slower */ - if ((flags & MAILBOX_SYNC_FLAG_NO_EXPUNGES) == 0 || - mail_modifylog_get_expunge_count(ibox->index->modifylog) == 0) - ret = index_storage_sync_modifylog(ibox, FALSE); - else - ret = TRUE; + if (ret < 0) + mail_storage_set_index_error(ibox); - index_storage_sync_size(ibox); - - if (!index_storage_lock(ibox, MAIL_LOCK_UNLOCK)) - return FALSE; - + mail_index_view_unlock(ibox->view); return ret; }
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-storage/index/index-transaction.c Tue Apr 27 23:25:52 2004 +0300 @@ -0,0 +1,44 @@ +/* Copyright (C) 2003 Timo Sirainen */ + +#include "lib.h" +#include "index-storage.h" + +static void index_transaction_free(struct index_transaction_context *t) +{ + mail_index_view_unlock(t->ibox->view); + + if (t->fetch_mail.pool != NULL) + index_mail_deinit(&t->fetch_mail); + i_free(t); +} + +int index_transaction_commit(struct mailbox_transaction_context *_t) +{ + struct index_transaction_context *t = + (struct index_transaction_context *)_t; + uint32_t seq; + uoff_t offset; + int ret; + + if (t->cache_trans != NULL) + (void)mail_cache_transaction_commit(t->cache_trans); + + ret = mail_index_transaction_commit(t->trans, &seq, &offset); + if (ret < 0) + mail_storage_set_index_error(t->ibox); + + t->ibox->commit_log_file_seq = seq; + t->ibox->commit_log_file_offset = offset; + + index_transaction_free(t); + return ret; +} + +void index_transaction_rollback(struct mailbox_transaction_context *_t) +{ + struct index_transaction_context *t = + (struct index_transaction_context *)_t; + + mail_index_transaction_rollback(t->trans); + index_transaction_free(t); +}
--- a/src/lib-storage/index/index-update-flags.c Tue Apr 27 23:14:15 2004 +0300 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,49 +0,0 @@ -/* Copyright (C) 2002 Timo Sirainen */ - -#include "lib.h" -#include "index-storage.h" -#include "index-messageset.h" -#include "mail-custom-flags.h" - -int index_storage_update_flags(struct mail *mail, - const struct mail_full_flags *flags, - enum modify_type modify_type) -{ - struct index_mail *imail = (struct index_mail *) mail; - struct index_mailbox *ibox = imail->ibox; - struct mail_storage *storage = mail->box->storage; - enum mail_flags modify_flags; - - if (mail->box->is_readonly(mail->box)) { - if (ibox->sent_readonly_flags_warning) - return TRUE; - ibox->sent_readonly_flags_warning = TRUE; - - storage->callbacks-> - notify_no(&ibox->box, - "Mailbox is read-only, ignoring flag changes", - storage->callback_context); - return TRUE; - } - - /* \Recent can't be changed */ - modify_flags = flags->flags & ~MAIL_RECENT; - - if (!index_mailbox_fix_custom_flags(ibox, &modify_flags, - flags->custom_flags, - flags->custom_flags_count)) - return FALSE; - - if (!ibox->index->update_flags(ibox->index, imail->data.rec, - imail->data.idx_seq, - modify_type, modify_flags, FALSE)) - return FALSE; - - if (mail_custom_flags_has_changes(ibox->index->custom_flags)) { - storage->callbacks->new_custom_flags(&ibox->box, - mail_custom_flags_list_get(ibox->index->custom_flags), - MAIL_CUSTOM_FLAGS_COUNT, storage->callback_context); - } - - return TRUE; -}
--- a/src/lib-storage/index/maildir/Makefile.am Tue Apr 27 23:14:15 2004 +0300 +++ b/src/lib-storage/index/maildir/Makefile.am Tue Apr 27 23:25:52 2004 +0300 @@ -11,10 +11,15 @@ libstorage_maildir_a_SOURCES = \ maildir-copy.c \ - maildir-expunge.c \ maildir-list.c \ + maildir-mail.c \ maildir-save.c \ - maildir-storage.c + maildir-storage.c \ + maildir-sync.c \ + maildir-transaction.c \ + maildir-uidlist.c \ + maildir-util.c noinst_HEADERS = \ - maildir-storage.h + maildir-storage.h \ + maildir-uidlist.h
--- a/src/lib-storage/index/maildir/maildir-copy.c Tue Apr 27 23:14:15 2004 +0300 +++ b/src/lib-storage/index/maildir/maildir-copy.c Tue Apr 27 23:25:52 2004 +0300 @@ -1,12 +1,9 @@ -/* Copyright (C) 2002 Timo Sirainen */ +/* Copyright (C) 2002-2004 Timo Sirainen */ #include "lib.h" #include "ioloop.h" -#include "maildir-index.h" #include "maildir-storage.h" -#include "mail-custom-flags.h" -#include "mail-index-util.h" -#include "index-messageset.h" +#include "mail-save.h" #include <stdlib.h> #include <unistd.h> @@ -17,8 +14,6 @@ pool_t pool; struct rollback *rollbacks; - - struct mail_copy_context *ctx; }; struct hardlink_ctx { @@ -31,7 +26,7 @@ const char *fname; }; -static int do_hardlink(struct mail_index *index, const char *path, +static int do_hardlink(struct index_mailbox *ibox, const char *path, void *context) { struct hardlink_ctx *ctx = context; @@ -41,14 +36,16 @@ return 0; if (ENOSPACE(errno)) { - index->nodiskspace = TRUE; + mail_storage_set_error(ibox->box.storage, + "Not enough disk space"); return -1; } if (errno == EACCES || errno == EXDEV) return 1; - index_set_error(index, "link(%s, %s) failed: %m", - path, ctx->dest_path); + mail_storage_set_critical(ibox->box.storage, + "link(%s, %s) failed: %m", + path, ctx->dest_path); return -1; } @@ -62,26 +59,24 @@ struct index_mail *imail = (struct index_mail *) mail; struct hardlink_ctx do_ctx; struct rollback *rb; + const struct mail_full_flags *flags; const char *dest_fname; + flags = mail->get_flags(mail); dest_fname = maildir_generate_tmp_filename(&ioloop_timeval); - dest_fname = maildir_filename_set_flags(dest_fname, - mail->get_flags(mail)->flags); + dest_fname = maildir_filename_set_flags(dest_fname, flags->flags, NULL); memset(&do_ctx, 0, sizeof(do_ctx)); - do_ctx.dest_path = t_strconcat(ctx->ibox->index->mailbox_path, "/new/", - dest_fname, NULL); + do_ctx.dest_path = + t_strconcat(ctx->ibox->path, "/new/", dest_fname, NULL); - if (!maildir_file_do(imail->ibox->index, imail->data.rec, - do_hardlink, &do_ctx)) + if (maildir_file_do(imail->ibox, imail->mail.uid, + do_hardlink, &do_ctx) < 0) return -1; if (!do_ctx.found) return 0; - if (ctx->pool == NULL) - ctx->pool = pool_alloconly_create("hard copy rollbacks", 2048); - rb = p_new(ctx->pool, struct rollback, 1); rb->fname = p_strdup(ctx->pool, dest_fname); @@ -90,68 +85,64 @@ return 1; } -struct mail_copy_context *maildir_storage_copy_init(struct mailbox *box) +static struct maildir_copy_context * +maildir_copy_init(struct index_mailbox *ibox) { - struct index_mailbox *ibox = (struct index_mailbox *) box; struct maildir_copy_context *ctx; + pool_t pool; - if (box->is_readonly(box)) { - mail_storage_set_error(box->storage, - "Destination mailbox is read-only"); - return NULL; - } + pool = pool_alloconly_create("maildir_copy_context", 2048); - ctx = i_new(struct maildir_copy_context, 1); + ctx = p_new(pool, struct maildir_copy_context, 1); + ctx->pool = pool; ctx->hardlink = getenv("MAILDIR_COPY_WITH_HARDLINKS") != NULL; ctx->ibox = ibox; - return (struct mail_copy_context *) ctx; + return ctx; +} + +int maildir_copy_commit(struct maildir_copy_context *ctx) +{ + pool_unref(ctx->pool); + return 0; } -int maildir_storage_copy_deinit(struct mail_copy_context *_ctx, int rollback) +void maildir_copy_rollback(struct maildir_copy_context *ctx) { - struct maildir_copy_context *ctx = (struct maildir_copy_context *) _ctx; struct rollback *rb; - int ret = TRUE; - if (ctx->ctx != NULL) - ret = index_storage_copy_deinit(ctx->ctx, rollback); - - if (rollback) { - for (rb = ctx->rollbacks; rb != NULL; rb = rb->next) { - t_push(); - (void)unlink(t_strconcat(ctx->ibox->index->mailbox_path, - "/new/", rb->fname, NULL)); - t_pop(); - } + for (rb = ctx->rollbacks; rb != NULL; rb = rb->next) { + t_push(); + (void)unlink(t_strconcat(ctx->ibox->path, + "/new/", rb->fname, NULL)); + t_pop(); } - if (ctx->pool != NULL) - pool_unref(ctx->pool); - - i_free(ctx); - return ret; + pool_unref(ctx->pool); } -int maildir_storage_copy(struct mail *mail, struct mail_copy_context *_ctx) +int maildir_copy(struct mailbox_transaction_context *_t, struct mail *mail) { - struct maildir_copy_context *ctx = (struct maildir_copy_context *) _ctx; + struct maildir_transaction_context *t = + (struct maildir_transaction_context *)_t; + struct maildir_copy_context *ctx; int ret; + if (t->copy_ctx == NULL) + t->copy_ctx = maildir_copy_init(t->ictx.ibox); + ctx = t->copy_ctx; + if (ctx->hardlink && mail->box->storage == ctx->ibox->box.storage) { t_push(); ret = maildir_copy_hardlink(mail, ctx); t_pop(); if (ret > 0) - return TRUE; + return 0; if (ret < 0) - return FALSE; + return -1; /* non-fatal hardlinking failure, try the slow way */ } - if (ctx->ctx == NULL) - ctx->ctx = index_storage_copy_init(&ctx->ibox->box); - - return index_storage_copy(mail, ctx->ctx); + return mail_storage_copy(_t, mail); }
--- a/src/lib-storage/index/maildir/maildir-expunge.c Tue Apr 27 23:14:15 2004 +0300 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,83 +0,0 @@ -/* Copyright (C) 2002-2003 Timo Sirainen */ - -#include "lib.h" -#include "index-expunge.h" -#include "maildir-index.h" -#include "maildir-storage.h" - -struct maildir_expunge_context { - struct mail_expunge_context *ctx; - int sent_access_warning; -}; - -struct mail_expunge_context * -maildir_storage_expunge_init(struct mailbox *box, - enum mail_fetch_field wanted_fields, - int expunge_all) -{ - struct maildir_expunge_context *ctx; - struct mail_expunge_context *mctx; - - mctx = index_storage_expunge_init(box, wanted_fields, expunge_all); - if (mctx == NULL) - return NULL; - - ctx = i_new(struct maildir_expunge_context, 1); - ctx->ctx = mctx; - return (struct mail_expunge_context *) ctx; -} - -int maildir_storage_expunge_deinit(struct mail_expunge_context *_ctx) -{ - struct maildir_expunge_context *ctx = - (struct maildir_expunge_context *) _ctx; - struct mail_expunge_context *mctx; - - mctx = ctx->ctx; - i_free(ctx); - return index_storage_expunge_deinit(mctx); -} - -struct mail * -maildir_storage_expunge_fetch_next(struct mail_expunge_context *_ctx) -{ - struct maildir_expunge_context *ctx = - (struct maildir_expunge_context *) _ctx; - - return index_storage_expunge_fetch_next(ctx->ctx); -} - -int maildir_storage_expunge(struct mail *mail, - struct mail_expunge_context *_ctx, - unsigned int *seq_r, int notify) -{ - struct maildir_expunge_context *ctx = - (struct maildir_expunge_context *) _ctx; - struct index_mail *imail = (struct index_mail *) mail; - int ret; - - if (mail->box->is_readonly(mail->box)) { - /* send warning */ - return index_storage_expunge(mail, ctx->ctx, seq_r, notify); - } - - t_push(); - ret = maildir_expunge_mail(imail->ibox->index, imail->data.rec); - t_pop(); - - if (!ret) { - if (errno != EACCES) - return FALSE; - - if (ctx->sent_access_warning) - return TRUE; - ctx->sent_access_warning = TRUE; - - mail->box->storage->callbacks->notify_no(mail->box, - "We didn't have permission to expunge all the mails", - mail->box->storage->callback_context); - return TRUE; - } - - return index_storage_expunge(mail, ctx->ctx, seq_r, notify); -}
--- a/src/lib-storage/index/maildir/maildir-list.c Tue Apr 27 23:14:15 2004 +0300 +++ b/src/lib-storage/index/maildir/maildir-list.c Tue Apr 27 23:25:52 2004 +0300 @@ -15,7 +15,8 @@ #define MAILBOX_FLAG_MATCHED 0x40000000 -struct mailbox_list_context { +struct maildir_list_context { + struct mailbox_list_context mailbox_ctx; pool_t pool; struct mail_storage *storage; @@ -48,7 +49,7 @@ } } -static int maildir_fill_readdir(struct mailbox_list_context *ctx, +static int maildir_fill_readdir(struct maildir_list_context *ctx, struct imap_match_glob *glob, int update_only) { DIR *dirp; @@ -180,15 +181,19 @@ return TRUE; } -static int maildir_fill_subscribed(struct mailbox_list_context *ctx, +static int maildir_fill_subscribed(struct maildir_list_context *ctx, struct imap_match_glob *glob) { + struct index_storage *istorage = (struct index_storage *)ctx->storage; struct subsfile_list_context *subsfile_ctx; - const char *name, *p; + const char *path, *name, *p; struct mailbox_node *node; int created; - subsfile_ctx = subsfile_list_init(ctx->storage); + path = t_strconcat(istorage->control_dir != NULL ? + istorage->control_dir : istorage->dir, + "/" SUBSCRIPTION_FILE_NAME, NULL); + subsfile_ctx = subsfile_list_init(ctx->storage, path); if (subsfile_ctx == NULL) return FALSE; @@ -227,10 +232,11 @@ } struct mailbox_list_context * -maildir_list_mailbox_init(struct mail_storage *storage, +maildir_mailbox_list_init(struct mail_storage *storage, const char *mask, enum mailbox_list_flags flags) { - struct mailbox_list_context *ctx; + struct index_storage *istorage = (struct index_storage *)storage; + struct maildir_list_context *ctx; struct imap_match_glob *glob; const char *dir, *p; pool_t pool; @@ -238,7 +244,7 @@ mail_storage_clear_error(storage); pool = pool_alloconly_create("maildir_list", 1024); - ctx = p_new(pool, struct mailbox_list_context, 1); + ctx = p_new(pool, struct maildir_list_context, 1); ctx->pool = pool; ctx->storage = storage; ctx->flags = flags; @@ -247,15 +253,15 @@ if (storage->hierarchy_sep != MAILDIR_FS_SEP && strchr(mask, MAILDIR_FS_SEP) != NULL) { /* this will never match, return nothing */ - return ctx; + return &ctx->mailbox_ctx; } - mask = maildir_fix_mailbox_name(storage, mask, FALSE); + mask = maildir_fix_mailbox_name(istorage, mask, FALSE); glob = imap_match_init(pool, mask, TRUE, MAILDIR_FS_SEP); - ctx->dir = storage->dir; + ctx->dir = istorage->dir; ctx->prefix = storage->namespace == NULL ? "" : - maildir_fix_mailbox_name(storage, storage->namespace, FALSE); + maildir_fix_mailbox_name(istorage, storage->namespace, FALSE); if ((flags & MAILBOX_LIST_SUBSCRIBED) != 0) { if (!maildir_fill_subscribed(ctx, glob)) { @@ -269,7 +275,7 @@ t_strdup_until(mask, p+1), NULL); if (*mask != '/' && *mask != '~') - dir = t_strconcat(storage->dir, "/", dir, NULL); + dir = t_strconcat(istorage->dir, "/", dir, NULL); ctx->dir = p_strdup(pool, home_expand(dir)); } @@ -286,11 +292,14 @@ ctx->prefix = p_strdup(pool, ctx->prefix); ctx->node_path = str_new(pool, 256); ctx->root = mailbox_tree_get(ctx->tree_ctx, NULL, NULL); - return ctx; + ctx->mailbox_ctx.storage = storage; + return &ctx->mailbox_ctx; } -int maildir_list_mailbox_deinit(struct mailbox_list_context *ctx) +int maildir_mailbox_list_deinit(struct mailbox_list_context *_ctx) { + struct maildir_list_context *ctx = (struct maildir_list_context *)_ctx; + mailbox_tree_deinit(ctx->tree_ctx); pool_unref(ctx->pool); return TRUE; @@ -327,8 +336,9 @@ } struct mailbox_list * -maildir_list_mailbox_next(struct mailbox_list_context *ctx) +maildir_mailbox_list_next(struct mailbox_list_context *_ctx) { + struct maildir_list_context *ctx = (struct maildir_list_context *)_ctx; struct mailbox_node *node; for (node = ctx->next_node; node != NULL; node = node->next) {
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-storage/index/maildir/maildir-mail.c Tue Apr 27 23:25:52 2004 +0300 @@ -0,0 +1,179 @@ +/* Copyright (C) 2003 Timo Sirainen */ + +#include "lib.h" +#include "istream.h" +#include "index-mail.h" +#include "maildir-storage.h" +#include "maildir-uidlist.h" + +#include <fcntl.h> +#include <unistd.h> +#include <sys/stat.h> + +static int do_open(struct index_mailbox *ibox, const char *path, void *context) +{ + int *fd = context; + + *fd = open(path, O_RDONLY); + if (*fd != -1) + return 1; + if (errno == ENOENT) + return 0; + + mail_storage_set_critical(ibox->box.storage, + "open(%s) failed: %m", path); + return -1; +} + +static int do_stat(struct index_mailbox *ibox, const char *path, void *context) +{ + struct stat *st = context; + + if (stat(path, st) == 0) + return 1; + if (errno == ENOENT) + return 0; + + mail_storage_set_critical(ibox->box.storage, + "stat(%s) failed: %m", path); + return -1; +} + +static struct istream * +maildir_open_mail(struct index_mailbox *ibox, uint32_t uid, int *deleted) +{ + int fd; + + *deleted = FALSE; + + fd = -1; + if (maildir_file_do(ibox, uid, do_open, &fd) < 0) + return NULL; + + if (fd == -1) { + *deleted = TRUE; + return NULL; + } + + if (ibox->mail_read_mmaped) { + return i_stream_create_mmap(fd, default_pool, + MAIL_MMAP_BLOCK_SIZE, 0, 0, TRUE); + } else { + return i_stream_create_file(fd, default_pool, + MAIL_READ_BLOCK_SIZE, TRUE); + } +} + +static time_t maildir_mail_get_received_date(struct mail *_mail) +{ + struct index_mail *mail = (struct index_mail *) _mail; + struct index_mail_data *data = &mail->data; + struct stat st; + int fd; + + if (data->received_date != (time_t)-1) + return data->received_date; + + if ((mail->wanted_fields & MAIL_FETCH_RECEIVED_DATE) == 0) { + data->received_date = index_mail_get_cached_received_date(mail); + if (data->received_date != (time_t)-1) + return data->received_date; + } + + if (data->stream != NULL) { + fd = i_stream_get_fd(data->stream); + i_assert(fd != -1); + + if (fstat(fd, &st) < 0) { + mail_storage_set_critical(mail->ibox->box.storage, + "fstat(maildir) failed: %m"); + return (time_t)-1; + } + } else { + if (maildir_file_do(mail->ibox, mail->mail.uid, + do_stat, &st) <= 0) + return (time_t)-1; + } + + data->received_date = st.st_mtime; + index_mail_cache_add(mail, MAIL_CACHE_RECEIVED_DATE, + &data->received_date, sizeof(data->received_date)); + return data->received_date; +} + +static uoff_t maildir_mail_get_size(struct mail *_mail) +{ + struct index_mail *mail = (struct index_mail *) _mail; + struct index_mail_data *data = &mail->data; + const char *fname, *p; + uoff_t virtual_size; + int new_dir; + + if (data->size != (uoff_t)-1) + return data->size; + + if ((mail->wanted_fields & MAIL_FETCH_SIZE) == 0) { + data->size = index_mail_get_cached_virtual_size(mail); + if (data->size != (uoff_t)-1) + return data->size; + } + + fname = maildir_uidlist_lookup(mail->ibox->uidlist, + mail->mail.uid, &new_dir); + if (fname == NULL) + return (uoff_t)-1; + + /* size can be included in filename */ + p = strstr(fname, ",W="); + if (p != NULL) { + p += 3; + virtual_size = 0; + while (*p >= '0' && *p <= '9') { + virtual_size = virtual_size * 10 + (*p - '0'); + p++; + } + + if (*p == ':' || *p == ',' || *p != '\0') { + index_mail_cache_add(mail, MAIL_CACHE_VIRTUAL_FULL_SIZE, + &virtual_size, + sizeof(virtual_size)); + return virtual_size; + } + } + + return index_mail_get_size(_mail); +} + +static struct istream *maildir_mail_get_stream(struct mail *_mail, + struct message_size *hdr_size, + struct message_size *body_size) +{ + struct index_mail *mail = (struct index_mail *) _mail; + struct index_mail_data *data = &mail->data; + int deleted; + + if (data->stream == NULL) { + data->stream = maildir_open_mail(mail->ibox, mail->mail.uid, + &deleted); + if (data->stream == NULL) + return NULL; + } + + return index_mail_init_stream(mail, hdr_size, body_size); +} + +struct mail maildir_mail = { + 0, 0, 0, 0, 0, 0, + + index_mail_get_flags, + index_mail_get_parts, + maildir_mail_get_received_date, + index_mail_get_date, + maildir_mail_get_size, + index_mail_get_header, + index_mail_get_headers, + maildir_mail_get_stream, + index_mail_get_special, + index_mail_update_flags, + index_mail_expunge +};
--- a/src/lib-storage/index/maildir/maildir-save.c Tue Apr 27 23:14:15 2004 +0300 +++ b/src/lib-storage/index/maildir/maildir-save.c Tue Apr 27 23:25:52 2004 +0300 @@ -1,9 +1,8 @@ -/* Copyright (C) 2002 Timo Sirainen */ +/* Copyright (C) 2002-2004 Timo Sirainen */ #include "lib.h" #include "ioloop.h" #include "ostream.h" -#include "maildir-index.h" #include "maildir-storage.h" #include "mail-save.h" @@ -14,19 +13,18 @@ #include <utime.h> #include <sys/stat.h> -struct mail_filename { - struct mail_filename *next; +struct maildir_filename { + struct maildir_filename *next; const char *src, *dest; }; -struct mail_save_context { +struct maildir_save_context { pool_t pool; struct index_mailbox *ibox; - int transaction; const char *tmpdir, *newdir; - struct mail_filename *files; + struct maildir_filename *files; }; static const char * @@ -37,8 +35,7 @@ struct ostream *output; int fd; - fd = maildir_create_tmp(ibox->index, dir, - ibox->index->mail_create_mode, &path); + fd = maildir_create_tmp(ibox, dir, ibox->mail_create_mode, &path); if (fd == -1) return NULL; @@ -46,12 +43,11 @@ i_assert(fname != NULL); fname++; - t_push(); output = o_stream_create_file(fd, pool_datastack_create(), 4096, FALSE); o_stream_set_blocking(output, 60000, NULL, NULL); - if (!mail_storage_save(ibox->box.storage, path, input, output, - getenv("MAIL_SAVE_CRLF") != NULL, NULL, NULL)) + if (mail_storage_save(ibox->box.storage, path, input, output, + getenv("MAIL_SAVE_CRLF") != NULL, NULL, NULL) < 0) fname = NULL; o_stream_unref(output); @@ -70,15 +66,14 @@ if (fname == NULL) (void)unlink(path); - t_pop(); return fname; } -static int maildir_copy(struct mail_save_context *ctx, - const char *src, const char *dest) +static int maildir_file_move(struct maildir_save_context *ctx, + const char *src, const char *dest) { const char *tmp_path, *new_path; - int failed; + int ret; t_push(); @@ -86,147 +81,147 @@ new_path = t_strconcat(ctx->newdir, "/", dest, NULL); if (link(tmp_path, new_path) == 0) - failed = FALSE; + ret = 0; else { - failed = TRUE; + ret = -1; if (ENOSPACE(errno)) { mail_storage_set_error(ctx->ibox->box.storage, "Not enough disk space"); } else { mail_storage_set_critical(ctx->ibox->box.storage, - "link(%s, %s) failed: %m", - tmp_path, new_path); + "link(%s, %s) failed: %m", tmp_path, new_path); } } - (void)unlink(tmp_path); + if (unlink(tmp_path) < 0 && errno != ENOENT) { + mail_storage_set_critical(ctx->ibox->box.storage, + "unlink(%s) failed: %m", tmp_path); + } t_pop(); - return !failed; + return ret; } -int maildir_storage_save_next(struct mail_save_context *ctx, - const struct mail_full_flags *flags, - time_t received_date, - int timezone_offset __attr_unused__, - struct istream *data) +static struct maildir_save_context * +mailbox_save_init(struct index_mailbox *ibox) { + struct maildir_save_context *ctx; + pool_t pool; + + pool = pool_alloconly_create("maildir_save_context", 4096); + ctx = p_new(pool, struct maildir_save_context, 1); + ctx->pool = pool; + ctx->ibox = ibox; + + ctx->tmpdir = p_strconcat(pool, ibox->path, "/tmp", NULL); + ctx->newdir = p_strconcat(pool, ibox->path, "/new", NULL); + return ctx; +} + +int maildir_save(struct mailbox_transaction_context *_t, + const struct mail_full_flags *flags, + time_t received_date, int timezone_offset __attr_unused__, + const char *from_envelope __attr_unused__, + struct istream *data) +{ + struct maildir_transaction_context *t = + (struct maildir_transaction_context *)_t; + struct maildir_save_context *ctx; + struct index_mailbox *ibox = t->ictx.ibox; + struct maildir_filename *mf; enum mail_flags mail_flags; struct utimbuf buf; const char *fname, *dest_fname, *tmp_path; - int failed; + + if (t->save_ctx == NULL) + t->save_ctx = mailbox_save_init(ibox); + ctx = t->save_ctx; mail_flags = flags->flags; - if (!index_mailbox_fix_custom_flags(ctx->ibox, &mail_flags, + /*FIXME:if (!index_mailbox_fix_custom_flags(ibox, &mail_flags, flags->custom_flags, flags->custom_flags_count)) - return FALSE; + return FALSE;*/ t_push(); /* create the file into tmp/ directory */ - fname = maildir_read_into_tmp(ctx->ibox, ctx->tmpdir, data); + fname = maildir_read_into_tmp(ibox, ctx->tmpdir, data); if (fname == NULL) { t_pop(); - return FALSE; + return -1; } tmp_path = t_strconcat(ctx->tmpdir, "/", fname, NULL); - /* set the received_date by modifying mtime */ - buf.actime = ioloop_time; - buf.modtime = received_date; - if (utime(tmp_path, &buf) < 0) { - mail_storage_set_critical(ctx->ibox->box.storage, - "utime() failed for %s: %m", - tmp_path); - t_pop(); - return FALSE; - } - - /* now, if we want to be able to rollback the whole append session, - we'll just store the name of this temp file and move it later - into new/ */ - dest_fname = mail_flags == 0 ? fname : - maildir_filename_set_flags(fname, mail_flags); - if (ctx->transaction) { - struct mail_filename *mf; - - mf = p_new(ctx->pool, struct mail_filename, 1); - mf->next = ctx->files; - mf->src = p_strdup(ctx->pool, fname); - mf->dest = p_strdup(ctx->pool, dest_fname); - ctx->files = mf; - - failed = FALSE; - } else { - failed = !maildir_copy(ctx, fname, dest_fname); - } - - t_pop(); - return !failed; -} - -struct mail_save_context * -maildir_storage_save_init(struct mailbox *box, int transaction) -{ - struct index_mailbox *ibox = (struct index_mailbox *) box; - struct mail_save_context *ctx; - pool_t pool; - - if (box->is_readonly(box)) { - mail_storage_set_error(box->storage, "Mailbox is read-only"); - return NULL; + if (received_date != (time_t)-1) { + /* set the received_date by modifying mtime */ + buf.actime = ioloop_time; + buf.modtime = received_date; + if (utime(tmp_path, &buf) < 0) { + mail_storage_set_critical(ibox->box.storage, + "utime(%s) failed: %m", tmp_path); + t_pop(); + return -1; + } } - pool = pool_alloconly_create("mail_save_context", 4096); - ctx = p_new(pool, struct mail_save_context, 1); - ctx->pool = pool; - ctx->ibox = ibox; - ctx->transaction = transaction; + /* now, we want to be able to rollback the whole append session, + so we'll just store the name of this temp file and move it later + into new/ */ + dest_fname = mail_flags == 0 ? fname : + maildir_filename_set_flags(fname, mail_flags, NULL); - ctx->tmpdir = p_strconcat(pool, ibox->index->mailbox_path, - "/tmp", NULL); - ctx->newdir = p_strconcat(pool, ibox->index->mailbox_path, - "/new", NULL); + mf = p_new(ctx->pool, struct maildir_filename, 1); + mf->next = ctx->files; + mf->src = p_strdup(ctx->pool, fname); + mf->dest = p_strdup(ctx->pool, dest_fname); + ctx->files = mf; - return ctx; + t_pop(); + return 0; } -int maildir_storage_save_deinit(struct mail_save_context *ctx, int rollback) +int maildir_save_commit(struct maildir_save_context *ctx) { - struct mail_filename *mf, *mf2; + struct maildir_filename *mf, *mf2; const char *path; - int failed = FALSE; + int ret = 0; - if (rollback) { - /* clean up the temp files */ - for (mf = ctx->files; mf != NULL; mf = mf->next) { + /* move them into new/ */ + for (mf = ctx->files; mf != NULL; mf = mf->next) { + if (maildir_file_move(ctx, mf->src, mf->dest) < 0) { + ret = -1; + break; + } + } + + if (ret < 0) { + /* failed, try to unlink the mails already moved */ + for (mf2 = ctx->files; mf2 != mf; mf2 = mf2->next) { t_push(); - path = t_strconcat(ctx->tmpdir, "/", mf->dest, NULL); + path = t_strconcat(ctx->newdir, "/", + mf2->dest, NULL); (void)unlink(path); t_pop(); } - } else { - /* move them into new/ */ - for (mf = ctx->files; mf != NULL; mf = mf->next) { - if (!maildir_copy(ctx, mf->src, mf->dest)) { - failed = TRUE; - break; - } - } - - if (failed) { - /* failed, try to unlink the mails already moved */ - for (mf2 = ctx->files; mf2 != mf; mf2 = mf2->next) { - t_push(); - path = t_strconcat(ctx->newdir, "/", - mf2->dest, NULL); - (void)unlink(path); - t_pop(); - } - } } pool_unref(ctx->pool); - return !failed; + return ret; } + +void maildir_save_rollback(struct maildir_save_context *ctx) +{ + struct maildir_filename *mf; + const char *path; + + /* clean up the temp files */ + for (mf = ctx->files; mf != NULL; mf = mf->next) { + t_push(); + path = t_strconcat(ctx->tmpdir, "/", mf->dest, NULL); + (void)unlink(path); + t_pop(); + } + + pool_unref(ctx->pool); +}
--- a/src/lib-storage/index/maildir/maildir-storage.c Tue Apr 27 23:14:15 2004 +0300 +++ b/src/lib-storage/index/maildir/maildir-storage.c Tue Apr 27 23:25:52 2004 +0300 @@ -5,8 +5,8 @@ #include "mkdir-parents.h" #include "unlink-directory.h" #include "subscription-file/subscription-file.h" -#include "maildir-index.h" #include "maildir-storage.h" +#include "maildir-uidlist.h" #include <stdio.h> #include <stdlib.h> @@ -30,7 +30,7 @@ maildir_create(const char *data, const char *user, const char *namespace, char hierarchy_sep) { - struct mail_storage *storage; + struct index_storage *storage; const char *root_dir, *inbox_dir, *index_dir, *control_dir; const char *home, *path, *p; size_t len; @@ -85,34 +85,37 @@ else if (strcmp(index_dir, "MEMORY") == 0) index_dir = NULL; - storage = i_new(struct mail_storage, 1); - memcpy(storage, &maildir_storage, sizeof(struct mail_storage)); + storage = i_new(struct index_storage, 1); + storage->storage = maildir_storage; if (hierarchy_sep != '\0') - storage->hierarchy_sep = hierarchy_sep; - storage->namespace = i_strdup(namespace); + storage->storage.hierarchy_sep = hierarchy_sep; + storage->storage.namespace = i_strdup(namespace); storage->dir = i_strdup(home_expand(root_dir)); - storage->inbox_file = i_strdup(home_expand(inbox_dir)); + storage->inbox_path = i_strdup(home_expand(inbox_dir)); storage->index_dir = i_strdup(home_expand(index_dir)); storage->control_dir = i_strdup(home_expand(control_dir)); storage->user = i_strdup(user); storage->callbacks = i_new(struct mail_storage_callbacks, 1); index_storage_init(storage); - return storage; + return &storage->storage; } -static void maildir_free(struct mail_storage *storage) +static void maildir_free(struct mail_storage *_storage) { + struct index_storage *storage = (struct index_storage *) _storage; + index_storage_deinit(storage); - i_free(storage->namespace); + i_free(storage->storage.namespace); + i_free(storage->storage.error); + i_free(storage->dir); - i_free(storage->inbox_file); + i_free(storage->inbox_path); i_free(storage->index_dir); i_free(storage->control_dir); i_free(storage->user); - i_free(storage->error); i_free(storage->callbacks); i_free(storage); } @@ -173,14 +176,14 @@ MAILDIR_FS_SEP_S, p+1, NULL); } -const char *maildir_fix_mailbox_name(struct mail_storage *storage, +const char *maildir_fix_mailbox_name(struct index_storage *storage, const char *name, int remove_namespace) { char *dup, *p, sep; size_t len; if (strncasecmp(name, "INBOX", 5) == 0 && - (name[5] == '\0' || name[5] == storage->hierarchy_sep)) { + (name[5] == '\0' || name[5] == storage->storage.hierarchy_sep)) { /* use same case with all INBOX folders or we'll get into trouble */ name = t_strconcat("INBOX", name+5, NULL); @@ -190,11 +193,11 @@ } } - if (storage->namespace != NULL && remove_namespace) { - len = strlen(storage->namespace); - if (strncmp(storage->namespace, name, len) != 0) { + if (storage->storage.namespace != NULL && remove_namespace) { + len = strlen(storage->storage.namespace); + if (strncmp(storage->storage.namespace, name, len) != 0) { i_panic("maildir: expecting namespace '%s' in name " - "'%s'", storage->namespace, name); + "'%s'", storage->storage.namespace, name); } name += len; } @@ -202,7 +205,7 @@ if (full_filesystem_access && (*name == '/' || *name == '~')) return name; - sep = storage->hierarchy_sep; + sep = storage->storage.hierarchy_sep; if (sep == MAILDIR_FS_SEP) return name; @@ -215,21 +218,21 @@ return dup; } -const char *maildir_get_path(struct mail_storage *storage, const char *name) +const char *maildir_get_path(struct index_storage *storage, const char *name) { if (full_filesystem_access && (*name == '/' || *name == '~')) return maildir_get_absolute_path(name, FALSE); if (strcmp(name, "INBOX") == 0) { - return storage->inbox_file != NULL ? - storage->inbox_file : storage->dir; + return storage->inbox_path != NULL ? + storage->inbox_path : storage->dir; } return t_strconcat(storage->dir, "/"MAILDIR_FS_SEP_S, name, NULL); } static const char * -maildir_get_unlink_path(struct mail_storage *storage, const char *name) +maildir_get_unlink_path(struct index_storage *storage, const char *name) { if (full_filesystem_access && (*name == '/' || *name == '~')) return maildir_get_absolute_path(name, TRUE); @@ -238,14 +241,14 @@ t_strconcat(MAILDIR_FS_SEP_S, name, NULL)); } -static const char *maildir_get_index_path(struct mail_storage *storage, +static const char *maildir_get_index_path(struct index_storage *storage, const char *name) { if (storage->index_dir == NULL) return NULL; - if (strcmp(name, "INBOX") == 0 && storage->inbox_file != NULL) - return storage->inbox_file; + if (strcmp(name, "INBOX") == 0 && storage->inbox_path != NULL) + return storage->inbox_path; if (full_filesystem_access && (*name == '/' || *name == '~')) return maildir_get_absolute_path(name, FALSE); @@ -253,7 +256,7 @@ return t_strconcat(storage->index_dir, "/"MAILDIR_FS_SEP_S, name, NULL); } -static const char *maildir_get_control_path(struct mail_storage *storage, +static const char *maildir_get_control_path(struct index_storage *storage, const char *name) { if (storage->control_dir == NULL) @@ -266,132 +269,132 @@ name, NULL); } -static int mkdir_verify(struct mail_storage *storage, +static int mkdir_verify(struct index_storage *storage, const char *dir, int verify) { struct stat st; if (verify) { if (lstat(dir, &st) == 0) - return TRUE; + return 0; if (errno != ENOENT) { - mail_storage_set_critical(storage, + mail_storage_set_critical(&storage->storage, "lstat(%s) failed: %m", dir); - return FALSE; + return -1; } } if (mkdir(dir, CREATE_MODE) < 0 && (errno != EEXIST || !verify)) { if (errno != EEXIST && (!verify || errno != ENOENT)) { - mail_storage_set_critical(storage, + mail_storage_set_critical(&storage->storage, "mkdir(%s) failed: %m", dir); } - return FALSE; + return -1; } - return TRUE; + return 0; } /* create or fix maildir, ignore if it already exists */ -static int create_maildir(struct mail_storage *storage, +static int create_maildir(struct index_storage *storage, const char *dir, int verify) { const char **tmp, *path; - if (!verify && !mkdir_verify(storage, dir, verify)) - return FALSE; + if (!verify && mkdir_verify(storage, dir, verify) < 0) + return -1; for (tmp = maildirs; *tmp != NULL; tmp++) { path = t_strconcat(dir, "/", *tmp, NULL); - if (!mkdir_verify(storage, path, verify)) { + if (mkdir_verify(storage, path, verify) < 0) { if (!verify || errno != ENOENT) - return FALSE; + return -1; /* small optimization. if we're verifying, we don't check that the root dir actually exists unless we fail here. */ - if (!mkdir_verify(storage, dir, verify)) - return FALSE; - if (!mkdir_verify(storage, path, verify)) - return FALSE; + if (mkdir_verify(storage, dir, verify) < 0) + return -1; + if (mkdir_verify(storage, path, verify) < 0) + return -1; } } - return TRUE; + return 0; } -static int create_index_dir(struct mail_storage *storage, const char *name) +static int create_index_dir(struct index_storage *storage, const char *name) { const char *dir; if (storage->index_dir == NULL) - return TRUE; + return 0; if (strcmp(storage->index_dir, storage->dir) == 0 || - (strcmp(name, "INBOX") == 0 && storage->inbox_file != NULL && - strcmp(storage->index_dir, storage->inbox_file) == 0)) - return TRUE; + (strcmp(name, "INBOX") == 0 && storage->inbox_path != NULL && + strcmp(storage->index_dir, storage->inbox_path) == 0)) + return 0; dir = t_strconcat(storage->index_dir, "/"MAILDIR_FS_SEP_S, name, NULL); if (mkdir_parents(dir, CREATE_MODE) == -1 && errno != EEXIST) { - mail_storage_set_critical(storage, "mkdir(%s) failed: %m", dir); - return FALSE; + mail_storage_set_critical(&storage->storage, + "mkdir(%s) failed: %m", dir); + return -1; } - return TRUE; + return 0; } -static int create_control_dir(struct mail_storage *storage, const char *name) +static int create_control_dir(struct index_storage *storage, const char *name) { const char *dir; if (storage->control_dir == NULL) - return TRUE; + return 0; dir = t_strconcat(storage->control_dir, "/"MAILDIR_FS_SEP_S, name, NULL); if (mkdir_parents(dir, CREATE_MODE) < 0 && errno != EEXIST) { - mail_storage_set_critical(storage, "mkdir(%s) failed: %m", dir); - return FALSE; + mail_storage_set_critical(&storage->storage, + "mkdir(%s) failed: %m", dir); + return -1; } - return TRUE; + return 0; } -static int verify_inbox(struct mail_storage *storage) +static int verify_inbox(struct index_storage *storage) { const char *inbox; - if (storage->inbox_file == NULL) { + if (storage->inbox_path == NULL) { /* first make sure the cur/ new/ and tmp/ dirs exist in root dir */ - if (!create_maildir(storage, storage->dir, TRUE)) - return FALSE; + if (create_maildir(storage, storage->dir, TRUE) < 0) + return -1; /* create the .INBOX directory */ inbox = t_strconcat(storage->dir, "/"MAILDIR_FS_SEP_S"INBOX", NULL); - if (!mkdir_verify(storage, inbox, TRUE)) - return FALSE; + if (mkdir_verify(storage, inbox, TRUE) < 0) + return -1; } else { - if (!create_maildir(storage, storage->inbox_file, TRUE)) - return FALSE; + if (create_maildir(storage, storage->inbox_path, TRUE) < 0) + return -1; } /* make sure the index directories exist */ - return create_index_dir(storage, "INBOX") && - create_control_dir(storage, "INBOX"); -} - -static void maildir_mail_init(struct index_mail *mail) -{ - mail->mail.expunge = maildir_storage_expunge; + if (create_index_dir(storage, "INBOX") < 0) + return -1; + if (create_control_dir(storage, "INBOX") < 0) + return -1; + return 0; } static struct mailbox * -maildir_open(struct mail_storage *storage, const char *name, +maildir_open(struct index_storage *storage, const char *name, enum mailbox_open_flags flags) { struct index_mailbox *ibox; @@ -403,113 +406,119 @@ index_dir = maildir_get_index_path(storage, name); control_dir = maildir_get_control_path(storage, name); - index = index_storage_lookup_ref(index_dir, path); - if (index == NULL) { - index = maildir_index_alloc(path, index_dir, control_dir); - index_storage_add(index); - } + index = index_storage_alloc(index_dir, path, MAILDIR_INDEX_PREFIX); + + ibox = index_storage_mailbox_init(storage, &maildir_mailbox, + index, name, flags); + if (ibox == NULL) + return NULL; + + ibox->path = i_strdup(path); + ibox->control_dir = i_strdup(control_dir); + + ibox->mail_interface = &maildir_mail; + ibox->uidlist = maildir_uidlist_init(ibox); /* for shared mailboxes get the create mode from the permissions of dovecot-shared file */ if (stat(t_strconcat(path, "/dovecot-shared", NULL), &st) < 0) - index->mail_create_mode = 0600; + ibox->mail_create_mode = 0600; else { - index->mail_create_mode = st.st_mode & 0666; - index->private_flags_mask = MAIL_SEEN; + ibox->mail_create_mode = st.st_mode & 0666; + ibox->private_flags_mask = MAIL_SEEN; } - ibox = index_storage_mailbox_init(storage, &maildir_mailbox, - index, name, flags); - if (ibox != NULL) - ibox->mail_init = maildir_mail_init; - - return (struct mailbox *) ibox; + return &ibox->box; } static struct mailbox * -maildir_open_mailbox(struct mail_storage *storage, +maildir_mailbox_open(struct mail_storage *_storage, const char *name, enum mailbox_open_flags flags) { + struct index_storage *storage = (struct index_storage *)_storage; const char *path; struct stat st; - mail_storage_clear_error(storage); + mail_storage_clear_error(_storage); name = maildir_fix_mailbox_name(storage, name, TRUE); if (strcmp(name, "INBOX") == 0) { - if (!verify_inbox(storage)) + if (verify_inbox(storage) < 0) return NULL; return maildir_open(storage, "INBOX", flags); } if (!maildir_is_valid_existing_name(name)) { - mail_storage_set_error(storage, "Invalid mailbox name"); - return FALSE; + mail_storage_set_error(_storage, "Invalid mailbox name"); + return NULL; } path = maildir_get_path(storage, name); if (stat(path, &st) == 0) { /* exists - make sure the required directories are also there */ - if (!create_maildir(storage, path, TRUE) || - !create_index_dir(storage, name) || - !create_control_dir(storage, name)) - return FALSE; + if (create_maildir(storage, path, TRUE) < 0 || + create_index_dir(storage, name) < 0 || + create_control_dir(storage, name) < 0) + return NULL; return maildir_open(storage, name, flags); } else if (errno == ENOENT) { - mail_storage_set_error(storage, "Mailbox doesn't exist: %s", + mail_storage_set_error(_storage, "Mailbox doesn't exist: %s", name); return NULL; } else { - mail_storage_set_critical(storage, "stat(%s) failed: %m", path); + mail_storage_set_critical(_storage, "stat(%s) failed: %m", + path); return NULL; } } -static int maildir_create_mailbox(struct mail_storage *storage, +static int maildir_mailbox_create(struct mail_storage *_storage, const char *name, int directory __attr_unused__) { + struct index_storage *storage = (struct index_storage *)_storage; const char *path; - mail_storage_clear_error(storage); + mail_storage_clear_error(_storage); name = maildir_fix_mailbox_name(storage, name, TRUE); if (!maildir_is_valid_create_name(name)) { - mail_storage_set_error(storage, "Invalid mailbox name"); - return FALSE; + mail_storage_set_error(_storage, "Invalid mailbox name"); + return -1; } path = maildir_get_path(storage, name); - if (!create_maildir(storage, path, FALSE)) { + if (create_maildir(storage, path, FALSE) < 0) { if (errno == EEXIST) { - mail_storage_set_error(storage, + mail_storage_set_error(_storage, "Mailbox already exists"); } - return FALSE; + return -1; } - return TRUE; + return 0; } -static int maildir_delete_mailbox(struct mail_storage *storage, +static int maildir_mailbox_delete(struct mail_storage *_storage, const char *name) { + struct index_storage *storage = (struct index_storage *)_storage; struct stat st; const char *src, *dest, *index_dir; int count; - mail_storage_clear_error(storage); + mail_storage_clear_error(_storage); name = maildir_fix_mailbox_name(storage, name, TRUE); if (strcmp(name, "INBOX") == 0) { - mail_storage_set_error(storage, "INBOX can't be deleted."); - return FALSE; + mail_storage_set_error(_storage, "INBOX can't be deleted."); + return -1; } if (!maildir_is_valid_existing_name(name)) { - mail_storage_set_error(storage, "Invalid mailbox name"); - return FALSE; + mail_storage_set_error(_storage, "Invalid mailbox name"); + return -1; } /* rename the .maildir into ..maildir which marks it as being @@ -518,9 +527,9 @@ src = maildir_get_path(storage, name); dest = maildir_get_unlink_path(storage, name); if (stat(src, &st) != 0 && errno == ENOENT) { - mail_storage_set_error(storage, "Mailbox doesn't exist: %s", + mail_storage_set_error(_storage, "Mailbox doesn't exist: %s", name); - return FALSE; + return -1; } if (storage->index_dir != NULL && *name != '/' && *name != '~' && @@ -533,48 +542,48 @@ opened by another session.. can't really help it. */ if (unlink_directory(index_dir, TRUE) < 0 && errno != ENOTEMPTY) { - mail_storage_set_critical(storage, + mail_storage_set_critical(_storage, "unlink_directory(%s) failed: %m", index_dir); - return FALSE; + return -1; } } count = 0; while (rename(src, dest) < 0 && count < 2) { if (errno != EEXIST && errno != ENOTEMPTY) { - mail_storage_set_critical(storage, + mail_storage_set_critical(_storage, "rename(%s, %s) failed: %m", src, dest); - return FALSE; + return -1; } /* ..dir already existed? delete it and try again */ if (unlink_directory(dest, TRUE) < 0) { - mail_storage_set_critical(storage, + mail_storage_set_critical(_storage, "unlink_directory(%s) failed: %m", dest); - return FALSE; + return -1; } count++; } if (unlink_directory(dest, TRUE) < 0 && errno != ENOTEMPTY) { - mail_storage_set_critical(storage, + mail_storage_set_critical(_storage, "unlink_directory(%s) failed: %m", dest); /* it's already renamed to ..dir, which means it's deleted as far as client is concerned. Report success. */ } - return TRUE; + return 0; } -static int rename_indexes(struct mail_storage *storage, +static int rename_indexes(struct index_storage *storage, const char *oldname, const char *newname) { const char *oldpath, *newpath; if (storage->index_dir == NULL || strcmp(storage->index_dir, storage->dir) == 0) - return TRUE; + return 0; /* Rename it's index. */ oldpath = t_strconcat(storage->index_dir, "/"MAILDIR_FS_SEP_S, @@ -583,15 +592,16 @@ newname, NULL); if (rename(oldpath, newpath) < 0 && errno != ENOENT) { - mail_storage_set_critical(storage, "rename(%s, %s) failed: %m", + mail_storage_set_critical(&storage->storage, + "rename(%s, %s) failed: %m", oldpath, newpath); - return FALSE; + return -1; } - return TRUE; + return 0; } -static int rename_subfolders(struct mail_storage *storage, +static int rename_subfolders(struct index_storage *storage, const char *oldname, const char *newname) { struct mailbox_list_context *ctx; @@ -603,12 +613,12 @@ ret = 0; oldnamelen = strlen(oldname); - mask = t_strdup_printf("%s%s%c*", storage->namespace != NULL ? - storage->namespace : "", oldname, - storage->hierarchy_sep); - ctx = storage->list_mailbox_init(storage, mask, - MAILBOX_LIST_FAST_FLAGS); - while ((list = maildir_list_mailbox_next(ctx)) != NULL) { + mask = t_strdup_printf("%s%s%c*", storage->storage.namespace != NULL ? + storage->storage.namespace : "", oldname, + storage->storage.hierarchy_sep); + ctx = maildir_mailbox_list_init(&storage->storage, mask, + MAILBOX_LIST_FAST_FLAGS); + while ((list = maildir_mailbox_list_next(ctx)) != NULL) { const char *list_name; t_push(); @@ -633,7 +643,7 @@ errno == EEXIST || errno == ENOTEMPTY) ret = 1; else { - mail_storage_set_critical(storage, + mail_storage_set_critical(&storage->storage, "rename(%s, %s) failed: %m", oldpath, newpath); ret = -1; @@ -645,32 +655,33 @@ t_pop(); } - if (!maildir_list_mailbox_deinit(ctx)) + if (maildir_mailbox_list_deinit(ctx) < 0) return -1; return ret; } -static int maildir_rename_mailbox(struct mail_storage *storage, +static int maildir_mailbox_rename(struct mail_storage *_storage, const char *oldname, const char *newname) { + struct index_storage *storage = (struct index_storage *)_storage; const char *oldpath, *newpath; int ret, found; - mail_storage_clear_error(storage); + mail_storage_clear_error(_storage); oldname = maildir_fix_mailbox_name(storage, oldname, TRUE); newname = maildir_fix_mailbox_name(storage, newname, TRUE); if (!maildir_is_valid_existing_name(oldname) || !maildir_is_valid_create_name(newname)) { - mail_storage_set_error(storage, "Invalid mailbox name"); - return FALSE; + mail_storage_set_error(_storage, "Invalid mailbox name"); + return -1; } if (strcmp(oldname, "INBOX") == 0) { - mail_storage_set_error(storage, + mail_storage_set_error(_storage, "Renaming INBOX isn't supported."); - return FALSE; + return -1; } /* NOTE: it's possible to rename a nonexisting folder which has @@ -685,89 +696,98 @@ found = ret == 0; ret = rename_subfolders(storage, oldname, newname); if (ret < 0) - return FALSE; + return -1; if (!found && ret == 0) { - mail_storage_set_error(storage, + mail_storage_set_error(_storage, "Mailbox doesn't exist"); - return FALSE; + return -1; } - return TRUE; + return 0; } if (errno == EEXIST) { - mail_storage_set_error(storage, + mail_storage_set_error(_storage, "Target mailbox already exists"); - return FALSE; + return -1; } else { - mail_storage_set_critical(storage, "rename(%s, %s) failed: %m", + mail_storage_set_critical(_storage, "rename(%s, %s) failed: %m", oldpath, newpath); - return FALSE; + return -1; } } -static int maildir_set_subscribed(struct mail_storage *storage, +static int maildir_set_subscribed(struct mail_storage *_storage, const char *name, int set) { + struct index_storage *storage = (struct index_storage *)_storage; + const char *path; + + path = t_strconcat(storage->control_dir != NULL ? + storage->control_dir : storage->dir, + "/" SUBSCRIPTION_FILE_NAME, NULL); + name = maildir_fix_mailbox_name(storage, name, FALSE); - return subsfile_set_subscribed(storage, name, set); + return subsfile_set_subscribed(_storage, path, name, set); } -static int maildir_get_mailbox_name_status(struct mail_storage *storage, +static int maildir_get_mailbox_name_status(struct mail_storage *_storage, const char *name, enum mailbox_name_status *status) { + struct index_storage *storage = (struct index_storage *)_storage; struct stat st; const char *path; - mail_storage_clear_error(storage); + mail_storage_clear_error(_storage); name = maildir_fix_mailbox_name(storage, name, TRUE); if (!maildir_is_valid_existing_name(name)) { *status = MAILBOX_NAME_INVALID; - return TRUE; + return 0; } path = maildir_get_path(storage, name); if (stat(path, &st) == 0) { *status = MAILBOX_NAME_EXISTS; - return TRUE; + return 0; } if (!maildir_is_valid_create_name(name)) { *status = MAILBOX_NAME_INVALID; - return TRUE; + return 0; } if (errno == ENOENT) { *status = MAILBOX_NAME_VALID; - return TRUE; + return 0; } else { - mail_storage_set_critical(storage, "stat(%s) failed: %m", path); - return FALSE; + mail_storage_set_critical(_storage, "stat(%s) failed: %m", + path); + return -1; } } static int maildir_storage_close(struct mailbox *box) { - struct index_mailbox *ibox = (struct index_mailbox *) box; - int failed = FALSE; + struct index_mailbox *ibox = (struct index_mailbox *)box; + int ret = 0; - index_storage_init_lock_notify(ibox); - if (!maildir_try_flush_dirty_flags(ibox->index, TRUE)) { + /*FIXME:if (!maildir_try_flush_dirty_flags(ibox->index, TRUE)) { mail_storage_set_index_error(ibox); - failed = TRUE; - } - ibox->index->set_lock_notify_callback(ibox->index, NULL, NULL); + ret = -1; + }*/ - return index_storage_mailbox_free(box) && !failed; + maildir_uidlist_deinit(ibox->uidlist); + index_storage_mailbox_free(box); + return ret; } static void maildir_storage_auto_sync(struct mailbox *box, enum mailbox_sync_flags flags, unsigned int min_newmail_notify_interval) { - struct index_mailbox *ibox = (struct index_mailbox *) box; + struct index_mailbox *ibox = (struct index_mailbox *)box; ibox->min_newmail_notify_interval = min_newmail_notify_interval; @@ -784,35 +804,9 @@ } index_mailbox_check_add(ibox, - t_strconcat(ibox->index->mailbox_path, "/new", NULL), TRUE); + t_strconcat(ibox->storage->dir, "/new", NULL), TRUE); index_mailbox_check_add(ibox, - t_strconcat(ibox->index->mailbox_path, "/cur", NULL), TRUE); -} - -static int maildir_storage_lock(struct mailbox *box, - enum mailbox_lock_type lock_type) -{ - struct index_mailbox *ibox = (struct index_mailbox *) box; - - if (lock_type == MAIL_LOCK_UNLOCK) { - ibox->lock_type = MAIL_LOCK_UNLOCK; - if (!index_storage_lock(ibox, MAIL_LOCK_UNLOCK)) - return FALSE; - return TRUE; - } - - i_assert(ibox->lock_type == MAIL_LOCK_UNLOCK); - - if ((lock_type & (MAILBOX_LOCK_EXPUNGE | MAILBOX_LOCK_FLAGS)) != 0) { - if (!index_storage_lock(ibox, MAIL_LOCK_EXCLUSIVE)) - return FALSE; - } else if ((lock_type & MAILBOX_LOCK_READ) != 0) { - if (!index_storage_lock(ibox, MAIL_LOCK_SHARED)) - return FALSE; - } - - ibox->lock_type = lock_type; - return TRUE; + t_strconcat(ibox->storage->dir, "/cur", NULL), TRUE); } struct mail_storage maildir_storage = { @@ -825,24 +819,18 @@ maildir_free, maildir_autodetect, index_storage_set_callbacks, - maildir_open_mailbox, - maildir_create_mailbox, - maildir_delete_mailbox, - maildir_rename_mailbox, - maildir_list_mailbox_init, - maildir_list_mailbox_deinit, - maildir_list_mailbox_next, + maildir_mailbox_open, + maildir_mailbox_create, + maildir_mailbox_delete, + maildir_mailbox_rename, + maildir_mailbox_list_init, + maildir_mailbox_list_next, + maildir_mailbox_list_deinit, maildir_set_subscribed, maildir_get_mailbox_name_status, mail_storage_get_last_error, NULL, - NULL, - NULL, - NULL, - NULL, - NULL, NULL, NULL, - 0 }; @@ -853,24 +841,19 @@ index_storage_is_readonly, index_storage_allow_new_custom_flags, maildir_storage_close, - maildir_storage_lock, index_storage_get_status, - index_storage_sync, + maildir_storage_sync, maildir_storage_auto_sync, - index_storage_fetch_uid, - index_storage_fetch_seq, + maildir_transaction_begin, + maildir_transaction_commit, + maildir_transaction_rollback, + index_storage_fetch, + index_storage_get_uids, index_storage_search_get_sorting, index_storage_search_init, index_storage_search_deinit, index_storage_search_next, - maildir_storage_save_init, - maildir_storage_save_deinit, - maildir_storage_save_next, - maildir_storage_copy_init, - maildir_storage_copy_deinit, - maildir_storage_copy, - maildir_storage_expunge_init, - maildir_storage_expunge_deinit, - maildir_storage_expunge_fetch_next, - index_storage_is_inconsistency_error + maildir_save, + maildir_copy, + index_storage_is_inconsistent };
--- a/src/lib-storage/index/maildir/maildir-storage.h Tue Apr 27 23:14:15 2004 +0300 +++ b/src/lib-storage/index/maildir/maildir-storage.h Tue Apr 27 23:25:52 2004 +0300 @@ -5,39 +5,73 @@ #define MAILDIR_FS_SEP '.' #define MAILDIR_FS_SEP_S "." +#define SUBSCRIPTION_FILE_NAME "subscriptions" +#define MAILDIR_INDEX_PREFIX "dovecot.index" + #include "index-storage.h" -struct mail_copy_context *maildir_storage_copy_init(struct mailbox *box); -int maildir_storage_copy_deinit(struct mail_copy_context *ctx, int rollback); -int maildir_storage_copy(struct mail *mail, struct mail_copy_context *ctx); +struct maildir_save_context; +struct maildir_copy_context; + +struct maildir_transaction_context { + struct index_transaction_context ictx; + struct maildir_save_context *save_ctx; + struct maildir_copy_context *copy_ctx; +}; -struct mail_save_context * -maildir_storage_save_init(struct mailbox *box, int transaction); -int maildir_storage_save_deinit(struct mail_save_context *ctx, int rollback); -int maildir_storage_save_next(struct mail_save_context *ctx, - const struct mail_full_flags *flags, - time_t received_date, int timezone_offset, - struct istream *data); +extern struct mail maildir_mail; + +/* Return -1 = error, 0 = file not found, 1 = ok */ +typedef int maildir_file_do_func(struct index_mailbox *ibox, + const char *path, void *context); + +int maildir_file_do(struct index_mailbox *ibox, uint32_t seq, + maildir_file_do_func *func, void *context); +const char *maildir_generate_tmp_filename(const struct timeval *tv); +int maildir_create_tmp(struct index_mailbox *ibox, const char *dir, + mode_t mode, const char **fname_r); struct mailbox_list_context * -maildir_list_mailbox_init(struct mail_storage *storage, +maildir_mailbox_list_init(struct mail_storage *storage, const char *mask, enum mailbox_list_flags flags); -int maildir_list_mailbox_deinit(struct mailbox_list_context *ctx); +int maildir_mailbox_list_deinit(struct mailbox_list_context *ctx); struct mailbox_list * -maildir_list_mailbox_next(struct mailbox_list_context *ctx); +maildir_mailbox_list_next(struct mailbox_list_context *ctx); + +int maildir_storage_sync(struct mailbox *box, enum mailbox_sync_flags flags); +int maildir_storage_sync_readonly(struct index_mailbox *ibox); + +struct mailbox_transaction_context * +maildir_transaction_begin(struct mailbox *box, int hide); +int maildir_transaction_commit(struct mailbox_transaction_context *t); +void maildir_transaction_rollback(struct mailbox_transaction_context *t); + +int maildir_save(struct mailbox_transaction_context *t, + const struct mail_full_flags *flags, + time_t received_date, int timezone_offset, + const char *from_envelope, struct istream *data); +int maildir_save_commit(struct maildir_save_context *ctx); +void maildir_save_rollback(struct maildir_save_context *ctx); -struct mail_expunge_context * -maildir_storage_expunge_init(struct mailbox *box, - enum mail_fetch_field wanted_fields, - int expunge_all); -int maildir_storage_expunge_deinit(struct mail_expunge_context *ctx); -struct mail * -maildir_storage_expunge_fetch_next(struct mail_expunge_context *ctx); -int maildir_storage_expunge(struct mail *mail, struct mail_expunge_context *ctx, - unsigned int *seq_r, int notify); +int maildir_copy(struct mailbox_transaction_context *t, struct mail *mail); +int maildir_copy_commit(struct maildir_copy_context *ctx); +void maildir_copy_rollback(struct maildir_copy_context *ctx); + +int maildir_storage_expunge(struct mail *mail, + struct mailbox_transaction_context *t); -const char *maildir_fix_mailbox_name(struct mail_storage *storage, +const char *maildir_fix_mailbox_name(struct index_storage *storage, const char *name, int remove_namespace); -const char *maildir_get_path(struct mail_storage *storage, const char *name); +const char *maildir_get_path(struct index_storage *storage, const char *name); + +int maildir_sync_last_commit(struct index_mailbox *ibox); + +int maildir_filename_get_flags(const char *fname, enum mail_flags *flags_r, + custom_flags_mask_t custom_flags_r); +const char *maildir_filename_set_flags(const char *fname, enum mail_flags flags, + custom_flags_mask_t custom_flags); + +unsigned int maildir_hash(const void *p); +int maildir_cmp(const void *p1, const void *p2); #endif
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-storage/index/maildir/maildir-sync.c Tue Apr 27 23:25:52 2004 +0300 @@ -0,0 +1,472 @@ +/* + 1. read files in maildir + 2. see if they're all found in uidlist read in memory + 3. if not, check if uidlist's mtime has changed and read it if so + 4. if still not, lock uidlist, sync it once more and generate UIDs for new + files + 5. apply changes in transaction log + 6. apply changes in maildir to index +*/ + +/* Copyright (C) 2004 Timo Sirainen */ + +#include "lib.h" +#include "ioloop.h" +#include "buffer.h" +#include "hash.h" +#include "str.h" +#include "maildir-storage.h" +#include "maildir-uidlist.h" + +#include <stdio.h> +#include <unistd.h> +#include <dirent.h> +#include <sys/stat.h> + +#define MAILDIR_SYNC_SECS 1 + +#define MAILDIR_FILENAME_FLAG_FOUND 128 + +struct maildir_sync_context { + struct index_mailbox *ibox; + const char *new_dir, *cur_dir; + + struct maildir_uidlist_sync_ctx *uidlist_sync_ctx; +}; + +static int maildir_expunge(struct index_mailbox *ibox, const char *path, + void *context __attr_unused__) +{ + if (unlink(path) == 0) + return 1; + if (errno == ENOENT) + return 0; + + mail_storage_set_critical(ibox->box.storage, + "unlink(%s) failed: %m", path); + return -1; +} + +static int maildir_sync_flags(struct index_mailbox *ibox, const char *path, + void *context) +{ + struct mail_index_sync_rec *syncrec = context; + const char *newpath; + enum mail_flags flags; + uint8_t flags8; + custom_flags_mask_t custom_flags; + + (void)maildir_filename_get_flags(path, &flags, custom_flags); + + flags8 = flags; + mail_index_sync_flags_apply(syncrec, &flags8, custom_flags); + + newpath = maildir_filename_set_flags(path, flags8, custom_flags); + if (rename(path, newpath) == 0) + return 1; + if (errno == ENOENT) + return 0; + + mail_storage_set_critical(ibox->box.storage, + "rename(%s, %s) failed: %m", path, newpath); + return -1; +} + +static int maildir_sync_record(struct index_mailbox *ibox, + struct mail_index_view *view, + struct mail_index_sync_rec *syncrec) +{ + const struct mail_index_record *rec; + uint32_t seq, uid; + + switch (syncrec->type) { + case MAIL_INDEX_SYNC_TYPE_APPEND: + break; + case MAIL_INDEX_SYNC_TYPE_EXPUNGE: + for (seq = syncrec->seq1; seq <= syncrec->seq2; seq++) { + if (mail_index_lookup_uid(view, seq, &uid) < 0) + return -1; + if (maildir_file_do(ibox, uid, maildir_expunge, + NULL) < 0) + return -1; + } + break; + case MAIL_INDEX_SYNC_TYPE_FLAGS: + for (seq = syncrec->seq1; seq <= syncrec->seq2; seq++) { + if (mail_index_lookup_uid(view, seq, &uid) < 0) + return -1; + if (maildir_file_do(ibox, uid, maildir_sync_flags, + syncrec) < 0) + return -1; + } + break; + } + + return 0; +} + +int maildir_sync_last_commit(struct index_mailbox *ibox) +{ + struct mail_index_view *view; + struct mail_index_sync_ctx *sync_ctx; + struct mail_index_sync_rec sync_rec; + int ret; + + if (ibox->commit_log_file_seq == 0) + return 0; + + ret = mail_index_sync_begin(ibox->index, &sync_ctx, &view, + ibox->commit_log_file_seq, + ibox->commit_log_file_offset); + if (ret > 0) { + while ((ret = mail_index_sync_next(sync_ctx, &sync_rec)) > 0) { + if (maildir_sync_record(ibox, view, &sync_rec) < 0) { + ret = -1; + break; + } + } + if (mail_index_sync_end(sync_ctx) < 0) + ret = -1; + } + + if (ret == 0) { + ibox->commit_log_file_seq = 0; + ibox->commit_log_file_offset = 0; + } else { + // FIXME: this is bad - we have to fix this in some way + mail_storage_set_index_error(ibox); + } + return ret; +} + +static struct maildir_sync_context * +maildir_sync_context_new(struct index_mailbox *ibox) +{ + struct maildir_sync_context *ctx; + + ctx = t_new(struct maildir_sync_context, 1); + ctx->ibox = ibox; + ctx->new_dir = t_strconcat(ibox->path, "/new", NULL); + ctx->cur_dir = t_strconcat(ibox->path, "/cur", NULL); + return ctx; +} + +static void maildir_sync_deinit(struct maildir_sync_context *ctx) +{ + if (ctx->uidlist_sync_ctx != NULL) + (void)maildir_uidlist_sync_deinit(ctx->uidlist_sync_ctx); +} + +static int maildir_fix_duplicate(struct index_mailbox *ibox, const char *dir, + const char *old_fname) +{ + const char *new_fname, *old_path, *new_path; + int ret = 0; + + t_push(); + + old_path = t_strconcat(dir, "/", old_fname, NULL); + new_fname = maildir_generate_tmp_filename(&ioloop_timeval); + new_path = t_strconcat(ibox->path, "/new/", new_fname, NULL); + + if (rename(old_path, new_path) == 0) { + i_warning("Fixed duplicate in %s: %s -> %s", + ibox->path, old_fname, new_fname); + } else if (errno != ENOENT) { + mail_storage_set_critical(ibox->box.storage, + "rename(%s, %s) failed: %m", old_path, new_path); + ret = -1; + } + t_pop(); + + return ret; +} + +static int maildir_scan_dir(struct maildir_sync_context *ctx, int new_dir) +{ + struct mail_storage *storage = ctx->ibox->box.storage; + const char *dir; + DIR *dirp; + string_t *src, *dest; + struct dirent *dp; + int move_new, this_new, ret = 1; + + src = t_str_new(1024); + dest = t_str_new(1024); + + dir = new_dir ? ctx->new_dir : ctx->cur_dir; + dirp = opendir(dir); + if (dirp == NULL) { + mail_storage_set_critical(storage, + "opendir(%s) failed: %m", dir); + return -1; + } + + move_new = new_dir; + while ((dp = readdir(dirp)) != NULL) { + if (dp->d_name[0] == '.') + continue; + + this_new = new_dir; + if (move_new) { + str_truncate(src, 0); + str_truncate(dest, 0); + str_printfa(src, "%s/%s", ctx->new_dir, dp->d_name); + str_printfa(dest, "%s/%s", ctx->cur_dir, dp->d_name); + if (rename(str_c(src), str_c(dest)) == 0 || + ENOTFOUND(errno)) { + /* moved - we'll look at it later in cur/ dir */ + this_new = FALSE; + continue; + } else if (ENOSPACE(errno)) { + /* not enough disk space, leave here */ + move_new = FALSE; + } else { + mail_storage_set_critical(storage, + "rename(%s, %s) failed: %m", + str_c(src), str_c(dest)); + } + } + + ret = maildir_uidlist_sync_next(ctx->uidlist_sync_ctx, + dp->d_name, this_new); + if (ret <= 0) { + if (ret < 0) + break; + + /* possibly duplicate - try fixing it */ + if (maildir_fix_duplicate(ctx->ibox, + dir, dp->d_name) < 0) { + ret = -1; + break; + } + } + } + + if (closedir(dirp) < 0) { + mail_storage_set_critical(storage, + "closedir(%s) failed: %m", dir); + } + return ret < 0 ? -1 : 0; +} + +static int maildir_sync_quick_check(struct maildir_sync_context *ctx, + int *new_changed_r, int *cur_changed_r) +{ + struct index_mailbox *ibox = ctx->ibox; + struct stat st; + time_t new_mtime, cur_mtime; + + *new_changed_r = *cur_changed_r = FALSE; + + if (stat(ctx->new_dir, &st) < 0) { + mail_storage_set_critical(ibox->box.storage, + "stat(%s) failed: %m", ctx->new_dir); + return -1; + } + new_mtime = st.st_mtime; + + if (stat(ctx->cur_dir, &st) < 0) { + mail_storage_set_critical(ibox->box.storage, + "stat(%s) failed: %m", ctx->cur_dir); + return -1; + } + cur_mtime = st.st_mtime; + + if (new_mtime != ibox->last_new_mtime || + new_mtime >= ibox->last_sync - MAILDIR_SYNC_SECS) { + *new_changed_r = TRUE; + ibox->last_new_mtime = new_mtime; + } + if (cur_mtime != ibox->last_cur_mtime || + (cur_mtime >= ibox->last_sync - MAILDIR_SYNC_SECS && + ioloop_time - ibox->last_sync > MAILDIR_SYNC_SECS)) { + /* cur/ changed, or delayed cur/ check */ + *cur_changed_r = TRUE; + ibox->last_cur_mtime = cur_mtime; + } + ibox->last_sync = ioloop_time; + + return 0; +} + +static int maildir_sync_index(struct maildir_sync_context *ctx) +{ + struct index_mailbox *ibox = ctx->ibox; + struct mail_index_sync_ctx *sync_ctx; + struct mail_index_sync_rec sync_rec; + struct maildir_uidlist_iter_ctx *iter; + struct mail_index_transaction *trans; + struct mail_index_view *view; + const struct mail_index_header *hdr; + const struct mail_index_record *rec; + uint32_t seq, uid, uflags; + const char *filename; + enum mail_flags flags; + custom_flags_mask_t custom_flags; + int ret = 0; + + if (mail_index_sync_begin(ibox->index, &sync_ctx, &view, + (uint32_t)-1, (uoff_t)-1) <= 0) { + // FIXME: ? + return -1; + } + + hdr = mail_index_get_header(view); + trans = mail_index_transaction_begin(view, FALSE); + + seq = 0; + iter = maildir_uidlist_iter_init(ibox->uidlist); + while (maildir_uidlist_iter_next(iter, &uid, &uflags, &filename)) { + maildir_filename_get_flags(filename, &flags, custom_flags); + + __again: + seq++; + if (seq > hdr->messages_count) { + mail_index_append(trans, uid, &seq); + mail_index_update_flags(trans, seq, MODIFY_REPLACE, + flags, custom_flags); + continue; + } + + if (mail_index_lookup(view, seq, &rec) < 0) { + mail_storage_set_index_error(ibox); + ret = -1; + break; + } + + if (rec->uid < uid) { + /* expunged */ + mail_index_expunge(trans, seq); + goto __again; + } + + if (rec->uid > uid) { + /* new UID in the middle of the mailbox - + shouldn't happen */ + mail_storage_set_critical(ibox->box.storage, + "Maildir sync: UID inserted in the middle " + "of mailbox (%u > %u)", rec->uid, uid); + (void)mail_index_reset(ibox->index); + ret = -1; + break; + } + + maildir_filename_get_flags(filename, &flags, custom_flags); + if (rec->flags & MAIL_RECENT) + flags |= MAIL_RECENT; + if ((uint8_t)flags != (rec->flags & MAIL_FLAGS_MASK) || + memcmp(custom_flags, rec->custom_flags, + INDEX_CUSTOM_FLAGS_BYTE_COUNT) != 0) { + mail_index_update_flags(trans, seq, MODIFY_REPLACE, + flags, custom_flags); + } + } + maildir_uidlist_iter_deinit(iter); + + if (ret < 0) + mail_index_transaction_rollback(trans); + else { + uint32_t seq; + uoff_t offset; + + if (mail_index_transaction_commit(trans, &seq, &offset) < 0) + mail_storage_set_index_error(ibox); + else { + ibox->commit_log_file_seq = seq; + ibox->commit_log_file_offset = offset; + } + } + + /* now, sync the index */ + while ((ret = mail_index_sync_next(sync_ctx, &sync_rec)) > 0) { + if (maildir_sync_record(ibox, view, &sync_rec) < 0) { + ret = -1; + break; + } + } + if (mail_index_sync_end(sync_ctx) < 0) + ret = -1; + + if (ret == 0) { + ibox->commit_log_file_seq = 0; + ibox->commit_log_file_offset = 0; + } else { + // FIXME: this is bad - we have to fix this in some way + mail_storage_set_index_error(ibox); + } + + return ret; +} + +static int maildir_sync_context(struct maildir_sync_context *ctx, + int *changes_r) +{ + int ret, new_changed, cur_changed; + + if (maildir_sync_quick_check(ctx, &new_changed, &cur_changed) < 0) + return -1; + + ctx->uidlist_sync_ctx = maildir_uidlist_sync_init(ctx->ibox->uidlist); + + if (maildir_scan_dir(ctx, TRUE) < 0) + return -1; + if (maildir_scan_dir(ctx, FALSE) < 0) + return -1; + + ret = maildir_uidlist_sync_deinit(ctx->uidlist_sync_ctx); + ctx->uidlist_sync_ctx = NULL; + + if (ret == 0) + ret = maildir_sync_index(ctx); + return ret; +} + +static int maildir_sync_context_readonly(struct maildir_sync_context *ctx) +{ + int ret; + + ctx->uidlist_sync_ctx = maildir_uidlist_sync_init(ctx->ibox->uidlist); + + if (maildir_scan_dir(ctx, TRUE) < 0) + return -1; + if (maildir_scan_dir(ctx, FALSE) < 0) + return -1; + + ret = maildir_uidlist_sync_deinit(ctx->uidlist_sync_ctx); + ctx->uidlist_sync_ctx = NULL; + + return ret; +} + +int maildir_storage_sync_readonly(struct index_mailbox *ibox) +{ + struct maildir_sync_context *ctx; + int ret; + + ctx = maildir_sync_context_new(ibox); + ret = maildir_sync_context_readonly(ctx); + maildir_sync_deinit(ctx); + return ret; +} + +int maildir_storage_sync(struct mailbox *box, enum mailbox_sync_flags flags) +{ + struct index_mailbox *ibox = (struct index_mailbox *)box; + struct maildir_sync_context *ctx; + int changes, ret; + + if ((flags & MAILBOX_SYNC_FLAG_FAST) == 0 || + ibox->sync_last_check + MAILBOX_FULL_SYNC_INTERVAL <= ioloop_time) { + ibox->sync_last_check = ioloop_time; + + ctx = maildir_sync_context_new(ibox); + ret = maildir_sync_context(ctx, &changes); + maildir_sync_deinit(ctx); + + if (ret < 0) + return -1; + } + + return index_storage_sync(box, flags); +}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-storage/index/maildir/maildir-transaction.c Tue Apr 27 23:25:52 2004 +0300 @@ -0,0 +1,51 @@ +/* Copyright (C) 2004 Timo Sirainen */ + +#include "lib.h" +#include "maildir-storage.h" + +struct mailbox_transaction_context * +maildir_transaction_begin(struct mailbox *box, int hide) +{ + struct index_mailbox *ibox = (struct index_mailbox *)box; + struct maildir_transaction_context *ctx; + + ctx = i_new(struct maildir_transaction_context, 1); + ctx->ictx.mailbox_ctx.box = box; + ctx->ictx.ibox = ibox; + ctx->ictx.trans = mail_index_transaction_begin(ibox->view, hide); + return &ctx->ictx.mailbox_ctx; +} + +int maildir_transaction_commit(struct mailbox_transaction_context *_t) +{ + struct maildir_transaction_context *t = + (struct maildir_transaction_context *)_t; + struct index_mailbox *ibox = t->ictx.ibox; + int ret = 0; + + if (t->save_ctx != NULL) { + if (maildir_save_commit(t->save_ctx) < 0) + ret = -1; + } + if (t->copy_ctx != NULL) { + if (maildir_copy_commit(t->copy_ctx) < 0) + ret = -1; + } + + if (index_transaction_commit(_t) < 0) + return -1; + + return ret < 0 ? -1 : maildir_sync_last_commit(ibox); +} + +void maildir_transaction_rollback(struct mailbox_transaction_context *_t) +{ + struct maildir_transaction_context *t = + (struct maildir_transaction_context *)_t; + + if (t->save_ctx != NULL) + maildir_save_rollback(t->save_ctx); + if (t->copy_ctx != NULL) + maildir_copy_rollback(t->copy_ctx); + index_transaction_rollback(_t); +}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-storage/index/maildir/maildir-uidlist.c Tue Apr 27 23:25:52 2004 +0300 @@ -0,0 +1,613 @@ +/* Copyright (C) 2003 Timo Sirainen */ + +#include "lib.h" +#include "ioloop.h" +#include "buffer.h" +#include "hash.h" +#include "istream.h" +#include "str.h" +#include "file-dotlock.h" +#include "write-full.h" +#include "maildir-storage.h" +#include "maildir-uidlist.h" + +#include <stdio.h> +#include <stdlib.h> +#include <sys/stat.h> +#include <utime.h> + +/* how many seconds to wait before overriding uidlist.lock */ +#define UIDLIST_LOCK_STALE_TIMEOUT (60*5) + +#define UIDLIST_IS_LOCKED(uidlist) \ + ((uidlist)->lock_fd != -1) + +#define MAILDIR_UIDLIST_REC_FLAG_NEW_DIR 0x01 + +struct maildir_uidlist_rec { + uint32_t uid; + uint32_t flags; + char *filename; +}; + +struct maildir_uidlist { + struct index_mailbox *ibox; + char *fname; + int lock_fd; + + time_t last_mtime; + + pool_t filename_pool; + buffer_t *record_buf; + struct hash_table *files; + + unsigned int version; + unsigned int uid_validity, next_uid, last_read_uid; +}; + +struct maildir_uidlist_sync_ctx { + struct maildir_uidlist *uidlist; + + pool_t filename_pool; + struct hash_table *files; + + struct maildir_uidlist_rec new_rec, cur_rec; + unsigned int new_files:1; + unsigned int synced:1; + unsigned int failed:1; +}; + +struct maildir_uidlist_iter_ctx { + const struct maildir_uidlist_rec *next, *end; +}; + +int maildir_uidlist_try_lock(struct maildir_uidlist *uidlist) +{ + const char *path; + mode_t old_mask; + int fd; + + if (UIDLIST_IS_LOCKED(uidlist)) + return 1; + + path = t_strconcat(uidlist->ibox->control_dir, + "/" MAILDIR_UIDLIST_NAME, NULL); + old_mask = umask(0777 & ~uidlist->ibox->mail_create_mode); + fd = file_dotlock_open(path, NULL, 0, 0, UIDLIST_LOCK_STALE_TIMEOUT, + NULL, NULL); + umask(old_mask); + if (fd == -1) { + if (errno == EAGAIN) + return 0; + return -1; + } + + uidlist->lock_fd = fd; + return 1; +} + +void maildir_uidlist_unlock(struct maildir_uidlist *uidlist) +{ + const char *path; + + if (!UIDLIST_IS_LOCKED(uidlist)) + return; + + path = t_strconcat(uidlist->ibox->control_dir, + "/" MAILDIR_UIDLIST_NAME, NULL); + (void)file_dotlock_delete(path, uidlist->lock_fd); + uidlist->lock_fd = -1; +} + +struct maildir_uidlist *maildir_uidlist_init(struct index_mailbox *ibox) +{ + struct maildir_uidlist *uidlist; + + uidlist = i_new(struct maildir_uidlist, 1); + uidlist->ibox = ibox; + uidlist->fname = + i_strconcat(ibox->control_dir, "/" MAILDIR_UIDLIST_NAME, NULL); + uidlist->lock_fd = -1; + uidlist->record_buf = + buffer_create_dynamic(default_pool, 512, (size_t)-1); + uidlist->files = hash_create(default_pool, default_pool, 4096, + maildir_hash, maildir_cmp); + + uidlist->uid_validity = ioloop_time; + uidlist->next_uid = 1; + + return uidlist; +} + +void maildir_uidlist_deinit(struct maildir_uidlist *uidlist) +{ + i_assert(!UIDLIST_IS_LOCKED(uidlist)); + + if (uidlist->filename_pool != NULL) + pool_unref(uidlist->filename_pool); + + hash_destroy(uidlist->files); + buffer_free(uidlist->record_buf); + i_free(uidlist->fname); + i_free(uidlist); +} + +static int maildir_uidlist_next(struct maildir_uidlist *uidlist, + const char *line) +{ + struct maildir_uidlist_rec *rec; + uint32_t uid, flags; + + uid = flags = 0; + while (*line >= '0' && *line <= '9') { + uid = uid*10 + (*line - '0'); + line++; + } + + if (uid == 0 || *line != ' ') { + /* invalid file */ + mail_storage_set_critical(uidlist->ibox->box.storage, + "Invalid data in file %s", uidlist->fname); + return 0; + } + if (uid <= uidlist->last_read_uid) { + mail_storage_set_critical(uidlist->ibox->box.storage, + "UIDs not ordered in file %s (%u > %u)", + uidlist->fname, uid, uidlist->last_read_uid); + return 0; + } + if (uid >= uidlist->next_uid) { + mail_storage_set_critical(uidlist->ibox->box.storage, + "UID larger than next_uid in file %s (%u >= %u)", + uidlist->fname, uid, uidlist->next_uid); + return 0; + } + + while (*line == ' ') line++; + + flags = 0; + if (uidlist->version > 1) { + while (*line != ' ') { + switch (*line) { + case 'N': + flags |= MAILDIR_UIDLIST_REC_FLAG_NEW_DIR; + break; + } + line++; + } + while (*line == ' ') line++; + } else { + /* old version, have to assume it's in new dir since we + don't know */ + flags |= MAILDIR_UIDLIST_REC_FLAG_NEW_DIR; + } + + if (hash_lookup(uidlist->files, line) != NULL) { + mail_storage_set_critical(uidlist->ibox->box.storage, + "Duplicate file in uidlist file %s: %s", + uidlist->fname, line); + return 0; + } + + rec = buffer_append_space_unsafe(uidlist->record_buf, sizeof(*rec)); + rec->uid = uid; + rec->flags = flags; + rec->filename = p_strdup(uidlist->filename_pool, line); + hash_insert(uidlist->files, rec->filename, rec); + return 1; +} + +int maildir_uidlist_update(struct maildir_uidlist *uidlist) +{ + struct mail_storage *storage = uidlist->ibox->box.storage; + const char *line; + struct istream *input; + struct stat st; + int fd, ret; + + if (uidlist->last_mtime != 0) { + if (stat(uidlist->fname, &st) < 0) { + if (errno != ENOENT) { + mail_storage_set_critical(storage, + "stat(%s) failed: %m", uidlist->fname); + return -1; + } + return 0; + } + + if (st.st_mtime == uidlist->last_mtime) { + /* unchanged */ + return 1; + } + } + + fd = open(uidlist->fname, O_RDONLY); + if (fd == -1) { + if (errno != ENOENT) { + mail_storage_set_critical(storage, + "open(%s) failed: %m", uidlist->fname); + return -1; + } + return 0; + } + + if (fstat(fd, &st) < 0) { + mail_storage_set_critical(storage, + "fstat(%s) failed: %m", uidlist->fname); + return -1; + } + + if (uidlist->filename_pool != NULL) + pool_unref(uidlist->filename_pool); + uidlist->filename_pool = + pool_alloconly_create("uidlist filename_pool", + nearest_power(st.st_size - + st.st_size/8)); + buffer_set_used_size(uidlist->record_buf, 0); + uidlist->version = 0; + + input = i_stream_create_file(fd, default_pool, 4096, TRUE); + + /* get header */ + line = i_stream_read_next_line(input); + if (line == NULL || sscanf(line, "%u %u %u", &uidlist->version, + &uidlist->uid_validity, + &uidlist->next_uid) != 3 || + uidlist->version < 1 || uidlist->version > 2) { + /* broken file */ + mail_storage_set_critical(storage, + "Corrupted header in file %s (version = %u)", + uidlist->fname, uidlist->version); + ret = 0; + } else { + ret = 1; + while ((line = i_stream_read_next_line(input)) != NULL) { + if (!maildir_uidlist_next(uidlist, line)) { + ret = 0; + break; + } + } + } + + if (ret != 0) + uidlist->last_mtime = st.st_mtime; + else { + (void)unlink(uidlist->fname); + uidlist->last_mtime = 0; + } + + i_stream_unref(input); + return ret; +} + +const char *maildir_uidlist_lookup(struct maildir_uidlist *uidlist, + uint32_t uid, int *new_dir_r) +{ + const struct maildir_uidlist_rec *rec; + unsigned int idx, left_idx, right_idx; + size_t size; + + i_assert(uidlist->last_mtime != 0); + + rec = buffer_get_data(uidlist->record_buf, &size); + size /= sizeof(*rec); + + idx = 0; + left_idx = 0; + right_idx = size; + + while (left_idx < right_idx) { + idx = (left_idx + right_idx) / 2; + + if (rec[idx].uid < uid) + left_idx = idx+1; + else if (rec[idx].uid > uid) + right_idx = idx; + else { + *new_dir_r = (rec[idx].flags & + MAILDIR_UIDLIST_REC_FLAG_NEW_DIR) != 0; + return rec[idx].filename; + } + } + + return NULL; +} + +static int maildir_uidlist_rewrite_fd(struct maildir_uidlist *uidlist, + const char *temp_path) +{ + struct mail_storage *storage = uidlist->ibox->box.storage; + struct maildir_uidlist_iter_ctx *iter; + struct utimbuf ut; + string_t *str; + uint32_t uid, flags; + const char *filename, *flags_str; + int ret = 0; + + uidlist->version = 2; + + str = t_str_new(4096); + str_printfa(str, "%u %u %u\n", uidlist->version, + uidlist->uid_validity, uidlist->next_uid); + + iter = maildir_uidlist_iter_init(uidlist->ibox->uidlist); + while (maildir_uidlist_iter_next(iter, &uid, &flags, &filename)) { + if (str_len(str) + MAX_INT_STRLEN + + strlen(filename) + 2 >= 4096) { + /* flush buffer */ + if (write_full(uidlist->lock_fd, + str_data(str), str_len(str)) < 0) { + mail_storage_set_critical(storage, + "write_full(%s) failed: %m", temp_path); + ret = -1; + break; + } + str_truncate(str, 0); + } + + flags_str = (flags & MAILDIR_UIDLIST_REC_FLAG_NEW_DIR) != 0 ? + "N" : "-"; + str_printfa(str, "%u %s %s\n", uid, flags_str, filename); + } + maildir_uidlist_iter_deinit(iter); + + if (ret < 0) + return -1; + + if (write_full(uidlist->lock_fd, str_data(str), str_len(str)) < 0) { + mail_storage_set_critical(storage, + "write_full(%s) failed: %m", temp_path); + return -1; + } + + /* uidlist's mtime must grow every time */ + uidlist->last_mtime = ioloop_time <= uidlist->last_mtime ? + uidlist->last_mtime + 1 : ioloop_time; + ut.actime = ioloop_time; + ut.modtime = uidlist->last_mtime; + if (utime(temp_path, &ut) < 0) { + mail_storage_set_critical(storage, + "utime(%s) failed: %m", temp_path); + return -1; + } + + if (fsync(uidlist->lock_fd) < 0) { + mail_storage_set_critical(storage, + "fsync(%s) failed: %m", temp_path); + return -1; + } + + return 0; +} + +static int maildir_uidlist_rewrite(struct maildir_uidlist *uidlist) +{ + struct index_mailbox *ibox = uidlist->ibox; + const char *temp_path, *db_path; + int ret; + + i_assert(UIDLIST_IS_LOCKED(uidlist)); + + temp_path = t_strconcat(ibox->control_dir, + "/" MAILDIR_UIDLIST_NAME ".lock", NULL); + ret = maildir_uidlist_rewrite_fd(uidlist, temp_path); + + if (ret == 0) { + db_path = t_strconcat(ibox->control_dir, + "/" MAILDIR_UIDLIST_NAME, NULL); + + if (file_dotlock_replace(db_path, uidlist->lock_fd, + FALSE) <= 0) { + mail_storage_set_critical(ibox->box.storage, + "file_dotlock_replace(%s) failed: %m", db_path); + ret = -1; + } + } else { + (void)close(uidlist->lock_fd); + } + uidlist->lock_fd = -1; + + if (ret < 0) + (void)unlink(temp_path); + return ret; +} + +struct maildir_uidlist_sync_ctx * +maildir_uidlist_sync_init(struct maildir_uidlist *uidlist) +{ + struct maildir_uidlist_sync_ctx *ctx; + + ctx = i_new(struct maildir_uidlist_sync_ctx, 1); + ctx->uidlist = uidlist; + ctx->filename_pool = + pool_alloconly_create("maildir_uidlist_sync", 16384); + ctx->files = hash_create(default_pool, ctx->filename_pool, 4096, + maildir_hash, maildir_cmp); + + if (uidlist->last_mtime == 0) { + /* uidlist not read yet, do it */ + if (maildir_uidlist_update(uidlist) < 0) + ctx->failed = TRUE; + } + return ctx; +} + +int maildir_uidlist_sync_next(struct maildir_uidlist_sync_ctx *ctx, + const char *filename, int new_dir) +{ + struct maildir_uidlist_rec *rec; + char *fname; + int ret; + + if (ctx->failed) + return -1; + + rec = hash_lookup(ctx->files, filename); + if (rec != NULL) { + if ((rec->flags & MAILDIR_UIDLIST_REC_FLAG_NEW_DIR) == 0) { + /* possibly duplicate */ + return 0; + } + + rec->flags &= ~MAILDIR_UIDLIST_REC_FLAG_NEW_DIR; + } else { + rec = hash_lookup(ctx->uidlist->files, filename); + if (rec == NULL && !ctx->synced) { + /* lock and update uidlist to see if it's just + been added */ + ret = maildir_uidlist_try_lock(ctx->uidlist); + if (ret <= 0) { + ctx->failed = TRUE; + return -1; + } + if (maildir_uidlist_update(ctx->uidlist) < 0) { + ctx->failed = TRUE; + return -1; + } + + ctx->synced = TRUE; + rec = hash_lookup(ctx->uidlist->files, filename); + } + + if (rec == NULL) { + ctx->new_files = TRUE; + rec = new_dir ? &ctx->new_rec : &ctx->cur_rec; + } + } + + fname = p_strdup(ctx->filename_pool, filename); + hash_insert(ctx->files, fname, rec); + return 1; +} + +static int maildir_time_cmp(const void *p1, const void *p2) +{ + const struct maildir_uidlist_rec *rec1 = p1, *rec2 = p2; + const char *s1 = rec1->filename, *s2 = rec2->filename; + time_t t1 = 0, t2 = 0; + + /* we have to do numeric comparision, strcmp() will break when + there's different amount of digits (mostly the 999999999 -> + 1000000000 change in Sep 9 2001) */ + while (*s1 >= '0' && *s1 <= '9') { + t1 = t1*10 + (*s1 - '0'); + s1++; + } + while (*s2 >= '0' && *s2 <= '9') { + t2 = t2*10 + (*s2 - '0'); + s2++; + } + + return t1 < t2 ? -1 : t1 > t2 ? 1 : 0; +} + +static void maildir_uidlist_swap(struct maildir_uidlist_sync_ctx *ctx) +{ + struct maildir_uidlist *uidlist = ctx->uidlist; + struct maildir_uidlist_rec *rec; + struct hash_iterate_context *iter; + void *key, *value; + size_t size; + unsigned int src, dest; + + rec = buffer_get_modifyable_data(uidlist->record_buf, &size); + size /= sizeof(*rec); + + /* update filename pointers, skip deleted messages */ + for (dest = src = 0; src < size; src++) { + if (hash_lookup_full(ctx->files, rec[src].filename, + &key, &value)) { + rec[dest].uid = rec[src].uid; + rec[dest].flags = rec[src].flags; + rec[dest].filename = key; + dest++; + } + } + buffer_set_used_size(uidlist->record_buf, dest * sizeof(*rec)); + + /* append new files */ + iter = hash_iterate_init(ctx->files); + while (hash_iterate(iter, &key, &value)) { + if (value == &ctx->new_rec || + value == &ctx->cur_rec) { + rec = buffer_append_space_unsafe(uidlist->record_buf, + sizeof(*rec)); + rec->flags = value == &ctx->cur_rec ? + 0 : MAILDIR_UIDLIST_REC_FLAG_NEW_DIR; + rec->filename = key; + hash_update(ctx->files, key, rec); + } + } + hash_iterate_deinit(iter); + + rec = buffer_get_modifyable_data(uidlist->record_buf, &size); + size /= sizeof(*rec); + + /* sort new files and assign UIDs for them */ + qsort(rec + dest, size - dest, sizeof(*rec), maildir_time_cmp); + for (; dest < size; dest++) + rec[dest].uid = uidlist->next_uid++; + + if (uidlist->filename_pool != NULL) + pool_unref(uidlist->filename_pool); + uidlist->filename_pool = ctx->filename_pool; + ctx->filename_pool = NULL; + + hash_destroy(uidlist->files); + uidlist->files = ctx->files; + ctx->files = NULL; +} + +int maildir_uidlist_sync_deinit(struct maildir_uidlist_sync_ctx *ctx) +{ + int ret; + + if (ctx->failed) + ret = -1; + else { + maildir_uidlist_swap(ctx); + if (!ctx->new_files) + ret = 0; + else + ret = maildir_uidlist_rewrite(ctx->uidlist); + } + + if (ctx->files != NULL) + hash_destroy(ctx->files); + if (ctx->filename_pool != NULL) + pool_unref(ctx->filename_pool); + i_free(ctx); + return ret; +} + +struct maildir_uidlist_iter_ctx * +maildir_uidlist_iter_init(struct maildir_uidlist *uidlist) +{ + struct maildir_uidlist_iter_ctx *ctx; + size_t size; + + ctx = i_new(struct maildir_uidlist_iter_ctx, 1); + ctx->next = buffer_get_data(uidlist->record_buf, &size); + size /= sizeof(*ctx->next); + ctx->end = ctx->next + size; + return ctx; +} + +int maildir_uidlist_iter_next(struct maildir_uidlist_iter_ctx *ctx, + uint32_t *uid_r, uint32_t *flags_r, + const char **filename_r) +{ + if (ctx->next == ctx->end) + return 0; + + *uid_r = ctx->next->uid; + *flags_r = ctx->next->flags; + *filename_r = ctx->next->filename; + ctx->next++; + return 1; +} + +void maildir_uidlist_iter_deinit(struct maildir_uidlist_iter_ctx *ctx) +{ + i_free(ctx); +}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-storage/index/maildir/maildir-uidlist.h Tue Apr 27 23:25:52 2004 +0300 @@ -0,0 +1,34 @@ +#ifndef __MAILDIR_UIDLIST_H +#define __MAILDIR_UIDLIST_H + +#define MAILDIR_UIDLIST_NAME "dovecot-uidlist" + +int maildir_uidlist_try_lock(struct maildir_uidlist *uidlist); +void maildir_uidlist_unlock(struct maildir_uidlist *uidlist); + +struct maildir_uidlist *maildir_uidlist_init(struct index_mailbox *ibox); +void maildir_uidlist_deinit(struct maildir_uidlist *uidlist); + +/* Returns -1 if error, 0 if file is broken or lost, 1 if ok. */ +int maildir_uidlist_update(struct maildir_uidlist *uidlist); + +/* Returns uidlist record for given filename, or NULL if not found. */ +const char *maildir_uidlist_lookup(struct maildir_uidlist *uidlist, + uint32_t uid, int *new_dir_r); + +/* Sync uidlist with what's actually on maildir. */ +struct maildir_uidlist_sync_ctx * +maildir_uidlist_sync_init(struct maildir_uidlist *uidlist); +int maildir_uidlist_sync_next(struct maildir_uidlist_sync_ctx *ctx, + const char *filename, int new_dir); +int maildir_uidlist_sync_deinit(struct maildir_uidlist_sync_ctx *ctx); + +/* List all maildir files. */ +struct maildir_uidlist_iter_ctx * +maildir_uidlist_iter_init(struct maildir_uidlist *uidlist); +int maildir_uidlist_iter_next(struct maildir_uidlist_iter_ctx *ctx, + uint32_t *uid_r, uint32_t *flags_r, + const char **filename_r); +void maildir_uidlist_iter_deinit(struct maildir_uidlist_iter_ctx *ctx); + +#endif
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-storage/index/maildir/maildir-util.c Tue Apr 27 23:25:52 2004 +0300 @@ -0,0 +1,287 @@ +/* Copyright (C) 2004 Timo Sirainen */ + +#include "lib.h" +#include "hostpid.h" +#include "ioloop.h" +#include "str.h" +#include "maildir-storage.h" +#include "maildir-uidlist.h" + +#include <unistd.h> +#include <fcntl.h> +#include <sys/stat.h> + +static int maildir_file_do_try(struct index_mailbox *ibox, uint32_t uid, + maildir_file_do_func *func, void *context) +{ + const char *fname, *path; + int ret, new_dir; + + fname = maildir_uidlist_lookup(ibox->uidlist, uid, &new_dir); + if (fname == NULL) + return -2; /* expunged */ + + if (new_dir) { + /* probably in new/ dir */ + path = t_strconcat(ibox->path, "/new/", fname, NULL); + ret = func(ibox, path, context); + if (ret != 0) + return ret; + } + + path = t_strconcat(ibox->path, "/cur/", fname, NULL); + return func(ibox, path, context); +} + +int maildir_file_do(struct index_mailbox *ibox, uint32_t uid, + maildir_file_do_func *func, void *context) +{ + int i, ret; + + ret = maildir_file_do_try(ibox, uid, func, context); + for (i = 0; i < 10 && ret == 0; i++) { + /* file is either renamed or deleted. sync the maildir and + see which one. if file appears to be renamed constantly, + don't try to open it more than 10 times. */ + if (maildir_storage_sync_readonly(ibox) < 0) + return -1; + + ret = maildir_file_do_try(ibox, uid, func, context); + } + + return ret == -2 ? 0 : ret; +} + +int maildir_filename_get_flags(const char *fname, enum mail_flags *flags_r, + custom_flags_mask_t custom_flags_r) +{ + const char *info; + unsigned int num; + + *flags_r = 0; + memset(custom_flags_r, 0, INDEX_CUSTOM_FLAGS_BYTE_COUNT); + + info = strchr(fname, ':'); + if (info == NULL || info[1] != '2' || info[2] != ',') + return 0; + + for (info += 3; *info != '\0' && *info != ','; info++) { + switch (*info) { + case 'R': /* replied */ + *flags_r |= MAIL_ANSWERED; + break; + case 'S': /* seen */ + *flags_r |= MAIL_SEEN; + break; + case 'T': /* trashed */ + *flags_r |= MAIL_DELETED; + break; + case 'D': /* draft */ + *flags_r |= MAIL_DRAFT; + break; + case 'F': /* flagged */ + *flags_r |= MAIL_FLAGGED; + break; + default: + if (*info >= 'a' && *info <= 'z') { + /* custom flag */ + num = (*info - 'a'); + custom_flags_r[num / CHAR_BIT] |= + num % CHAR_BIT; + break; + } + + /* unknown flag - ignore */ + break; + } + } + + return 1; +} + +const char *maildir_filename_set_flags(const char *fname, enum mail_flags flags, + custom_flags_mask_t custom_flags) +{ + string_t *flags_str; + const char *info, *oldflags; + int i, nextflag; + + if (custom_flags != NULL) { + /* see if any custom flags are given */ + for (i = 0; i < INDEX_CUSTOM_FLAGS_BYTE_COUNT; i++) { + if (custom_flags[i] != 0) + break; + } + if (i == INDEX_CUSTOM_FLAGS_BYTE_COUNT) + custom_flags = NULL; + } + + /* remove the old :info from file name, and get the old flags */ + info = strrchr(fname, ':'); + if (info != NULL && strrchr(fname, '/') > info) + info = NULL; + + oldflags = ""; + if (info != NULL) { + fname = t_strdup_until(fname, info); + if (info[1] == '2' && info[2] == ',') + oldflags = info+3; + } + + /* insert the new flags between old flags. flags must be sorted by + their ASCII code. unknown flags are kept. */ + flags_str = t_str_new(256); + str_append(flags_str, fname); + str_append(flags_str, ":2,"); + for (;;) { + /* skip all known flags */ + while (*oldflags == 'D' || *oldflags == 'F' || + *oldflags == 'R' || *oldflags == 'S' || + *oldflags == 'T' || + (*oldflags >= 'a' && *oldflags <= 'z')) + oldflags++; + + nextflag = *oldflags == '\0' || *oldflags == ',' ? 256 : + (unsigned char) *oldflags; + + if ((flags & MAIL_DRAFT) && nextflag > 'D') { + str_append_c(flags_str, 'D'); + flags &= ~MAIL_DRAFT; + } + if ((flags & MAIL_FLAGGED) && nextflag > 'F') { + str_append_c(flags_str, 'F'); + flags &= ~MAIL_FLAGGED; + } + if ((flags & MAIL_ANSWERED) && nextflag > 'R') { + str_append_c(flags_str, 'R'); + flags &= ~MAIL_ANSWERED; + } + if ((flags & MAIL_SEEN) && nextflag > 'S') { + str_append_c(flags_str, 'S'); + flags &= ~MAIL_SEEN; + } + if ((flags & MAIL_DELETED) && nextflag > 'T') { + str_append_c(flags_str, 'T'); + flags &= ~MAIL_DELETED; + } + + if (custom_flags != NULL && nextflag > 'a') { + for (i = 0; i < INDEX_CUSTOM_FLAGS_COUNT; i++) { + if ((custom_flags[i / CHAR_BIT] & + (1 << (i % CHAR_BIT))) != 0) + str_append_c(flags_str, 'a' + i); + } + custom_flags = NULL; + } + + if (*oldflags == '\0' || *oldflags == ',') + break; + + str_append_c(flags_str, *oldflags); + oldflags++; + } + + if (*oldflags == ',') { + /* another flagset, we don't know about these, just keep them */ + while (*oldflags != '\0') + str_append_c(flags_str, *oldflags++); + } + + return str_c(flags_str); +} + +const char *maildir_generate_tmp_filename(const struct timeval *tv) +{ + static unsigned int create_count = 0; + static time_t first_stamp = 0; + + if (first_stamp == 0 || first_stamp == ioloop_time) { + /* it's possible that within last second another process had + the same UID as us. Use usecs to make sure we don't create + duplicate base name. */ + first_stamp = ioloop_time; + return t_strdup_printf("%s.P%sQ%uM%s.%s", + dec2str(tv->tv_sec), my_pid, + create_count++, + dec2str(tv->tv_usec), my_hostname); + } else { + /* Don't bother with usecs. Saves a bit space :) */ + return t_strdup_printf("%s.P%sQ%u.%s", + dec2str(tv->tv_sec), my_pid, + create_count++, my_hostname); + } +} + +int maildir_create_tmp(struct index_mailbox *ibox, const char *dir, + mode_t mode, const char **fname_r) +{ + const char *path, *tmp_fname; + struct stat st; + struct timeval *tv, tv_now; + pool_t pool; + int fd; + + tv = &ioloop_timeval; + pool = pool_alloconly_create("maildir_tmp", 4096); + for (;;) { + p_clear(pool); + tmp_fname = maildir_generate_tmp_filename(tv); + + path = p_strconcat(pool, dir, "/", tmp_fname, NULL); + if (stat(path, &st) < 0 && errno == ENOENT) { + /* doesn't exist */ + mode_t old_mask = umask(0); + fd = open(path, O_WRONLY | O_CREAT | O_EXCL, mode); + umask(old_mask); + if (fd != -1 || errno != EEXIST) + break; + } + + /* wait and try again - very unlikely */ + sleep(2); + tv = &tv_now; + if (gettimeofday(&tv_now, NULL) < 0) + i_fatal("gettimeofday(): %m"); + } + + *fname_r = t_strdup(path); + if (fd == -1) { + mail_storage_set_critical(ibox->box.storage, + "open(%s) failed: %m", path); + } + + pool_unref(pool); + return fd; +} + +/* a char* hash function from ASU -- from glib */ +unsigned int maildir_hash(const void *p) +{ + const unsigned char *s = p; + unsigned int g, h = 0; + + while (*s != ':' && *s != '\0') { + h = (h << 4) + *s; + if ((g = h & 0xf0000000UL)) { + h = h ^ (g >> 24); + h = h ^ g; + } + + s++; + } + + return h; +} + +int maildir_cmp(const void *p1, const void *p2) +{ + const char *s1 = p1, *s2 = p2; + + while (*s1 == *s2 && *s1 != ':' && *s1 != '\0') { + s1++; s2++; + } + if ((*s1 == '\0' || *s1 == ':') && + (*s2 == '\0' || *s2 == ':')) + return 0; + return *s1 - *s2; +}
--- a/src/lib-storage/index/mbox/Makefile.am Tue Apr 27 23:14:15 2004 +0300 +++ b/src/lib-storage/index/mbox/Makefile.am Tue Apr 27 23:25:52 2004 +0300 @@ -10,10 +10,19 @@ -I$(top_srcdir)/src/lib-storage/index libstorage_mbox_a_SOURCES = \ + istream-raw-mbox.c \ mbox-expunge.c \ + mbox-from.c \ mbox-list.c \ mbox-save.c \ + mbox-sync-parse.c \ + mbox-sync-rewrite.c \ + mbox-sync-update.c \ + mbox-sync.c mbox-storage.c noinst_HEADERS = \ - mbox-storage.h + istream-raw-mbox.h \ + mbox-from.h \ + mbox-storage.h \ + mbox-sync-private.h
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-storage/index/mbox/istream-raw-mbox.c Tue Apr 27 23:25:52 2004 +0300 @@ -0,0 +1,279 @@ +/* Copyright (C) 2003 Timo Sirainen */ + +#include "lib.h" +#include "buffer.h" +#include "istream-internal.h" +#include "istream-raw-mbox.h" +#include "mbox-from.h" + +struct raw_mbox_istream { + struct _istream istream; + + time_t received_time, next_received_time; + uoff_t from_offset, body_size; + struct istream *input; +}; + +static void _close(struct _iostream *stream __attr_unused__) +{ +} + +static void _destroy(struct _iostream *stream) +{ + struct raw_mbox_istream *rstream = (struct raw_mbox_istream *)stream; + + i_stream_seek(rstream->input, rstream->istream.istream.v_offset); + i_stream_unref(rstream->input); +} + +static void _set_max_buffer_size(struct _iostream *stream, size_t max_size) +{ + struct raw_mbox_istream *rstream = (struct raw_mbox_istream *)stream; + + i_stream_set_max_buffer_size(rstream->input, max_size); +} + +static void _set_blocking(struct _iostream *stream, int timeout_msecs, + void (*timeout_cb)(void *), void *context) +{ + struct raw_mbox_istream *rstream = (struct raw_mbox_istream *)stream; + + i_stream_set_blocking(rstream->input, timeout_msecs, + timeout_cb, context); +} + +static ssize_t _read(struct _istream *stream) +{ + static const char *mbox_from = "\nFrom "; + struct raw_mbox_istream *rstream = (struct raw_mbox_istream *)stream; + const unsigned char *buf, *p; + const char *fromp; + time_t received_time; + size_t i, pos; + ssize_t ret; + + i_stream_seek(rstream->input, stream->istream.v_offset); + + stream->pos -= stream->skip; + stream->skip = 0; + stream->buffer = NULL; + + do { + ret = i_stream_read(rstream->input); + buf = i_stream_get_data(rstream->input, &pos); + } while (ret > 0 && pos <= 6); + + if (pos == 1 && buf[0] == '\n') { + /* EOF */ + stream->pos = 0; + stream->istream.eof = TRUE; + return -1; + } + + if (stream->istream.v_offset == rstream->from_offset) { + /* read the full From-line */ + int skip = rstream->from_offset != 0; + size_t line_pos; + + while ((p = memchr(buf+skip, '\n', pos-skip)) == NULL) { + if (i_stream_read(rstream->input) < 0) { + /* EOF - shouldn't happen */ + stream->pos = 0; + stream->istream.eof = TRUE; + return -1; + } + buf = i_stream_get_data(rstream->input, &pos); + } + line_pos = (size_t)(p - buf); + + if (rstream->from_offset != 0) { + buf++; + pos--; + } + + /* beginning of mbox */ + if (memcmp(buf, "From ", 5) != 0) + received_time = (time_t)-1; + else + received_time = mbox_from_parse_date(buf+5, pos-5); + + if (received_time == (time_t)-1) { + /* broken From - should happen only at beginning of + file if this isn't a mbox.. */ + stream->pos = 0; + stream->istream.eof = TRUE; + return -1; + } + + if (rstream->from_offset == 0) + rstream->received_time = received_time; + else + rstream->next_received_time = received_time; + + /* we'll skip over From-line and try again */ + stream->istream.v_offset += line_pos+1; + return _read(stream); + } + + if (pos >= 31) { + if (memcmp(buf, "\nFrom ", 6) == 0) { + received_time = mbox_from_parse_date(buf+6, pos-6); + if (received_time != (time_t)-1) { + rstream->next_received_time = received_time; + i_assert(stream->pos == 0); + return -1; + } + } + } else if (ret == -1) { + /* last few bytes, can't contain From-line */ + ret = pos <= stream->pos ? -1 : + (ssize_t) (pos - stream->pos); + + stream->buffer = buf; + stream->pos = pos; + stream->istream.eof = ret == -1; + return ret; + } + + /* See if we have From-line here - note that it works right only + because all characters are different in mbox_from. */ + for (i = 0, fromp = mbox_from; i < pos; i++) { + if (buf[i] == *fromp) { + if (*++fromp == '\0') { + /* potential From-line - stop here */ + i++; + break; + } + } else { + fromp = mbox_from; + if (buf[i] == *fromp) + fromp++; + } + } + pos = i - (fromp - mbox_from); + + ret = pos <= stream->pos ? -1 : + (ssize_t) (pos - stream->pos); + stream->buffer = buf; + stream->pos = pos; + return ret; +} + +static void _seek(struct _istream *stream, uoff_t v_offset) +{ + stream->istream.v_offset = v_offset; + stream->skip = stream->pos = 0; + stream->buffer = NULL; +} + +struct istream *i_stream_create_raw_mbox(pool_t pool, struct istream *input) +{ + struct raw_mbox_istream *rstream; + + i_stream_ref(input); + + rstream = p_new(pool, struct raw_mbox_istream, 1); + + rstream->input = input; + rstream->body_size = (uoff_t)-1; + + rstream->istream.iostream.close = _close; + rstream->istream.iostream.destroy = _destroy; + rstream->istream.iostream.set_max_buffer_size = _set_max_buffer_size; + rstream->istream.iostream.set_blocking = _set_blocking; + + rstream->istream.read = _read; + rstream->istream.seek = _seek; + + return _i_stream_create(&rstream->istream, pool, -1, + input->real_stream->abs_start_offset); +} + +static int istream_raw_mbox_is_valid_from(struct raw_mbox_istream *rstream) +{ + const unsigned char *data; + size_t size; + time_t received_time; + + /* minimal: "From x Thu Nov 29 22:33:52 2001" = 31 chars */ + if (i_stream_read_data(rstream->input, &data, &size, 30) == -1) + return -1; + + if (size == 1 && data[0] == '\n') { + /* EOF */ + return TRUE; + } + + if (size < 31 || memcmp(data, "\nFrom ", 6) != 0) + return FALSE; + + while (memchr(data+1, '\n', size-1) == NULL) { + if (i_stream_read_data(rstream->input, &data, &size, size) < 0) + break; + } + + received_time = mbox_from_parse_date(data+6, size-6); + if (received_time == (time_t)-1) + return FALSE; + + rstream->next_received_time = received_time; + return TRUE; +} + +uoff_t istream_raw_mbox_get_size(struct istream *stream, uoff_t body_size) +{ + struct raw_mbox_istream *rstream = + (struct raw_mbox_istream *)stream->real_stream; + uoff_t old_offset; + const unsigned char *data; + size_t size; + + if (rstream->body_size != (uoff_t)-1) + return rstream->body_size; + + if (body_size != (uoff_t)-1) { + i_stream_seek(rstream->input, rstream->from_offset + body_size); + if (istream_raw_mbox_is_valid_from(rstream) > 0) { + rstream->body_size = body_size; + return body_size; + } + } + + old_offset = stream->v_offset; + + /* have to read through the message body */ + while (i_stream_read_data(stream, &data, &size, 0) > 0) + i_stream_skip(stream, size); + + rstream->body_size = stream->v_offset - old_offset; + i_stream_seek(stream, old_offset); + return rstream->body_size; +} + +void istream_raw_mbox_next(struct istream *stream, uoff_t body_size) +{ + struct raw_mbox_istream *rstream = + (struct raw_mbox_istream *)stream->real_stream; + + body_size = istream_raw_mbox_get_size(stream, body_size); + rstream->body_size = (uoff_t)-1; + + rstream->received_time = rstream->next_received_time; + rstream->next_received_time = (time_t)-1; + + rstream->from_offset = stream->v_offset + body_size; + i_stream_seek(rstream->input, rstream->from_offset); +} + +void istream_raw_mbox_flush(struct istream *stream) +{ + struct raw_mbox_istream *rstream = + (struct raw_mbox_istream *)stream->real_stream; + + /* kludgy */ + rstream->input->real_stream->skip = 0; + rstream->input->real_stream->pos = 0; + + rstream->istream.skip = 0; + rstream->istream.pos = 0; +}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-storage/index/mbox/istream-raw-mbox.h Tue Apr 27 23:25:52 2004 +0300 @@ -0,0 +1,20 @@ +#ifndef __ISTREAM_RAW_MBOX_H +#define __ISTREAM_RAW_MBOX_H + +/* Create a mbox stream for parsing mbox. Reading stops before From-line, + you'll have to call istream_raw_mbox_next() to get to next message. */ +struct istream *i_stream_create_raw_mbox(pool_t pool, struct istream *input); + +/* Return number of bytes in this message after current offset. + If body_size isn't (uoff_t)-1, we'll use it as potentially valid body size + to avoid actually reading through the whole message. */ +uoff_t istream_raw_mbox_get_size(struct istream *stream, uoff_t body_size); + +/* Jump to next message. If body_size isn't (uoff_t)-1, we'll use it as + potentially valid body size. */ +void istream_raw_mbox_next(struct istream *stream, uoff_t body_size); + +/* Flush all buffering. Call if you modify the mbox. */ +void istream_raw_mbox_flush(struct istream *stream); + +#endif
--- a/src/lib-storage/index/mbox/mbox-expunge.c Tue Apr 27 23:14:15 2004 +0300 +++ b/src/lib-storage/index/mbox/mbox-expunge.c Tue Apr 27 23:25:52 2004 +0300 @@ -1,5 +1,6 @@ /* Copyright (C) 2002-2003 Timo Sirainen */ +#if 0 #include "lib.h" #include "istream.h" #include "ostream.h" @@ -210,3 +211,4 @@ return index_storage_expunge(mail, ctx->ctx, seq_r, notify); } +#endif
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-storage/index/mbox/mbox-from.c Tue Apr 27 23:25:52 2004 +0300 @@ -0,0 +1,197 @@ +/* Copyright (C) 2002 Timo Sirainen */ + +#include "lib.h" +#include "str.h" +#include "utc-mktime.h" +#include "mbox-from.h" + +#include <time.h> +#include <ctype.h> + +static const char *weekdays[] = { + "Sun", "Mon", "Tue", "Wed", "Thu", "Fri", "Sat" +}; + +static const char *months[] = { + "Jan", "Feb", "Mar", "Apr", "May", "Jun", + "Jul", "Aug", "Sep", "Oct", "Nov", "Dec" +}; + +time_t mbox_from_parse_date(const unsigned char *msg, size_t size) +{ + const unsigned char *msg_end; + struct tm tm; + int i, timezone; + time_t t; + + /* <sender> <date> <moreinfo> */ + msg_end = msg + size; + + /* skip sender */ + while (msg < msg_end && *msg != ' ') { + if (*msg == '\r' || *msg == '\n') + return (time_t)-1; + msg++; + } + while (msg < msg_end && *msg == ' ') msg++; + + /* next 24 chars should be in the date in asctime() format, eg. + "Thu Nov 29 22:33:52 2001 +0300" + + Some also include named timezone, which we ignore: + + "Thu Nov 29 22:33:52 EEST 2001" + */ + if (msg+24 > msg_end) + return (time_t)-1; + + memset(&tm, 0, sizeof(tm)); + + /* skip weekday */ + msg += 4; + + /* month */ + for (i = 0; i < 12; i++) { + if (memcasecmp(months[i], msg, 3) == 0) { + tm.tm_mon = i; + break; + } + } + + if (i == 12 && memcmp(msg, "???", 3) == 0) { + /* just a hack to parse one special mbox I have :) */ + i = 0; + } + + if (i == 12 || msg[3] != ' ') + return (time_t)-1; + msg += 4; + + /* day */ + if (msg[0] == ' ') { + if (!i_isdigit(msg[1]) || msg[2] != ' ') + return (time_t)-1; + tm.tm_mday = msg[1]-'0'; + } else { + if (!i_isdigit(msg[0]) || !i_isdigit(msg[1]) || msg[2] != ' ') + return (time_t)-1; + tm.tm_mday = (msg[0]-'0') * 10 + (msg[1]-'0'); + } + if (tm.tm_mday == 0) + tm.tm_mday = 1; + msg += 3; + + /* hour */ + if (!i_isdigit(msg[0]) || !i_isdigit(msg[1]) || msg[2] != ':') + return (time_t)-1; + tm.tm_hour = (msg[0]-'0') * 10 + (msg[1]-'0'); + msg += 3; + + /* minute */ + if (!i_isdigit(msg[0]) || !i_isdigit(msg[1]) || msg[2] != ':') + return (time_t)-1; + tm.tm_min = (msg[0]-'0') * 10 + (msg[1]-'0'); + msg += 3; + + /* second */ + if (!i_isdigit(msg[0]) || !i_isdigit(msg[1]) || msg[2] != ' ') + return (time_t)-1; + tm.tm_sec = (msg[0]-'0') * 10 + (msg[1]-'0'); + msg += 3; + + /* optional named timezone */ + if (!i_isdigit(msg[0]) || !i_isdigit(msg[1]) || + !i_isdigit(msg[2]) || !i_isdigit(msg[3])) { + /* skip to next space */ + while (msg < msg_end && *msg != ' ') { + if (*msg == '\r' || *msg == '\n') + return (time_t)-1; + msg++; + } + if (msg+5 > msg_end) + return (time_t)-1; + msg++; + } + + /* year */ + if (!i_isdigit(msg[0]) || !i_isdigit(msg[1]) || + !i_isdigit(msg[2]) || !i_isdigit(msg[3])) + return (time_t)-1; + + tm.tm_year = (msg[0]-'0') * 1000 + (msg[1]-'0') * 100 + + (msg[2]-'0') * 10 + (msg[3]-'0') - 1900; + msg += 4; + + tm.tm_isdst = -1; + if (msg[0] == ' ' && (msg[1] == '-' || msg[1] == '+') && + i_isdigit(msg[2]) && i_isdigit(msg[3]) && + i_isdigit(msg[4]) && i_isdigit(msg[5])) { + timezone = (msg[2]-'0') * 1000 + (msg[3]-'0') * 100 + + (msg[4]-'0') * 10 +(msg[5]-'0'); + if (msg[1] == '-') timezone = -timezone; + + t = utc_mktime(&tm); + if (t == (time_t)-1) + return (time_t)-1; + + t -= timezone * 60; + return t; + } else { + /* assume local timezone */ + return mktime(&tm); + } +} + +const char *mbox_from_create(const char *sender, time_t time) +{ + string_t *str; + struct tm *tm; + int year; + + str = t_str_new(256); + str_append(str, "From "); + str_append(str, sender); + str_append(str, " "); + + /* we could use simply asctime(), but i18n etc. may break it. + Example: "Thu Nov 29 22:33:52 2001" */ + tm = localtime(&time); + + /* week day */ + str_append(str, weekdays[tm->tm_wday]); + str_append_c(str, ' '); + + /* month */ + str_append(str, months[tm->tm_mon]); + str_append_c(str, ' '); + + /* day */ + str_append_c(str, (tm->tm_mday / 10) + '0'); + str_append_c(str, (tm->tm_mday % 10) + '0'); + str_append_c(str, ' '); + + /* hour */ + str_append_c(str, (tm->tm_hour / 10) + '0'); + str_append_c(str, (tm->tm_hour % 10) + '0'); + str_append_c(str, ':'); + + /* minute */ + str_append_c(str, (tm->tm_min / 10) + '0'); + str_append_c(str, (tm->tm_min % 10) + '0'); + str_append_c(str, ':'); + + /* second */ + str_append_c(str, (tm->tm_sec / 10) + '0'); + str_append_c(str, (tm->tm_sec % 10) + '0'); + str_append_c(str, ' '); + + /* year */ + year = tm->tm_year + 1900; + str_append_c(str, (year / 1000) + '0'); + str_append_c(str, ((year / 100) % 10) + '0'); + str_append_c(str, ((year / 10) % 10) + '0'); + str_append_c(str, (year % 10) + '0'); + + str_append_c(str, '\n'); + return str_c(str); +}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-storage/index/mbox/mbox-from.h Tue Apr 27 23:25:52 2004 +0300 @@ -0,0 +1,7 @@ +#ifndef __MBOX_FROM_H +#define __MBOX_FROM_H + +time_t mbox_from_parse_date(const unsigned char *msg, size_t size); +const char *mbox_from_create(const char *sender, time_t time); + +#endif
--- a/src/lib-storage/index/mbox/mbox-list.c Tue Apr 27 23:14:15 2004 +0300 +++ b/src/lib-storage/index/mbox/mbox-list.c Tue Apr 27 23:25:52 2004 +0300 @@ -1,5 +1,6 @@ /* Copyright (C) 2002-2003 Timo Sirainen */ +#if 0 #include "lib.h" #include "unlink-directory.h" #include "imap-match.h" @@ -453,3 +454,4 @@ { return NULL; } +#endif
--- a/src/lib-storage/index/mbox/mbox-save.c Tue Apr 27 23:14:15 2004 +0300 +++ b/src/lib-storage/index/mbox/mbox-save.c Tue Apr 27 23:25:52 2004 +0300 @@ -1,5 +1,6 @@ /* Copyright (C) 2002 Timo Sirainen */ +#if 0 #include "lib.h" #include "hostpid.h" #include "ostream.h" @@ -371,3 +372,4 @@ i_free(ctx); return !failed; } +#endif
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-storage/index/mbox/mbox-sync-header.c Tue Apr 27 23:25:52 2004 +0300 @@ -0,0 +1,491 @@ +/* Copyright (C) 2004 Timo Sirainen */ + +#include "lib.h" +#include "buffer.h" +#include "istream.h" +#include "str.h" +#include "write-full.h" +#include "message-parser.h" +#include "mail-index.h" +#include "mbox-sync-private.h" + +#include <unistd.h> + +struct mbox_flag_type { + char chr; + enum mail_flags flag; +}; + +#define MBOX_NONRECENT MAIL_RECENT /* kludgy */ + +#define STATUS_FLAGS_MASK (MAIL_SEEN|MBOX_NONRECENT) +static struct mbox_flag_type status_flags[] = { + { 'R', MAIL_SEEN }, + { 'O', MBOX_NONRECENT }, + { 0, 0 } +}; + +#define XSTATUS_FLAGS_MASK (MAIL_ANSWERED|MAIL_FLAGGED|MAIL_DRAFT|MAIL_DELETED) +static struct mbox_flag_type xstatus_flags[] = { + { 'A', MAIL_ANSWERED }, + { 'F', MAIL_FLAGGED }, + { 'T', MAIL_DRAFT }, + { 'D', MAIL_DELETED }, + { 0, 0 } +}; + +struct header_func { + const char *header; + int (*func)(struct mbox_sync_mail_context *ctx, + struct message_header_line *hdr); +}; + +static enum mail_flags mbox_flag_find(struct mbox_flag_type *flags, char chr) +{ + int i; + + for (i = 0; flags[i].chr != 0; i++) { + if (flags[i].chr == chr) + return flags[i].flag; + } + + return 0; +} + +static void status_flags_append(struct mbox_sync_mail_context *ctx, + struct mbox_flag_type *flags_list) +{ + int i; + + for (i = 0; flags_list[i].chr != 0; i++) { + if ((ctx->mail_flags & flags_list[i].flag) != 0) { + str_append_c(ctx->header, flags_list[i].chr); + ctx->mail_flags &= ~flags_list[i].flag; + } + } +} + +static int parse_status_flags(struct mbox_sync_mail_context *ctx, + struct message_header_line *hdr, + struct mbox_flag_type *flags_list) +{ + size_t i, start, end; + int j; + enum mail_flags flags, flags_mask; + + flags = 0; + for (i = 0; i < hdr->full_value_len; i++) + flags |= mbox_flag_find(flags_list, hdr->full_value[i]); + + flags_mask = 0; + for (j = 0; flags_list[j].chr != 0; j++) + flags_mask |= flags_list[j].flag; + + /* see if anything changed */ + if (flags == (ctx->mail_flags & flags_mask)) { + ctx->mail_flags &= ~flags_mask; + return FALSE; + } + + start = str_len(ctx->header); + str_append(ctx->header, hdr->name); + str_append(ctx->header, ": "); + end = str_len(ctx->header); + + status_flags_append(ctx, flags_list); + + for (i = 0; i < hdr->full_value_len; i++) { + switch (hdr->full_value[i]) { + case ' ': + case '\t': + case '\r': + case '\n': + break; + default: + if (mbox_flag_find(flags_list, hdr->full_value[i]) != 0) + break; + + /* unknown, keep it */ + str_append_c(ctx->header, hdr->full_value[i]); + break; + } + } + + if (str_len(ctx->header) != end) + str_append_c(ctx->header, '\n'); + else + str_truncate(ctx->header, start); + return TRUE; +} + +static int parse_status(struct mbox_sync_mail_context *ctx, + struct message_header_line *hdr) +{ + return parse_status_flags(ctx, hdr, status_flags); +} + +static int parse_x_status(struct mbox_sync_mail_context *ctx, + struct message_header_line *hdr) +{ + return parse_status_flags(ctx, hdr, xstatus_flags); +} + +static int parse_x_imap_base(struct mbox_sync_mail_context *ctx, + struct message_header_line *hdr) +{ + ctx->ximapbase_pos = buffer_get_used_size(ctx->header); + // FIXME: check it + //ctx->extra_space += 1; + return FALSE; +} + +static int parse_x_keywords(struct mbox_sync_mail_context *ctx, + struct message_header_line *hdr) +{ + ctx->xkeywords_pos = buffer_get_used_size(ctx->header); + // FIXME: update it + //ctx->extra_space += 1; + return FALSE; +} + +static int parse_content_length(struct mbox_sync_mail_context *ctx, + struct message_header_line *hdr) +{ + uoff_t value = 0; + size_t i; + + if (ctx->content_length != (uoff_t)-1) { + /* duplicate */ + return TRUE; + } + + for (i = 0; i < hdr->full_value_len; i++) { + if (hdr->full_value[i] < '0' || hdr->full_value[i] > '9') + break; + value = value*10 + (hdr->full_value[i] - '0'); + } + + for (; i < hdr->full_value_len; i++) { + if (hdr->full_value[i] != ' ' && hdr->full_value[i] != '\t') { + /* broken value */ + return TRUE; + } + } + + ctx->content_length = value; + return FALSE; +} + +static int parse_x_uid(struct mbox_sync_mail_context *ctx, + struct message_header_line *hdr) +{ + uint32_t value = 0; + size_t i, extra_space = 0; + + if (ctx->uid != 0) { + /* duplicate */ + return TRUE; + } + + for (i = 0; i < hdr->full_value_len; i++) { + if (hdr->full_value[i] < '0' || hdr->full_value[i] > '9') + break; + value = value*10 + (hdr->full_value[i] - '0'); + } + + for (; i < hdr->full_value_len; i++) { + if (hdr->full_value[i] != ' ' && hdr->full_value[i] != '\t') { + /* broken value */ + return TRUE; + } + extra_space++; + } + + if (value <= ctx->parent->prev_msg_uid) { + /* broken - UIDs must be growing */ + return TRUE; + } + + ctx->uid = value; + ctx->extra_space += extra_space; + ctx->xuid_pos = buffer_get_used_size(ctx->header); + return FALSE; +} + +static struct header_func header_funcs[] = { + { "Content-Length", parse_content_length }, + { "Status", parse_status }, + { "X-IMAPbase", parse_x_imap_base }, + { "X-Keywords", parse_x_keywords }, + { "X-Status", parse_x_status }, + { "X-UID", parse_x_uid }, + { NULL, NULL } +}; + +static struct header_func *header_func_find(const char *header) +{ + int i; + + for (i = 0; header_funcs[i].header != NULL; i++) { + if (strcasecmp(header_funcs[i].header, header) == 0) + return &header_funcs[i]; + } + return NULL; +} + +void mbox_sync_mail_parse_headers(struct mbox_sync_mail_context *ctx) +{ + struct message_header_parser_ctx *hdr_ctx; + struct message_header_line *hdr; + struct header_func *func; + + ctx->header_first_change = (size_t)-1; + ctx->header_last_change = (size_t)-1; + + ctx->xuid_pos = (size_t)-1; + ctx->xkeywords_pos = (size_t)-1; + + ctx->content_length = (uoff_t)-1; + str_truncate(ctx->header, 0); + + hdr_ctx = message_parse_header_init(ctx->parent->input, NULL); + while ((hdr = message_parse_header_next(hdr_ctx)) != NULL) { + if (hdr->eoh) { + ctx->have_eoh = 1; + break; + } + + func = header_func_find(hdr->name); + if (func != NULL) { + if (hdr->continues) { + hdr->use_full_value = TRUE; + continue; + } + if (func->func(ctx, hdr)) { + /* we modified this header */ + if (ctx->header_first_change == (size_t)-1) { + ctx->header_first_change = + buffer_get_used_size(ctx->header); + } + ctx->header_last_change = (size_t)-1; + } else { + func = NULL; + } + } + + if (func == NULL) { + if (ctx->header_last_change == (size_t)-1) { + /* we may be able to stop rewriting here */ + ctx->header_last_change = + buffer_get_used_size(ctx->header); + } + if (!hdr->continued) { + str_append(ctx->header, hdr->name); + str_append(ctx->header, ": "); + } + buffer_append(ctx->header, hdr->full_value, + hdr->full_value_len); + if (!hdr->no_newline) + str_append_c(ctx->header, '\n'); + } + } + message_parse_header_deinit(hdr_ctx); +} + +void mbox_sync_mail_add_missing_headers(struct mbox_sync_mail_context *ctx) +{ + size_t old_hdr_size, new_hdr_size, size; + const char *str; + void *p; + int changed; + + old_hdr_size = ctx->body_offset - ctx->hdr_offset; + new_hdr_size = str_len(ctx->header) + ctx->have_eoh; + + changed = FALSE; + if (ctx->uid == 0) { + ctx->xuid_pos = buffer_get_used_size(ctx->header); + str_printfa(ctx->header, "X-UID: %u\n", + ctx->parent->next_uid++); + } + + if ((ctx->mail_flags & STATUS_FLAGS_MASK) != 0) { + str_append(ctx->header, "Status: "); + status_flags_append(ctx, status_flags); + str_append_c(ctx->header, '\n'); + } + + if ((ctx->mail_flags & XSTATUS_FLAGS_MASK) != 0) { + str_append(ctx->header, "X-Status: "); + status_flags_append(ctx, xstatus_flags); + str_append_c(ctx->header, '\n'); + } + + if (ctx->seq == 1 && ctx->base_uidvalidity == 0) { + ctx->ximapbase_pos = buffer_get_used_size(ctx->header); + str_printfa(ctx->header, "X-IMAPbase: %u %u", + ctx->parent->hdr->uid_validity, + ctx->parent->next_uid); + + /* if we can get away by adding only a little space, do it. + otherwise give a lot of extra */ + size = str_len(ctx->header) + ctx->have_eoh + 1 - + ctx->extra_space/2; + if (size <= old_hdr_size) + size = old_hdr_size - size; + else + size = 256; + + p = buffer_append_space_unsafe(ctx->header, size); + memset(p, ' ', size); + str_append_c(ctx->header, '\n'); + } + + /* write Content-Length if we have space */ + if (ctx->content_length == (uoff_t)-1) { + str = t_strdup_printf("Content-Length: %"PRIuUOFF_T"\n", + ctx->body_size); + size = buffer_get_used_size(ctx->header) + ctx->have_eoh - + ctx->extra_space; + if (size > old_hdr_size || size + strlen(str) <= old_hdr_size) + str_append(ctx->header, str); + } + + /* Create X-Keywords header if it's not there and we have space */ + if (ctx->xkeywords_pos == (size_t)-1) { + size = buffer_get_used_size(ctx->header) + ctx->have_eoh - + ctx->extra_space; + if (size > old_hdr_size || + size + sizeof("X-Keywords: ") <= old_hdr_size) { + ctx->xkeywords_pos = buffer_get_used_size(ctx->header); + str_append(ctx->header, "X-Keywords: \n"); + } + } + + if (buffer_get_used_size(ctx->header) != new_hdr_size) { + if (ctx->header_first_change == (size_t)-1) + ctx->header_first_change = new_hdr_size; + ctx->header_last_change = (size_t)-1; + new_hdr_size = buffer_get_used_size(ctx->header) + + ctx->have_eoh; + } + + if (ctx->header_first_change == (size_t)-1) { + /* no headers had to be modified */ + return; + } + + if (ctx->have_eoh) + str_append_c(ctx->header, '\n'); +} + +static void mbox_sync_headers_add_space(struct mbox_sync_mail_context *ctx, + size_t size) +{ + size_t data_size, pos; + const unsigned char *data; + void *p; + + /* Append at the end of X-Keywords header, + or X-UID if it doesn't exist */ + pos = ctx->xkeywords_pos != (size_t)-1 ? + ctx->xkeywords_pos : ctx->xuid_pos; + + data = buffer_get_data(ctx->header, &data_size); + while (pos < data_size && data[pos] != '\n') + pos++; + + buffer_copy(ctx->header, pos + size, + ctx->header, pos, (size_t)-1); + p = buffer_get_space_unsafe(ctx->header, pos, size); + memset(p, ' ', size); + ctx->extra_space += size; + + if (ctx->header_first_change > pos) + ctx->header_first_change = pos; + ctx->header_last_change = (size_t)-1; +} + +static void mbox_sync_header_remove_space(struct mbox_sync_mail_context *ctx, + size_t pos, size_t *size) +{ + const unsigned char *data; + size_t data_size, end, nonspace; + + /* find the end of the lwsp */ + nonspace = pos; + data = str_data(ctx->header); + data_size = str_len(ctx->header); + for (end = pos; end < data_size; end++) { + if (data[end] == '\n') { + if (end+1 == data_size || !IS_LWSP(data[end+1])) + break; + } else { + if (!IS_LWSP(data[end])) + nonspace = end; + } + } + + /* and remove what we can */ + nonspace++; + if (end-nonspace < *size) { + str_delete(ctx->header, nonspace, end-nonspace); + *size -= end-nonspace; + } else { + str_delete(ctx->header, nonspace, *size); + *size = 0; + } +} + +static void mbox_sync_headers_remove_space(struct mbox_sync_mail_context *ctx, + size_t size) +{ + if (ctx->xkeywords_pos != (size_t)-1) + mbox_sync_header_remove_space(ctx, ctx->xkeywords_pos, &size); + if (ctx->xuid_pos != (size_t)-1 && size > 0) + mbox_sync_header_remove_space(ctx, ctx->xuid_pos, &size); + if (ctx->ximapbase_pos != (size_t)-1 && size > 0) + mbox_sync_header_remove_space(ctx, ctx->ximapbase_pos, &size); + i_assert(size == 0); +} + +int mbox_sync_try_rewrite_headers(struct mbox_sync_mail_context *ctx, + uoff_t *missing_space_r) +{ + size_t old_hdr_size, new_hdr_size; + const unsigned char *data; + + old_hdr_size = ctx->body_offset - ctx->hdr_offset; + new_hdr_size = str_len(ctx->header); + + /* do we have enough space? */ + if (new_hdr_size < old_hdr_size) + mbox_sync_headers_add_space(ctx, old_hdr_size - new_hdr_size); + else if (new_hdr_size > old_hdr_size) { + if (ctx->extra_space < new_hdr_size - old_hdr_size) { + *missing_space_r = new_hdr_size - old_hdr_size - + ctx->extra_space; + return 0; + } + + ctx->extra_space -= new_hdr_size - old_hdr_size; + mbox_sync_headers_remove_space(ctx, new_hdr_size - + old_hdr_size); + } + + i_assert(ctx->header_first_change != (size_t)-1); + + if (ctx->header_last_change != (size_t)-1) + str_truncate(ctx->header, ctx->header_last_change); + + data = str_data(ctx->header); + new_hdr_size = str_len(ctx->header); + if (pwrite_full(ctx->parent->fd, data + ctx->header_first_change, + new_hdr_size, + ctx->hdr_offset + ctx->header_first_change) < 0) { + // FIXME: error handling + return -1; + } + *missing_space_r = 0; + return 0; +}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-storage/index/mbox/mbox-sync-parse.c Tue Apr 27 23:25:52 2004 +0300 @@ -0,0 +1,296 @@ +/* Copyright (C) 2004 Timo Sirainen */ + +#include "lib.h" +#include "buffer.h" +#include "istream.h" +#include "str.h" +#include "write-full.h" +#include "message-parser.h" +#include "mail-index.h" +#include "mbox-sync-private.h" + +#include <stdlib.h> + +#define IS_LWSP_LF(c) (IS_LWSP(c) || (c) == '\n') + +struct mbox_flag_type mbox_status_flags[] = { + { 'R', MAIL_SEEN }, + { 'O', MBOX_NONRECENT }, + { 0, 0 } +}; + +struct mbox_flag_type mbox_xstatus_flags[] = { + { 'A', MAIL_ANSWERED }, + { 'F', MAIL_FLAGGED }, + { 'T', MAIL_DRAFT }, + { 'D', MAIL_DELETED }, + { 0, 0 } +}; + +struct header_func { + const char *header; + int (*func)(struct mbox_sync_mail_context *ctx, + struct message_header_line *hdr); +}; + +static enum mail_flags mbox_flag_find(struct mbox_flag_type *flags, char chr) +{ + int i; + + for (i = 0; flags[i].chr != 0; i++) { + if (flags[i].chr == chr) + return flags[i].flag; + } + + return 0; +} + +static void parse_status_flags(struct mbox_sync_mail_context *ctx, + struct message_header_line *hdr, + struct mbox_flag_type *flags_list) +{ + size_t i; + + for (i = 0; i < hdr->full_value_len; i++) { + ctx->mail->flags |= + mbox_flag_find(flags_list, hdr->full_value[i]); + } +} + +static int parse_status(struct mbox_sync_mail_context *ctx, + struct message_header_line *hdr) +{ + parse_status_flags(ctx, hdr, mbox_status_flags); + ctx->hdr_pos[MBOX_HDR_STATUS] = str_len(ctx->header); + return TRUE; +} + +static int parse_x_status(struct mbox_sync_mail_context *ctx, + struct message_header_line *hdr) +{ + parse_status_flags(ctx, hdr, mbox_xstatus_flags); + ctx->hdr_pos[MBOX_HDR_X_STATUS] = str_len(ctx->header); + return TRUE; +} + +static int parse_x_imap_base(struct mbox_sync_mail_context *ctx, + struct message_header_line *hdr) +{ + const char *str; + char *end; + size_t pos; + + if (ctx->seq != 1 || ctx->base_uid_validity != 0) { + /* Valid only in first message */ + return FALSE; + } + + t_push(); + + /* <uid validity> <last uid> */ + str = t_strndup(hdr->full_value, hdr->full_value_len); + ctx->base_uid_validity = strtoul(str, &end, 10); + ctx->base_uid_last = strtoul(end, &end, 10); + pos = end - str; + + while (pos < hdr->full_value_len && IS_LWSP_LF(hdr->full_value[pos])) + pos++; + + if (ctx->base_uid_validity == 0) { + /* broken */ + t_pop(); + return FALSE; + } + + if (pos == hdr->full_value_len) { + t_pop(); + return TRUE; + } + + // FIXME: save keywords + + ctx->hdr_pos[MBOX_HDR_X_IMAPBASE] = str_len(ctx->header); + return TRUE; +} + +static int parse_x_keywords(struct mbox_sync_mail_context *ctx, + struct message_header_line *hdr) +{ + size_t i, space = 0; + + for (i = hdr->full_value_len; i > 0; i++) { + if (!IS_LWSP_LF(hdr->full_value[i-1])) + break; + space++; + } + + if (space > ctx->mail->space) { + ctx->mail->space_offset = hdr->full_value_offset + i; + ctx->mail->space = space; + } + + // FIXME: parse them + + ctx->hdr_pos[MBOX_HDR_X_KEYWORDS] = str_len(ctx->header); + return TRUE; +} + +static int parse_x_uid(struct mbox_sync_mail_context *ctx, + struct message_header_line *hdr) +{ + uint32_t value = 0; + size_t i, space_pos, extra_space = 0; + + if (ctx->mail->uid != 0) { + /* duplicate */ + return FALSE; + } + + for (i = 0; i < hdr->full_value_len; i++) { + if (hdr->full_value[i] < '0' || hdr->full_value[i] > '9') + break; + value = value*10 + (hdr->full_value[i] - '0'); + } + + space_pos = i; + for (; i < hdr->full_value_len; i++) { + if (!IS_LWSP_LF(hdr->full_value[i])) { + /* broken value */ + return FALSE; + } + extra_space++; + } + + if (value <= ctx->sync_ctx->prev_msg_uid) { + /* broken - UIDs must be growing */ + return FALSE; + } + + ctx->hdr_pos[MBOX_HDR_X_UID] = str_len(ctx->header); + + ctx->mail->uid = value; + if (ctx->mail->space == 0) { + /* set it only if X-Keywords hasn't been seen. spaces in X-UID + should be removed when writing X-Keywords. */ + ctx->mail->space_offset = hdr->full_value_offset + space_pos; + ctx->mail->space = extra_space; + } + return TRUE; +} + +static int parse_content_length(struct mbox_sync_mail_context *ctx, + struct message_header_line *hdr) +{ + uoff_t value = 0; + size_t i; + + if (ctx->content_length != (uoff_t)-1) { + /* duplicate */ + return FALSE; + } + + for (i = 0; i < hdr->full_value_len; i++) { + if (hdr->full_value[i] < '0' || hdr->full_value[i] > '9') + break; + value = value*10 + (hdr->full_value[i] - '0'); + } + + for (; i < hdr->full_value_len; i++) { + if (!IS_LWSP_LF(hdr->full_value[i])) { + /* broken value */ + return FALSE; + } + } + + ctx->content_length = value; + return TRUE; +} + +static struct header_func header_funcs[] = { + { "Content-Length", parse_content_length }, + { "Status", parse_status }, + { "X-IMAPbase", parse_x_imap_base }, + { "X-Keywords", parse_x_keywords }, + { "X-Status", parse_x_status }, + { "X-UID", parse_x_uid }, + { NULL, NULL } +}; + +static struct header_func *header_func_find(const char *header) +{ + int i; + + for (i = 0; header_funcs[i].header != NULL; i++) { + if (strcasecmp(header_funcs[i].header, header) == 0) + return &header_funcs[i]; + } + return NULL; +} + +void mbox_sync_parse_next_mail(struct istream *input, + struct mbox_sync_mail_context *ctx) +{ + struct message_header_parser_ctx *hdr_ctx; + struct message_header_line *hdr; + struct header_func *func; + size_t line_start_pos; + int i; + + ctx->hdr_offset = input->v_offset; + ctx->mail->space_offset = input->v_offset; + + ctx->header_first_change = (size_t)-1; + ctx->header_last_change = (size_t)-1; + + for (i = 0; i < MBOX_HDR_COUNT; i++) + ctx->hdr_pos[i] = (size_t)-1; + + ctx->content_length = (uoff_t)-1; + str_truncate(ctx->header, 0); + + line_start_pos = 0; + hdr_ctx = message_parse_header_init(input, NULL); + while ((hdr = message_parse_header_next(hdr_ctx)) != NULL) { + if (hdr->eoh) { + ctx->have_eoh = TRUE; + break; + } + + func = header_func_find(hdr->name); + if (func != NULL) { + if (hdr->continues) + hdr->use_full_value = TRUE; + else if (!func->func(ctx, hdr)) { + /* this header is broken, remove it */ + ctx->need_rewrite = TRUE; + if (hdr->continued) { + str_truncate(ctx->header, + line_start_pos); + } + if (ctx->header_first_change == (size_t)-1) { + ctx->header_first_change = + str_len(ctx->header); + } + continue; + } + } + + if (!hdr->continued) { + line_start_pos = str_len(ctx->header); + str_append(ctx->header, hdr->name); + str_append(ctx->header, ": "); + } + buffer_append(ctx->header, hdr->full_value, + hdr->full_value_len); + if (!hdr->no_newline) + str_append_c(ctx->header, '\n'); + } + message_parse_header_deinit(hdr_ctx); + + if (ctx->seq == 1 && ctx->base_uid_validity == 0) { + /* missing X-IMAPbase */ + ctx->need_rewrite = TRUE; + } + + ctx->body_offset = input->v_offset; +}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-storage/index/mbox/mbox-sync-private.h Tue Apr 27 23:25:52 2004 +0300 @@ -0,0 +1,78 @@ +#ifndef __MBOX_SYNC_PRIVATE_H +#define __MBOX_SYNC_PRIVATE_H + +#include "mail-index.h" + +struct mbox_flag_type { + char chr; + enum mail_flags flag; +}; + +enum header_position { + MBOX_HDR_STATUS, + MBOX_HDR_X_IMAPBASE, + MBOX_HDR_X_KEYWORDS, + MBOX_HDR_X_STATUS, + MBOX_HDR_X_UID, + + MBOX_HDR_COUNT +}; + +#define MBOX_NONRECENT MAIL_RECENT /* kludgy */ + +#define STATUS_FLAGS_MASK (MAIL_SEEN|MBOX_NONRECENT) +#define XSTATUS_FLAGS_MASK (MAIL_ANSWERED|MAIL_FLAGGED|MAIL_DRAFT|MAIL_DELETED) +extern struct mbox_flag_type mbox_status_flags[]; +extern struct mbox_flag_type mbox_xstatus_flags[]; + +struct mbox_mail { + uint32_t uid; + uint8_t flags; + custom_flags_mask_t custom_flags; + + uoff_t space_offset; /* if space is negative, points to beginning */ + off_t space; + uoff_t body_size; +}; + +struct mbox_sync_mail_context { + struct mbox_sync_context *sync_ctx; + struct mbox_mail *mail; + + uint32_t seq; + uoff_t hdr_offset, body_offset; + + size_t header_first_change, header_last_change; + string_t *header; + + uint32_t base_uid_validity, base_uid_last; + uoff_t content_length; + + size_t hdr_pos[MBOX_HDR_COUNT]; + + unsigned int have_eoh:1; + unsigned int need_rewrite:1; +}; + +struct mbox_sync_context { + struct istream *file_input; + struct istream *input; + int fd; + + const struct mail_index_header *hdr; + + uint32_t prev_msg_uid, next_uid; +}; + +void mbox_sync_parse_next_mail(struct istream *input, + struct mbox_sync_mail_context *ctx); +void mbox_sync_update_header(struct mbox_sync_mail_context *ctx, + struct mail_index_sync_rec *update); +int mbox_sync_try_rewrite(struct mbox_sync_mail_context *ctx); +int mbox_sync_rewrite(struct mbox_sync_context *sync_ctx, buffer_t *mails_buf, + uint32_t first_seq, uint32_t last_seq, off_t extra_space); + +int mbox_move(struct mbox_sync_context *sync_ctx, + uoff_t dest, uoff_t source, uoff_t size); + +#endif
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-storage/index/mbox/mbox-sync-rewrite.c Tue Apr 27 23:25:52 2004 +0300 @@ -0,0 +1,183 @@ +#include "lib.h" +#include "buffer.h" +#include "istream.h" +#include "ostream.h" +#include "str.h" +#include "write-full.h" +#include "message-parser.h" +#include "mbox-sync-private.h" +#include "istream-raw-mbox.h" + +int mbox_move(struct mbox_sync_context *sync_ctx, + uoff_t dest, uoff_t source, uoff_t size) +{ + struct istream *input; + struct ostream *output; + off_t ret; + + output = o_stream_create_file(sync_ctx->fd, default_pool, 4096, FALSE); + i_stream_seek(sync_ctx->file_input, source); + o_stream_seek(output, dest); + + istream_raw_mbox_flush(sync_ctx->input); + + if (size == (uoff_t)-1) { + input = sync_ctx->file_input; + return o_stream_send_istream(output, input) < 0 ? -1 : 0; + } else { + input = i_stream_create_limit(default_pool, + sync_ctx->file_input, + source, size); + ret = o_stream_send_istream(output, input); + i_stream_unref(input); + return ret == (off_t)size ? 0 : -1; + } +} + +static void mbox_sync_headers_add_space(struct mbox_sync_mail_context *ctx, + size_t size) +{ + size_t data_size, pos; + const unsigned char *data; + void *p; + + /* Append at the end of X-Keywords header, + or X-UID if it doesn't exist */ + pos = ctx->hdr_pos[MBOX_HDR_X_KEYWORDS] != (size_t)-1 ? + ctx->hdr_pos[MBOX_HDR_X_KEYWORDS] : + ctx->hdr_pos[MBOX_HDR_X_UID]; + + data = buffer_get_data(ctx->header, &data_size); + while (pos < data_size && data[pos] != '\n') + pos++; + + buffer_copy(ctx->header, pos + size, + ctx->header, pos, (size_t)-1); + p = buffer_get_space_unsafe(ctx->header, pos, size); + memset(p, ' ', size); + + if (ctx->header_first_change > pos) + ctx->header_first_change = pos; + ctx->header_last_change = (size_t)-1; +} + +static void mbox_sync_header_remove_space(struct mbox_sync_mail_context *ctx, + size_t pos, size_t *size) +{ + const unsigned char *data; + size_t data_size, end, nonspace; + + /* find the end of the lwsp */ + nonspace = pos; + data = str_data(ctx->header); + data_size = str_len(ctx->header); + for (end = pos; end < data_size; end++) { + if (data[end] == '\n') { + if (end+1 == data_size || !IS_LWSP(data[end+1])) + break; + } else { + if (!IS_LWSP(data[end])) + nonspace = end; + } + } + + /* and remove what we can */ + nonspace++; + if (end-nonspace < *size) { + str_delete(ctx->header, nonspace, end-nonspace); + *size -= end-nonspace; + } else { + str_delete(ctx->header, nonspace, *size); + *size = 0; + } +} + +static void mbox_sync_headers_remove_space(struct mbox_sync_mail_context *ctx, + size_t size) +{ + static enum header_position space_positions[] = { + MBOX_HDR_X_KEYWORDS, + MBOX_HDR_X_UID, + MBOX_HDR_X_IMAPBASE + }; + enum header_position pos; + int i; + + for (i = 0; i < 3 && size > 0; i++) { + pos = space_positions[i]; + if (ctx->hdr_pos[pos] != (size_t)-1) { + mbox_sync_header_remove_space(ctx, ctx->hdr_pos[pos], + &size); + } + } + + i_assert(size == 0); +} + +int mbox_sync_try_rewrite(struct mbox_sync_mail_context *ctx) +{ + size_t old_hdr_size, new_hdr_size; + const unsigned char *data; + + old_hdr_size = ctx->body_offset - ctx->hdr_offset; + new_hdr_size = str_len(ctx->header); + + /* do we have enough space? */ + if (new_hdr_size < old_hdr_size) { + mbox_sync_headers_add_space(ctx, old_hdr_size - new_hdr_size); + ctx->mail->space += old_hdr_size - new_hdr_size; + } else if (new_hdr_size > old_hdr_size) { + size_t needed = new_hdr_size - old_hdr_size; + if (ctx->mail->space < needed) { + ctx->mail->space -= needed; + return 0; + } + + ctx->mail->space -= needed; + mbox_sync_headers_remove_space(ctx, needed); + } + + i_assert(ctx->header_first_change != (size_t)-1); + + if (ctx->header_last_change != (size_t)-1) + str_truncate(ctx->header, ctx->header_last_change); + + data = str_data(ctx->header); + new_hdr_size = str_len(ctx->header); + if (pwrite_full(ctx->sync_ctx->fd, data + ctx->header_first_change, + new_hdr_size, + ctx->hdr_offset + ctx->header_first_change) < 0) { + // FIXME: error handling + return -1; + } + istream_raw_mbox_flush(ctx->sync_ctx->input); + return 1; +} + +int mbox_sync_rewrite(struct mbox_sync_context *sync_ctx, buffer_t *mails_buf, + uint32_t first_seq, uint32_t last_seq, off_t extra_space) +{ + struct mbox_mail *mails; + size_t size; + uint32_t first_idx, last_idx, extra_per_mail; + + first_idx = first_seq-1; + last_idx = last_seq-1; + + mails = buffer_get_modifyable_data(mails_buf, &size); + size /= sizeof(*mails); + + /* FIXME: see if we can be faster by going back a few mails + (update first_seq and last_seq) */ + /*while (mails[last_idx].space > 0) { + }*/ + +#if 0 + /* start moving backwards */ + extra_per_mail = (extra_space / (last_seq - first_seq + 1)) + 1; + space_diff = 0; + while (last_seq > first_seq) { + dest = mails[last_seq].space_offset + mails[last_seq].space + } +#endif +}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-storage/index/mbox/mbox-sync-update.c Tue Apr 27 23:25:52 2004 +0300 @@ -0,0 +1,138 @@ +#include "lib.h" +#include "buffer.h" +#include "str.h" +#include "message-parser.h" +#include "mbox-sync-private.h" + +static void status_flags_append(struct mbox_sync_mail_context *ctx, + struct mbox_flag_type *flags_list) +{ + int i; + + for (i = 0; flags_list[i].chr != 0; i++) { + if ((ctx->mail->flags & flags_list[i].flag) != 0) + str_append_c(ctx->header, flags_list[i].chr); + } +} +static void keywords_append(struct mbox_sync_mail_context *ctx, + custom_flags_mask_t custom_flags) +{ + // FIXME +} + +static void mbox_sync_add_missing_headers(struct mbox_sync_mail_context *ctx) +{ + size_t old_hdr_size, new_hdr_size; + int i, have_keywords; + + old_hdr_size = ctx->body_offset - ctx->hdr_offset; + new_hdr_size = str_len(ctx->header) + ctx->have_eoh; + + if (ctx->seq == 1 && ctx->base_uid_validity == 0) { + ctx->hdr_pos[MBOX_HDR_X_IMAPBASE] = str_len(ctx->header); + str_printfa(ctx->header, "X-IMAPbase: %u %u", + ctx->sync_ctx->hdr->uid_validity, + ctx->sync_ctx->next_uid); + //FIXME:keywords_append(ctx, all_custom_flags); + str_append_c(ctx->header, '\n'); + } + + if (ctx->mail->uid == 0) { + ctx->hdr_pos[MBOX_HDR_X_UID] = str_len(ctx->header); + str_printfa(ctx->header, "X-UID: %u\n", + ctx->sync_ctx->next_uid++); + } + + if (ctx->hdr_pos[MBOX_HDR_STATUS] == (size_t)-1 && + (ctx->mail->flags & STATUS_FLAGS_MASK) != 0) { + ctx->hdr_pos[MBOX_HDR_STATUS] = str_len(ctx->header); + str_append(ctx->header, "Status: "); + status_flags_append(ctx, mbox_status_flags); + str_append_c(ctx->header, '\n'); + } + + if (ctx->hdr_pos[MBOX_HDR_X_STATUS] == (size_t)-1 && + (ctx->mail->flags & XSTATUS_FLAGS_MASK) != 0) { + ctx->hdr_pos[MBOX_HDR_X_STATUS] = str_len(ctx->header); + str_append(ctx->header, "X-Status: "); + status_flags_append(ctx, mbox_xstatus_flags); + str_append_c(ctx->header, '\n'); + } + + have_keywords = FALSE; + for (i = 0; i < INDEX_CUSTOM_FLAGS_BYTE_COUNT; i++) { + if (ctx->mail->custom_flags[i] != 0) { + have_keywords = TRUE; + break; + } + } + + if (ctx->hdr_pos[MBOX_HDR_X_KEYWORDS] == (size_t)-1 && have_keywords) { + ctx->hdr_pos[MBOX_HDR_X_KEYWORDS] = str_len(ctx->header); + str_append(ctx->header, "X-Keywords: "); + keywords_append(ctx, ctx->mail->custom_flags); + str_append_c(ctx->header, '\n'); + } + + if (ctx->content_length == (uoff_t)-1) { + str_printfa(ctx->header, "Content-Length: %"PRIuUOFF_T"\n", + ctx->mail->body_size); + } + + if (str_len(ctx->header) != new_hdr_size) { + if (ctx->header_first_change == (size_t)-1) + ctx->header_first_change = new_hdr_size; + ctx->header_last_change = (size_t)-1; + ctx->mail->space -= str_len(ctx->header) - + (new_hdr_size - ctx->have_eoh); + new_hdr_size = str_len(ctx->header) + ctx->have_eoh; + } + + if (ctx->header_first_change == (size_t)-1) { + /* no headers had to be modified */ + return; + } + + if (ctx->have_eoh) + str_append_c(ctx->header, '\n'); +} + +static void mbox_sync_update_status(struct mbox_sync_mail_context *ctx) +{ +} + +static void mbox_sync_update_xstatus(struct mbox_sync_mail_context *ctx) +{ +} + +static void mbox_sync_update_xkeywords(struct mbox_sync_mail_context *ctx) +{ +} + +void mbox_sync_update_header(struct mbox_sync_mail_context *ctx, + struct mail_index_sync_rec *update) +{ + uint8_t old_flags; + custom_flags_mask_t old_custom_flags; + + if (update != NULL) { + old_flags = ctx->mail->flags; + memcpy(old_custom_flags, ctx->mail->custom_flags, + sizeof(old_custom_flags)); + + mail_index_sync_flags_apply(update, &ctx->mail->flags, + ctx->mail->custom_flags); + + if ((old_flags & STATUS_FLAGS_MASK) != + (ctx->mail->flags & STATUS_FLAGS_MASK)) + mbox_sync_update_status(ctx); + if ((old_flags & XSTATUS_FLAGS_MASK) != + (ctx->mail->flags & XSTATUS_FLAGS_MASK)) + mbox_sync_update_xstatus(ctx); + if (memcmp(old_custom_flags, ctx->mail->custom_flags, + sizeof(old_custom_flags)) != 0) + mbox_sync_update_xkeywords(ctx); + } + + mbox_sync_add_missing_headers(ctx); +}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-storage/index/mbox/mbox-sync.c Tue Apr 27 23:25:52 2004 +0300 @@ -0,0 +1,267 @@ +/* Copyright (C) 2004 Timo Sirainen */ + +/* + Modifying mbox can be slow, so we try to do it all at once minimizing the + required disk I/O. We may need to: + + - Update message flags in Status, X-Status and X-Keywords headers + - Write missing X-UID and X-IMAPbase headers + - Write missing or broken Content-Length header if there's space + - Expunge specified messages + + Here's how we do it: + + - Start reading the mails mail headers from the beginning + - X-Keywords and X-UID headers may contain extra spaces at the end of them, + remember how much extra each message has and offset to beginning of the + spaces + - If message flags are dirty and there's enough space to write them, do it + - If we didn't have enough space, remember how much was missing and keep + the total amount of them + - When we encounter expunged message, check if the amount of empty space in + previous messages plus size of expunged message is enough to cover the + missing space. If yes, + - execute the rewrite plan + - forget all the messages before the expunged message. only remember + how much data we still have to move to cover the expunged message + - If we encounter end of file, grow the file and execute the rewrite plan + + Rewrite plan goes: + + - Start from the first message that needs more space + - If there's expunged messages before us, we have to write over them. + - Move all messages after it backwards to fill it + - Each moved message's X-Keywords header should have n bytes extra + space, unless there's not enough space to do it. + - If there's no expunged messages, we can move data either forward or + backward to get it. Calculate which requires less moving. Forward + counting may encounter more messages which require extra space, count + that too. + - If we decide to move forwards and we had to go through dirty + messages, do the moving from last to first dirty message + - If we encounter end of file, grow the file enough to get the required + amount of space plus enough space to fill X-Keywords headers full of + spaces. +*/ + +#include "lib.h" +#include "buffer.h" +#include "istream.h" +#include "file-set-size.h" +#include "str.h" +#include "write-full.h" +#include "istream-raw-mbox.h" +#include "mbox-sync-private.h" + +static int mbox_sync_grow_file(struct mbox_sync_context *sync_ctx, + struct mbox_mail *mail, uoff_t body_offset, + uoff_t grow_size) +{ + char spaces[1024]; + uoff_t offset, size; + + i_assert(grow_size > 0); + + memset(spaces, ' ', sizeof(spaces)); + + size = sync_ctx->input->v_offset + grow_size; + if (file_set_size(sync_ctx->fd, size) < 0) + return -1; + + if (mail->space_offset == 0) { + /* no X-Keywords header - place it at the end. */ + grow_size += 13; + + offset = body_offset-1; + if (mbox_move(sync_ctx, body_offset-1 + size, + offset, (uoff_t)-1) < 0) + return -1; + if (pwrite_full(sync_ctx->fd, "X-Keywords: ", 12, offset) < 0) + return -1; + if (pwrite_full(sync_ctx->fd, "\n", 1, + offset + grow_size-1) < 0) + return -1; + grow_size -= 13; offset += 12; + + /* FIXME: can this break anything? X-Keywords text might + have been already included in space calculation. now we + have more.. */ + mail->space_offset = offset; + mail->space += grow_size; + } else { + offset = mail->space_offset; + if (mbox_move(sync_ctx, mail->space_offset + grow_size, + offset, (uoff_t)-1) < 0) + return -1; + } + + while (grow_size >= sizeof(spaces)) { + if (pwrite_full(sync_ctx->fd, spaces, + sizeof(spaces), offset) < 0) + return -1; + grow_size -= sizeof(spaces); + offset += sizeof(spaces); + } + + if (grow_size > 0) { + if (pwrite_full(sync_ctx->fd, spaces, grow_size, offset) < 0) + return -1; + } + + istream_raw_mbox_flush(sync_ctx->input); + return 0; +} + +int mbox_sync(struct istream *input) +{ + struct mbox_sync_context sync_ctx; + struct mbox_sync_mail_context mail_ctx; + struct mbox_mail mail; + uint32_t seq, need_space_seq; + off_t space_diff; + buffer_t *mails; + int ret = 0; + + mails = buffer_create_dynamic(default_pool, 4096, (size_t)-1); + + memset(&sync_ctx, 0, sizeof(sync_ctx)); + sync_ctx.file_input = input; + sync_ctx.input = i_stream_create_raw_mbox(default_pool, input); + sync_ctx.fd = i_stream_get_fd(input); + //sync_ctx.hdr = ; + + input = sync_ctx.input; + + space_diff = 0; need_space_seq = 0; seq = 1; + for (seq = 1; !input->eof; seq++) { + memset(&mail, 0, sizeof(mail)); + memset(&mail_ctx, 0, sizeof(mail_ctx)); + mail_ctx.sync_ctx = &sync_ctx; + mail_ctx.mail = &mail; + mail_ctx.seq = seq; + + mbox_sync_parse_next_mail(input, &mail_ctx); + mail.body_size = + istream_raw_mbox_get_size(input, + mail_ctx.content_length); + buffer_append(mails, &mail, sizeof(mail)); + + if (mail_ctx.need_rewrite) { + mbox_sync_update_header(&mail_ctx, NULL); + if ((ret = mbox_sync_try_rewrite(&mail_ctx)) < 0) + break; + } else { + ret = 1; + } + + if (ret == 0 && need_space_seq == 0) { + /* didn't have space to write it */ + need_space_seq = seq; + space_diff = mail.space; + } else if (need_space_seq != 0) { + space_diff += mail.space; + if (space_diff >= 0) { + /* we have enough space now */ + if (mbox_sync_rewrite(&sync_ctx, mails, + need_space_seq, seq, + space_diff) < 0) { + ret = -1; + break; + } + need_space_seq = 0; + } + } + + istream_raw_mbox_next(input, mail.body_size); + } + + if (need_space_seq != 0) { + i_assert(space_diff < 0); + if (mbox_sync_grow_file(&sync_ctx, &mail, mail_ctx.body_offset, + -space_diff) < 0) + ret = -1; + else if (mbox_sync_rewrite(&sync_ctx, mails, need_space_seq, + seq-1, space_diff) < 0) + ret = -1; + } + + i_stream_unref(input); + return ret < 0 ? -1 : 0; +} + +#if 0 +int mbox_sync(void) +{ + struct mail_index_view *sync_view; + struct mail_index_sync_ctx *sync_ctx; + struct mail_index_sync_rec sync_rec; + struct mbox_sync_context ctx; + struct mbox_sync_mail_context mail_ctx; + struct mbox_mail mail; + string_t *header; + uint32_t seq; + unsigned int need_space_seq; + uoff_t missing_space; + buffer_t *mails; + int ret; + + memset(&ctx, 0, sizeof(ctx)); + /*ctx.index = storage->index; + ctx.input = storage->input;*/ + ctx.fd = i_stream_get_fd(ctx.input); + + header = str_new(default_pool, 4096); + + if (mail_index_sync_begin(ctx.index, &sync_ctx, &sync_view, 0, 0) < 0) + return -1; + + ctx.hdr = mail_index_get_header(sync_view); + ctx.next_uid = ctx.hdr->next_uid; + + seq = 1; + while ((ret = mail_index_sync_next(sync_ctx, &sync_rec)) > 0) { + while (seq < sync_rec.seq1) { + seq++; + } + switch (sync_rec.type) { + case MAIL_INDEX_SYNC_TYPE_EXPUNGE: + break; + case MAIL_INDEX_SYNC_TYPE_FLAGS: + break; + } + } + + while (!ctx.input->eof) { + memset(&mail_ctx, 0, sizeof(mail_ctx)); + mail_ctx.parent = &ctx; + mail_ctx.header = header; + mail_ctx.seq = seq; + + mail_ctx.hdr_offset = ctx.input->v_offset; + mbox_sync_mail_parse_headers(&mail_ctx); + mail_ctx.body_offset = ctx.input->v_offset; + mail_ctx.body_size = + istream_raw_mbox_get_size(ctx.input, + mail_ctx.content_length); + + mbox_sync_mail_add_missing_headers(&mail_ctx); + + ret = mbox_sync_try_rewrite_headers(&mail_ctx, &missing_space); + if (ret < 0) + break; + if (missing_space != 0) { + ctx.space_diff -= missing_space; + } else { + ctx.space_diff += mail_ctx.extra_space; + } + + if (ctx.first_spacy_msg_offset == 0) + ctx.first_spacy_msg_offset = mail_ctx.hdr_offset; + + ctx.prev_msg_uid = mail_ctx.uid; + istream_raw_mbox_next(ctx.input, mail_ctx.content_length); + } + str_free(header); + return 0; +} +#endif
--- a/src/lib-storage/mail-save.c Tue Apr 27 23:14:15 2004 +0300 +++ b/src/lib-storage/mail-save.c Tue Apr 27 23:25:52 2004 +0300 @@ -4,7 +4,7 @@ #include "istream.h" #include "ostream.h" #include "message-parser.h" -#include "mail-storage.h" +#include "mail-storage-private.h" #include "mail-save.h" static int write_with_crlf(struct ostream *output, const void *v_data, @@ -94,16 +94,14 @@ { struct message_header_parser_ctx *hdr_ctx; struct message_header_line *hdr; - int ret, failed = FALSE; + int ret = 0; hdr_ctx = message_parse_header_init(input, NULL); while ((hdr = message_parse_header_next(hdr_ctx)) != NULL) { ret = header_callback(hdr->name, write_func, context); if (ret <= 0) { - if (ret < 0) { - failed = TRUE; + if (ret < 0) break; - } continue; } @@ -118,16 +116,17 @@ write_func(output, "\n", 1); } } - if (!failed) { + + if (ret >= 0) { if (header_callback(NULL, write_func, context) < 0) - failed = TRUE; + ret = -1; /* end of headers */ write_func(output, "\n", 1); } message_parse_header_deinit(hdr_ctx); - return !failed; + return ret < 0 ? -1 : 0; } int mail_storage_save(struct mail_storage *storage, const char *path, @@ -143,9 +142,9 @@ write_func = crlf ? write_with_crlf : write_with_lf; if (header_callback != NULL) { - if (!save_headers(input, output, header_callback, - context, write_func)) - return FALSE; + if (save_headers(input, output, header_callback, + context, write_func) < 0) + return -1; } failed = FALSE; @@ -186,5 +185,19 @@ } } - return !failed; + return failed ? -1 : 0; } + +int mail_storage_copy(struct mailbox_transaction_context *t, struct mail *mail) +{ + struct istream *input; + + input = mail->get_stream(mail, NULL, NULL); + if (input == NULL) + return -1; + + return mailbox_save(t, mail->get_flags(mail), + mail->get_received_date(mail), 0, + mail->get_special(mail, MAIL_FETCH_FROM_ENVELOPE), + input); +}
--- a/src/lib-storage/mail-save.h Tue Apr 27 23:14:15 2004 +0300 +++ b/src/lib-storage/mail-save.h Tue Apr 27 23:25:52 2004 +0300 @@ -11,4 +11,6 @@ struct istream *input, struct ostream *output, int crlf, header_callback_t *header_callback, void *context); +int mail_storage_copy(struct mailbox_transaction_context *t, struct mail *mail); + #endif
--- a/src/lib-storage/mail-search.h Tue Apr 27 23:14:15 2004 +0300 +++ b/src/lib-storage/mail-search.h Tue Apr 27 23:25:52 2004 +0300 @@ -5,10 +5,9 @@ SEARCH_OR, SEARCH_SUB, - /* message sets */ + /* sequence sets */ SEARCH_ALL, - SEARCH_SET, - SEARCH_UID, + SEARCH_SEQSET, /* flags */ SEARCH_ANSWERED, @@ -40,12 +39,18 @@ SEARCH_TEXT }; +struct mail_search_seqset { + uint32_t seq1, seq2; + struct mail_search_seqset *next; +}; + struct mail_search_arg { struct mail_search_arg *next; enum mail_search_arg_type type; - union { + struct { struct mail_search_arg *subargs; + struct mail_search_seqset *seqset; const char *str; } value;
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-storage/mail-storage-private.h Tue Apr 27 23:25:52 2004 +0300 @@ -0,0 +1,128 @@ +#ifndef __MAIL_STORAGE_PRIVATE_H +#define __MAIL_STORAGE_PRIVATE_H + +#include "mail-storage.h" + +struct mail_storage { + char *name; + char *namespace; + char hierarchy_sep; + + struct mail_storage *(*create)(const char *data, const char *user, + const char *namespace, + char hierarchy_sep); + void (*destroy)(struct mail_storage *storage); + + int (*autodetect)(const char *data); + + void (*set_callbacks)(struct mail_storage *storage, + struct mail_storage_callbacks *callbacks, + void *context); + + struct mailbox *(*mailbox_open)(struct mail_storage *storage, + const char *name, + enum mailbox_open_flags flags); + + int (*mailbox_create)(struct mail_storage *storage, const char *name, + int directory); + int (*mailbox_delete)(struct mail_storage *storage, const char *name); + int (*mailbox_rename)(struct mail_storage *storage, const char *oldname, + const char *newname); + + struct mailbox_list_context * + (*mailbox_list_init)(struct mail_storage *storage, + const char *mask, + enum mailbox_list_flags flags); + struct mailbox_list * + (*mailbox_list_next)(struct mailbox_list_context *ctx); + int (*mailbox_list_deinit)(struct mailbox_list_context *ctx); + + int (*set_subscribed)(struct mail_storage *storage, + const char *name, int set); + + int (*get_mailbox_name_status)(struct mail_storage *storage, + const char *name, + enum mailbox_name_status *status); + + const char *(*get_last_error)(struct mail_storage *storage, + int *syntax_error_r); + +/* private: */ + char *error; + + unsigned int syntax_error:1; /* Give a BAD reply instead of NO */ +}; + +struct mailbox { + char *name; + + struct mail_storage *storage; + + int (*is_readonly)(struct mailbox *box); + int (*allow_new_custom_flags)(struct mailbox *box); + + int (*close)(struct mailbox *box); + + int (*get_status)(struct mailbox *box, enum mailbox_status_items items, + struct mailbox_status *status); + + int (*sync)(struct mailbox *box, enum mailbox_sync_flags flags); + void (*auto_sync)(struct mailbox *box, enum mailbox_sync_flags flags, + unsigned int min_newmail_notify_interval); + + struct mailbox_transaction_context * + (*transaction_begin)(struct mailbox *box, int hide); + int (*transaction_commit)(struct mailbox_transaction_context *t); + void (*transaction_rollback)(struct mailbox_transaction_context *t); + + struct mail *(*fetch)(struct mailbox_transaction_context *t, + uint32_t seq, + enum mail_fetch_field wanted_fields); + int (*get_uids)(struct mailbox *box, uint32_t uid1, uint32_t uid2, + uint32_t *seq1_r, uint32_t *seq2_r); + + int (*search_get_sorting)(struct mailbox *box, + enum mail_sort_type *sort_program); + struct mail_search_context * + (*search_init)(struct mailbox_transaction_context *t, + const char *charset, + struct mail_search_arg *args, + const enum mail_sort_type *sort_program, + enum mail_fetch_field wanted_fields, + const char *const wanted_headers[]); + int (*search_deinit)(struct mail_search_context *ctx); + struct mail *(*search_next)(struct mail_search_context *ctx); + + int (*save)(struct mailbox_transaction_context *t, + const struct mail_full_flags *flags, + time_t received_date, int timezone_offset, + const char *from_envelope, struct istream *data); + int (*copy)(struct mailbox_transaction_context *t, struct mail *mail); + + int (*is_inconsistent)(struct mailbox *box); +}; + +struct mailbox_list_context { + struct mail_storage *storage; +}; + +struct mailbox_transaction_context { + struct mailbox *box; +}; + +struct mail_search_context { + struct mailbox *box; +}; + +/* Set error message in storage. Critical errors are logged with i_error(), + but user sees only "internal error" message. */ +void mail_storage_clear_error(struct mail_storage *storage); +void mail_storage_set_error(struct mail_storage *storage, + const char *fmt, ...) __attr_format__(2, 3); +void mail_storage_set_syntax_error(struct mail_storage *storage, + const char *fmt, ...) __attr_format__(2, 3); +void mail_storage_set_critical(struct mail_storage *storage, + const char *fmt, ...) __attr_format__(2, 3); +void mail_storage_set_internal_error(struct mail_storage *storage); + +#endif
--- a/src/lib-storage/mail-storage.c Tue Apr 27 23:14:15 2004 +0300 +++ b/src/lib-storage/mail-storage.c Tue Apr 27 23:25:52 2004 +0300 @@ -1,8 +1,8 @@ -/* Copyright (C) 2002 Timo Sirainen */ +/* Copyright (C) 2002-2003 Timo Sirainen */ #include "lib.h" #include "ioloop.h" -#include "mail-storage.h" +#include "mail-storage-private.h" #include <stdlib.h> #include <time.h> @@ -25,7 +25,6 @@ struct client_workaround_list client_workaround_list[] = { { "oe6-fetch-no-newmail", WORKAROUND_OE6_FETCH_NO_NEWMAIL }, - { "oe6-fetch-redundant-msgset", WORKAROUND_OE6_FETCH_REDUNDANT_MSGSET }, { "outlook-idle", WORKAROUND_OUTLOOK_IDLE }, { NULL, 0 } }; @@ -93,7 +92,7 @@ if ((*list)->storage == storage_class) { next = (*list)->next; - (*list)->storage->free((*list)->storage); + mail_storage_destroy((*list)->storage); i_free(*list); *list = next; @@ -184,7 +183,7 @@ { i_assert(storage != NULL); - storage->free(storage); + storage->destroy(storage); } void mail_storage_clear_error(struct mail_storage *storage) @@ -262,10 +261,193 @@ } } -const char *mail_storage_get_last_error(struct mail_storage *storage, - int *syntax) +char mail_storage_get_hierarchy_sep(struct mail_storage *storage) +{ + return storage->hierarchy_sep; +} + +void mail_storage_set_callbacks(struct mail_storage *storage, + struct mail_storage_callbacks *callbacks, + void *context) +{ + storage->set_callbacks(storage, callbacks, context); +} + +int mail_storage_mailbox_create(struct mail_storage *storage, const char *name, + int directory) +{ + return storage->mailbox_create(storage, name, directory); +} + +int mail_storage_mailbox_delete(struct mail_storage *storage, const char *name) +{ + return storage->mailbox_delete(storage, name); +} + +int mail_storage_mailbox_rename(struct mail_storage *storage, + const char *oldname, const char *newname) { - if (syntax != NULL) - *syntax = storage->syntax_error; + return storage->mailbox_rename(storage, oldname, newname); +} + +struct mailbox_list_context * +mail_storage_mailbox_list_init(struct mail_storage *storage, + const char *mask, + enum mailbox_list_flags flags) +{ + return storage->mailbox_list_init(storage, mask, flags); +} + +struct mailbox_list * +mail_storage_mailbox_list_next(struct mailbox_list_context *ctx) +{ + return ctx->storage->mailbox_list_next(ctx); +} + +int mail_storage_mailbox_list_deinit(struct mailbox_list_context *ctx) +{ + return ctx->storage->mailbox_list_deinit(ctx); +} + +int mail_storage_set_subscribed(struct mail_storage *storage, + const char *name, int set) +{ + return storage->set_subscribed(storage, name, set); +} + +int mail_storage_get_mailbox_name_status(struct mail_storage *storage, + const char *name, + enum mailbox_name_status *status) +{ + return storage->get_mailbox_name_status(storage, name, status); +} + +const char *mail_storage_get_last_error(struct mail_storage *storage, + int *syntax_error_r) +{ + *syntax_error_r = storage->syntax_error; return storage->error; } + +struct mailbox *mailbox_open(struct mail_storage *storage, + const char *name, enum mailbox_open_flags flags) +{ + return storage->mailbox_open(storage, name, flags); +} + +int mailbox_close(struct mailbox *box) +{ + return box->close(box); +} + +struct mail_storage *mailbox_get_storage(struct mailbox *box) +{ + return box->storage; +} + +const char *mailbox_get_name(struct mailbox *box) +{ + return box->name; +} + +int mailbox_is_readonly(struct mailbox *box) +{ + return box->is_readonly(box); +} + +int mailbox_allow_new_custom_flags(struct mailbox *box) +{ + return box->allow_new_custom_flags(box); +} + +int mailbox_get_status(struct mailbox *box, + enum mailbox_status_items items, + struct mailbox_status *status) +{ + return box->get_status(box, items, status); +} + +int mailbox_sync(struct mailbox *box, enum mailbox_sync_flags flags) +{ + return box->sync(box, flags); +} + +void mailbox_auto_sync(struct mailbox *box, enum mailbox_sync_flags flags, + unsigned int min_newmail_notify_interval) +{ + box->auto_sync(box, flags, min_newmail_notify_interval); +} + +struct mail *mailbox_fetch(struct mailbox_transaction_context *t, uint32_t seq, + enum mail_fetch_field wanted_fields) +{ + return t->box->fetch(t, seq, wanted_fields); +} + +int mailbox_get_uids(struct mailbox *box, uint32_t uid1, uint32_t uid2, + uint32_t *seq1_r, uint32_t *seq2_r) +{ + return box->get_uids(box, uid1, uid2, seq1_r, seq2_r); +} + +int mailbox_search_get_sorting(struct mailbox *box, + enum mail_sort_type *sort_program) +{ + return box->search_get_sorting(box, sort_program); +} + +struct mail_search_context * +mailbox_search_init(struct mailbox_transaction_context *t, + const char *charset, struct mail_search_arg *args, + const enum mail_sort_type *sort_program, + enum mail_fetch_field wanted_fields, + const char *const wanted_headers[]) +{ + return t->box->search_init(t, charset, args, sort_program, + wanted_fields, wanted_headers); +} + +int mailbox_search_deinit(struct mail_search_context *ctx) +{ + return ctx->box->search_deinit(ctx); +} + +struct mail *mailbox_search_next(struct mail_search_context *ctx) +{ + return ctx->box->search_next(ctx); +} + +struct mailbox_transaction_context * +mailbox_transaction_begin(struct mailbox *box, int hide) +{ + return box->transaction_begin(box, hide); +} + +int mailbox_transaction_commit(struct mailbox_transaction_context *t) +{ + return t->box->transaction_commit(t); +} + +void mailbox_transaction_rollback(struct mailbox_transaction_context *t) +{ + t->box->transaction_rollback(t); +} + +int mailbox_save(struct mailbox_transaction_context *t, + const struct mail_full_flags *flags, + time_t received_date, int timezone_offset, + const char *from_envelope, struct istream *data) +{ + return t->box->save(t, flags, received_date, timezone_offset, + from_envelope, data); +} + +int mailbox_copy(struct mailbox_transaction_context *t, struct mail *mail) +{ + return t->box->copy(t, mail); +} + +int mailbox_is_inconsistent(struct mailbox *box) +{ + return box->is_inconsistent(box); +}
--- a/src/lib-storage/mail-storage.h Tue Apr 27 23:14:15 2004 +0300 +++ b/src/lib-storage/mail-storage.h Tue Apr 27 23:25:52 2004 +0300 @@ -3,7 +3,7 @@ struct message_size; -#include "imap-util.h" +#include "mail-types.h" enum mailbox_open_flags { MAILBOX_OPEN_READONLY = 0x01, @@ -47,14 +47,6 @@ MAILBOX_NAME_NOINFERIORS }; -enum mailbox_lock_type { - MAILBOX_LOCK_UNLOCK = 0x00, - MAILBOX_LOCK_READ = 0x01, - MAILBOX_LOCK_FLAGS = 0x02, - MAILBOX_LOCK_EXPUNGE = 0x04, - MAILBOX_LOCK_SAVE = 0x08 -}; - enum mail_sort_type { /* Maximum size for sort program, 2x for reverse + END */ #define MAX_SORT_PROGRAM_SIZE (2*7 + 1) @@ -92,20 +84,19 @@ /* specials: */ MAIL_FETCH_IMAP_BODY = 0x1000, MAIL_FETCH_IMAP_BODYSTRUCTURE = 0x2000, - MAIL_FETCH_IMAP_ENVELOPE = 0x4000 + MAIL_FETCH_IMAP_ENVELOPE = 0x4000, + MAIL_FETCH_FROM_ENVELOPE = 0x8000 }; enum mailbox_sync_flags { - MAILBOX_SYNC_NONE = 0x00, - MAILBOX_SYNC_FULL = 0x01, - MAILBOX_SYNC_FAST = 0x02, - MAILBOX_SYNC_FLAG_NO_EXPUNGES = 0x04 + MAILBOX_SYNC_FLAG_FAST = 0x01, + MAILBOX_SYNC_FLAG_NO_EXPUNGES = 0x02, + MAILBOX_SYNC_AUTO_STOP = 0x04 }; enum client_workarounds { WORKAROUND_OE6_FETCH_NO_NEWMAIL = 0x01, - WORKAROUND_OE6_FETCH_REDUNDANT_MSGSET = 0x02, - WORKAROUND_OUTLOOK_IDLE = 0x04 + WORKAROUND_OUTLOOK_IDLE = 0x02 }; struct mail_storage; @@ -115,230 +106,250 @@ struct mail_search_arg; struct fetch_context; struct search_context; - -/* All methods returning int return either TRUE or FALSE. */ -struct mail_storage { - char *name; - char *namespace; - - char hierarchy_sep; +struct mail; +struct mailbox; +struct mailbox_list_context; +struct mailbox_transaction_context; - /* Create new instance. If namespace is non-NULL, all mailbox names - are expected to begin with it. hierarchy_sep overrides the default - separator if it's not '\0'. */ - struct mail_storage *(*create)(const char *data, const char *user, - const char *namespace, - char hierarchy_sep); - - /* Free this instance */ - void (*free)(struct mail_storage *storage); - - /* Returns TRUE if this storage would accept the given data - as a valid parameter to create(). */ - int (*autodetect)(const char *data); +struct mailbox_list { + const char *name; + enum mailbox_flags flags; +}; - /* Set storage callback functions to use. */ - void (*set_callbacks)(struct mail_storage *storage, - struct mail_storage_callbacks *callbacks, - void *context); +struct mailbox_status { + uint32_t messages; + uint32_t recent; + uint32_t unseen; - /* Open a mailbox. If readonly is TRUE, mailbox must not be - modified in any way even when it's asked. If fast is TRUE, - any extra time consuming operations shouldn't be performed - (eg. when opening mailbox just for STATUS). + uint32_t uidvalidity; + uint32_t uidnext; - Note that append and copy may open the selected mailbox again - with possibly different readonly-state. */ - struct mailbox *(*open_mailbox)(struct mail_storage *storage, - const char *name, - enum mailbox_open_flags flags); + uint32_t first_unseen_seq; + + unsigned int diskspace_full:1; - /* name is allowed to contain multiple new hierarchy levels. - If directory is TRUE, the mailbox should be created so that it - can contain children. The mailbox itself doesn't have to be - created as long as it shows in LIST. */ - int (*create_mailbox)(struct mail_storage *storage, const char *name, - int directory); - - /* Only the specified mailbox is deleted, ie. folders under the - specified mailbox must not be deleted. */ - int (*delete_mailbox)(struct mail_storage *storage, const char *name); + /* may be allocated from data stack */ + unsigned int custom_flags_count; + const char **custom_flags; +}; - /* If the name has inferior hierarchical names, then the inferior - hierarchical names MUST also be renamed (ie. foo -> bar renames - also foo/bar -> bar/bar). newname may contain multiple new - hierarchies. - - If oldname is case-insensitively "INBOX", the mails are moved - into new folder but the INBOX folder must not be deleted. */ - int (*rename_mailbox)(struct mail_storage *storage, const char *oldname, - const char *newname); - - /* Initialize new mailbox list request. mask may contain '%' and '*' - wildcards as defined in RFC2060. Matching against "INBOX" is - case-insensitive, but anything else is not. */ - struct mailbox_list_context * - (*list_mailbox_init)(struct mail_storage *storage, - const char *mask, - enum mailbox_list_flags flags); - /* Deinitialize mailbox list request. Returns FALSE if some error - occured while listing. */ - int (*list_mailbox_deinit)(struct mailbox_list_context *ctx); - /* Get next mailbox. Returns the mailbox name */ - struct mailbox_list * - (*list_mailbox_next)(struct mailbox_list_context *ctx); +struct mail_storage_callbacks { + /* Alert: Not enough disk space */ + void (*alert_no_diskspace)(struct mailbox *mailbox, void *context); + /* "* OK <text>" */ + void (*notify_ok)(struct mailbox *mailbox, const char *text, + void *context); + /* "* NO <text>" */ + void (*notify_no)(struct mailbox *mailbox, const char *text, + void *context); - /* Subscribe/unsubscribe mailbox. There should be no error when - subscribing to already subscribed mailbox. Subscribing to - unexisting mailboxes is optional. */ - int (*set_subscribed)(struct mail_storage *storage, - const char *name, int set); - - /* Returns mailbox name status */ - int (*get_mailbox_name_status)(struct mail_storage *storage, - const char *name, - enum mailbox_name_status *status); + /* EXPUNGE */ + void (*expunge)(struct mailbox *mailbox, unsigned int seq, + void *context); + /* FETCH FLAGS */ + void (*update_flags)(struct mailbox *mailbox, unsigned int seq, + const struct mail_full_flags *flags, + void *context); - /* Returns the error message of last occured error. */ - const char *(*get_last_error)(struct mail_storage *storage, - int *syntax_error); + /* EXISTS, RECENT */ + void (*new_messages)(struct mailbox *mailbox, + unsigned int messages_count, + unsigned int recent_count, void *context); + /* FLAGS, PERMANENTFLAGS */ + void (*new_custom_flags)(struct mailbox *mailbox, + const char *custom_flags[], + unsigned int custom_flags_count, + void *context); -/* private: */ - char *dir; /* root directory */ - char *inbox_file; /* INBOX file for mbox */ - char *index_dir; - char *control_dir; - - char *user; /* name of user accessing the storage */ - char *error; - - struct mail_storage_callbacks *callbacks; - void *callback_context; - - unsigned int syntax_error:1; /* Give a BAD reply instead of NO */ }; -struct mailbox { - char *name; +extern enum client_workarounds client_workarounds; +extern int full_filesystem_access; - struct mail_storage *storage; +void mail_storage_init(void); +void mail_storage_deinit(void); + +/* register all mail storages */ +void mail_storage_register_all(void); - /* Returns TRUE if mailbox is read-only. */ - int (*is_readonly)(struct mailbox *box); +/* Register mail storage class with given name - all methods that are NULL + are set to default methods */ +void mail_storage_class_register(struct mail_storage *storage_class); +void mail_storage_class_unregister(struct mail_storage *storage_class); - /* Returns TRUE if mailbox supports adding custom flags. */ - int (*allow_new_custom_flags)(struct mailbox *box); +/* Create a new instance of registered mail storage class with given + storage-specific data. If data is NULL, it tries to use defaults. + May return NULL if anything fails. - /* Close the box. Returns FALSE if some cleanup errors occured, but - the mailbox was closed anyway. */ - int (*close)(struct mailbox *box); + If namespace is non-NULL, all mailbox names are expected to begin with it. + hierarchy_sep overrides the default separator if it's not '\0'. */ +struct mail_storage * +mail_storage_create(const char *name, const char *data, const char *user, + const char *namespace, char hierarchy_sep); +void mail_storage_destroy(struct mail_storage *storage); - /* Explicitly lock the mailbox. If not used, all the methods below - use the minimum locking requirements. This allows you to for - example use the update_flags() method in struct mail. The mailbox - stays locked until you unlock it. Note that if you call a method - which wants more locks than you've given here, the call will fail - (to avoid deadlocks). */ - int (*lock)(struct mailbox *box, enum mailbox_lock_type lock_type); +struct mail_storage * +mail_storage_create_default(const char *user, + const char *namespace, char hierarchy_sep); +struct mail_storage * +mail_storage_create_with_data(const char *data, const char *user, + const char *namespace, char hierarchy_sep); - /* Gets the mailbox status information. */ - int (*get_status)(struct mailbox *box, enum mailbox_status_items items, - struct mailbox_status *status); +char mail_storage_get_hierarchy_sep(struct mail_storage *storage); + +/* Set storage callback functions to use. */ +void mail_storage_set_callbacks(struct mail_storage *storage, + struct mail_storage_callbacks *callbacks, + void *context); - /* Synchronize the mailbox. */ - int (*sync)(struct mailbox *box, enum mailbox_sync_flags flags); +/* name is allowed to contain multiple new hierarchy levels. + If directory is TRUE, the mailbox should be created so that it + can contain children. The mailbox itself doesn't have to be + created as long as it shows in LIST. */ +int mail_storage_mailbox_create(struct mail_storage *storage, const char *name, + int directory); +/* Only the specified mailbox is deleted, ie. folders under the + specified mailbox must not be deleted. */ +int mail_storage_mailbox_delete(struct mail_storage *storage, const char *name); +/* If the name has inferior hierarchical names, then the inferior + hierarchical names MUST also be renamed (ie. foo -> bar renames + also foo/bar -> bar/bar). newname may contain multiple new + hierarchies. - /* Synchronize mailbox in background. It's done until this function is - called with flags = MAILBOX_SYNC_NONE. */ - void (*auto_sync)(struct mailbox *box, enum mailbox_sync_flags flags, - unsigned int min_newmail_notify_interval); + If oldname is case-insensitively "INBOX", the mails are moved + into new folder but the INBOX folder must not be deleted. */ +int mail_storage_mailbox_rename(struct mail_storage *storage, + const char *oldname, const char *newname); - /* Simplified fetching for a single UID or sequence. Must be called - between fetch_init() .. fetch_deinit() or - search_init() .. search_deinit() */ - struct mail *(*fetch_uid)(struct mailbox *box, unsigned int uid, - enum mail_fetch_field wanted_fields); - struct mail *(*fetch_seq)(struct mailbox *box, unsigned int seq, - enum mail_fetch_field wanted_fields); +/* Initialize new mailbox list request. mask may contain '%' and '*' + wildcards as defined in RFC2060. Matching against "INBOX" is + case-insensitive, but anything else is not. */ +struct mailbox_list_context * +mail_storage_mailbox_list_init(struct mail_storage *storage, + const char *mask, + enum mailbox_list_flags flags); +/* Get next mailbox. Returns the mailbox name */ +struct mailbox_list * +mail_storage_mailbox_list_next(struct mailbox_list_context *ctx); +/* Deinitialize mailbox list request. Returns FALSE if some error + occured while listing. */ +int mail_storage_mailbox_list_deinit(struct mailbox_list_context *ctx); - /* Modify sort_program to specify a sort program acceptable for - search_init(). If mailbox supports no sorting, it's simply set to - {MAIL_SORT_END}. */ - int (*search_get_sorting)(struct mailbox *box, - enum mail_sort_type *sort_program); - /* Initialize new search request. Search arguments are given so that - the storage can optimize the searching as it wants. +/* Subscribe/unsubscribe mailbox. There should be no error when + subscribing to already subscribed mailbox. Subscribing to + unexisting mailboxes is optional. */ +int mail_storage_set_subscribed(struct mail_storage *storage, + const char *name, int set); - If sort_program is non-NULL, it requests that the returned messages - are sorted by the given criteria. sort_program must have gone - through search_get_sorting(). +/* Returns mailbox name status */ +int mail_storage_get_mailbox_name_status(struct mail_storage *storage, + const char *name, + enum mailbox_name_status *status); + +/* Returns the error message of last occured error. */ +const char *mail_storage_get_last_error(struct mail_storage *storage, + int *syntax_error_r); - wanted_fields and wanted_headers aren't required, but they can be - used for optimizations. */ - struct mail_search_context * - (*search_init)(struct mailbox *box, const char *charset, - struct mail_search_arg *args, - const enum mail_sort_type *sort_program, - enum mail_fetch_field wanted_fields, - const char *const wanted_headers[]); - /* Deinitialize search request. all_found is set to TRUE if all of the - messages in search range were found. */ - int (*search_deinit)(struct mail_search_context *ctx, int *all_found); - /* Search the next message. Returned mail object can be used until - the next call to search_next() or search_deinit(). */ - struct mail *(*search_next)(struct mail_search_context *ctx); +/* Open a mailbox. If readonly is TRUE, mailbox must not be + modified in any way even when it's asked. If fast is TRUE, + any extra time consuming operations shouldn't be performed + (eg. when opening mailbox just for STATUS). + + Note that append and copy may open the selected mailbox again + with possibly different readonly-state. */ +struct mailbox *mailbox_open(struct mail_storage *storage, + const char *name, enum mailbox_open_flags flags); +/* Close the box. Returns FALSE if some cleanup errors occured, but + the mailbox was closed anyway. */ +int mailbox_close(struct mailbox *box); + +/* Returns storage of given mailbox */ +struct mail_storage *mailbox_get_storage(struct mailbox *box); + +/* Returns name of given mailbox */ +const char *mailbox_get_name(struct mailbox *box); + +/* Returns TRUE if mailbox is read-only. */ +int mailbox_is_readonly(struct mailbox *box); + +/* Returns TRUE if mailbox currently supports adding custom flags. */ +int mailbox_allow_new_custom_flags(struct mailbox *box); + +/* Gets the mailbox status information. */ +int mailbox_get_status(struct mailbox *box, + enum mailbox_status_items items, + struct mailbox_status *status); + +/* Synchronize the mailbox. */ +int mailbox_sync(struct mailbox *box, enum mailbox_sync_flags flags); + +/* Synchronize mailbox in background. It's done until this function is + called with flags = MAILBOX_SYNC_AUTO_STOP. */ +void mailbox_auto_sync(struct mailbox *box, enum mailbox_sync_flags flags, + unsigned int min_newmail_notify_interval); + +struct mailbox_transaction_context * +mailbox_transaction_begin(struct mailbox *box, int hide); +int mailbox_transaction_commit(struct mailbox_transaction_context *t); +void mailbox_transaction_rollback(struct mailbox_transaction_context *t); + +/* Simplified fetching for a single sequence. */ +struct mail *mailbox_fetch(struct mailbox_transaction_context *t, uint32_t seq, + enum mail_fetch_field wanted_fields); - /* Initialize saving one or more mails. If transaction is TRUE, all - the saved mails are deleted if an error occurs or save_deinit() - is called with rollback TRUE. */ - struct mail_save_context *(*save_init)(struct mailbox *box, - int transaction); - /* Deinitialize saving. rollback has effect only if save_init() was - called with transaction being TRUE. If rollback is FALSE but - committing the changes fails, all the commits are rollbacked if - possible. */ - int (*save_deinit)(struct mail_save_context *ctx, int rollback); - /* Save a mail into mailbox. timezone_offset specifies the timezone in - minutes in which received_date was originally given with. */ - int (*save_next)(struct mail_save_context *ctx, - const struct mail_full_flags *flags, - time_t received_date, int timezone_offset, - struct istream *data); +/* Convert uid range to sequence range. */ +int mailbox_get_uids(struct mailbox *box, uint32_t uid1, uint32_t uid2, + uint32_t *seq1_r, uint32_t *seq2_r); + +/* Modify sort_program to specify a sort program acceptable for + search_init(). If mailbox supports no sorting, it's simply set to + {MAIL_SORT_END}. */ +int mailbox_search_get_sorting(struct mailbox *box, + enum mail_sort_type *sort_program); +/* Initialize new search request. Search arguments are given so that + the storage can optimize the searching as it wants. + + If sort_program is non-NULL, it requests that the returned messages + are sorted by the given criteria. sort_program must have gone + through search_get_sorting(). - /* Initialize copying operation to this mailbox. The actual copying - can be done by fetching or searching mails and calling mail's - copy() method. */ - struct mail_copy_context *(*copy_init)(struct mailbox *box); - /* Finish copying. */ - int (*copy_deinit)(struct mail_copy_context *ctx, int rollback); - /* Copy given message. */ - int (*copy)(struct mail *mail, struct mail_copy_context *ctx); + wanted_fields and wanted_headers aren't required, but they can be + used for optimizations. */ +struct mail_search_context * +mailbox_search_init(struct mailbox_transaction_context *t, + const char *charset, struct mail_search_arg *args, + const enum mail_sort_type *sort_program, + enum mail_fetch_field wanted_fields, + const char *const wanted_headers[]); +/* Deinitialize search request. */ +int mailbox_search_deinit(struct mail_search_context *ctx); +/* Search the next message. Returned mail object can be used until + the next call to search_next() or search_deinit(). */ +struct mail *mailbox_search_next(struct mail_search_context *ctx); - /* Initialize expunging operation to this mailbox. If expunge_all - is TRUE, all messages are returned rather than just deleted. */ - struct mail_expunge_context * - (*expunge_init)(struct mailbox *box, - enum mail_fetch_field wanted_fields, - int expunge_all); - /* Finish expunging. */ - int (*expunge_deinit)(struct mail_expunge_context *ctx); - /* Fetch next mail. */ - struct mail *(*expunge_fetch_next)(struct mail_expunge_context *ctx); +/* Save a mail into mailbox. timezone_offset specifies the timezone in + minutes in which received_date was originally given with. To use + current time, set received_date to (time_t)-1. */ +int mailbox_save(struct mailbox_transaction_context *t, + const struct mail_full_flags *flags, + time_t received_date, int timezone_offset, + const char *from_envelope, struct istream *data); +/* Copy given message. */ +int mailbox_copy(struct mailbox_transaction_context *t, struct mail *mail); - /* Returns TRUE if mailbox is now in inconsistent state, meaning that - the message IDs etc. may have changed - only way to recover this - would be to fully close the mailbox and reopen it. With IMAP - connection this would mean a forced disconnection since we can't - do forced CLOSE. */ - int (*is_inconsistency_error)(struct mailbox *box); -}; +/* Returns TRUE if mailbox is now in inconsistent state, meaning that + the message IDs etc. may have changed - only way to recover this + would be to fully close the mailbox and reopen it. With IMAP + connection this would mean a forced disconnection since we can't + do forced CLOSE. */ +int mailbox_is_inconsistent(struct mailbox *box); struct mail { /* always set */ struct mailbox *box; - unsigned int seq; - unsigned int uid; + uint32_t seq, uid; + unsigned int expunged:1; unsigned int has_nuls:1; /* message data is known to contain NULs */ unsigned int has_no_nuls:1; /* -''- known to not contain NULs */ @@ -378,116 +389,8 @@ const struct mail_full_flags *flags, enum modify_type modify_type); - /* Expunge this message. Note that the actual message may or may not - be really expunged until expunge_deinit() is called. In any case, - after this call you must not try to access this mail, or any other - mail you've previously fetched. - - Since you can't be sure when the message is really expunged, you - can't be sure what it's sequence number is from client's point of - view. seq_r is set to that sequence number. - - This call is allowed only for mails fetched with - expunge_fetch_next(). Otherwise the sequence number updates would - get too tricky. */ - int (*expunge)(struct mail *mail, struct mail_expunge_context *ctx, - unsigned int *seq_r, int notify); -}; - -struct mailbox_list { - const char *name; - enum mailbox_flags flags; -}; - -struct mailbox_status { - unsigned int messages; - unsigned int recent; - unsigned int unseen; - - unsigned int uidvalidity; - unsigned int uidnext; - - unsigned int first_unseen_seq; - - unsigned int diskspace_full:1; - - /* may be allocated from data stack */ - unsigned int custom_flags_count; - const char **custom_flags; + /* Expunge this message. Sequence numbers don't change until commit. */ + int (*expunge)(struct mail *mail); }; -struct mail_storage_callbacks { - /* Alert: Not enough disk space */ - void (*alert_no_diskspace)(struct mailbox *mailbox, void *context); - /* "* OK <text>" */ - void (*notify_ok)(struct mailbox *mailbox, const char *text, - void *context); - /* "* NO <text>" */ - void (*notify_no)(struct mailbox *mailbox, const char *text, - void *context); - - /* EXPUNGE */ - void (*expunge)(struct mailbox *mailbox, unsigned int seq, - void *context); - /* FETCH FLAGS */ - void (*update_flags)(struct mailbox *mailbox, - unsigned int seq, unsigned int uid, - const struct mail_full_flags *flags, - void *context); - - /* EXISTS, RECENT */ - void (*new_messages)(struct mailbox *mailbox, - unsigned int messages_count, - unsigned int recent_count, void *context); - /* FLAGS, PERMANENTFLAGS */ - void (*new_custom_flags)(struct mailbox *mailbox, - const char *custom_flags[], - unsigned int custom_flags_count, - void *context); - -}; - -extern enum client_workarounds client_workarounds; -extern int full_filesystem_access; - -void mail_storage_init(void); -void mail_storage_deinit(void); - -/* register all mail storages */ -void mail_storage_register_all(void); - -/* Register mail storage class with given name - all methods that are NULL - are set to default methods */ -void mail_storage_class_register(struct mail_storage *storage_class); -void mail_storage_class_unregister(struct mail_storage *storage_class); - -/* Create a new instance of registered mail storage class with given - storage-specific data. If data is NULL, it tries to use defaults. - May return NULL if anything fails. */ -struct mail_storage * -mail_storage_create(const char *name, const char *data, const char *user, - const char *namespace, char hierarchy_sep); -void mail_storage_destroy(struct mail_storage *storage); - -struct mail_storage * -mail_storage_create_default(const char *user, - const char *namespace, char hierarchy_sep); -struct mail_storage * -mail_storage_create_with_data(const char *data, const char *user, - const char *namespace, char hierarchy_sep); - -/* Set error message in storage. Critical errors are logged with i_error(), - but user sees only "internal error" message. */ -void mail_storage_clear_error(struct mail_storage *storage); -void mail_storage_set_error(struct mail_storage *storage, - const char *fmt, ...) __attr_format__(2, 3); -void mail_storage_set_syntax_error(struct mail_storage *storage, - const char *fmt, ...) __attr_format__(2, 3); -void mail_storage_set_critical(struct mail_storage *storage, - const char *fmt, ...) __attr_format__(2, 3); -void mail_storage_set_internal_error(struct mail_storage *storage); - -const char *mail_storage_get_last_error(struct mail_storage *storage, - int *syntax); - #endif
--- a/src/lib-storage/proxy-mail-storage.c Tue Apr 27 23:14:15 2004 +0300 +++ b/src/lib-storage/proxy-mail-storage.c Tue Apr 27 23:25:52 2004 +0300 @@ -3,11 +3,11 @@ #include "lib.h" #include "proxy-mail-storage.h" -static void _free(struct mail_storage *storage) +static void _destroy(struct mail_storage *storage) { struct proxy_mail_storage *s = (struct proxy_mail_storage *) storage; - s->storage->free(s->storage); + return s->storage->destroy(s->storage); } static void _set_callbacks(struct mail_storage *storage, @@ -19,45 +19,45 @@ s->storage->set_callbacks(s->storage, callbacks, context); } -static struct mailbox *_open_mailbox(struct mail_storage *storage, +static struct mailbox *_mailbox_open(struct mail_storage *storage, const char *name, enum mailbox_open_flags flags) { struct proxy_mail_storage *s = (struct proxy_mail_storage *) storage; - return s->storage->open_mailbox(s->storage, name, flags); + return s->storage->mailbox_open(s->storage, name, flags); } -static int _create_mailbox(struct mail_storage *storage, const char *name, +static int _mailbox_create(struct mail_storage *storage, const char *name, int only_hierarchy) { struct proxy_mail_storage *s = (struct proxy_mail_storage *) storage; - return s->storage->create_mailbox(s->storage, name, only_hierarchy); + return s->storage->mailbox_create(s->storage, name, only_hierarchy); } -static int _delete_mailbox(struct mail_storage *storage, const char *name) +static int _mailbox_delete(struct mail_storage *storage, const char *name) { struct proxy_mail_storage *s = (struct proxy_mail_storage *) storage; - return s->storage->delete_mailbox(s->storage, name); + return s->storage->mailbox_delete(s->storage, name); } -static int _rename_mailbox(struct mail_storage *storage, const char *oldname, +static int _mailbox_rename(struct mail_storage *storage, const char *oldname, const char *newname) { struct proxy_mail_storage *s = (struct proxy_mail_storage *) storage; - return s->storage->rename_mailbox(s->storage, oldname, newname); + return s->storage->mailbox_rename(s->storage, oldname, newname); } static struct mailbox_list_context * -_list_mailbox_init(struct mail_storage *storage, const char *mask, +_mailbox_list_init(struct mail_storage *storage, const char *mask, enum mailbox_list_flags flags) { struct proxy_mail_storage *s = (struct proxy_mail_storage *) storage; - return s->storage->list_mailbox_init(s->storage, mask, flags); + return s->storage->mailbox_list_init(s->storage, mask, flags); } static int _set_subscribed(struct mail_storage *storage, @@ -97,16 +97,16 @@ ps->create = storage->create; ps->autodetect = storage->autodetect; - ps->list_mailbox_deinit = storage->list_mailbox_deinit; - ps->list_mailbox_next = storage->list_mailbox_next; + ps->mailbox_list_deinit = storage->mailbox_list_deinit; + ps->mailbox_list_next = storage->mailbox_list_next; - ps->free = _free; + ps->destroy = _destroy; ps->set_callbacks = _set_callbacks; - ps->open_mailbox = _open_mailbox; - ps->create_mailbox = _create_mailbox; - ps->delete_mailbox = _delete_mailbox; - ps->rename_mailbox = _rename_mailbox; - ps->list_mailbox_init = _list_mailbox_init; + ps->mailbox_open = _mailbox_open; + ps->mailbox_create = _mailbox_create; + ps->mailbox_delete = _mailbox_delete; + ps->mailbox_rename = _mailbox_rename; + ps->mailbox_list_init = _mailbox_list_init; ps->set_subscribed = _set_subscribed; ps->get_mailbox_name_status = _get_mailbox_name_status; ps->get_last_error = _get_last_error;
--- a/src/lib-storage/proxy-mail-storage.h Tue Apr 27 23:14:15 2004 +0300 +++ b/src/lib-storage/proxy-mail-storage.h Tue Apr 27 23:25:52 2004 +0300 @@ -1,7 +1,7 @@ #ifndef __PROXY_MAIL_STORAGE_H #define __PROXY_MAIL_STORAGE_H -#include "mail-storage.h" +#include "mail-storage-private.h" struct proxy_mail_storage { struct mail_storage proxy_storage;
--- a/src/lib-storage/proxy-mail.c Tue Apr 27 23:14:15 2004 +0300 +++ b/src/lib-storage/proxy-mail.c Tue Apr 27 23:25:52 2004 +0300 @@ -61,7 +61,8 @@ return p->mail->get_special(p->mail, field); } -static int _update_flags(struct mail *mail, const struct mail_full_flags *flags, +static int _update_flags(struct mail *mail, + const struct mail_full_flags *flags, enum modify_type modify_type) { struct proxy_mail *p = (struct proxy_mail *) mail; @@ -69,12 +70,11 @@ return p->mail->update_flags(p->mail, flags, modify_type); } -static int _expunge(struct mail *mail, struct mail_expunge_context *ctx, - unsigned int *seq_r, int notify) +static int _expunge(struct mail *mail) { struct proxy_mail *p = (struct proxy_mail *) mail; - return p->mail->expunge(p->mail, ctx, seq_r, notify); + return p->mail->expunge(p->mail); } void proxy_mail_init(struct proxy_mail *proxy, struct mail *mail)
--- a/src/lib-storage/proxy-mailbox.c Tue Apr 27 23:14:15 2004 +0300 +++ b/src/lib-storage/proxy-mailbox.c Tue Apr 27 23:25:52 2004 +0300 @@ -3,6 +3,7 @@ #include "lib.h" #include "proxy-mailbox.h" +#if 0 static int _is_readonly(struct mailbox *box) { struct proxy_mailbox *p = (struct proxy_mailbox *) box; @@ -24,13 +25,6 @@ return p->box->close(p->box); } -static int _lock(struct mailbox *box, enum mailbox_lock_type lock_type) -{ - struct proxy_mailbox *p = (struct proxy_mailbox *) box; - - return p->box->lock(p->box, lock_type); -} - static int _get_status(struct mailbox *box, enum mailbox_status_items items, struct mailbox_status *status) { @@ -54,20 +48,21 @@ p->box->auto_sync(p->box, flags, min_newmail_notify_interval); } -static struct mail *_fetch_uid(struct mailbox *box, unsigned int uid, - enum mail_fetch_field wanted_fields) +static struct mail *_fetch(struct mailbox_transaction_context *t, uint32_t seq, + enum mail_fetch_field wanted_fields) { - struct proxy_mailbox *p = (struct proxy_mailbox *) box; + struct proxy_mailbox *p = (struct proxy_mailbox *) t->box; - return p->box->fetch_uid(p->box, uid, wanted_fields); + return box->fetch(t, seq, wanted_fields); } -static struct mail *_fetch_seq(struct mailbox *box, unsigned int seq, - enum mail_fetch_field wanted_fields) +static int _get_uids(struct mailbox_transaction_context *t, + uint32_t uid1, uint32_t uid2, + uint32_t *seq1_r, uint32_t *seq2_r) { - struct proxy_mailbox *p = (struct proxy_mailbox *) box; + struct proxy_mailbox *p = (struct proxy_mailbox *) t->box; - return p->box->fetch_uid(p->box, seq, wanted_fields); + return p->box->get_uids(p->box, uid1, uid2, seq1_r, seq2_r); } static int _search_get_sorting(struct mailbox *box, @@ -91,35 +86,19 @@ wanted_fields, wanted_headers); } -static struct mail_save_context * -_save_init(struct mailbox *box, int transaction) -{ - struct proxy_mailbox *p = (struct proxy_mailbox *) box; - - return p->box->save_init(p->box, transaction); -} - -static struct mail_copy_context *_copy_init(struct mailbox *box) +static struct mailbox_transaction_context * +_transaction_begin(struct mailbox *box) { struct proxy_mailbox *p = (struct proxy_mailbox *) box; - return p->box->copy_init(p->box); + return p->box->transaction_begin(p->box); } -static struct mail_expunge_context * -_expunge_init(struct mailbox *box, enum mail_fetch_field wanted_fields, - int expunge_all) +static int _is_inconsistent(struct mailbox *box) { struct proxy_mailbox *p = (struct proxy_mailbox *) box; - return p->box->expunge_init(p->box, wanted_fields, expunge_all); -} - -static int _is_inconsistency_error(struct mailbox *box) -{ - struct proxy_mailbox *p = (struct proxy_mailbox *) box; - - return p->box->is_inconsistency_error(p->box); + return p->box->is_inconsistent(p->box); } void proxy_mailbox_init(struct proxy_mailbox *proxy, struct mailbox *box) @@ -131,27 +110,27 @@ pb->name = box->name; pb->storage = box->storage; - pb->search_deinit = box->search_deinit; - pb->search_next = box->search_next; - pb->save_deinit = box->save_deinit; - pb->save_next = box->save_next; - pb->copy_deinit = box->copy_deinit; - pb->expunge_deinit = box->expunge_deinit; - pb->expunge_fetch_next = box->expunge_fetch_next; - pb->is_readonly = _is_readonly; pb->allow_new_custom_flags = _allow_new_custom_flags; pb->close = _close; - pb->lock = _lock; pb->get_status = _get_status; pb->sync = _sync; pb->auto_sync = _auto_sync; - pb->fetch_uid = _fetch_uid; - pb->fetch_seq = _fetch_seq; + pb->fetch = box->fetch; + pb->get_uids = box->get_uids; + pb->search_get_sorting = _search_get_sorting; - pb->search_init = _search_init; - pb->save_init = _save_init; - pb->copy_init = _copy_init; - pb->expunge_init = _expunge_init; - pb->is_inconsistency_error = _is_inconsistency_error; + pb->search_init = box->search_init; + pb->search_next = box->search_next; + pb->search_deinit = box->search_deinit; + + pb->transaction_begin = _transaction_begin; + pb->transaction_commit = box->transaction_commit; + pb->transaction_rollback = box->transaction_rollback; + + pb->save = box->save; + pb->copy = box->copy; + + pb->is_inconsistent = _is_inconsistent; } +#endif
--- a/src/lib-storage/proxy-mailbox.h Tue Apr 27 23:14:15 2004 +0300 +++ b/src/lib-storage/proxy-mailbox.h Tue Apr 27 23:25:52 2004 +0300 @@ -1,7 +1,7 @@ #ifndef __PROXY_MAILBOX_H #define __PROXY_MAILBOX_H -#include "mail-storage.h" +#include "mail-storage-private.h" struct proxy_mailbox { struct mailbox proxy_box;
--- a/src/lib-storage/register/Makefile.am Tue Apr 27 23:14:15 2004 +0300 +++ b/src/lib-storage/register/Makefile.am Tue Apr 27 23:25:52 2004 +0300 @@ -2,7 +2,7 @@ INCLUDES = \ -I$(top_srcdir)/src/lib \ - -I$(top_srcdir)/src/lib-imap \ + -I$(top_srcdir)/src/lib-mail \ -I$(top_srcdir)/src/lib-storage libstorage_register_a_SOURCES = \
--- a/src/lib-storage/subscription-file/Makefile.am Tue Apr 27 23:14:15 2004 +0300 +++ b/src/lib-storage/subscription-file/Makefile.am Tue Apr 27 23:25:52 2004 +0300 @@ -3,7 +3,7 @@ INCLUDES = \ -I$(top_srcdir)/src/lib \ -I$(top_srcdir)/src/lib-storage \ - -I$(top_srcdir)/src/lib-imap + -I$(top_srcdir)/src/lib-mail libstorage_subscription_file_a_SOURCES = \ subscription-file.c
--- a/src/lib-storage/subscription-file/subscription-file.c Tue Apr 27 23:14:15 2004 +0300 +++ b/src/lib-storage/subscription-file/subscription-file.c Tue Apr 27 23:25:52 2004 +0300 @@ -4,13 +4,12 @@ #include "istream.h" #include "ostream.h" #include "file-dotlock.h" -#include "mail-storage.h" +#include "mail-storage-private.h" #include "subscription-file.h" #include <unistd.h> #include <fcntl.h> -#define SUBSCRIPTION_FILE_NAME ".subscriptions" #define MAX_MAILBOX_LENGTH PATH_MAX #define SUBSCRIPTION_FILE_LOCK_TIMEOUT 120 @@ -27,19 +26,18 @@ int failed; }; -static int subsfile_set_syscall_error(struct mail_storage *storage, - const char *function, const char *path) +static void subsfile_set_syscall_error(struct mail_storage *storage, + const char *function, const char *path) { i_assert(function != NULL); - if (errno == EACCES) { + if (errno == EACCES) mail_storage_set_error(storage, "Permission denied"); - return FALSE; + else { + mail_storage_set_critical(storage, + "%s failed with subscription file %s: %m", + function, path); } - - mail_storage_set_critical(storage, - "%s failed with subscription file %s: %m", function, path); - return FALSE; } static const char *next_line(struct mail_storage *storage, const char *path, @@ -69,10 +67,10 @@ return line; } -int subsfile_set_subscribed(struct mail_storage *storage, +int subsfile_set_subscribed(struct mail_storage *storage, const char *path, const char *name, int set) { - const char *path, *line; + const char *line; struct istream *input; struct ostream *output; int fd_in, fd_out, found, failed = FALSE; @@ -80,9 +78,6 @@ if (strcasecmp(name, "INBOX") == 0) name = "INBOX"; - path = t_strconcat(storage->control_dir != NULL ? - storage->control_dir : storage->dir, - "/" SUBSCRIPTION_FILE_NAME, NULL); /* FIXME: set lock notification callback */ fd_out = file_dotlock_open(path, NULL, SUBSCRIPTION_FILE_LOCK_TIMEOUT, SUBSCRIPTION_FILE_CHANGE_TIMEOUT, @@ -96,14 +91,14 @@ subsfile_set_syscall_error(storage, "file_dotlock_open()", path); } - return FALSE; + return -1; } fd_in = open(path, O_RDONLY); if (fd_in == -1 && errno != ENOENT) { subsfile_set_syscall_error(storage, "open()", path); file_dotlock_delete(path, fd_out); - return FALSE; + return -1; } input = fd_in == -1 ? NULL : @@ -154,20 +149,16 @@ failed = TRUE; } } - return !failed; + return failed ? -1 : 0; } struct subsfile_list_context * -subsfile_list_init(struct mail_storage *storage) +subsfile_list_init(struct mail_storage *storage, const char *path) { struct subsfile_list_context *ctx; pool_t pool; - const char *path; int fd; - path = t_strconcat(storage->control_dir != NULL ? - storage->control_dir : storage->dir, - "/" SUBSCRIPTION_FILE_NAME, NULL); fd = open(path, O_RDONLY); if (fd == -1 && errno != ENOENT) { subsfile_set_syscall_error(storage, "open()", path); @@ -194,7 +185,7 @@ i_stream_unref(ctx->input); pool_unref(ctx->pool); - return !failed; + return failed ? -1 : 0; } const char *subsfile_list_next(struct subsfile_list_context *ctx)
--- a/src/lib-storage/subscription-file/subscription-file.h Tue Apr 27 23:14:15 2004 +0300 +++ b/src/lib-storage/subscription-file/subscription-file.h Tue Apr 27 23:25:52 2004 +0300 @@ -5,7 +5,7 @@ /* Initialize new subscription file listing. Returns NULL if failed. */ struct subsfile_list_context * -subsfile_list_init(struct mail_storage *storage); +subsfile_list_init(struct mail_storage *storage, const char *path); /* Deinitialize subscription file listing. Returns FALSE if some error occured while listing. */ @@ -13,7 +13,7 @@ /* Returns the next subscribed mailbox, or NULL. */ const char *subsfile_list_next(struct subsfile_list_context *ctx); -int subsfile_set_subscribed(struct mail_storage *storage, +int subsfile_set_subscribed(struct mail_storage *storage, const char *path, const char *name, int set); #endif
--- a/src/lib/Makefile.am Tue Apr 27 23:14:15 2004 +0300 +++ b/src/lib/Makefile.am Tue Apr 27 23:25:52 2004 +0300 @@ -48,6 +48,7 @@ printf-upper-bound.c \ process-title.c \ randgen.c \ + read-full.c \ restrict-access.c \ restrict-process-size.c \ safe-memset.c \ @@ -102,6 +103,7 @@ printf-upper-bound.h \ process-title.h \ randgen.h \ + read-full.h \ restrict-access.h \ restrict-process-size.h \ safe-memset.h \
--- a/src/lib/compat.h Tue Apr 27 23:14:15 2004 +0300 +++ b/src/lib/compat.h Tue Apr 27 23:25:52 2004 +0300 @@ -81,6 +81,11 @@ # define fdatasync fsync #endif +struct const_iovec { + const void *iov_base; + size_t iov_len; +}; + #ifndef HAVE_STRUCT_IOVEC struct iovec { void *iov_base;
--- a/src/lib/file-dotlock.c Tue Apr 27 23:14:15 2004 +0300 +++ b/src/lib/file-dotlock.c Tue Apr 27 23:25:52 2004 +0300 @@ -399,6 +399,7 @@ if (dotlock->ino != st.st_ino || !CMP_DEV_T(dotlock->dev, st.st_dev)) { i_warning("Our dotlock file %s was overridden", lock_path); + errno = EEXIST; return 0; } @@ -447,12 +448,15 @@ { struct stat st, st2; const char *lock_path; + int old_errno; lock_path = t_strconcat(path, ".lock", NULL); if (verify_owner) { if (fstat(fd, &st) < 0) { + old_errno = errno; i_error("fstat(%s) failed: %m", lock_path); (void)close(fd); + errno = old_errno; return -1; } } @@ -471,6 +475,7 @@ !CMP_DEV_T(st.st_dev, st2.st_dev)) { i_warning("Our dotlock file %s was overridden", lock_path); + errno = EEXIST; return 0; } } @@ -486,11 +491,14 @@ { struct dotlock dotlock; struct stat st; + int old_errno; if (fstat(fd, &st) < 0) { + old_errno = errno; i_error("fstat(%s) failed: %m", t_strconcat(path, ".lock", NULL)); (void)close(fd); + errno = old_errno; return -1; }
--- a/src/lib/file-lock.c Tue Apr 27 23:14:15 2004 +0300 +++ b/src/lib/file-lock.c Tue Apr 27 23:25:52 2004 +0300 @@ -30,6 +30,7 @@ else { alarm_hup_init(); timeout_time = time(NULL) + timeout; + alarm(timeout); } fl.l_type = lock_type;
--- a/src/lib/istream-data.c Tue Apr 27 23:14:15 2004 +0300 +++ b/src/lib/istream-data.c Tue Apr 27 23:25:52 2004 +0300 @@ -23,9 +23,8 @@ { } -static ssize_t _read(struct _istream *stream) +static ssize_t _read(struct _istream *stream __attr_unused__) { - stream->istream.eof = TRUE; return -1; }
--- a/src/lib/istream-file.c Tue Apr 27 23:14:15 2004 +0300 +++ b/src/lib/istream-file.c Tue Apr 27 23:25:52 2004 +0300 @@ -152,7 +152,8 @@ } if (ret == 0) { /* EOF */ - stream->istream.eof = TRUE; + if (!fstream->file) + stream->istream.eof = TRUE; return -1; }
--- a/src/lib/istream-limit.c Tue Apr 27 23:14:15 2004 +0300 +++ b/src/lib/istream-limit.c Tue Apr 27 23:25:52 2004 +0300 @@ -61,8 +61,8 @@ if (i_stream_read(lstream->input) == -2 && stream->buffer != NULL) { if (stream->skip == 0) return -2; - stream->istream.eof = lstream->input->eof; } + stream->istream.eof = lstream->input->eof; stream->pos -= stream->skip; stream->skip = 0;
--- a/src/lib/istream-mmap.c Tue Apr 27 23:14:15 2004 +0300 +++ b/src/lib/istream-mmap.c Tue Apr 27 23:25:52 2004 +0300 @@ -90,10 +90,8 @@ return stream->pos - stream->skip; } - if (stream->istream.v_offset >= mstream->v_size) { - stream->istream.eof = TRUE; + if (stream->istream.v_offset >= mstream->v_size) return -1; - } aligned_skip = stream->skip & ~mmap_pagemask; if (aligned_skip == 0 && mstream->mmap_base != NULL) {
--- a/src/lib/macros.h Tue Apr 27 23:14:15 2004 +0300 +++ b/src/lib/macros.h Tue Apr 27 23:25:52 2004 +0300 @@ -21,6 +21,11 @@ #define MEM_ALIGN(size) \ (((size) + MEM_ALIGN_SIZE-1) & ~((unsigned int) MEM_ALIGN_SIZE-1)) +#define PTR_OFFSET(ptr, offset) \ + ((void *) (((unsigned char *) (ptr)) + (offset))) +#define CONST_PTR_OFFSET(ptr, offset) \ + ((const void *) (((const unsigned char *) (ptr)) + (offset))) + /* Don't use simply MIN/MAX, as they're often defined elsewhere in include files that are included after this file generating tons of warnings. */ #define I_MIN(a, b) (((a) < (b)) ? (a) : (b))
--- a/src/lib/network.c Tue Apr 27 23:14:15 2004 +0300 +++ b/src/lib/network.c Tue Apr 27 23:25:52 2004 +0300 @@ -579,8 +579,12 @@ int net_hosterror_notfound(int error) { #ifdef HAVE_IPV6 +#ifdef EAI_NODATA /* NODATA is depricated */ return error != 1 && (error == EAI_NONAME || error == EAI_NODATA); #else + return error != 1 && (error == EAI_NONAME); +#endif +#else return error == HOST_NOT_FOUND || error == NO_ADDRESS; #endif }
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib/read-full.c Tue Apr 27 23:25:52 2004 +0300 @@ -0,0 +1,38 @@ +/* Copyright (c) 2003 Timo Sirainen */ + +#include "lib.h" +#include "read-full.h" + +#include <unistd.h> + +int read_full(int fd, void *data, size_t size) +{ + ssize_t ret; + + while (size > 0) { + ret = read(fd, data, size < SSIZE_T_MAX ? size : SSIZE_T_MAX); + if (ret <= 0) + return ret; + + size -= ret; + } + + return 1; +} + +int pread_full(int fd, void *data, size_t size, off_t offset) +{ + ssize_t ret; + + while (size > 0) { + ret = pread(fd, data, size < SSIZE_T_MAX ? + size : SSIZE_T_MAX, offset); + if (ret <= 0) + return ret; + + size -= ret; + offset += ret; + } + + return 1; +}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib/read-full.h Tue Apr 27 23:25:52 2004 +0300 @@ -0,0 +1,9 @@ +#ifndef __READ_FULL_H +#define __READ_FULL_H + +/* Read data from file. Returns -1 if error occured, or 0 if EOF came before + everything was read, or 1 if all was ok. */ +int read_full(int fd, void *data, size_t size); +int pread_full(int fd, void *data, size_t size, off_t offset); + +#endif
--- a/src/lib/write-full.c Tue Apr 27 23:14:15 2004 +0300 +++ b/src/lib/write-full.c Tue Apr 27 23:25:52 2004 +0300 @@ -25,3 +25,26 @@ return 0; } + +int pwrite_full(int fd, const void *data, size_t size, off_t offset) +{ + ssize_t ret; + + while (size > 0) { + ret = pwrite(fd, data, size < SSIZE_T_MAX ? + size : SSIZE_T_MAX, offset); + if (ret < 0) + return -1; + + if (ret == 0) { + /* nothing was written, only reason for this should + be out of disk space */ + errno = ENOSPC; + return -1; + } + size -= ret; + offset += ret; + } + + return 0; +}
--- a/src/lib/write-full.h Tue Apr 27 23:14:15 2004 +0300 +++ b/src/lib/write-full.h Tue Apr 27 23:25:52 2004 +0300 @@ -5,5 +5,6 @@ If there's not enough space in device, -1 with ENOSPC is returned, and it's unspecified how much data was actually written. */ int write_full(int fd, const void *data, size_t size); +int pwrite_full(int fd, const void *data, size_t size, off_t offset); #endif
--- a/src/pop3/client.c Tue Apr 27 23:14:15 2004 +0300 +++ b/src/pop3/client.c Tue Apr 27 23:25:52 2004 +0300 @@ -41,14 +41,15 @@ static int init_mailbox(struct client *client) { struct mail_search_arg search_arg; + struct mailbox_transaction_context *t; struct mail_search_context *ctx; struct mail *mail; struct mailbox_status status; - int i, all_found, failed; + int i, failed; - if (!client->mailbox->get_status(client->mailbox, - STATUS_MESSAGES | STATUS_UIDVALIDITY, - &status)) { + if (mailbox_get_status(client->mailbox, + STATUS_MESSAGES | STATUS_UIDVALIDITY, + &status) < 0) { client_send_storage_error(client); return FALSE; } @@ -62,19 +63,21 @@ memset(&search_arg, 0, sizeof(search_arg)); search_arg.type = SEARCH_ALL; + t = mailbox_transaction_begin(client->mailbox, FALSE); + client->message_sizes = i_new(uoff_t, client->messages_count); for (i = 0; i < 2; i++) { - ctx = client->mailbox->search_init(client->mailbox, NULL, - &search_arg, NULL, - MAIL_FETCH_SIZE, NULL); + ctx = mailbox_search_init(t, NULL, &search_arg, NULL, + MAIL_FETCH_SIZE, NULL); if (ctx == NULL) { client_send_storage_error(client); + mailbox_transaction_rollback(t); return FALSE; } client->total_size = 0; failed = FALSE; - while ((mail = client->mailbox->search_next(ctx)) != NULL) { + while ((mail = mailbox_search_next(ctx)) != NULL) { uoff_t size = mail->get_size(mail); if (size == (uoff_t)-1) { @@ -87,20 +90,23 @@ client->message_sizes[mail->seq-1] = size; } - if (!client->mailbox->search_deinit(ctx, &all_found)) { + if (mailbox_search_deinit(ctx) < 0) { client_send_storage_error(client); + mailbox_transaction_rollback(t); return FALSE; } - if (!failed && all_found) + if (!failed) return TRUE; /* well, sync and try again */ - if (!client->mailbox->sync(client->mailbox, TRUE)) { + if (mailbox_sync(client->mailbox, 0) < 0) { client_send_storage_error(client); + mailbox_transaction_rollback(t); return FALSE; } } + mailbox_transaction_commit(t); client_send_line(client, "-ERR [IN-USE] Couldn't sync mailbox."); return FALSE; @@ -124,11 +130,11 @@ client->last_input = ioloop_time; client->storage = storage; - storage->set_callbacks(storage, &mail_storage_callbacks, client); + mail_storage_set_callbacks(storage, &mail_storage_callbacks, client); flags = getenv("MMAP_INVALIDATE") != NULL ? MAILBOX_OPEN_MMAP_INVALIDATE : 0; - client->mailbox = storage->open_mailbox(storage, "INBOX", flags); + client->mailbox = mailbox_open(storage, "INBOX", flags); if (client->mailbox == NULL) { client_send_line(client, "-ERR No INBOX for user."); return NULL; @@ -152,7 +158,7 @@ o_stream_flush(client->output); if (client->mailbox != NULL) - client->mailbox->close(client->mailbox); + mailbox_close(client->mailbox); mail_storage_destroy(client->storage); i_free(client->message_sizes); @@ -195,14 +201,14 @@ { const char *error; - if (client->mailbox->is_inconsistency_error(client->mailbox)) { + if (mailbox_is_inconsistent(client->mailbox)) { client_send_line(client, "-ERR Mailbox is in inconsistent " "state, please relogin."); client_disconnect(client); return; } - error = client->storage->get_last_error(client->storage, NULL); + error = mail_storage_get_last_error(client->storage, NULL); client_send_line(client, "-ERR %s", error != NULL ? error : "BUG: Unknown error"); }
--- a/src/pop3/commands.c Tue Apr 27 23:14:15 2004 +0300 +++ b/src/pop3/commands.c Tue Apr 27 23:25:52 2004 +0300 @@ -142,34 +142,39 @@ static int expunge_mails(struct client *client, struct mailbox *box) { - struct mail_expunge_context *ctx; + struct mail_search_arg search_arg; + struct mailbox_transaction_context *t; + struct mail_search_context *ctx; struct mail *mail; - unsigned int i, j; + uint32_t i; int failed = FALSE; - /* NOTE: if there's any external expunges, they'll get synced here. - Currently we update only the deleted_bitmask[] so we don't end up - expunging wrong messages, but message_sizes[] isn't updated. */ - ctx = box->expunge_init(box, 0, TRUE); - if (ctx == NULL) + memset(&search_arg, 0, sizeof(search_arg)); + search_arg.type = SEARCH_ALL; + + t = mailbox_transaction_begin(box, FALSE); + ctx = mailbox_search_init(t, NULL, &search_arg, NULL, + MAIL_FETCH_SIZE, NULL); + if (ctx == NULL) { + mailbox_transaction_rollback(t); return FALSE; + } - i = j = 0; - while ((mail = box->expunge_fetch_next(ctx)) != NULL) { - if ((client->deleted_bitmask[i] & (1 << j)) != 0) { - if (!mail->expunge(mail, ctx, NULL, FALSE)) { + while ((mail = mailbox_search_next(ctx)) != NULL) { + i = mail->seq-1; + if ((client->deleted_bitmask[i >> CHAR_BIT] & + (1 << (i % CHAR_BIT))) != 0) { + if (mail->expunge(mail) < 0) { failed = TRUE; break; } } - if (++j == CHAR_BIT) { - j = 0; i++; - } } - if (!box->expunge_deinit(ctx)) + if (mailbox_search_deinit(ctx) < 0) return FALSE; + mailbox_transaction_commit(t); return !failed; } @@ -257,24 +262,29 @@ uoff_t body_lines) { struct mail_search_arg search_arg; + struct mail_search_seqset seqset; + struct mailbox_transaction_context *t; struct mail_search_context *ctx; struct mail *mail; struct istream *stream; + seqset.seq1 = seqset.seq2 = msgnum+1; + memset(&search_arg, 0, sizeof(search_arg)); - search_arg.type = SEARCH_SET; - search_arg.value.str = dec2str(msgnum+1); + search_arg.type = SEARCH_SEQSET; + search_arg.value.seqset = &seqset; - ctx = client->mailbox->search_init(client->mailbox, NULL, - &search_arg, NULL, - MAIL_FETCH_STREAM_HEADER | - MAIL_FETCH_STREAM_BODY, NULL); + t = mailbox_transaction_begin(client->mailbox, FALSE); + ctx = mailbox_search_init(t, NULL, &search_arg, NULL, + MAIL_FETCH_STREAM_HEADER | + MAIL_FETCH_STREAM_BODY, NULL); if (ctx == NULL) { + mailbox_transaction_rollback(t); client_send_storage_error(client); return; } - mail = client->mailbox->search_next(ctx); + mail = mailbox_search_next(ctx); stream = mail == NULL ? NULL : mail->get_stream(mail, NULL, NULL); if (stream == NULL) client_send_line(client, "-ERR Message not found."); @@ -290,7 +300,8 @@ client_send_line(client, "."); } - (void)client->mailbox->search_deinit(ctx, NULL); + (void)mailbox_search_deinit(ctx); + (void)mailbox_transaction_commit(t); } static int cmd_retr(struct client *client, const char *args) @@ -340,6 +351,8 @@ static void list_uids(struct client *client, unsigned int message) { struct mail_search_arg search_arg; + struct mail_search_seqset seqset; + struct mailbox_transaction_context *t; struct mail_search_context *ctx; struct mail *mail; int found = FALSE; @@ -351,25 +364,28 @@ if (message == 0) search_arg.type = SEARCH_ALL; else { - search_arg.type = SEARCH_SET; - search_arg.value.str = dec2str(message); + seqset.seq1 = seqset.seq2 = message; + search_arg.type = SEARCH_SEQSET; + search_arg.value.seqset = &seqset; } - ctx = client->mailbox->search_init(client->mailbox, NULL, - &search_arg, NULL, 0, NULL); + t = mailbox_transaction_begin(client->mailbox, FALSE); + ctx = mailbox_search_init(t, NULL, &search_arg, NULL, 0, NULL); if (ctx == NULL) { + mailbox_transaction_rollback(t); client_send_storage_error(client); return; } - while ((mail = client->mailbox->search_next(ctx)) != NULL) { + while ((mail = mailbox_search_next(ctx)) != NULL) { client_send_line(client, message == 0 ? "%u %u.%u" : "+OK %u %u.%u", mail->seq, client->uidvalidity, mail->uid); found = TRUE; } - (void)client->mailbox->search_deinit(ctx, NULL); + (void)mailbox_search_deinit(ctx); + (void)mailbox_transaction_commit(t); if (!found && message != 0) client_send_line(client, "-ERR Message not found.");
--- a/src/pop3/mail-storage-callbacks.c Tue Apr 27 23:14:15 2004 +0300 +++ b/src/pop3/mail-storage-callbacks.c Tue Apr 27 23:25:52 2004 +0300 @@ -54,7 +54,6 @@ static void update_flags(struct mailbox *mailbox __attr_unused__, unsigned int seq __attr_unused__, - unsigned int uid __attr_unused__, const struct mail_full_flags *flags __attr_unused__, void *context __attr_unused__) {