# HG changeset patch # User Timo Sirainen # Date 1357341251 -7200 # Node ID 13e74bd5ac8c05e5f2f54822af8d3fdaa470ef18 # Parent 922049229f7fdfc0f6dd27f71ca8805d6d7204d5# Parent 3eeb5270963b701b442e6c1207a25440c2b3637f Merged changes from v2.1 tree. diff -r 922049229f7f -r 13e74bd5ac8c .hgsigs --- a/.hgsigs Sat Jan 05 00:37:26 2013 +0200 +++ b/.hgsigs Sat Jan 05 01:14:11 2013 +0200 @@ -51,3 +51,5 @@ bc86680293d256d5a8009690caeb73ab2e34e359 0 iEYEABECAAYFAlAZaTUACgkQyUhSUUBVisnTAACfU1pB34RrXEyLnpnL4Ee/oeNBYcoAnRWxTqx870Efjwf+eBPzafO0C/NU 1a6c3b4e92e4174d3b1eb0a7c841f97e8fb9e590 0 iEYEABECAAYFAlBYwJMACgkQyUhSUUBVisn2PwCeIJxfB5ebXlAbtMcjrZBCmB8Kg1sAn39BC9rQoR/wjD2/ix1JaxH7gHOT f5941f3ac7622361634b6cba464da79cc883d1bb 0 iEYEABECAAYFAlCO62QACgkQyUhSUUBViskUtQCffWRQpSqaf+iCOipsTWE1D93TwVEAnAhxx1aezuqDVAsjWoYZkrIufO28 +741d800a192fa23572bb14196df2a8917cf20614 0 iEYEABECAAYFAlC3A5EACgkQyUhSUUBVisnmlACcCm6jc7NRoTkBtrJLcz+P325U1xcAn2+0eghqEMiP+rzRJC55oQxV00Zy +75bfda4a7c6c9aa04b6a6ef233fc527356171a06 0 iEYEABECAAYFAlC4WKwACgkQyUhSUUBViskaOACgmcwWV8hgsCOWvkbdh0OIw1ImSQYAn1RcTL0CG3M8+XG7QrrxSfQ7+V99 diff -r 922049229f7f -r 13e74bd5ac8c .hgtags --- a/.hgtags Sat Jan 05 00:37:26 2013 +0200 +++ b/.hgtags Sat Jan 05 01:14:11 2013 +0200 @@ -88,3 +88,5 @@ bc86680293d256d5a8009690caeb73ab2e34e359 2.1.9 1a6c3b4e92e4174d3b1eb0a7c841f97e8fb9e590 2.1.10 f5941f3ac7622361634b6cba464da79cc883d1bb 2.2.alpha1 +741d800a192fa23572bb14196df2a8917cf20614 2.1.11 +75bfda4a7c6c9aa04b6a6ef233fc527356171a06 2.1.12 diff -r 922049229f7f -r 13e74bd5ac8c Makefile.am --- a/Makefile.am Sat Jan 05 00:37:26 2013 +0200 +++ b/Makefile.am Sat Jan 05 01:14:11 2013 +0200 @@ -64,12 +64,12 @@ grep -v '^LIBDOVECOT_.*_INCLUDE' dovecot-config | \ grep -v '^LIBDOVECOT.*_DEPS' | sed \ -e "s|^$LIBDOVECOT$=.*$$|\1='-L$(pkglibdir) -ldovecot'|" \ - -e "s|^$LIBDOVECOT_LOGIN$=.*$$|\1=-ldovecot-login|" \ + -e "s|^$LIBDOVECOT_LOGIN$=.*$$|\1='-ldovecot-login $(SSL_LIBS)'|" \ -e "s|^$LIBDOVECOT_SQL$=.*$$|\1=-ldovecot-sql|" \ -e "s|^$LIBDOVECOT_SSL$=.*$$|\1=-ldovecot-ssl|" \ -e "s|^$LIBDOVECOT_COMPRESS$=.*$$|\1=-ldovecot-compression|" \ -e "s|^$LIBDOVECOT_LDA$=.*$$|\1=-ldovecot-lda|" \ - -e "s|^$LIBDOVECOT_STORAGE$=.*$$|\1=-ldovecot-storage|" \ + -e "s|^$LIBDOVECOT_STORAGE$=.*$$|\1='-ldovecot-storage $(LINKED_STORAGE_LDADD)'|" \ -e "s|^$LIBDOVECOT_INCLUDE$=.*$$|\1=-I$(pkgincludedir)|" \ > $(DESTDIR)$(pkglibdir)/dovecot-config diff -r 922049229f7f -r 13e74bd5ac8c NEWS --- a/NEWS Sat Jan 05 00:37:26 2013 +0200 +++ b/NEWS Sat Jan 05 01:14:11 2013 +0200 @@ -30,6 +30,29 @@ + LMTP proxy: Implemented XCLIENT extension for passing remote IP address through proxy. +v2.1.12 2012-11-30 Timo Sirainen + + - dovecot-config in v2.1.11 caused build problems with Pigeonhole + +v2.1.11 2012-11-29 Timo Sirainen + + * lmtp/lda: dovecot.index.cache file is no longer fully mapped to + memory, allowing mail deliveries to work even if the file is huge. + * auth: userdb passwd lookups are now done by auth worker processes + instead of auth master process (as it was documented, but + accidentally didn't work that way). + + + lmtp: lmtp_rcpt_check_quota=yes setting checks quota on RCPT TO. + - lmtp: After successful proxying RCPT TO, the next one to a + nonexistent user gave tempfail error instead of "user not found". + - lmtp proxy: Fixed hanging if remote server was down. + - imap: Fixed crash when SEARCH contained multiple KEYWORD parameters. + - doveadm: Various fixes to handling doveadm-server connections. + - -i parameter for Dovecot tools didn't work correctly. + - director was somewhat broken in v2.1.10. This version also includes + various reliability enhancements. + - auth: passdb imap was broken in v2.1.10. + v2.1.10 2012-09-18 Timo Sirainen + imap: Implemented THREAD=ORDEREDSUBJECT extension. diff -r 922049229f7f -r 13e74bd5ac8c configure.ac diff -r 922049229f7f -r 13e74bd5ac8c doc/example-config/Makefile.am --- a/doc/example-config/Makefile.am Sat Jan 05 00:37:26 2013 +0200 +++ b/doc/example-config/Makefile.am Sat Jan 05 01:14:11 2013 +0200 @@ -13,6 +13,7 @@ example_DATA = \ dovecot.conf \ dovecot-db.conf.ext \ + dovecot-dict-auth.conf.ext \ dovecot-dict-sql.conf.ext \ dovecot-ldap.conf.ext \ dovecot-sql.conf.ext diff -r 922049229f7f -r 13e74bd5ac8c doc/man/doveadm-expunge.1.in --- a/doc/man/doveadm-expunge.1.in Sat Jan 05 00:37:26 2013 +0200 +++ b/doc/man/doveadm-expunge.1.in Sat Jan 05 01:14:11 2013 +0200 @@ -1,5 +1,5 @@ -.\" Copyright (c) 2010 Dovecot authors, see the included COPYING file -.TH DOVEADM\-EXPUNGE 1 "2010-11-25" "Dovecot v2.1" "Dovecot" +.\" Copyright (c) 2010-2012 Dovecot authors, see the included COPYING file +.TH DOVEADM\-EXPUNGE 1 "2012-11-27" "Dovecot v2.1" "Dovecot" .SH NAME doveadm\-expunge \- Expunge messages matching given search query .\"------------------------------------------------------------------------ @@ -47,6 +47,10 @@ .\"------------------------------------- @INCLUDE:option-A@ .\"------------------------------------- +.TP +.B \-d +Delete the mailbox if it is empty after expunging. +.\"------------------------------------- @INCLUDE:option-S-socket@ .\"------------------------------------- @INCLUDE:option-u-user@ diff -r 922049229f7f -r 13e74bd5ac8c src/auth/auth-master-connection.c --- a/src/auth/auth-master-connection.c Sat Jan 05 00:37:26 2013 +0200 +++ b/src/auth/auth-master-connection.c Sat Jan 05 01:14:11 2013 +0200 @@ -240,7 +240,8 @@ auth_request_log_error(auth_request, "userdb", "client doesn't have lookup permissions for this user: %s " - "(change userdb socket permissions)", reason); + "(to bypass this check, set: service auth { unix_listener %s { mode=0777 } })", + reason, conn->path); return -1; } diff -r 922049229f7f -r 13e74bd5ac8c src/auth/auth-request.c diff -r 922049229f7f -r 13e74bd5ac8c src/auth/auth-stream.c diff -r 922049229f7f -r 13e74bd5ac8c src/auth/auth-stream.h diff -r 922049229f7f -r 13e74bd5ac8c src/auth/db-ldap.c --- a/src/auth/db-ldap.c Sat Jan 05 00:37:26 2013 +0200 +++ b/src/auth/db-ldap.c Sat Jan 05 01:14:11 2013 +0200 @@ -990,7 +990,7 @@ char *ldap_attr; if (*data != '\0') { - ldap_attr = p_strdup(ctx->pool, data); + ldap_attr = p_strdup(ctx->pool, t_strcut(data, ':')); array_append(&ctx->attr_names, &ldap_attr, 1); } return NULL; @@ -1198,28 +1198,42 @@ return ctx; } +static const char *db_ldap_field_get_default(const char *data) +{ + const char *p; + + p = strchr(data, ':'); + if (p == NULL) + return ""; + else { + /* default value given */ + return p+1; + } +} + static const char *db_ldap_field_expand(const char *data, void *context) { struct db_ldap_result_iterate_context *ctx = context; struct db_ldap_value *ldap_value; + const char *field_name = t_strcut(data, ':'); - ldap_value = hash_table_lookup(ctx->ldap_attrs, data); + ldap_value = hash_table_lookup(ctx->ldap_attrs, field_name); if (ldap_value == NULL) { - /* ldap attribute wasn't requested */ + /* requested ldap attribute wasn't returned at all */ if (ctx->debug) - str_printfa(ctx->debug, "; %s missing", data); - return ""; + str_printfa(ctx->debug, "; %s missing", field_name); + return db_ldap_field_get_default(data); } ldap_value->used = TRUE; if (ldap_value->values[0] == NULL) { /* no value for ldap attribute */ - return ""; + return db_ldap_field_get_default(data); } if (ldap_value->values[1] != NULL) { auth_request_log_warning(ctx->auth_request, "ldap", "Multiple values found for '%s', using value '%s'", - data, ldap_value->values[0]); + field_name, ldap_value->values[0]); } return ldap_value->values[0]; } diff -r 922049229f7f -r 13e74bd5ac8c src/auth/mech.c diff -r 922049229f7f -r 13e74bd5ac8c src/auth/password-scheme.c --- a/src/auth/password-scheme.c Sat Jan 05 00:37:26 2013 +0200 +++ b/src/auth/password-scheme.c Sat Jan 05 01:14:11 2013 +0200 @@ -818,6 +818,7 @@ { "SSHA256", PW_ENCODING_BASE64, 0, ssha256_verify, ssha256_generate }, { "SSHA512", PW_ENCODING_BASE64, 0, ssha512_verify, ssha512_generate }, { "PLAIN", PW_ENCODING_NONE, 0, NULL, plain_generate }, + { "CLEAR", PW_ENCODING_NONE, 0, NULL, plain_generate }, { "CLEARTEXT", PW_ENCODING_NONE, 0, NULL, plain_generate }, { "PLAIN-TRUNC", PW_ENCODING_NONE, 0, plain_trunc_verify, plain_generate }, { "CRAM-MD5", PW_ENCODING_HEX, CRAM_MD5_CONTEXTLEN, diff -r 922049229f7f -r 13e74bd5ac8c src/auth/userdb-passwd.c --- a/src/auth/userdb-passwd.c Sat Jan 05 00:37:26 2013 +0200 +++ b/src/auth/userdb-passwd.c Sat Jan 05 01:14:11 2013 +0200 @@ -217,11 +217,10 @@ module = p_new(pool, struct passwd_userdb_module, 1); module->module.cache_key = USER_CACHE_KEY; module->tmpl = userdb_template_build(pool, "passwd", args); + module->module.blocking = TRUE; - if (userdb_template_remove(module->tmpl, "blocking", &value)) { - module->module.blocking = value == NULL || - strcasecmp(value, "yes") == 0; - } + if (userdb_template_remove(module->tmpl, "blocking", &value)) + module->module.blocking = strcasecmp(value, "yes") == 0; /* FIXME: backwards compatibility */ if (!userdb_template_is_empty(module->tmpl)) i_warning("userdb passwd: Move templates args to override_fields setting"); diff -r 922049229f7f -r 13e74bd5ac8c src/config/main.c --- a/src/config/main.c Sat Jan 05 00:37:26 2013 +0200 +++ b/src/config/main.c Sat Jan 05 01:14:11 2013 +0200 @@ -26,13 +26,17 @@ restrict_access_by_env(NULL, FALSE); restrict_access_allow_coredumps(TRUE); - master_service_init_finish(master_service); config_parse_load_modules(); path = master_service_get_config_path(master_service); if (config_parse_file(path, TRUE, NULL, &error) <= 0) i_fatal("%s", error); + /* notify about our success only after successfully parsing the + config file, so if the parsing fails, master won't immediately + just recreate this process (and fail again and so on). */ + master_service_init_finish(master_service); + master_service_run(master_service, client_connected); config_connections_destroy_all(); diff -r 922049229f7f -r 13e74bd5ac8c src/doveadm/client-connection.c --- a/src/doveadm/client-connection.c Sat Jan 05 00:37:26 2013 +0200 +++ b/src/doveadm/client-connection.c Sat Jan 05 01:14:11 2013 +0200 @@ -57,6 +57,7 @@ ctx = doveadm_mail_cmd_init(cmd, set); ctx->full_args = (const void *)(argv + 1); + ctx->proxying = TRUE; ctx->service_flags |= MAIL_STORAGE_SERVICE_FLAG_NO_LOG_INIT | @@ -139,6 +140,10 @@ o_stream_nsend(conn->output, "\n+\n", 3); } pool_unref(&ctx->pool); + + /* clear all headers */ + doveadm_print_deinit(); + doveadm_print_init(DOVEADM_PRINT_TYPE_SERVER); } static bool client_is_allowed_command(const struct doveadm_settings *set, @@ -239,8 +244,11 @@ const unsigned char *data; size_t size; - if ((line = i_stream_read_next_line(conn->input)) == NULL) + if ((line = i_stream_read_next_line(conn->input)) == NULL) { + if (conn->input->eof) + return -1; return 0; + } if (*conn->set->doveadm_password == '\0') { i_error("doveadm_password not set, " diff -r 922049229f7f -r 13e74bd5ac8c src/doveadm/doveadm-mail-server.c --- a/src/doveadm/doveadm-mail-server.c Sat Jan 05 00:37:26 2013 +0200 +++ b/src/doveadm/doveadm-mail-server.c Sat Jan 05 01:14:11 2013 +0200 @@ -22,6 +22,11 @@ #define DOVEADM_MAIL_SERVER_FAILED() \ (internal_failure || master_service_is_killed(master_service)) +struct doveadm_mail_server_cmd { + struct server_connection *conn; + char *username; +}; + static HASH_TABLE(char *, struct doveadm_server *) servers; static pool_t server_pool; static struct doveadm_mail_cmd_context *cmd_ctx; @@ -78,16 +83,22 @@ static void doveadm_cmd_callback(enum server_cmd_reply reply, void *context) { - struct server_connection *conn = context; - struct doveadm_server *server; + struct doveadm_mail_server_cmd *servercmd = context; + struct doveadm_server *server = + server_connection_get_server(servercmd->conn); + const char *username = t_strdup(servercmd->username); + + i_free(servercmd->username); + i_free(servercmd); switch (reply) { case SERVER_CMD_REPLY_INTERNAL_FAILURE: + i_error("%s: Internal failure for %s", server->name, username); internal_failure = TRUE; master_service_stop(master_service); return; case SERVER_CMD_REPLY_UNKNOWN_USER: - i_error("No such user"); + i_error("%s: No such user: %s", server->name, username); if (cmd_ctx->exit_code == 0) cmd_ctx->exit_code = EX_NOUSER; break; @@ -98,8 +109,8 @@ break; } - server = server_connection_get_server(conn); if (array_count(&server->queue) > 0) { + struct server_connection *conn; char *const *usernamep = array_idx(&server->queue, 0); char *username = *usernamep; @@ -117,6 +128,7 @@ static void doveadm_mail_server_handle(struct server_connection *conn, const char *username) { + struct doveadm_mail_server_cmd *servercmd; string_t *cmd; unsigned int i; @@ -136,7 +148,12 @@ str_append_tabescaped(cmd, cmd_ctx->full_args[i]); } str_append_c(cmd, '\n'); - server_connection_cmd(conn, str_c(cmd), doveadm_cmd_callback, conn); + + servercmd = i_new(struct doveadm_mail_server_cmd, 1); + servercmd->conn = conn; + servercmd->username = i_strdup(username); + server_connection_cmd(conn, str_c(cmd), + doveadm_cmd_callback, servercmd); } static void doveadm_server_flush_one(struct doveadm_server *server) @@ -158,7 +175,7 @@ struct auth_master_connection *auth_conn; struct auth_user_info info; pool_t pool; - const char *proxy_host, *const *fields; + const char *auth_socket_path, *proxy_host, *const *fields; unsigned int i; bool proxying; int ret; @@ -176,14 +193,15 @@ pool = pool_alloconly_create("auth lookup", 1024); auth_conn = mail_storage_service_get_auth_conn(ctx->storage_service); + auth_socket_path = auth_master_get_socket_path(auth_conn); ret = auth_master_pass_lookup(auth_conn, input->username, &info, pool, &fields); if (ret < 0) { *error_r = fields[0] != NULL ? t_strdup(fields[0]) : "passdb lookup failed"; - *error_r = t_strdup_printf("%s (to see if user is proxied, " + *error_r = t_strdup_printf("%s: %s (to see if user is proxied, " "because doveadm_proxy_port is set)", - *error_r); + auth_socket_path, *error_r); } else if (ret == 0) { /* user not found from passdb. it could be in userdb though, so just continue with the default host */ @@ -199,7 +217,13 @@ if (!proxying) ret = 0; else if (proxy_host == NULL) { - *error_r = "Proxy is missing destination host"; + *error_r = t_strdup_printf("%s: Proxy is missing destination host", + auth_socket_path); + if (strstr(auth_socket_path, "/auth-userdb") != NULL) { + *error_r = t_strdup_printf( + "%s (maybe set auth_socket_path=director-userdb)", + *error_r); + } ret = -1; } else { *host_r = t_strdup_printf("%s:%u", proxy_host, diff -r 922049229f7f -r 13e74bd5ac8c src/doveadm/doveadm-mail.h --- a/src/doveadm/doveadm-mail.h Sat Jan 05 00:37:26 2013 +0200 +++ b/src/doveadm/doveadm-mail.h Sat Jan 05 01:14:11 2013 +0200 @@ -62,6 +62,8 @@ /* if non-zero, exit with this code */ int exit_code; + /* This command is being called by a remote doveadm client. */ + unsigned int proxying:1; /* We're handling only a single user */ unsigned int iterate_single_user:1; /* We're going through all users (not set for wildcard usernames) */ diff -r 922049229f7f -r 13e74bd5ac8c src/doveadm/doveadm-print-flow.c --- a/src/doveadm/doveadm-print-flow.c Sat Jan 05 00:37:26 2013 +0200 +++ b/src/doveadm/doveadm-print-flow.c Sat Jan 05 01:14:11 2013 +0200 @@ -63,7 +63,7 @@ if ((hdr->flags & DOVEADM_PRINT_HEADER_FLAG_HIDE_TITLE) == 0) printf("%s=", hdr->title); } - printf("%.*s", (int)size, value); + fwrite(value, 1, size, stdout); if (size == 0) { flow_next_hdr(); ctx->streaming = FALSE; diff -r 922049229f7f -r 13e74bd5ac8c src/doveadm/doveadm-print-pager.c --- a/src/doveadm/doveadm-print-pager.c Sat Jan 05 00:37:26 2013 +0200 +++ b/src/doveadm/doveadm-print-pager.c Sat Jan 05 01:14:11 2013 +0200 @@ -56,7 +56,7 @@ ctx->streaming = TRUE; printf("%s:\n", hdr->title); } - printf("%.*s", (int)size, value); + fwrite(value, 1, size, stdout); if (size == 0) { pager_next_hdr(); ctx->streaming = FALSE; diff -r 922049229f7f -r 13e74bd5ac8c src/doveadm/doveadm-print-tab.c --- a/src/doveadm/doveadm-print-tab.c Sat Jan 05 00:37:26 2013 +0200 +++ b/src/doveadm/doveadm-print-tab.c Sat Jan 05 01:14:11 2013 +0200 @@ -51,7 +51,7 @@ } if (ctx.header_idx > 0) printf("\t"); - printf("%.*s", (int)size, value); + fwrite(value, 1, size, stdout); } static void doveadm_print_tab_flush(void) diff -r 922049229f7f -r 13e74bd5ac8c src/doveadm/doveadm-print-table.c --- a/src/doveadm/doveadm-print-table.c Sat Jan 05 00:37:26 2013 +0200 +++ b/src/doveadm/doveadm-print-table.c Sat Jan 05 01:14:11 2013 +0200 @@ -2,6 +2,7 @@ #include "lib.h" #include "array.h" +#include "str.h" #include "doveadm-print-private.h" #include @@ -24,6 +25,7 @@ pool_t pool; ARRAY(struct doveadm_print_table_header) headers; ARRAY_TYPE(const_string) buffered_values; + string_t *stream; unsigned int hdr_idx; unsigned int columns; @@ -179,10 +181,17 @@ } static void -doveadm_print_table_print_stream(const unsigned char *value ATTR_UNUSED, - size_t size ATTR_UNUSED) +doveadm_print_table_print_stream(const unsigned char *value, size_t size) { - i_fatal("table formatter doesn't support multi-line values"); + if (memchr(value, '\n', size) != NULL) + i_fatal("table formatter doesn't support multi-line values"); + + if (size != 0) + str_append_n(ctx->stream, value, size); + else { + doveadm_print_table_print(str_c(ctx->stream)); + str_truncate(ctx->stream, 0); + } } static void doveadm_print_table_flush(void) @@ -199,6 +208,7 @@ pool = pool_alloconly_create("doveadm print table", 2048); ctx = p_new(pool, struct doveadm_print_table_context, 1); ctx->pool = pool; + ctx->stream = str_new(default_pool, 128); p_array_init(&ctx->headers, pool, 16); i_array_init(&ctx->buffered_values, 64); ctx->columns = DEFAULT_COLUMNS; @@ -211,6 +221,7 @@ static void doveadm_print_table_deinit(void) { + str_free(&ctx->stream); array_free(&ctx->buffered_values); pool_unref(&ctx->pool); ctx = NULL; diff -r 922049229f7f -r 13e74bd5ac8c src/doveadm/server-connection.c --- a/src/doveadm/server-connection.c Sat Jan 05 00:37:26 2013 +0200 +++ b/src/doveadm/server-connection.c Sat Jan 05 01:14:11 2013 +0200 @@ -269,8 +269,10 @@ server_connection_callback(conn, reply); } else i_error("doveadm server sent broken input"); - /* we're finished, close the connection */ - server_connection_destroy(&conn); + if (conn->callback == NULL) { + /* we're finished, close the connection */ + server_connection_destroy(&conn); + } break; } } diff -r 922049229f7f -r 13e74bd5ac8c src/imap/cmd-append.c --- a/src/imap/cmd-append.c Sat Jan 05 00:37:26 2013 +0200 +++ b/src/imap/cmd-append.c Sat Jan 05 01:14:11 2013 +0200 @@ -597,7 +597,7 @@ msg = t_str_new(256); save_count = seq_range_count(&changes.saved_uids); - if (save_count == 0) { + if (save_count == 0 || changes.no_read_perm) { /* not supported by backend (virtual) */ str_append(msg, "OK Append completed."); } else { diff -r 922049229f7f -r 13e74bd5ac8c src/imap/cmd-copy.c --- a/src/imap/cmd-copy.c Sat Jan 05 00:37:26 2013 +0200 +++ b/src/imap/cmd-copy.c Sat Jan 05 01:14:11 2013 +0200 @@ -132,8 +132,10 @@ else if (copy_count == 0) { str_append(msg, "OK No messages found."); pool_unref(&changes.pool); - } else if (seq_range_count(&changes.saved_uids) == 0) { - /* not supported by backend (virtual) */ + } else if (seq_range_count(&changes.saved_uids) == 0 || + changes.no_read_perm) { + /* not supported by backend (virtual) or no read permissions + for mailbox */ str_append(msg, move ? "OK Move completed." : "OK Copy completed."); pool_unref(&changes.pool); diff -r 922049229f7f -r 13e74bd5ac8c src/imap/imap-sync.c diff -r 922049229f7f -r 13e74bd5ac8c src/indexer/master-connection.c --- a/src/indexer/master-connection.c Sat Jan 05 00:37:26 2013 +0200 +++ b/src/indexer/master-connection.c Sat Jan 05 01:14:11 2013 +0200 @@ -134,7 +134,7 @@ mailbox, mailbox_get_last_error(box, NULL)); return -1; } - i_info("Indexes disabled for Mailbox %s, skipping", mailbox); + i_info("Indexes disabled for mailbox %s, skipping", mailbox); return 0; } ret = 0; diff -r 922049229f7f -r 13e74bd5ac8c src/lib-auth/auth-master.c --- a/src/lib-auth/auth-master.c Sat Jan 05 00:37:26 2013 +0200 +++ b/src/lib-auth/auth-master.c Sat Jan 05 01:14:11 2013 +0200 @@ -107,6 +107,11 @@ i_free(conn); } +const char *auth_master_get_socket_path(struct auth_master_connection *conn) +{ + return conn->auth_socket_path; +} + static void auth_request_lookup_abort(struct auth_master_connection *conn) { io_loop_stop(conn->ioloop); diff -r 922049229f7f -r 13e74bd5ac8c src/lib-auth/auth-master.h --- a/src/lib-auth/auth-master.h Sat Jan 05 00:37:26 2013 +0200 +++ b/src/lib-auth/auth-master.h Sat Jan 05 01:14:11 2013 +0200 @@ -28,6 +28,9 @@ auth_master_init(const char *auth_socket_path, enum auth_master_flags flags); void auth_master_deinit(struct auth_master_connection **conn); +/* Returns the auth_socket_path */ +const char *auth_master_get_socket_path(struct auth_master_connection *conn); + /* Do a USER lookup. Returns -1 = error, 0 = user not found, 1 = ok. When returning -1 and fields[0] isn't NULL, it contains an error message that should be shown to user. */ diff -r 922049229f7f -r 13e74bd5ac8c src/lib-dict/dict-file.c --- a/src/lib-dict/dict-file.c Sat Jan 05 00:37:26 2013 +0200 +++ b/src/lib-dict/dict-file.c Sat Jan 05 01:14:11 2013 +0200 @@ -3,6 +3,7 @@ #include "lib.h" #include "array.h" #include "hash.h" +#include "mkdir-parents.h" #include "file-lock.h" #include "file-dotlock.h" #include "nfs-workarounds.h" @@ -420,6 +421,33 @@ return fd_copy_stat_permissions(&src_st, dest_fd, dest_path); } +static int file_dict_mkdir(struct file_dict *dict) +{ + const char *path, *p, *root; + struct stat st; + mode_t mode = 0700; + + p = strrchr(dict->path, '/'); + if (p == NULL) + return 0; + path = t_strdup_until(dict->path, p); + + if (stat_first_parent(path, &root, &st) < 0) { + i_error("stat(%s) failed: %m", root); + return -1; + } + if ((st.st_mode & S_ISGID) != 0) { + /* preserve parent's permissions when it has setgid bit */ + mode = st.st_mode; + } + + if (mkdir_parents(path, mode) < 0) { + i_error("mkdir_parents(%s) failed: %m", path); + return -1; + } + return 0; +} + static int file_dict_lock(struct file_dict *dict, struct file_lock **lock_r) { @@ -431,6 +459,11 @@ if (dict->fd == -1) { /* quota file doesn't exist yet, we need to create it */ dict->fd = open(dict->path, O_CREAT | O_RDWR, 0600); + if (dict->fd == -1 && errno == ENOENT) { + if (file_dict_mkdir(dict) < 0) + return -1; + dict->fd = open(dict->path, O_CREAT | O_RDWR, 0600); + } if (dict->fd == -1) { i_error("creat(%s) failed: %m", dict->path); return -1; @@ -484,6 +517,12 @@ case FILE_LOCK_METHOD_DOTLOCK: fd = file_dotlock_open(&file_dict_dotlock_settings, dict->path, 0, &dotlock); + if (fd == -1 && errno == ENOENT) { + if (file_dict_mkdir(dict) < 0) + return -1; + fd = file_dotlock_open(&file_dict_dotlock_settings, + dict->path, 0, &dotlock); + } if (fd == -1) { i_error("file dict commit: file_dotlock_open(%s) failed: %m", dict->path); diff -r 922049229f7f -r 13e74bd5ac8c src/lib-index/mail-cache-compress.c --- a/src/lib-index/mail-cache-compress.c Sat Jan 05 00:37:26 2013 +0200 +++ b/src/lib-index/mail-cache-compress.c Sat Jan 05 01:14:11 2013 +0200 @@ -458,7 +458,13 @@ return 0; /* compression isn't very efficient with small read()s */ - cache->map_with_read = FALSE; + if (cache->map_with_read) { + cache->map_with_read = FALSE; + if (cache->read_buf != NULL) + buffer_set_used_size(cache->read_buf, 0); + cache->hdr = NULL; + cache->mmap_length = 0; + } if (cache->index->lock_method == FILE_LOCK_METHOD_DOTLOCK) { /* we're using dotlocking, cache file creation itself creates diff -r 922049229f7f -r 13e74bd5ac8c src/lib-index/mail-cache-fields.c --- a/src/lib-index/mail-cache-fields.c Sat Jan 05 00:37:26 2013 +0200 +++ b/src/lib-index/mail-cache-fields.c Sat Jan 05 01:14:11 2013 +0200 @@ -204,7 +204,7 @@ const struct mail_cache_header_fields *field_hdr; struct mail_cache_header_fields tmp_field_hdr; const void *data; - uint32_t offset = 0, next_offset; + uint32_t offset = 0, next_offset, field_hdr_size; unsigned int next_count = 0; int ret; @@ -272,13 +272,15 @@ cache->need_compress_file_seq = cache->hdr->file_seq; if (field_hdr_r != NULL) { + /* detect corrupted size later */ + field_hdr_size = I_MAX(field_hdr->size, sizeof(*field_hdr)); if (cache->file_cache != NULL) { /* invalidate the cache fields area to make sure we get the latest cache decisions/last_used fields */ file_cache_invalidate(cache->file_cache, offset, - field_hdr->size); + field_hdr_size); } - ret = mail_cache_map(cache, offset, field_hdr->size, &data); + ret = mail_cache_map(cache, offset, field_hdr_size, &data); if (ret < 0) return -1; if (ret == 0) { diff -r 922049229f7f -r 13e74bd5ac8c src/lib-index/mail-cache-private.h diff -r 922049229f7f -r 13e74bd5ac8c src/lib-index/mail-cache-transaction.c diff -r 922049229f7f -r 13e74bd5ac8c src/lib-index/mail-cache.c --- a/src/lib-index/mail-cache.c Sat Jan 05 00:37:26 2013 +0200 +++ b/src/lib-index/mail-cache.c Sat Jan 05 01:14:11 2013 +0200 @@ -294,6 +294,7 @@ !MAIL_CACHE_IS_UNUSABLE(cache) && cache->hdr->file_seq != 0 ? cache->hdr->file_seq : 0; + cache->hdr = NULL; return -1; } } @@ -309,6 +310,7 @@ } else { i_assert(cache->hdr != NULL); } + i_assert(cache->hdr->file_seq != 0); if (offset + size > cache->mmap_length) return 0; @@ -357,11 +359,12 @@ buffer_set_used_size(cache->read_buf, ret); cache->read_offset = offset; - cache->mmap_length = offset + size; + cache->mmap_length = offset + cache->read_buf->used; *data_r = data; hdr_data = offset == 0 ? *data_r : NULL; - return mail_cache_map_finish(cache, offset, size, hdr_data, TRUE); + return mail_cache_map_finish(cache, offset, + cache->read_buf->used, hdr_data, TRUE); } int mail_cache_map(struct mail_cache *cache, size_t offset, size_t size, @@ -378,8 +381,6 @@ return mail_cache_map_with_read(cache, offset, size, data_r); if (cache->file_cache != NULL) { - cache->hdr = NULL; - ret = file_cache_read(cache->file_cache, offset, size); if (ret < 0) { /* In case of ESTALE we'll simply fail without error @@ -391,6 +392,7 @@ offsets. */ if (errno != ESTALE) mail_cache_set_syscall_error(cache, "read()"); + cache->hdr = NULL; return -1; } @@ -398,12 +400,14 @@ &cache->mmap_length); *data_r = offset > cache->mmap_length ? NULL : CONST_PTR_OFFSET(data, offset); - return mail_cache_map_finish(cache, offset, size, data, TRUE); + return mail_cache_map_finish(cache, offset, size, + offset == 0 ? data : NULL, TRUE); } if (offset < cache->mmap_length && size <= cache->mmap_length - offset) { /* already mapped */ + i_assert(cache->mmap_base != NULL); *data_r = CONST_PTR_OFFSET(cache->mmap_base, offset); return 1; } @@ -547,6 +551,8 @@ mail_index_unregister_expunge_handler(cache->index, cache->ext_id); mail_cache_file_close(cache); + if (cache->read_buf != NULL) + buffer_free(&cache->read_buf); hash_table_destroy(&cache->field_name_hash); pool_unref(&cache->field_pool); i_free(cache->field_file_map); @@ -748,6 +754,8 @@ if (cache->file_cache != NULL) file_cache_write(cache->file_cache, data, size, offset); + if (cache->read_buf != NULL) + buffer_set_used_size(cache->read_buf, 0); return 0; } diff -r 922049229f7f -r 13e74bd5ac8c src/lib-index/mail-index-alloc-cache.c --- a/src/lib-index/mail-index-alloc-cache.c Sat Jan 05 00:37:26 2013 +0200 +++ b/src/lib-index/mail-index-alloc-cache.c Sat Jan 05 01:14:11 2013 +0200 @@ -22,6 +22,7 @@ struct mail_index *index; char *mailbox_path; int refcount; + bool referenced; dev_t index_dir_dev; ino_t index_dir_ino; @@ -58,7 +59,7 @@ static void mail_index_alloc_cache_list_free(struct mail_index_alloc_cache_list *list) { - if (list->index->open_count > 0) + if (list->referenced) mail_index_close(list->index); mail_index_free(&list->index); i_free(list->mailbox_path); @@ -166,6 +167,15 @@ } else { if (rec->refcount == 0) seen_ref0 = TRUE; + if (all && rec->index->open_count == 1 && + rec->referenced) { + /* we're the only one keeping this index open. + we might be here, because the caller is + deleting this mailbox and wants its indexes + to be closed. so close it. */ + rec->referenced = FALSE; + mail_index_close(rec->index); + } list = &(*list)->next; } } @@ -229,8 +239,9 @@ list->index_dir_dev = st.st_dev; } } - if (list != NULL) { + if (list != NULL && !list->referenced) { /* keep it referenced for ourself */ + list->referenced = TRUE; index->open_count++; } } diff -r 922049229f7f -r 13e74bd5ac8c src/lib-index/mail-index-transaction-view.c --- a/src/lib-index/mail-index-transaction-view.c Sat Jan 05 00:37:26 2013 +0200 +++ b/src/lib-index/mail-index-transaction-view.c Sat Jan 05 01:14:11 2013 +0200 @@ -203,11 +203,17 @@ if (first_uid <= rec->uid) break; } - if (seq > tview->t->last_new_seq) { + if (seq > tview->t->last_new_seq || rec->uid > last_uid) { /* no messages in range */ return; } *first_seq_r = seq; + + if (rec->uid == last_uid) { + /* one seq in range */ + *last_seq_r = seq; + return; + } } seq = tview->t->last_new_seq; diff -r 922049229f7f -r 13e74bd5ac8c src/lib-lda/mail-deliver.c --- a/src/lib-lda/mail-deliver.c Sat Jan 05 00:37:26 2013 +0200 +++ b/src/lib-lda/mail-deliver.c Sat Jan 05 01:14:11 2013 +0200 @@ -325,7 +325,8 @@ ctx->saved_mail = TRUE; mail_deliver_log(ctx, "saved mail to %s", mailbox_name); - if (ctx->save_dest_mail && mailbox_sync(box, 0) == 0) { + if (ctx->save_dest_mail && + mailbox_sync(box, MAILBOX_SYNC_FLAG_FAST) == 0) { range = array_idx(&changes.saved_uids, 0); i_assert(range[0].seq1 == range[0].seq2); diff -r 922049229f7f -r 13e74bd5ac8c src/lib-master/mountpoint-list.c --- a/src/lib-master/mountpoint-list.c Sat Jan 05 00:37:26 2013 +0200 +++ b/src/lib-master/mountpoint-list.c Sat Jan 05 01:14:11 2013 +0200 @@ -57,6 +57,10 @@ "/proc", "/var/run", "/run", +#ifdef __APPLE__ + "/Volumes", + "/private/tmp", +#endif NULL }; diff -r 922049229f7f -r 13e74bd5ac8c src/lib-ssl-iostream/iostream-openssl.c diff -r 922049229f7f -r 13e74bd5ac8c src/lib-storage/index/dbox-multi/mdbox-map.c --- a/src/lib-storage/index/dbox-multi/mdbox-map.c Sat Jan 05 00:37:26 2013 +0200 +++ b/src/lib-storage/index/dbox-multi/mdbox-map.c Sat Jan 05 01:14:11 2013 +0200 @@ -208,7 +208,8 @@ int mdbox_map_refresh(struct mdbox_map *map) { struct mail_index_view_sync_ctx *ctx; - bool delayed_expunges; + bool delayed_expunges, fscked; + int ret = 0; /* some open files may have read partially written mails. now that map syncing makes the new mails visible, we need to make sure the @@ -227,14 +228,15 @@ ctx = mail_index_view_sync_begin(map->view, MAIL_INDEX_VIEW_SYNC_FLAG_FIX_INCONSISTENT); - if (mail_index_reset_fscked(map->view->index)) - mdbox_storage_set_corrupted(map->storage); + fscked = mail_index_reset_fscked(map->view->index); if (mail_index_view_sync_commit(&ctx, &delayed_expunges) < 0) { mail_storage_set_internal_error(MAP_STORAGE(map)); mail_index_reset_error(map->index); - return -1; + ret = -1; } - return 0; + if (fscked) + mdbox_storage_set_corrupted(map->storage); + return ret; } static void diff -r 922049229f7f -r 13e74bd5ac8c src/lib-storage/index/dbox-multi/mdbox-save.c --- a/src/lib-storage/index/dbox-multi/mdbox-save.c Sat Jan 05 00:37:26 2013 +0200 +++ b/src/lib-storage/index/dbox-multi/mdbox-save.c Sat Jan 05 01:14:11 2013 +0200 @@ -295,16 +295,6 @@ mdbox_transaction_save_rollback(_ctx); return -1; } - - /* assign map UIDs for newly saved messages. they're written to - transaction log immediately within this function, but the map - is left locked. */ - if (mdbox_map_append_assign_map_uids(ctx->append_ctx, &first_map_uid, - &last_map_uid) < 0) { - mdbox_transaction_save_rollback(_ctx); - return -1; - } - /* lock the mailbox after map to avoid deadlocks. if we've noticed any corruption, deal with it later, otherwise we won't have up-to-date atomic->sync_view */ @@ -317,6 +307,16 @@ return -1; } + /* assign map UIDs for newly saved messages after we've successfully + acquired all the locks. the transaction is now very unlikely to + fail. the UIDs are written to the transaction log immediately within + this function, but the map is left locked. */ + if (mdbox_map_append_assign_map_uids(ctx->append_ctx, &first_map_uid, + &last_map_uid) < 0) { + mdbox_transaction_save_rollback(_ctx); + return -1; + } + /* assign UIDs for new messages */ hdr = mail_index_get_header(ctx->sync_ctx->sync_view); mail_index_append_finish_uids(ctx->ctx.trans, hdr->next_uid, diff -r 922049229f7f -r 13e74bd5ac8c src/lib-storage/index/dbox-multi/mdbox-storage-rebuild.c --- a/src/lib-storage/index/dbox-multi/mdbox-storage-rebuild.c Sat Jan 05 00:37:26 2013 +0200 +++ b/src/lib-storage/index/dbox-multi/mdbox-storage-rebuild.c Sat Jan 05 01:14:11 2013 +0200 @@ -838,6 +838,12 @@ if (mdbox_map_atomic_lock(ctx->atomic) < 0) return -1; + /* fsck the map just in case its UIDs are broken */ + if (mail_index_fsck(ctx->storage->map->index) < 0) { + mail_storage_set_internal_error(&ctx->storage->storage.storage); + return -1; + } + /* get old map header */ mail_index_get_header_ext(ctx->atomic->sync_view, ctx->storage->map->map_ext_id, diff -r 922049229f7f -r 13e74bd5ac8c src/lib-storage/index/index-sync.c --- a/src/lib-storage/index/index-sync.c Sat Jan 05 00:37:26 2013 +0200 +++ b/src/lib-storage/index/index-sync.c Sat Jan 05 01:14:11 2013 +0200 @@ -31,6 +31,17 @@ ioloop_time < ibox->sync_last_check + MAILBOX_FULL_SYNC_INTERVAL) return FALSE; + if ((flags & MAILBOX_SYNC_FLAG_FAST) != 0 && + (box->flags & MAILBOX_FLAG_SAVEONLY) != 0) { + /* lib-lda is syncing the mailbox after saving a mail. + it only wants to find the new mail for potentially copying + to other mailboxes. that's mainly an optimization, and since + the mail was most likely already added to index we don't + need to do a full sync to find it. the main benefit here is + to avoid a very costly sync with a large Maildir/new/ */ + return FALSE; + } + if (ibox->notify_to != NULL) timeout_reset(ibox->notify_to); ibox->sync_last_check = ioloop_time; diff -r 922049229f7f -r 13e74bd5ac8c src/lib-storage/index/raw/raw-mail.c --- a/src/lib-storage/index/raw/raw-mail.c Sat Jan 05 00:37:26 2013 +0200 +++ b/src/lib-storage/index/raw/raw-mail.c Sat Jan 05 01:14:11 2013 +0200 @@ -102,7 +102,8 @@ switch (field) { case MAIL_FETCH_FROM_ENVELOPE: - *value_r = mbox->envelope_sender; + *value_r = mbox->envelope_sender != NULL ? + mbox->envelope_sender : ""; return 0; case MAIL_FETCH_UIDL_FILE_NAME: *value_r = mbox->have_filename ? diff -r 922049229f7f -r 13e74bd5ac8c src/lib-storage/index/shared/shared-storage.c diff -r 922049229f7f -r 13e74bd5ac8c src/lib-storage/mail-storage.h --- a/src/lib-storage/mail-storage.h Sat Jan 05 00:37:26 2013 +0200 +++ b/src/lib-storage/mail-storage.h Sat Jan 05 01:14:11 2013 +0200 @@ -291,6 +291,9 @@ /* TRUE if anything actually changed with this commit */ bool changed; + /* User doesn't have read ACL for the mailbox, so don't show the + uid_validity / saved_uids. */ + bool no_read_perm; }; struct mailbox_sync_rec { diff -r 922049229f7f -r 13e74bd5ac8c src/lib-storage/mail-user.c diff -r 922049229f7f -r 13e74bd5ac8c src/lib-storage/mail-user.h diff -r 922049229f7f -r 13e74bd5ac8c src/lib-storage/mailbox-list.c --- a/src/lib-storage/mailbox-list.c Sat Jan 05 00:37:26 2013 +0200 +++ b/src/lib-storage/mailbox-list.c Sat Jan 05 01:14:11 2013 +0200 @@ -921,27 +921,6 @@ } } -static int -mailbox_list_stat_parent(const char *path, const char **root_dir_r, - struct stat *st_r, const char **error_r) -{ - const char *p; - - while (stat(path, st_r) < 0) { - if (errno != ENOENT || strcmp(path, "/") == 0) { - *error_r = t_strdup_printf("stat(%s) failed: %m", path); - return -1; - } - p = strrchr(path, '/'); - if (p == NULL) - path = "/"; - else - path = t_strdup_until(path, p); - } - *root_dir_r = path; - return 0; -} - static const char * get_expanded_path(const char *unexpanded_start, const char *unexpanded_stop, const char *expanded_full) @@ -1020,8 +999,10 @@ } /* get the first existing parent directory's permissions */ - if (mailbox_list_stat_parent(expanded, &root_dir, &st, error_r) < 0) + if (stat_first_parent(expanded, &root_dir, &st) < 0) { + *error_r = t_strdup_printf("stat(%s) failed: %m", root_dir); return -1; + } /* if the parent directory doesn't have setgid-bit enabled, we don't copy any permissions from it. */ diff -r 922049229f7f -r 13e74bd5ac8c src/lib/Makefile.am --- a/src/lib/Makefile.am Sat Jan 05 00:37:26 2013 +0200 +++ b/src/lib/Makefile.am Sat Jan 05 01:14:11 2013 +0200 @@ -291,6 +291,7 @@ test-str-find.c \ test-str-sanitize.c \ test-time-util.c \ + test-unichar.c \ test-utc-mktime.c \ test-var-expand.c diff -r 922049229f7f -r 13e74bd5ac8c src/lib/buffer.c --- a/src/lib/buffer.c Sat Jan 05 00:37:26 2013 +0200 +++ b/src/lib/buffer.c Sat Jan 05 01:14:11 2013 +0200 @@ -319,3 +319,17 @@ return memcmp(buf1->data, buf2->data, buf1->used) == 0; } + +void buffer_verify_pool(buffer_t *_buf) +{ + const struct real_buffer *buf = (const struct real_buffer *)_buf; + void *ret; + + if (buf->pool->datastack_pool) { + /* this doesn't really do anything except verify the + stack frame */ + ret = p_realloc(buf->pool, buf->w_buffer, + buf->alloc, buf->alloc); + i_assert(ret == buf->w_buffer); + } +} diff -r 922049229f7f -r 13e74bd5ac8c src/lib/buffer.h --- a/src/lib/buffer.h Sat Jan 05 00:37:26 2013 +0200 +++ b/src/lib/buffer.h Sat Jan 05 01:14:11 2013 +0200 @@ -104,4 +104,10 @@ return buf->used; } +/* Crash if buffer was allocated from data stack and stack frame has changed. + This can be used as an assert-like check to verify that it's valid to + increase the buffer size here, instead of crashing only randomly when the + buffer needs to be increased. */ +void buffer_verify_pool(buffer_t *buf); + #endif diff -r 922049229f7f -r 13e74bd5ac8c src/lib/mkdir-parents.c --- a/src/lib/mkdir-parents.c Sat Jan 05 00:37:26 2013 +0200 +++ b/src/lib/mkdir-parents.c Sat Jan 05 01:14:11 2013 +0200 @@ -137,3 +137,23 @@ { return mkdir_parents_chown(path, mode, (uid_t)-1, (gid_t)-1); } + +int stat_first_parent(const char *path, const char **root_dir_r, + struct stat *st_r) +{ + const char *p; + + while (stat(path, st_r) < 0) { + if (errno != ENOENT || strcmp(path, "/") == 0) { + *root_dir_r = path; + return -1; + } + p = strrchr(path, '/'); + if (p == NULL) + path = "/"; + else + path = t_strdup_until(path, p); + } + *root_dir_r = path; + return 0; +} diff -r 922049229f7f -r 13e74bd5ac8c src/lib/mkdir-parents.h --- a/src/lib/mkdir-parents.h Sat Jan 05 00:37:26 2013 +0200 +++ b/src/lib/mkdir-parents.h Sat Jan 05 01:14:11 2013 +0200 @@ -1,6 +1,8 @@ #ifndef MKDIR_PARENTS_H #define MKDIR_PARENTS_H +#include + /* Create path and all the directories under it if needed. Permissions for existing directories isn't changed. Returns 0 if ok. If directory already exists, returns -1 with errno=EEXIST. */ @@ -22,4 +24,10 @@ int mkdir_chgrp(const char *path, mode_t mode, gid_t gid, const char *gid_origin); +/* stat() the path or its first parent that exists. Returns 0 if ok, -1 if + failed. root_dir is set to the last stat()ed directory (on success and + on failure). */ +int stat_first_parent(const char *path, const char **root_dir_r, + struct stat *st_r); + #endif diff -r 922049229f7f -r 13e74bd5ac8c src/lib/str.c --- a/src/lib/str.c Sat Jan 05 00:37:26 2013 +0200 +++ b/src/lib/str.c Sat Jan 05 01:14:11 2013 +0200 @@ -45,6 +45,9 @@ size_t len = str_len(str); size_t alloc = buffer_get_size(str); +#ifdef DEBUG + buffer_verify_pool(str); +#endif if (len == alloc || data[len] != '\0') { buffer_write(str, len, "", 1); /* remove the \0 - we don't want to keep it */ diff -r 922049229f7f -r 13e74bd5ac8c src/lib/test-lib.c --- a/src/lib/test-lib.c Sat Jan 05 00:37:26 2013 +0200 +++ b/src/lib/test-lib.c Sat Jan 05 01:14:11 2013 +0200 @@ -34,6 +34,7 @@ test_str_find, test_str_sanitize, test_time_util, + test_unichar, test_utc_mktime, test_var_expand, NULL diff -r 922049229f7f -r 13e74bd5ac8c src/lib/test-lib.h --- a/src/lib/test-lib.h Sat Jan 05 00:37:26 2013 +0200 +++ b/src/lib/test-lib.h Sat Jan 05 01:14:11 2013 +0200 @@ -33,6 +33,7 @@ void test_str_find(void); void test_str_sanitize(void); void test_time_util(void); +void test_unichar(void); void test_utc_mktime(void); void test_var_expand(void); diff -r 922049229f7f -r 13e74bd5ac8c src/lib/test-unichar.c --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib/test-unichar.c Sat Jan 05 01:14:11 2013 +0200 @@ -0,0 +1,24 @@ +/* Copyright (c) 2007-2012 Dovecot authors, see the included COPYING file */ + +#include "test-lib.h" +#include "str.h" +#include "unichar.h" + +void test_unichar(void) +{ + static const char *overlong_utf8 = "\xf8\x80\x95\x81\xa1"; + unichar_t chr, chr2; + string_t *str = t_str_new(16); + + test_begin("unichars"); + for (chr = 0; chr <= 0x10ffff; chr++) { + str_truncate(str, 0); + uni_ucs4_to_utf8_c(chr, str); + test_assert(uni_utf8_str_is_valid(str_c(str))); + test_assert(uni_utf8_get_char(str_c(str), &chr2) > 0); + test_assert(chr2 == chr); + } + test_assert(!uni_utf8_str_is_valid(overlong_utf8)); + test_assert(uni_utf8_get_char(overlong_utf8, &chr2) < 0); + test_end(); +} diff -r 922049229f7f -r 13e74bd5ac8c src/lib/unichar.c --- a/src/lib/unichar.c Sat Jan 05 00:37:26 2013 +0200 +++ b/src/lib/unichar.c Sat Jan 05 01:14:11 2013 +0200 @@ -37,8 +37,10 @@ int uni_utf8_get_char_n(const void *_input, size_t max_len, unichar_t *chr_r) { + static unichar_t lowest_valid_chr_table[] = + { 0, 0, 0x80, 0x800, 0x10000, 0x20000, 0x40000 }; const unsigned char *input = _input; - unichar_t chr; + unichar_t chr, lowest_valid_chr; unsigned int i, len; int ret; @@ -75,10 +77,12 @@ return -1; } - if (len <= max_len) + if (len <= max_len) { + lowest_valid_chr = lowest_valid_chr_table[len]; ret = 1; - else { + } else { /* check first if the input is invalid before returning 0 */ + lowest_valid_chr = 0; ret = 0; len = max_len; } @@ -91,6 +95,10 @@ chr <<= 6; chr |= input[i] & 0x3f; } + if (chr < lowest_valid_chr) { + /* overlong encoding */ + return -1; + } *chr_r = chr; return ret; @@ -340,19 +348,11 @@ static inline unsigned int is_valid_utf8_seq(const unsigned char *input, unsigned int size) { - unsigned int i, len; - - len = uni_utf8_char_bytes(input[0]); - if (unlikely(len > size || len == 1)) - return 0; + unichar_t chr; - /* the rest of the chars should be in 0x80..0xbf range. - anything else is start of a sequence or invalid */ - for (i = 1; i < len; i++) { - if (unlikely(input[i] < 0x80 || input[i] > 0xbf)) - return 0; - } - return len; + if (uni_utf8_get_char_n(input, size, &chr) <= 0) + return 0; + return uni_utf8_char_bytes(input[0]); } static int uni_utf8_find_invalid_pos(const unsigned char *input, size_t size, diff -r 922049229f7f -r 13e74bd5ac8c src/lib/unichar.h diff -r 922049229f7f -r 13e74bd5ac8c src/lib/var-expand.c --- a/src/lib/var-expand.c Sat Jan 05 00:37:26 2013 +0200 +++ b/src/lib/var-expand.c Sat Jan 05 01:14:11 2013 +0200 @@ -160,7 +160,7 @@ const void *key_start, unsigned int key_len, void *context) { const struct var_expand_table *t; - const char *value = NULL; + const char *key, *value = NULL; if (table != NULL) { for (t = table; !TABLE_LAST(t); t++) { @@ -171,35 +171,33 @@ } } } + key = t_strndup(key_start, key_len); /* built-in variables: */ - T_BEGIN { - const char *key = t_strndup(key_start, key_len); + switch (key_len) { + case 3: + if (strcmp(key, "pid") == 0) + value = my_pid; + else if (strcmp(key, "uid") == 0) + value = dec2str(geteuid()); + else if (strcmp(key, "gid") == 0) + value = dec2str(getegid()); + break; + case 8: + if (strcmp(key, "hostname") == 0) + value = my_hostname; + break; + } - switch (key_len) { - case 3: - if (strcmp(key, "pid") == 0) - value = my_pid; - else if (strcmp(key, "uid") == 0) - value = dec2str(geteuid()); - else if (strcmp(key, "gid") == 0) - value = dec2str(getegid()); - break; - case 8: - if (strcmp(key, "hostname") == 0) - value = my_hostname; - break; - } - if (value == NULL) { - const char *data = strchr(key, ':'); + if (value == NULL) { + const char *data = strchr(key, ':'); - if (data != NULL) - key = t_strdup_until(key, data++); - else - data = ""; - value = var_expand_func(func_table, key, data, context); - } - } T_END; + if (data != NULL) + key = t_strdup_until(key, data++); + else + data = ""; + value = var_expand_func(func_table, key, data, context); + } return value; } diff -r 922049229f7f -r 13e74bd5ac8c src/login-common/login-proxy.c --- a/src/login-common/login-proxy.c Sat Jan 05 00:37:26 2013 +0200 +++ b/src/login-common/login-proxy.c Sat Jan 05 01:14:11 2013 +0200 @@ -5,6 +5,7 @@ #include "istream.h" #include "ostream.h" #include "llist.h" +#include "str.h" #include "str-sanitize.h" #include "time-util.h" #include "master-service.h" @@ -194,16 +195,33 @@ proxy->state_rec = NULL; } +static void +proxy_log_connect_error(struct login_proxy *proxy) +{ + string_t *str = t_str_new(128); + struct ip_addr local_ip; + unsigned int local_port; + + str_printfa(str, "proxy(%s): connect(%s, %u) failed: %m (after %u secs", + proxy->client->virtual_user, + proxy->host, proxy->port, + (unsigned int)(ioloop_time - proxy->created.tv_sec)); + + if (proxy->server_fd != -1 && + net_getsockname(proxy->server_fd, &local_ip, &local_port) == 0) { + str_printfa(str, ", local=%s:%u", + net_ip2addr(&local_ip), local_port); + } + + str_append_c(str, ')'); + i_error("%s", str_c(str)); +} + static void proxy_wait_connect(struct login_proxy *proxy) { - int err; - - err = net_geterror(proxy->server_fd); - if (err != 0) { - i_error("proxy(%s): connect(%s, %u) failed: %s (after %u secs)", - proxy->client->virtual_user, - proxy->host, proxy->port, strerror(err), - (unsigned int)(ioloop_time - proxy->created.tv_sec)); + errno = net_geterror(proxy->server_fd); + if (errno != 0) { + proxy_log_connect_error(proxy); proxy_fail_connect(proxy); login_proxy_free(&proxy); return; @@ -229,8 +247,8 @@ static void proxy_connect_timeout(struct login_proxy *proxy) { - i_error("proxy(%s): connect(%s, %u) timed out", - proxy->client->virtual_user, proxy->host, proxy->port); + errno = ETIMEDOUT; + proxy_log_connect_error(proxy); proxy_fail_connect(proxy); login_proxy_free(&proxy); } @@ -252,8 +270,7 @@ proxy->server_fd = net_connect_ip(&proxy->ip, proxy->port, NULL); if (proxy->server_fd == -1) { - i_error("proxy(%s): connect(%s, %u) failed: %m", - proxy->client->virtual_user, proxy->host, proxy->port); + proxy_log_connect_error(proxy); login_proxy_free(&proxy); return -1; } diff -r 922049229f7f -r 13e74bd5ac8c src/login-common/ssl-proxy-openssl.c diff -r 922049229f7f -r 13e74bd5ac8c src/plugins/acl/acl-mailbox.c --- a/src/plugins/acl/acl-mailbox.c Sat Jan 05 00:37:26 2013 +0200 +++ b/src/plugins/acl/acl-mailbox.c Sat Jan 05 01:14:11 2013 +0200 @@ -21,6 +21,7 @@ struct acl_object *aclobj; bool skip_acl_checks; bool acl_enabled; + bool no_read_right; }; struct acl_transaction_context { @@ -412,13 +413,19 @@ { struct acl_mailbox *abox = ACL_CONTEXT(ctx->box); void *at = ACL_CONTEXT(ctx); + int ret; if (at != NULL) { abox->module_ctx.super.transaction_rollback(ctx); return -1; } - return abox->module_ctx.super.transaction_commit(ctx, changes_r); + ret = abox->module_ctx.super.transaction_commit(ctx, changes_r); + if (abox->no_read_right) { + /* don't allow IMAP client to see what UIDs the messages got */ + changes_r->no_read_perm = TRUE; + } + return ret; } static int acl_mailbox_exists(struct mailbox *box, bool auto_boxes, @@ -477,6 +484,14 @@ } return -1; } + if (open_right != ACL_STORAGE_RIGHT_READ) { + ret = acl_object_have_right(abox->aclobj, + idx_arr[ACL_STORAGE_RIGHT_READ]); + if (ret < 0) + return -1; + if (ret == 0) + abox->no_read_right = TRUE; + } return 0; } diff -r 922049229f7f -r 13e74bd5ac8c src/plugins/acl/acl-storage.c --- a/src/plugins/acl/acl-storage.c Sat Jan 05 00:37:26 2013 +0200 +++ b/src/plugins/acl/acl-storage.c Sat Jan 05 01:14:11 2013 +0200 @@ -52,7 +52,7 @@ const char *env; env = mail_user_plugin_getenv(user, "acl"); - if (env != NULL) + if (env != NULL && *env != '\0') acl_mail_user_create(user, env); else { if (user->mail_debug) diff -r 922049229f7f -r 13e74bd5ac8c src/plugins/expire/doveadm-expire.c --- a/src/plugins/expire/doveadm-expire.c Sat Jan 05 00:37:26 2013 +0200 +++ b/src/plugins/expire/doveadm-expire.c Sat Jan 05 01:14:11 2013 +0200 @@ -381,7 +381,10 @@ if (expire_dict == NULL) return; - if (ctx->iterate_single_user) { + /* doveadm proxying uses expire database only locally. the remote + doveadm handles each user one at a time (even though + iterate_single_user=FALSE) */ + if (ctx->iterate_single_user || ctx->proxying) { if (doveadm_debug) { i_debug("expire: Iterating only a single user, " "ignoring expire database"); diff -r 922049229f7f -r 13e74bd5ac8c src/plugins/fts-solr/fts-backend-solr-old.c diff -r 922049229f7f -r 13e74bd5ac8c src/plugins/fts-solr/fts-backend-solr.c --- a/src/plugins/fts-solr/fts-backend-solr.c Sat Jan 05 00:37:26 2013 +0200 +++ b/src/plugins/fts-solr/fts-backend-solr.c Sat Jan 05 01:14:11 2013 +0200 @@ -305,7 +305,7 @@ } array_foreach_modifiable(&ctx->fields, field) { str_printfa(ctx->cmd, "", field->key); - str_append_str(ctx->cmd, field->value); + xml_encode_data(ctx->cmd, str_data(field->value), str_len(field->value)); str_append(ctx->cmd, ""); str_truncate(field->value, 0); } @@ -352,8 +352,7 @@ visible to the following search */ if (ctx->expunges) fts_backend_solr_expunge_flush(ctx); - str = t_strdup_printf("", + str = t_strdup_printf("", ctx->documents_added ? "true" : "false"); if (solr_connection_post(solr_conn, str) < 0) ret = -1; diff -r 922049229f7f -r 13e74bd5ac8c src/plugins/fts-solr/solr-connection.c --- a/src/plugins/fts-solr/solr-connection.c Sat Jan 05 00:37:26 2013 +0200 +++ b/src/plugins/fts-solr/solr-connection.c Sat Jan 05 01:14:11 2013 +0200 @@ -192,6 +192,7 @@ curl_slist_free_all(conn->headers_post); curl_multi_cleanup(conn->curlm); curl_easy_cleanup(conn->curl); + XML_ParserFree(conn->xml_parser); i_free(conn->last_sent_url); i_free(conn->url); i_free(conn); diff -r 922049229f7f -r 13e74bd5ac8c src/plugins/fts/fts-api-private.h diff -r 922049229f7f -r 13e74bd5ac8c src/plugins/fts/fts-build-mail.c diff -r 922049229f7f -r 13e74bd5ac8c src/plugins/fts/fts-parser-script.c --- a/src/plugins/fts/fts-parser-script.c Sat Jan 05 00:37:26 2013 +0200 +++ b/src/plugins/fts/fts-parser-script.c Sat Jan 05 01:14:11 2013 +0200 @@ -99,10 +99,13 @@ content->content_type = args[0]; content->extensions = (const void *)(args+1); } + if (!eof_seen) { + if (input->v_offset == 0) + i_error("parser script didn't send any data"); + else + i_error("parser script didn't send empty EOF line"); + } i_stream_destroy(&input); - - if (!eof_seen) - i_error("parser script didn't send empty EOF line"); return 0; } diff -r 922049229f7f -r 13e74bd5ac8c src/plugins/fts/fts-parser.c diff -r 922049229f7f -r 13e74bd5ac8c src/plugins/fts/fts-parser.h diff -r 922049229f7f -r 13e74bd5ac8c src/plugins/lazy-expunge/lazy-expunge-plugin.c --- a/src/plugins/lazy-expunge/lazy-expunge-plugin.c Sat Jan 05 00:37:26 2013 +0200 +++ b/src/plugins/lazy-expunge/lazy-expunge-plugin.c Sat Jan 05 01:14:11 2013 +0200 @@ -121,9 +121,20 @@ union mail_module_context *mmail = LAZY_EXPUNGE_MAIL_CONTEXT(mail); struct lazy_expunge_transaction *lt = LAZY_EXPUNGE_CONTEXT(_mail->transaction); + struct lazy_expunge_mailbox_list *llist; + struct mailbox *real_box; struct mail_save_context *save_ctx; const char *error; + /* don't copy the mail if we're expunging from lazy_expunge + namespace (even if it's via a virtual mailbox) */ + real_box = mail_get_real_mail(_mail)->box; + llist = LAZY_EXPUNGE_LIST_CONTEXT(real_box->list); + if (llist != NULL && llist->internal_namespace) { + mmail->super.expunge(_mail); + return; + } + if (lt->dest_box == NULL) { lt->dest_box = mailbox_open_or_create(luser->lazy_ns->list, _mail->box, &error); diff -r 922049229f7f -r 13e74bd5ac8c src/plugins/mailbox-alias/mailbox-alias-plugin.c --- a/src/plugins/mailbox-alias/mailbox-alias-plugin.c Sat Jan 05 00:37:26 2013 +0200 +++ b/src/plugins/mailbox-alias/mailbox-alias-plugin.c Sat Jan 05 01:14:11 2013 +0200 @@ -119,7 +119,7 @@ if (mailbox_symlink_exists(list, alias->new_vname, &existence) < 0) ret = -1; - if (existence == MAILBOX_SYMLINK_EXISTENCE_SYMLINK) + else if (existence == MAILBOX_SYMLINK_EXISTENCE_SYMLINK) return 1; } } diff -r 922049229f7f -r 13e74bd5ac8c src/plugins/quota/quota-storage.c --- a/src/plugins/quota/quota-storage.c Sat Jan 05 00:37:26 2013 +0200 +++ b/src/plugins/quota/quota-storage.c Sat Jan 05 01:14:11 2013 +0200 @@ -87,7 +87,7 @@ if ((items & ~STATUS_CHECK_OVER_QUOTA) == 0) { /* don't bother calling parent, it may unnecessarily try to open the mailbox */ - return ret; + return ret < 0 ? -1 : 0; } } @@ -533,6 +533,7 @@ struct quota_mailbox_list *qlist; struct quota *quota = NULL; struct quota_root *root; + struct mail_user *quota_user; bool add; if (QUOTA_USER_CONTEXT(list->ns->user) == NULL) @@ -541,8 +542,14 @@ /* see if we have a quota explicitly defined for this namespace */ quota = quota_get_mail_user_quota(list->ns->user); root = quota_find_root_for_ns(quota, list->ns); - if (root != NULL) + if (root != NULL) { + /* explicit quota root */ root->ns = list->ns; + quota_user = list->ns->user; + } else { + quota_user = list->ns->owner != NULL ? + list->ns->owner : list->ns->user; + } if ((list->ns->flags & NAMESPACE_FLAG_NOQUOTA) != 0) add = FALSE; @@ -551,7 +558,9 @@ explicitly defined for it */ add = root != NULL; } else { - add = TRUE; + /* for shared namespaces add only if the owner has quota + enabled */ + add = QUOTA_USER_CONTEXT(quota_user) != NULL; } if (add) { @@ -563,10 +572,7 @@ v->deinit = quota_mailbox_list_deinit; MODULE_CONTEXT_SET(list, quota_mailbox_list_module, qlist); - /* register to owner's quota roots */ - quota = list->ns->owner != NULL ? - quota_get_mail_user_quota(list->ns->owner) : - quota_get_mail_user_quota(list->ns->user); + quota = quota_get_mail_user_quota(quota_user); quota_add_user_namespace(quota, list->ns); } } diff -r 922049229f7f -r 13e74bd5ac8c src/plugins/stats/stats-plugin.c --- a/src/plugins/stats/stats-plugin.c Sat Jan 05 00:37:26 2013 +0200 +++ b/src/plugins/stats/stats-plugin.c Sat Jan 05 01:14:11 2013 +0200 @@ -587,9 +587,13 @@ stats_global_user = user; } else if (stats_user_count == 1) { /* second user connection. we'll need to start doing - per-io callback tracking now. */ - stats_add_session(stats_global_user); - stats_global_user = NULL; + per-io callback tracking now. (we might have been doing it + also previously but just temporarily quickly dropped to + having 1 user, in which case stats_global_user=NULL) */ + if (stats_global_user != NULL) { + stats_add_session(stats_global_user); + stats_global_user = NULL; + } } stats_user_count++; diff -r 922049229f7f -r 13e74bd5ac8c src/plugins/virtual/virtual-config.c --- a/src/plugins/virtual/virtual-config.c Sat Jan 05 00:37:26 2013 +0200 +++ b/src/plugins/virtual/virtual-config.c Sat Jan 05 01:14:11 2013 +0200 @@ -28,6 +28,7 @@ char sep; bool have_wildcards; + bool have_mailbox_defines; }; static struct mail_search_args * @@ -170,6 +171,7 @@ bbox->name++; ctx->mbox->save_bbox = bbox; } + ctx->have_mailbox_defines = TRUE; array_append(&ctx->mbox->backend_boxes, &bbox, 1); return 0; } @@ -420,7 +422,7 @@ if (ret == 0 && ctx.have_wildcards) ret = virtual_config_expand_wildcards(&ctx); - if (ret == 0 && array_count(&mbox->backend_boxes) == 0) { + if (ret == 0 && !ctx.have_mailbox_defines) { mail_storage_set_critical(storage, "%s: No mailboxes defined", path); ret = -1; diff -r 922049229f7f -r 13e74bd5ac8c src/plugins/virtual/virtual-mail.c --- a/src/plugins/virtual/virtual-mail.c Sat Jan 05 00:37:26 2013 +0200 +++ b/src/plugins/virtual/virtual-mail.c Sat Jan 05 01:14:11 2013 +0200 @@ -293,15 +293,17 @@ { struct virtual_mail *vmail = (struct virtual_mail *)mail; struct mail_private *p = (struct mail_private *)vmail->backend_mail; + int ret; if (virtual_mail_handle_lost(vmail) < 0) return -1; - if (p->v.get_first_header(vmail->backend_mail, field, - decode_to_utf8, value_r) < 0) { + ret = p->v.get_first_header(vmail->backend_mail, field, + decode_to_utf8, value_r); + if (ret < 0) { virtual_box_copy_error(mail->box, vmail->backend_mail->box); return -1; } - return 0; + return ret; } static int diff -r 922049229f7f -r 13e74bd5ac8c src/stats/mail-session.c --- a/src/stats/mail-session.c Sat Jan 05 00:37:26 2013 +0200 +++ b/src/stats/mail-session.c Sat Jan 05 01:14:11 2013 +0200 @@ -264,13 +264,16 @@ return -1; if (mail_stats_parse(args+1, &stats, error_r) < 0) { - *error_r = t_strconcat("UPDATE-SESSION: ", *error_r, NULL); + *error_r = t_strdup_printf("UPDATE-SESSION %s %s: %s", + session->user->name, + session->service, *error_r); return -1; } if (!mail_stats_diff(&session->stats, &stats, &diff_stats, &error)) { - *error_r = t_strconcat("UPDATE-SESSION: stats shrank: ", - error, NULL); + *error_r = t_strdup_printf("UPDATE-SESSION %s %s: stats shrank: %s", + session->user->name, + session->service, error); return -1; } mail_session_refresh(session, &diff_stats);