Mercurial > dovecot > core-2.2
changeset 14576:fbb1ecb9b888
Merged changes from v2.1 tree.
line wrap: on
line diff
--- a/.hgignore Sat May 19 22:40:08 2012 +0300 +++ b/.hgignore Sun May 20 03:25:04 2012 +0300 @@ -85,6 +85,8 @@ src/plugins/fts-squat/squat-test src/pop3-login/pop3-login src/pop3/pop3 +src/replication/replicator/replicator +src/replication/aggregator/aggregator src/util/gdbhelper src/util/listview src/util/maildirlock
--- a/.hgsigs Sat May 19 22:40:08 2012 +0300 +++ b/.hgsigs Sun May 20 03:25:04 2012 +0300 @@ -41,3 +41,8 @@ 736f1b7af190ea68e65719277bd8d3d4682e0844 0 iEYEABECAAYFAk87Kz4ACgkQyUhSUUBVismKXACeME7EBYoEuoLLELp0uX6B/lkgRWQAoJ2OAgz4mmluGbi0Db8grDDWSsTj e2cd03cc9c690c4989fb53a1871b7109c547388a 0 iEYEABECAAYFAk89MXsACgkQyUhSUUBVisnPmwCgiAr4OfQX1uAjhuqj5X0xbd8O1NQAn2bQW+h8QPAbqN6dQRNTm82D2hNF 04b0acc03f1eaa0353888a75a452e5c8e61e4c94 0 iEYEABECAAYFAk9F+k0ACgkQyUhSUUBViskLmwCfUt/aex6wOIEohJKnRGA4diF5WxoAn2zlMxSaPX/b0LBmV1P46GAMqZbO +744e0d7f1b255a9339060f761b850303121f14df 0 iEYEABECAAYFAk9h/8oACgkQyUhSUUBVism2OQCfWh62w8pMxJaf1oYx2A+2PxQvBocAn29RFDgZblGRLn7iMCPw6We1yiIw +b9adfd52cb66d5d89d291b76b9845d6361216d12 0 iEYEABECAAYFAk9jbugACgkQyUhSUUBVislSgwCgpo3f0bsSujItBum/M6js8SzF06YAmwftHlwaOstKeALdjLR5vtF2c5F7 +2c21c940e19d97a772128a7f281cea302e157d73 0 iEYEABECAAYFAk+CtkYACgkQyUhSUUBVisknxgCfTJw2YPGJ17HbHRGmbwmCyLqepbMAn17j7IYzUfEU0xkXhCgwEAmk7XO4 +469cee314d9c54d2d7101ec9e38579fdc9610eaf 0 iEYEABECAAYFAk+VWqkACgkQyUhSUUBVislnXACfVjPqMmPUvYtXQXwqff0h7N76mZUAn02lPeUCyuyr1TF9e1hGM/sKgmko +7c249e2a82a9cd33ae15ead6443c3499e16da623 0 iEYEABECAAYFAk+nX2sACgkQyUhSUUBVisn7uwCbBD3boxBOGEJ8OYsIJ57n5Cr09FAAoIvhxL6EHRB15AMOw4sPaALJ3/bB
--- a/.hgtags Sat May 19 22:40:08 2012 +0300 +++ b/.hgtags Sun May 20 03:25:04 2012 +0300 @@ -78,3 +78,8 @@ 736f1b7af190ea68e65719277bd8d3d4682e0844 2.1.rc7 e2cd03cc9c690c4989fb53a1871b7109c547388a 2.1.0 04b0acc03f1eaa0353888a75a452e5c8e61e4c94 2.1.1 +744e0d7f1b255a9339060f761b850303121f14df 2.1.2 +b9adfd52cb66d5d89d291b76b9845d6361216d12 2.1.3 +2c21c940e19d97a772128a7f281cea302e157d73 2.1.4 +469cee314d9c54d2d7101ec9e38579fdc9610eaf 2.1.5 +7c249e2a82a9cd33ae15ead6443c3499e16da623 2.1.6
--- a/Makefile.am Sat May 19 22:40:08 2012 +0300 +++ b/Makefile.am Sun May 20 03:25:04 2012 +0300 @@ -70,12 +70,17 @@ -e "s|^\(LIBDOVECOT_INCLUDE\)=.*$$|\1=-I$(pkgincludedir)|" \ > $(DESTDIR)$(pkglibdir)/dovecot-config +uninstall-hook: + rm $(DESTDIR)$(pkglibdir)/dovecot-config + CLEANFILES = $(datafiles) if HAVE_SYSTEMD CLEANFILES += $systedmsystemunit_DATA endif -DISTCLEANFILES = $(top_builddir)/dovecot-version.h +DISTCLEANFILES = \ + $(top_builddir)/dovecot-version.h \ + $(top_builddir)/dovecot-config distcheck-hook: if which scan-build > /dev/null; then \
--- a/NEWS Sat May 19 22:40:08 2012 +0300 +++ b/NEWS Sun May 20 03:25:04 2012 +0300 @@ -1,3 +1,96 @@ +v2.1.6 2012-05-07 Timo Sirainen <tss@iki.fi> + + * Session ID is now included by default in auth and login process + log lines. It can be added to mail processes also by adding + %{session} to mail_log_prefix. + + + Added ssl_require_crl setting, which specifies if CRL check must + be successful when verifying client certificates. + + Added mail_shared_explicit_inbox setting to specify if a shared INBOX + should be accessible as "shared/$user" or "shared/$user/INBOX". + - v2.1.5: Using "~/" as mail_location or elsewhere failed to actually + expand it to home directory. + - dbox: Fixed potential assert-crash when reading dbox files. + - trash plugin: Fixed behavior when quota is already over limit. + - mail_log plugin: Logging "copy" event didn't work. + - Proxying to backend server with SSL: Verifying server certificate + name always failed, because it was compared to an IP address. + +v2.1.5 2012-04-23 Timo Sirainen <tss@iki.fi> + + * IMAP: When neither the session nor the mailbox has modseq tracking + enabled, return the mailbox as having NOMODSEQ in SELECT/EXAMINE + reply. Old versions in this situation always simply returned + HIGHESTMODSEQ as 1, which could have broken some clients. + + + dict file: Added optional fcntl/flock locking (default is dotlock) + + fts-solr: doveadm fts rescan now resets indexes, which allows + reindexing mails. (This isn't a full rescan implementation like + fts-lucene has.) + + doveadm expunge: Added -d parameter to delete mailbox if it's + empty after expunging. + - IMAP: Several fixes related to mailbox listing in some configs + - director: A lot of fixes and performance improvements + - v2.1.4 didn't work without a mail home directory set + - mbox: Deleting a mailbox didn't delete its index files. + - pop3c: TOP command was sent incorrectly + - trash plugin didn't work properly + - LMTP: Don't add a duplicate Return-Path: header when proxying. + - listescape: Don't unescape namespace prefixes. + +v2.1.4 2012-04-09 Timo Sirainen <tss@iki.fi> + + + Added mail_temp_scan_interval setting and changed its default value + from 8 hours to 1 week. + + Added pop3-migration plugin for easily doing a transparent IMAP+POP3 + migration to Dovecot: http://wiki2.dovecot.org/Migration/Dsync + + doveadm user: Added -m parameter to show some of the mail settings. + - Proxying SSL connections crashed in v2.1.[23] + - fts-solr: Indexing mail bodies was broken. + - director: Several changes to significantly improve error handling + - doveadm import didn't import messages' flags + - mail_full_filesystem_access=yes was broken + - Make sure IMAP clients can't create directories when accessing + nonexistent users' mailboxes via shared namespace. + - Dovecot auth clients authenticating via TCP socket could have failed + with bogus "PID already in use" errors. + +v2.1.3 2012-03-16 Timo Sirainen <tss@iki.fi> + + - mdbox was broken in v2.1.2 + +v2.1.2 2012-03-15 Timo Sirainen <tss@iki.fi> + + + Initial implementation of dsync-based replication. For now this + should be used only on non-critical systems. + + Proxying: POP3 now supports sending remote IP+port from proxy to + backend server via Dovecot-specific XCLIENT extension. + + Proxying: proxy_maybe=yes with host=<hostname> (instead of IP) + works now properly. + + Proxying: Added auth_proxy_self setting + + Proxying: Added proxy_always extra field (see wiki docs) + + Added director_username_hash setting to specify what part of the + username is hashed. This can be used to implement per-domain + backends (which allows safely accessing shared mailboxes within + domain). + + Added a "session ID" string for imap/pop3 connections, available + in %{session} variable. The session ID passes through Dovecot + IMAP/POP3 proxying to backend server. The same session ID is can be + reused after a long time (currently a bit under 9 years). + + passdb checkpassword: Support "credentials lookups" (for + non-plaintext auth and for lmtp_proxy lookups) + + fts: Added fts_index_timeout setting to abort search if indexing + hasn't finished by then (default is to wait forever). + - doveadm sync: If mailbox was expunged empty, messages may have + become back instead of also being expunged in the other side. + - director: If user logged into two directors while near user + expiration, the directors might have redirected the user to two + different backends. + - imap_id_* settings were ignored before login. + - Several fixes to mailbox_list_index=yes + - Previous v2.1.x didn't log all messages at shutdown. + - mbox: Fixed accessing Dovecot v1.x mbox index files without errors. + v2.1.1 2012-02-23 Timo Sirainen <tss@iki.fi> + dsync: If message with same GUID is saved multiple times in session,
--- a/TODO Sat May 19 22:40:08 2012 +0300 +++ b/TODO Sun May 20 03:25:04 2012 +0300 @@ -46,7 +46,6 @@ non-userdb_* extra fields too? - imap, pop3: if client init fails, wait a second or two before disconnecting client. - - doveadm expunge parameter to delete empty mailboxes (for lazy-expunge) - doveadm search savedbefore 7d could be optimized in large mailboxes.. - mdbox: storage rebuilding could log about changes it does - mdbox: broken extrefs header keeps causing index rebuilds
--- a/configure.in Sat May 19 22:40:08 2012 +0300 +++ b/configure.in Sun May 20 03:25:04 2012 +0300 @@ -2490,10 +2490,13 @@ want_ssl_libs=yes fi done +LINKED_STORAGE_LDADD= if test "$want_ssl_libs" = yes; then LINKED_STORAGE_LIBS="$LINKED_STORAGE_LIBS \$(top_builddir)/src/lib-ssl-iostream/libssl_iostream.la" + LINKED_STORAGE_LDADD="$SSL_LIBS" fi AC_SUBST(LINKED_STORAGE_LIBS) +AC_SUBST(LINKED_STORAGE_LDADD) AC_SUBST(mailbox_list_drivers) AC_DEFINE_UNQUOTED(MAIL_STORAGES, "$mail_storages", List of compiled in mail storages) @@ -2509,7 +2512,7 @@ if test "$want_shared_libs" = "yes"; then LIBDOVECOT_DEPS='$(top_builddir)/src/lib-dovecot/libdovecot.la' LIBDOVECOT="$LIBDOVECOT_DEPS" - LIBDOVECOT_STORAGE='$(top_builddir)/src/lib-storage/libdovecot-storage.la' + LIBDOVECOT_STORAGE_DEPS='$(top_builddir)/src/lib-storage/libdovecot-storage.la' LIBDOVECOT_LOGIN='$(top_builddir)/src/login-common/libdovecot-login.la' LIBDOVECOT_LDA='$(top_builddir)/src/lib-lda/libdovecot-lda.la' else @@ -2517,14 +2520,16 @@ LIBDOVECOT="$LIBDOVECOT_DEPS \$(LIBICONV)" LIBDOVECOT_STORAGE_LAST='$(top_builddir)/src/lib-storage/list/libstorage_list.la $(top_builddir)/src/lib-storage/index/libstorage_index.la $(top_builddir)/src/lib-storage/libstorage.la $(top_builddir)/src/lib-index/libindex.la' LIBDOVECOT_STORAGE_FIRST='$(top_builddir)/src/lib-storage/libstorage_service.la $(top_builddir)/src/lib-storage/register/libstorage_register.la' - LIBDOVECOT_STORAGE="$LIBDOVECOT_STORAGE_FIRST $LINKED_STORAGE_LIBS $LIBDOVECOT_STORAGE_LAST" + LIBDOVECOT_STORAGE_DEPS="$LIBDOVECOT_STORAGE_FIRST $LINKED_STORAGE_LIBS $LIBDOVECOT_STORAGE_LAST" LIBDOVECOT_LOGIN='$(top_builddir)/src/login-common/liblogin.la $(top_builddir)/src/lib-ssl-iostream/libssl_iostream.la' LIBDOVECOT_LDA='$(top_builddir)/src/lib-lda/liblda.la' fi +LIBDOVECOT_STORAGE="$LIBDOVECOT_STORAGE_DEPS $LINKED_STORAGE_LDADD" LIBDOVECOT_SQL='$(top_builddir)/src/lib-sql/libsql.la' AC_SUBST(LIBDOVECOT) AC_SUBST(LIBDOVECOT_DEPS) AC_SUBST(LIBDOVECOT_STORAGE) +AC_SUBST(LIBDOVECOT_STORAGE_DEPS) AC_SUBST(LIBDOVECOT_LOGIN) AC_SUBST(LIBDOVECOT_SQL) AC_SUBST(LIBDOVECOT_LDA) @@ -2699,7 +2704,7 @@ dnl IDLE doesn't really belong to banner. It's there just to make Blackberries dnl happy, because otherwise BIS server disables push email. capability_banner="IMAP4rev1 LITERAL+ SASL-IR LOGIN-REFERRALS ID ENABLE IDLE" -capability="$capability_banner SORT SORT=DISPLAY THREAD=REFERENCES THREAD=REFS MULTIAPPEND UNSELECT CHILDREN NAMESPACE UIDPLUS LIST-EXTENDED I18NLEVEL=1 CONDSTORE QRESYNC ESEARCH ESORT SEARCHRES WITHIN CONTEXT=SEARCH LIST-STATUS SEARCH=FUZZY SPECIAL-USE" +capability="$capability_banner SORT SORT=DISPLAY THREAD=REFERENCES THREAD=REFS MULTIAPPEND UNSELECT CHILDREN NAMESPACE UIDPLUS LIST-EXTENDED I18NLEVEL=1 CONDSTORE QRESYNC ESEARCH ESORT SEARCHRES WITHIN CONTEXT=SEARCH LIST-STATUS SPECIAL-USE" AC_DEFINE_UNQUOTED(CAPABILITY_STRING, "$capability", IMAP capabilities) AC_DEFINE_UNQUOTED(CAPABILITY_BANNER_STRING, "$capability_banner", IMAP capabilities advertised in banner) @@ -2787,6 +2792,9 @@ src/master/Makefile src/pop3/Makefile src/pop3-login/Makefile +src/replication/Makefile +src/replication/aggregator/Makefile +src/replication/replicator/Makefile src/ssl-params/Makefile src/stats/Makefile src/util/Makefile @@ -2803,8 +2811,10 @@ src/plugins/listescape/Makefile src/plugins/mail-log/Makefile src/plugins/notify/Makefile +src/plugins/pop3-migration/Makefile src/plugins/quota/Makefile src/plugins/imap-quota/Makefile +src/plugins/replication/Makefile src/plugins/snarf/Makefile src/plugins/stats/Makefile src/plugins/imap-stats/Makefile
--- a/doc/example-config/conf.d/10-mail.conf Sat May 19 22:40:08 2012 +0300 +++ b/doc/example-config/conf.d/10-mail.conf Sun May 20 03:25:04 2012 +0300 @@ -97,6 +97,8 @@ # List the shared/ namespace only if there are visible shared mailboxes. #list = children #} +# Should shared INBOX be visible as "shared/user" or "shared/user/INBOX"? +#mail_shared_explicit_inbox = yes # System user and group used to access mails. If you use multiple, userdb # can override these by returning uid or gid fields. You can use either numbers @@ -227,6 +229,10 @@ # some mailbox formats and/or operating systems. #mail_prefetch_count = 0 +# How often to scan for stale temporary files and delete them (0 = never). +# These should exist only after Dovecot dies in the middle of saving mails. +#mail_temp_scan_interval = 1w + ## ## Maildir-specific settings ##
--- a/doc/example-config/conf.d/10-ssl.conf Sat May 19 22:40:08 2012 +0300 +++ b/doc/example-config/conf.d/10-ssl.conf Sun May 20 03:25:04 2012 +0300 @@ -23,6 +23,9 @@ # followed by the matching CRL(s). (e.g. ssl_ca = </etc/ssl/certs/ca.pem) #ssl_ca = +# Require that CRL check succeeds for client certificates. +#ssl_require_crl = yes + # Request client to send a certificate. If you also want to require it, set # auth_ssl_require_client_cert=yes in auth section. #ssl_verify_client_cert = no
--- a/doc/example-config/conf.d/20-pop3.conf Sat May 19 22:40:08 2012 +0300 +++ b/doc/example-config/conf.d/20-pop3.conf Sun May 20 03:25:04 2012 +0300 @@ -54,6 +54,11 @@ # won't change those UIDLs. Currently this works only with Maildir. #pop3_save_uidl = no + # What to do about duplicate UIDLs if they exist? + # allow: Show duplicates to clients. + # rename: Append a temporary -2, -3, etc. counter after the UIDL. + #pop3_uidl_duplicates = allow + # POP3 logout format string: # %i - total number of bytes read from client # %o - total number of bytes sent to client
--- a/doc/example-config/dovecot.conf Sat May 19 22:40:08 2012 +0300 +++ b/doc/example-config/dovecot.conf Sun May 20 03:25:04 2012 +0300 @@ -46,6 +46,11 @@ # Sepace separated list of login access check sockets (e.g. tcpwrap) #login_access_sockets = +# With proxy_maybe=yes if proxy destination matches any of these IPs, don't do +# proxying. This isn't necessary normally, but may be useful if the destination +# IP is e.g. a load balancer's IP. +#auth_proxy_self = + # Show more verbose process titles (in ps). Currently shows user name and # IP address. Useful for seeing who are actually using the IMAP processes # (eg. shared mailboxes or if same uid is used for multiple accounts).
--- a/dovecot-config.in.in Sat May 19 22:40:08 2012 +0300 +++ b/dovecot-config.in.in Sun May 20 03:25:04 2012 +0300 @@ -13,7 +13,7 @@ LIBDOVECOT_LOGIN_DEPS="@LIBDOVECOT_LOGIN@" LIBDOVECOT_SQL_DEPS="@LIBDOVECOT_SQL@" LIBDOVECOT_LDA_DEPS="@LIBDOVECOT_LDA@" -LIBDOVECOT_STORAGE_DEPS="@LIBDOVECOT_STORAGE@" +LIBDOVECOT_STORAGE_DEPS="@LIBDOVECOT_STORAGE_DEPS@" LIBDOVECOT_INCLUDE="-I$(incdir) -I$(incdir)/src/lib -I$(incdir)/src/lib-dict -I$(incdir)/src/lib-dns -I$(incdir)/src/lib-mail -I$(incdir)/src/lib-imap -I$(incdir)/src/lib-fs -I$(incdir)/src/lib-charset -I$(incdir)/src/lib-auth -I$(incdir)/src/lib-master -I$(incdir)/src/lib-settings" LIBDOVECOT_LDA_INCLUDE="-I$(incdir)/src/lib-lda -I$(incdir)/src/lda"
--- a/src/Makefile.am Sat May 19 22:40:08 2012 +0300 +++ b/src/Makefile.am Sun May 20 03:25:04 2012 +0300 @@ -39,6 +39,7 @@ log \ config \ director \ + replication \ util \ doveadm \ ssl-params \
--- a/src/anvil/anvil-connection.c Sat May 19 22:40:08 2012 +0300 +++ b/src/anvil/anvil-connection.c Sun May 20 03:25:04 2012 +0300 @@ -40,7 +40,7 @@ const char *line; line = i_stream_next_line(conn->input); - return line == NULL ? NULL : t_strsplit(line, "\t"); + return line == NULL ? NULL : t_strsplit_tab(line); } static int
--- a/src/anvil/penalty.c Sat May 19 22:40:08 2012 +0300 +++ b/src/anvil/penalty.c Sun May 20 03:25:04 2012 +0300 @@ -136,9 +136,9 @@ rec->checksum_is_pointer = TRUE; } - memcpy(rec->checksum.value_ptr + 1, rec->checksum.value_ptr, - sizeof(rec->checksum.value_ptr[0]) * - (CHECKSUM_VALUE_PTR_COUNT-1)); + memmove(rec->checksum.value_ptr + 1, rec->checksum.value_ptr, + sizeof(rec->checksum.value_ptr[0]) * + (CHECKSUM_VALUE_PTR_COUNT-1)); rec->checksum.value_ptr[0] = checksum; }
--- a/src/auth/Makefile.am Sat May 19 22:40:08 2012 +0300 +++ b/src/auth/Makefile.am Sun May 20 03:25:04 2012 +0300 @@ -24,6 +24,7 @@ AM_CPPFLAGS = \ -I$(top_srcdir)/src/lib \ -I$(top_srcdir)/src/lib-auth \ + -I$(top_srcdir)/src/lib-test \ -I$(top_srcdir)/src/lib-dns \ -I$(top_srcdir)/src/lib-sql \ -I$(top_srcdir)/src/lib-settings \ @@ -167,7 +168,7 @@ libauthdb_imap_la_LIBADD = \ ../lib-imap-client/libimap_client.la \ ../lib-ssl-iostream/libssl_iostream.la \ - $(LIBDOVECOT) + $(LIBDOVECOT) $(SSL_LIBS) libauthdb_imap_la_CPPFLAGS = \ $(AM_CPPFLAGS) \ -I$(top_srcdir)/src/lib-imap \ @@ -182,3 +183,22 @@ checkpassword_reply_sources = \ checkpassword-reply.c + +test_programs = \ + test-auth-cache + +noinst_PROGRAMS = $(test_programs) + +test_libs = \ + ../lib-test/libtest.la \ + ../lib/liblib.la + +test_auth_cache_SOURCES = test-auth-cache.c +test_auth_cache_LDADD = auth-cache.o $(test_libs) +test_auth_cache_DEPENDENCIES = auth-cache.o $(test_libs) + +check: check-am check-test +check-test: all-am + for bin in $(test_programs); do \ + if ! $(RUN_TEST) ./$$bin; then exit 1; fi; \ + done
--- a/src/auth/auth-cache.c Sat May 19 22:40:08 2012 +0300 +++ b/src/auth/auth-cache.c Sun May 20 03:25:04 2012 +0300 @@ -23,29 +23,75 @@ unsigned long long pos_size, neg_size; }; +static const struct var_expand_table * +auth_request_var_expand_tab_find(const char *key, unsigned int size) +{ + const struct var_expand_table *tab = auth_request_var_expand_static_tab; + unsigned int i; + + for (i = 0; tab[i].key != '\0' || tab[i].long_key != NULL; i++) { + if (size == 1) { + if (key[0] == tab[i].key) + return &tab[i]; + } else if (tab[i].long_key != NULL) { + if (strncmp(key, tab[i].long_key, size) == 0 && + tab[i].long_key[size] == '\0') + return &tab[i]; + } + } + return NULL; +} + char *auth_cache_parse_key(pool_t pool, const char *query) { + const struct var_expand_table *tab; string_t *str; - char key_seen[256]; - uint8_t key; + bool key_seen[100]; + unsigned int idx, size, tab_idx; + bool add_key; memset(key_seen, 0, sizeof(key_seen)); str = str_new(pool, 32); - for (; *query != '\0'; query++) { - if (*query == '%' && query[1] != '\0') { + for (; *query != '\0'; ) { + if (*query != '%') { query++; - key = var_get_key(query); - if (key != '\0' && key != '%' && !key_seen[key]) { - if (str_len(str) != 0) - str_append_c(str, '\t'); - str_append_c(str, '%'); - str_append_c(str, key); + continue; + } + + var_get_key_range(++query, &idx, &size); + if (size == 0) { + /* broken %variable ending too early */ + break; + } + query += idx; - /* @UNSAFE */ - key_seen[key] = 1; + tab = auth_request_var_expand_tab_find(query, size); + if (tab == NULL) { + /* just add the key. it would be nice to prevent + duplicates here as well, but that's just too + much trouble and probably very rare. */ + add_key = TRUE; + } else { + tab_idx = tab - auth_request_var_expand_static_tab; + i_assert(tab_idx < N_ELEMENTS(key_seen)); + /* @UNSAFE */ + add_key = !key_seen[tab_idx]; + key_seen[tab_idx] = TRUE; + } + if (add_key) { + if (str_len(str) != 0) + str_append_c(str, '\t'); + str_append_c(str, '%'); + if (size == 1) + str_append_c(str, query[0]); + else { + str_append_c(str, '{'); + str_append_n(str, query, size); + str_append_c(str, '}'); } } + query += size; } return str_free_without_data(&str); } @@ -161,6 +207,15 @@ hash_table_clear(cache->hash, FALSE); } +static const char * +auth_cache_escape(const char *string, + const struct auth_request *auth_request ATTR_UNUSED) +{ + /* cache key %variables are separated by tabs, make sure that there + are no tabs in the string */ + return str_tabescape(string); +} + const char * auth_cache_lookup(struct auth_cache *cache, const struct auth_request *request, const char *key, struct auth_cache_node **node_r, @@ -179,7 +234,7 @@ str = t_str_new(256); var_expand(str, t_strconcat(request->userdb_lookup ? "U" : "P", "%!/", key, NULL), - auth_request_get_var_expand_table(request, NULL)); + auth_request_get_var_expand_table(request, auth_cache_escape)); node = hash_table_lookup(cache->hash, str_c(str)); if (node == NULL) { @@ -235,7 +290,7 @@ str = t_str_new(256); var_expand(str, t_strconcat(request->userdb_lookup ? "U" : "P", "%!/", key, NULL), - auth_request_get_var_expand_table(request, NULL)); + auth_request_get_var_expand_table(request, auth_cache_escape)); request->user = current_username; @@ -284,7 +339,7 @@ str = t_str_new(256); var_expand(str, key, - auth_request_get_var_expand_table(request, NULL)); + auth_request_get_var_expand_table(request, auth_cache_escape)); node = hash_table_lookup(cache->hash, str_c(str)); if (node == NULL)
--- a/src/auth/auth-client-connection.c Sat May 19 22:40:08 2012 +0300 +++ b/src/auth/auth-client-connection.c Sun May 20 03:25:04 2012 +0300 @@ -95,7 +95,16 @@ return FALSE; } - old = auth_client_connection_lookup(pid); + if (conn->login_requests) + old = auth_client_connection_lookup(pid); + else { + /* the client is only authenticating, not logging in. + the PID isn't necessary, and since we allow authentication + via TCP sockets the PIDs may conflict, so ignore them. */ + old = NULL; + pid = 0; + } + if (old != NULL) { /* already exists. it's possible that it just reconnected, see if the old connection is still there. */
--- a/src/auth/auth-master-connection.c Sat May 19 22:40:08 2012 +0300 +++ b/src/auth/auth-master-connection.c Sun May 20 03:25:04 2012 +0300 @@ -100,7 +100,7 @@ buffer_t buf; /* <id> <client-pid> <client-id> <cookie> */ - list = t_strsplit(args, "\t"); + list = t_strsplit_tab(args); if (str_array_length(list) < 4 || str_to_uint(list[0], &id) < 0 || str_to_uint(list[1], &client_pid) < 0 || @@ -146,7 +146,7 @@ unsigned int id; /* <id> <userid> [<parameters>] */ - list = t_strsplit(args, "\t"); + list = t_strsplit_tab(args); if (list[0] == NULL || list[1] == NULL || str_to_uint(list[0], &id) < 0) { i_error("BUG: Master sent broken %s", cmd); @@ -485,7 +485,7 @@ unsigned int id; /* <id> [<parameters>] */ - list = t_strsplit(args, "\t"); + list = t_strsplit_tab(args); if (list[0] == NULL || str_to_uint(list[0], &id) < 0) { i_error("BUG: Master sent broken LIST"); return -1;
--- a/src/auth/auth-request-handler.c Sat May 19 22:40:08 2012 +0300 +++ b/src/auth/auth-request-handler.c Sun May 20 03:25:04 2012 +0300 @@ -162,15 +162,14 @@ extra_fields = auth_stream_reply_export(request->extra_fields); - if (!request->proxy) { - /* we only wish to remove all fields prefixed with "userdb_" */ - if (strstr(extra_fields, "userdb_") == NULL) { - auth_stream_reply_import(reply, extra_fields); - return; - } + if (!request->proxy && strstr(extra_fields, "userdb_") == NULL) { + /* optimization: there are no userdb_* fields, we can just + import */ + auth_stream_reply_import(reply, extra_fields); + return; } - fields = t_strsplit(extra_fields, "\t"); + fields = t_strsplit_tab(extra_fields); for (src = 0; fields[src] != NULL; src++) { if (strncmp(fields[src], "userdb_", 7) != 0) { if (!seen_pass && strncmp(fields[src], "pass=", 5) == 0) @@ -288,6 +287,21 @@ auth_stream_reply_add(reply, "nodelay", NULL); get_client_extra_fields(request, reply); + switch (request->passdb_result) { + case PASSDB_RESULT_INTERNAL_FAILURE: + case PASSDB_RESULT_SCHEME_NOT_AVAILABLE: + case PASSDB_RESULT_USER_UNKNOWN: + case PASSDB_RESULT_PASSWORD_MISMATCH: + case PASSDB_RESULT_OK: + break; + case PASSDB_RESULT_USER_DISABLED: + auth_stream_reply_add(reply, "user_disabled", NULL); + break; + case PASSDB_RESULT_PASS_EXPIRED: + auth_stream_reply_add(reply, "pass_expired", NULL); + break; + } + auth_request_handle_failure(request, reply); } @@ -439,7 +453,7 @@ i_assert(!handler->destroyed); /* <id> <mechanism> [...] */ - list = t_strsplit(args, "\t"); + list = t_strsplit_tab(args); if (list[0] == NULL || list[1] == NULL || str_to_uint(list[0], &id) < 0) { i_error("BUG: Authentication client %u "
--- a/src/auth/auth-request.c Sat May 19 22:40:08 2012 +0300 +++ b/src/auth/auth-request.c Sun May 20 03:25:04 2012 +0300 @@ -245,6 +245,8 @@ request->local_port = atoi(value); else if (strcmp(key, "rport") == 0) request->remote_port = atoi(value); + else if (strcmp(key, "session") == 0) + request->session_id = p_strdup(request->pool, value); else return FALSE; return TRUE; @@ -525,6 +527,10 @@ request->passdb = request->passdb->next; request->passdb_password = NULL; + request->proxy = FALSE; + request->proxy_maybe = FALSE; + request->proxy_always = FALSE; + if (*result == PASSDB_RESULT_USER_UNKNOWN) { /* remember that we did at least one successful passdb lookup */ @@ -559,6 +565,7 @@ request->private_callback.verify_plain); } else { auth_request_ref(request); + request->passdb_result = result; request->private_callback.verify_plain(result, request); auth_request_unref(&request); } @@ -610,6 +617,20 @@ return FALSE; } +static bool auth_request_is_disabled_master_user(struct auth_request *request) +{ + if (request->passdb != NULL) + return FALSE; + + /* no masterdbs, master logins not supported */ + i_assert(request->requested_login_user != NULL); + auth_request_log_info(request, "passdb", + "Attempted master login with no master passdbs " + "(trying to log in as user: %s)", + request->requested_login_user); + return TRUE; +} + void auth_request_verify_plain(struct auth_request *request, const char *password, verify_plain_callback_t *callback) @@ -620,13 +641,7 @@ i_assert(request->state == AUTH_REQUEST_STATE_MECH_CONTINUE); - if (request->passdb == NULL) { - /* no masterdbs, master logins not supported */ - i_assert(request->requested_login_user != NULL); - auth_request_log_info(request, "passdb", - "Attempted master login with no master passdbs " - "(trying to log in as user: %s)", - request->requested_login_user); + if (auth_request_is_disabled_master_user(request)) { callback(PASSDB_RESULT_USER_UNKNOWN, request); return; } @@ -693,6 +708,7 @@ but the user was unknown there */ result = PASSDB_RESULT_USER_UNKNOWN; } + request->passdb_result = result; request->private_callback. lookup_credentials(result, credentials, size, request); } @@ -746,6 +762,11 @@ i_assert(request->state == AUTH_REQUEST_STATE_MECH_CONTINUE); + if (auth_request_is_disabled_master_user(request)) { + callback(PASSDB_RESULT_USER_UNKNOWN, NULL, 0, request); + return; + } + request->credentials_scheme = p_strdup(request->pool, scheme); request->private_callback.lookup_credentials = callback; @@ -1295,25 +1316,31 @@ } } +void auth_request_set_field_keyvalue(struct auth_request *request, + const char *field, + const char *default_scheme) +{ + const char *key, *value; + + value = strchr(field, '='); + if (value == NULL) { + key = field; + value = ""; + } else { + key = t_strdup_until(field, value); + value++; + } + auth_request_set_field(request, key, value, default_scheme); +} + void auth_request_set_fields(struct auth_request *request, const char *const *fields, const char *default_scheme) { - const char *key, *value; - for (; *fields != NULL; fields++) { if (**fields == '\0') continue; - - value = strchr(*fields, '='); - if (value == NULL) { - key = *fields; - value = ""; - } else { - key = t_strdup_until(*fields, value); - value++; - } - auth_request_set_field(request, key, value, default_scheme); + auth_request_set_field_keyvalue(request, *fields, default_scheme); } } @@ -1484,11 +1511,14 @@ } else { /* proxying to ourself - log in without proxying by dropping all the proxying fields. */ + bool proxy_always = request->proxy_always; + auth_request_proxy_finish_failure(request); - if (request->proxy_always) { + if (proxy_always) { /* director adds the host */ auth_stream_reply_add(request->extra_fields, "proxy", NULL); + request->proxy = TRUE; } } } @@ -1518,8 +1548,8 @@ "DNS lookup for %s took %u.%03u s", host, result->msecs/1000, result->msecs % 1000); } - auth_stream_reply_remove(request->extra_fields, "host"); - auth_stream_reply_add(request->extra_fields, "host", + auth_stream_reply_remove(request->extra_fields, "hostip"); + auth_stream_reply_add(request->extra_fields, "hostip", net_ip2addr(&result->ips[0])); for (i = 0; i < result->ips_count; i++) { if (auth_request_proxy_ip_is_self(request, @@ -1606,6 +1636,10 @@ auth_stream_reply_remove(request->extra_fields, "host"); auth_stream_reply_remove(request->extra_fields, "port"); auth_stream_reply_remove(request->extra_fields, "destuser"); + + request->proxy = FALSE; + request->proxy_maybe = FALSE; + request->proxy_always = FALSE; } static void log_password_failure(struct auth_request *request, @@ -1745,38 +1779,41 @@ return str_escape(string); } +const struct var_expand_table auth_request_var_expand_static_tab[] = { + { 'u', NULL, "user" }, + { 'n', NULL, "username" }, + { 'd', NULL, "domain" }, + { 's', NULL, "service" }, + { 'h', NULL, "home" }, + { 'l', NULL, "lip" }, + { 'r', NULL, "rip" }, + { 'p', NULL, "pid" }, + { 'w', NULL, "password" }, + { '!', NULL, NULL }, + { 'm', NULL, "mech" }, + { 'c', NULL, "secured" }, + { 'a', NULL, "lport" }, + { 'b', NULL, "rport" }, + { 'k', NULL, "cert" }, + { '\0', NULL, "login_user" }, + { '\0', NULL, "login_username" }, + { '\0', NULL, "login_domain" }, + { '\0', NULL, "session" }, + { '\0', NULL, NULL } +}; + const struct var_expand_table * auth_request_get_var_expand_table(const struct auth_request *auth_request, auth_request_escape_func_t *escape_func) { - static struct var_expand_table static_tab[] = { - { 'u', NULL, "user" }, - { 'n', NULL, "username" }, - { 'd', NULL, "domain" }, - { 's', NULL, "service" }, - { 'h', NULL, "home" }, - { 'l', NULL, "lip" }, - { 'r', NULL, "rip" }, - { 'p', NULL, "pid" }, - { 'w', NULL, "password" }, - { '!', NULL, NULL }, - { 'm', NULL, "mech" }, - { 'c', NULL, "secured" }, - { 'a', NULL, "lport" }, - { 'b', NULL, "rport" }, - { 'k', NULL, "cert" }, - { '\0', NULL, "login_user" }, - { '\0', NULL, "login_username" }, - { '\0', NULL, "login_domain" }, - { '\0', NULL, NULL } - }; struct var_expand_table *tab; if (escape_func == NULL) escape_func = escape_none; - tab = t_malloc(sizeof(static_tab)); - memcpy(tab, static_tab, sizeof(static_tab)); + tab = t_malloc(sizeof(auth_request_var_expand_static_tab)); + memcpy(tab, auth_request_var_expand_static_tab, + sizeof(auth_request_var_expand_static_tab)); tab[0].value = escape_func(auth_request->user, auth_request); tab[1].value = escape_func(t_strcut(auth_request->user, '@'), @@ -1821,6 +1858,8 @@ auth_request); } } + tab[18].value = auth_request->session_id == NULL ? NULL : + escape_func(auth_request->session_id, auth_request); return tab; } @@ -1847,6 +1886,8 @@ } if (auth_request->requested_login_user != NULL) str_append(str, ",master"); + if (auth_request->session_id != NULL) + str_printfa(str, ",<%s>", auth_request->session_id); str_append(str, "): "); }
--- a/src/auth/auth-request.h Sat May 19 22:40:08 2012 +0300 +++ b/src/auth/auth-request.h Sun May 20 03:25:04 2012 +0300 @@ -2,6 +2,7 @@ #define AUTH_REQUEST_H #include "network.h" +#include "var-expand.h" #include "mech.h" #include "userdb.h" #include "passdb.h" @@ -56,6 +57,8 @@ /* the whole userdb result reply */ struct auth_stream_reply *userdb_reply; struct auth_request_proxy_dns_lookup_ctx *dns_lookup_ctx; + /* Result of passdb lookup */ + enum passdb_result passdb_result; const struct mech_module *mech; const struct auth_settings *set; @@ -71,7 +74,7 @@ unsigned int id; time_t last_access; - const char *service, *mech_name; + const char *service, *mech_name, *session_id; struct ip_addr local_ip, remote_ip; unsigned int local_port, remote_port; @@ -123,6 +126,7 @@ typedef void auth_request_proxy_cb_t(bool success, struct auth_request *); extern unsigned int auth_request_state_count[AUTH_REQUEST_STATE_MAX]; +extern const struct var_expand_table auth_request_var_expand_static_tab[]; struct auth_request * auth_request_new(const struct mech_module *mech); @@ -172,6 +176,9 @@ void auth_request_set_field(struct auth_request *request, const char *name, const char *value, const char *default_scheme); +void auth_request_set_field_keyvalue(struct auth_request *request, + const char *field, + const char *default_scheme); void auth_request_set_fields(struct auth_request *request, const char *const *fields, const char *default_scheme); @@ -208,7 +215,7 @@ const char *format, ...) ATTR_FORMAT(3, 4); void auth_request_log_warning(struct auth_request *auth_request, const char *subsystem, - const char *format, ...); + const char *format, ...) ATTR_FORMAT(3, 4); void auth_request_log_error(struct auth_request *auth_request, const char *subsystem, const char *format, ...) ATTR_FORMAT(3, 4);
--- a/src/auth/auth-settings.c Sat May 19 22:40:08 2012 +0300 +++ b/src/auth/auth-settings.c Sun May 20 03:25:04 2012 +0300 @@ -19,7 +19,7 @@ { "login/login", 0666, "", "" }, { "auth-login", 0600, "$default_internal_user", "" }, { "auth-client", 0600, "", "" }, - { "auth-userdb", 0666, "", "" }, + { "auth-userdb", 0666, "$default_internal_user", "" }, { "auth-master", 0600, "", "" } }; static struct file_listener_settings *auth_unix_listeners[] = {
--- a/src/auth/auth-stream.c Sat May 19 22:40:08 2012 +0300 +++ b/src/auth/auth-stream.c Sun May 20 03:25:04 2012 +0300 @@ -140,7 +140,7 @@ const char *const *auth_stream_split(struct auth_stream_reply *reply) { - return t_strsplit(str_c(reply->str), "\t"); + return t_strsplit_tab(str_c(reply->str)); } string_t *auth_stream_reply_get_str(struct auth_stream_reply *reply)
--- a/src/auth/auth-worker-client.c Sat May 19 22:40:08 2012 +0300 +++ b/src/auth/auth-worker-client.c Sun May 20 03:25:04 2012 +0300 @@ -19,6 +19,7 @@ #define CLIENT_STATE_HANDSHAKE "handshaking" #define CLIENT_STATE_IDLE "idling" +#define CLIENT_STATE_STOP "waiting for shutdown" struct auth_worker_client { int refcount; @@ -546,7 +547,7 @@ unsigned int id; bool ret = FALSE; - args = t_strsplit(line, "\t"); + args = t_strsplit_tab(line); if (args[0] == NULL || args[1] == NULL || args[2] == NULL || str_to_uint(args[0], &id) < 0) { i_error("BUG: Invalid input: %s", line); @@ -595,9 +596,6 @@ char *line; bool ret; - if (client->to_idle != NULL) - timeout_reset(client->to_idle); - switch (i_stream_read(client->input)) { case 0: return; @@ -655,6 +653,8 @@ break; } } + if (client->to_idle != NULL) + timeout_reset(client->to_idle); auth_worker_client_unref(&client); } @@ -776,4 +776,5 @@ { if (auth_worker_client != NULL) o_stream_send_str(auth_worker_client->output, "SHUTDOWN\n"); + auth_worker_refresh_proctitle(CLIENT_STATE_STOP); }
--- a/src/auth/auth.c Sat May 19 22:40:08 2012 +0300 +++ b/src/auth/auth.c Sun May 20 03:25:04 2012 +0300 @@ -194,7 +194,6 @@ /* use a dummy userdb static. */ auth_userdb_preinit(auth, &userdb_dummy_set); } - auth_mech_list_verify_passdb(auth); return auth; } @@ -251,9 +250,10 @@ { struct master_service_settings_output set_output; const struct auth_settings *service_set; - struct auth *auth; + struct auth *auth, *const *authp; unsigned int i; const char *not_service = NULL; + bool check_default = TRUE; i_array_init(&auths, 8); @@ -274,6 +274,14 @@ auth = auth_preinit(service_set, services[i], pool, reg); array_append(&auths, &auth, 1); } + + if (not_service != NULL && str_array_find(services, not_service+1)) + check_default = FALSE; + + array_foreach(&auths, authp) { + if ((*authp)->service != NULL || check_default) + auth_mech_list_verify_passdb(*authp); + } } void auths_init(void)
--- a/src/auth/checkpassword-reply.c Sat May 19 22:40:08 2012 +0300 +++ b/src/auth/checkpassword-reply.c Sun May 20 03:25:04 2012 +0300 @@ -2,6 +2,7 @@ #include "lib.h" #include "str.h" +#include "strescape.h" #include "write-full.h" #include <stdlib.h> @@ -23,7 +24,9 @@ i_error("checkpassword: USER contains TAB"); return 1; } - str_printfa(str, "user=%s\t", user); + str_printfa(str, "user="); + str_tabescape_write(str, user); + str_append_c(str, '\t'); } home = getenv("HOME"); @@ -32,7 +35,9 @@ i_error("checkpassword: HOME contains TAB"); return 1; } - str_printfa(str, "userdb_home=%s\t", home); + str_printfa(str, "userdb_home="); + str_tabescape_write(str, home); + str_append_c(str, '\t'); } extra_env = getenv("EXTRA"); @@ -45,7 +50,10 @@ uid_found = TRUE; else if (strcmp(key, "userdb_gid") == 0) gid_found = TRUE; - str_printfa(str, "%s=%s\t", key, value); + str_tabescape_write(str, key); + str_append_c(str, '='); + str_tabescape_write(str, value); + str_append_c(str, '\t'); } } } @@ -54,10 +62,21 @@ if (!gid_found) str_printfa(str, "userdb_gid=%s\t", dec2str(getgid())); + i_assert(str_len(str) > 0); + if (write_full(4, str_data(str), str_len(str)) < 0) { i_error("checkpassword: write_full() failed: %m"); exit(111); } authorized = getenv("AUTHORIZED"); - return authorized != NULL && strcmp(authorized, "2") == 0 ? 2 : 0; + if (authorized == NULL) { + /* authentication */ + return 0; + } else if (strcmp(authorized, "2") == 0) { + /* successful passdb/userdb lookup */ + return 2; + } else { + i_error("checkpassword: Script doesn't support passdb/userdb lookup"); + return 111; + } }
--- a/src/auth/db-checkpassword.c Sat May 19 22:40:08 2012 +0300 +++ b/src/auth/db-checkpassword.c Sun May 20 03:25:04 2012 +0300 @@ -4,15 +4,57 @@ #if defined(PASSDB_CHECKPASSWORD) || defined(USERDB_CHECKPASSWORD) +#include "lib-signals.h" +#include "buffer.h" +#include "str.h" +#include "ioloop.h" +#include "hash.h" +#include "execv-const.h" +#include "env-util.h" +#include "safe-memset.h" +#include "strescape.h" +#include "child-wait.h" #include "var-expand.h" #include "db-checkpassword.h" +#include <stdlib.h> +#include <unistd.h> +#include <sys/wait.h> + +#define CHECKPASSWORD_MAX_REQUEST_LEN 512 + +struct chkpw_auth_request { + struct db_checkpassword *db; + struct auth_request *request; + char *auth_password; + + db_checkpassword_callback_t *callback; + void (*request_callback)(); + + pid_t pid; + int fd_out, fd_in; + struct io *io_out, *io_in; + + string_t *input_buf; + unsigned int output_pos, output_len; + + int exit_status; + unsigned int exited:1; +}; + +struct db_checkpassword { + char *checkpassword_path, *checkpassword_reply_path; + + struct hash_table *clients; + struct child_wait *child_wait; +}; + static void env_put_extra_fields(const char *extra_fields) { const char *const *tmp; const char *key, *p; - for (tmp = t_strsplit(extra_fields, "\t"); *tmp != NULL; tmp++) { + for (tmp = t_strsplit_tab(extra_fields); *tmp != NULL; tmp++) { key = t_str_ucase(t_strcut(*tmp, '=')); p = strchr(*tmp, '='); if (p == NULL) @@ -40,52 +82,144 @@ } } -void checkpassword_request_free(struct chkpw_auth_request *request) +static void checkpassword_request_free(struct chkpw_auth_request **_request) { + struct chkpw_auth_request *request = *_request; + + *_request = NULL; + + if (!request->exited) { + hash_table_remove(request->db->clients, + POINTER_CAST(request->pid)); + child_wait_remove_pid(request->db->child_wait, request->pid); + } checkpassword_request_close(request); - if (request->input_buf != NULL) - str_free(&request->input_buf); - if (request->password != NULL) { - safe_memset(request->password, 0, strlen(request->password)); - i_free(request->password); + if (request->auth_password != NULL) { + safe_memset(request->auth_password, 0, + strlen(request->auth_password)); + i_free(request->auth_password); } + auth_request_unref(&request->request); + str_free(&request->input_buf); i_free(request); } -enum checkpassword_sigchld_handler_result -checkpassword_sigchld_handler(const struct child_wait_status *child_wait_status, - struct chkpw_auth_request *request) +static void checkpassword_finish(struct chkpw_auth_request **_request, + enum db_checkpassword_status status) +{ + struct chkpw_auth_request *request = *_request; + const char *const *extra_fields; + + *_request = NULL; + + extra_fields = t_strsplit_tabescaped(str_c(request->input_buf)); + request->callback(request->request, status, extra_fields, + request->request_callback); + checkpassword_request_free(&request); +} + +static void checkpassword_internal_failure(struct chkpw_auth_request **request) { - int status = child_wait_status->status; - pid_t pid = child_wait_status->pid; + checkpassword_finish(request, DB_CHECKPASSWORD_STATUS_INTERNAL_FAILURE); +} - if (request == NULL) { - i_error("checkpassword: sighandler called for unknown child %s", - dec2str(pid)); - return SIGCHLD_RESULT_UNKNOWN_CHILD; - } +static void +checkpassword_request_finish_auth(struct chkpw_auth_request *request) +{ + switch (request->exit_status) { + /* vpopmail exit codes: */ + case 3: /* password fail / vpopmail user not found */ + case 12: /* null user name given */ + case 13: /* null password given */ + case 15: /* user has no password */ + case 20: /* invalid user/domain characters */ + case 21: /* system user not found */ + case 22: /* system user shadow entry not found */ + case 23: /* system password fail */ - if (WIFSIGNALED(status)) { - i_error("checkpassword: Child %s died with signal %d", - dec2str(pid), WTERMSIG(status)); - return SIGCHLD_RESULT_DEAD_CHILD; - } else if (WIFEXITED(status)) { - request->exited = TRUE; - request->exit_status = WEXITSTATUS(status); + /* standard checkpassword exit codes: */ + case 1: + /* (1 is additionally defined in vpopmail for + "pop/smtp/webmal/ imap/access denied") */ + auth_request_log_info(request->request, "checkpassword", + "Login failed (status=%d)", + request->exit_status); + checkpassword_finish(&request, DB_CHECKPASSWORD_STATUS_FAILURE); + break; + case 0: + if (request->input_buf->used == 0) { + auth_request_log_error(request->request, "checkpassword", + "Received no input"); + checkpassword_internal_failure(&request); + break; + } + checkpassword_finish(&request, DB_CHECKPASSWORD_STATUS_OK); + break; + case 2: + /* checkpassword is called with wrong parameters? unlikely */ + auth_request_log_error(request->request, "checkpassword", + "Child %s exited with status 2 (tried to use " + "userdb-only checkpassword program for passdb?)", + dec2str(request->pid)); + checkpassword_internal_failure(&request); + break; + case 111: + /* temporary problem, treat as internal error */ + default: + /* whatever error.. */ + auth_request_log_error(request->request, "checkpassword", + "Child %s exited with status %d", + dec2str(request->pid), request->exit_status); + checkpassword_internal_failure(&request); + break; + } +} - auth_request_log_debug(request->request, - "checkpassword", "exit_status=%d", - request->exit_status); - return SIGCHLD_RESULT_OK; - } else { - /* shouldn't happen */ - auth_request_log_debug(request->request, "checkpassword", - "Child exited with status=%d", status); - return SIGCHLD_RESULT_UNKNOWN_ERROR; +static void +checkpassword_request_finish_lookup(struct chkpw_auth_request *request) +{ + switch (request->exit_status) { + case 3: + /* User does not exist. */ + auth_request_log_info(request->request, "userdb-checkpassword", + "User unknown"); + checkpassword_finish(&request, DB_CHECKPASSWORD_STATUS_FAILURE); + break; + case 2: + /* This is intentionally not 0. checkpassword-reply exits with + 2 on success when AUTHORIZED is set. */ + if (request->input_buf->used == 0) { + auth_request_log_error(request->request, "checkpassword", + "Received no input"); + checkpassword_internal_failure(&request); + break; + } + checkpassword_finish(&request, DB_CHECKPASSWORD_STATUS_OK); + break; + default: + /* whatever error... */ + auth_request_log_error(request->request, "userdb-checkpassword", + "Child %s exited with status %d", + dec2str(request->pid), request->exit_status); + checkpassword_internal_failure(&request); + break; } } +static void +checkpassword_request_half_finish(struct chkpw_auth_request *request) +{ + /* the process must have exited, and the input fd must have closed */ + if (!request->exited || request->fd_in != -1) + return; + + if (request->auth_password != NULL) + checkpassword_request_finish_auth(request); + else + checkpassword_request_finish_lookup(request); +} + static void env_put_auth_vars(struct auth_request *request) { const struct var_expand_table *tab; @@ -101,7 +235,7 @@ } } -void checkpassword_setup_env(struct auth_request *request) +static void checkpassword_setup_env(struct auth_request *request) { /* Besides passing the standard username and password in a pipe, also pass some other possibly interesting information @@ -146,7 +280,7 @@ env_put_auth_vars(request); } -const char * +static const char * checkpassword_get_cmd(struct auth_request *request, const char *args, const char *checkpassword_reply_path) { @@ -158,33 +292,34 @@ return t_strconcat(str_c(str), " ", checkpassword_reply_path, NULL); } -void checkpassword_child_input(struct chkpw_auth_request *request) +static void checkpassword_child_input(struct chkpw_auth_request *request) { unsigned char buf[1024]; ssize_t ret; ret = read(request->fd_in, buf, sizeof(buf)); - if (ret <= 0) { - if (ret < 0) { - auth_request_log_error(request->request, - "checkpassword", "read() failed: %m"); - } + if (ret > 0) { + str_append_n(request->input_buf, buf, ret); + return; + } - auth_request_log_debug(request->request, "checkpassword", - "Received no input"); - checkpassword_request_close(request); - request->half_finish_callback(request); + if (ret < 0) { + auth_request_log_error(request->request, + "checkpassword", "read() failed: %m"); + checkpassword_internal_failure(&request); + } else if (strchr(str_c(request->input_buf), '\n') != NULL) { + auth_request_log_error(request->request, "checkpassword", + "LF characters in checkpassword reply"); + checkpassword_internal_failure(&request); } else { - if (request->input_buf == NULL) - request->input_buf = str_new(default_pool, 512); - str_append_n(request->input_buf, buf, ret); - auth_request_log_debug(request->request, "checkpassword", "Received input: %s", str_c(request->input_buf)); + checkpassword_request_close(request); + checkpassword_request_half_finish(request); } } -void checkpassword_child_output(struct chkpw_auth_request *request) +static void checkpassword_child_output(struct chkpw_auth_request *request) { /* Send: username \0 password \0 timestamp \0. Must be 512 bytes or less. The "timestamp" parameter is actually @@ -196,39 +331,41 @@ size_t size; ssize_t ret; - buf = buffer_create_dynamic(pool_datastack_create(), 512+1); + buf = buffer_create_dynamic(pool_datastack_create(), + CHECKPASSWORD_MAX_REQUEST_LEN); buffer_append(buf, auth_request->user, strlen(auth_request->user)+1); - if (request->password != NULL) - buffer_append(buf, request->password, strlen(request->password)+1); - else - buffer_append_c(buf, '\0'); + if (request->auth_password != NULL) { + buffer_append(buf, request->auth_password, + strlen(request->auth_password)+1); + } else { + buffer_append_c(buf, '\0'); + } buffer_append_c(buf, '\0'); data = buffer_get_data(buf, &size); - if (size > 512) { - auth_request_log_error(request->request, "checkpassword", - "output larger than 512 bytes: %"PRIuSIZE_T, size); - request->finish_callback(request, - request->internal_failure_code); - return; - } + i_assert(size == request->output_len); + /* already checked this */ + i_assert(size <= CHECKPASSWORD_MAX_REQUEST_LEN); - ret = write(request->fd_out, data + request->write_pos, - size - request->write_pos); + ret = write(request->fd_out, data + request->output_pos, + size - request->output_pos); if (ret <= 0) { if (ret < 0) { auth_request_log_error(request->request, "checkpassword", "write() failed: %m"); + } else { + auth_request_log_error(request->request, + "checkpassword", "write() returned 0"); } - request->finish_callback(request, - request->internal_failure_code); + checkpassword_internal_failure(&request); return; } - request->write_pos += ret; - if (request->write_pos < size) + request->output_pos += ret; + if (request->output_pos < size) return; + /* finished sending the data */ io_remove(&request->io_out); if (close(request->fd_out) < 0) @@ -236,4 +373,205 @@ request->fd_out = -1; } +static void ATTR_NORETURN +checkpassword_exec(struct db_checkpassword *db, struct auth_request *request, + int fd_in, int fd_out, bool authenticate) +{ + const char *cmd, *const *args; + + /* fd 3 is used to send the username+password for the script + fd 4 is used to communicate with checkpassword-reply */ + if (dup2(fd_out, 3) < 0 || dup2(fd_in, 4) < 0) { + auth_request_log_error(request, "checkpassword", + "dup2() failed: %m"); + exit(111); + } + + if (!authenticate) { + /* We want to retrieve passdb/userdb data and don't do + authorization, so we need to signalize the + checkpassword program that the password shall be + ignored by setting AUTHORIZED. This needs a + special checkpassword program which knows how to + handle this. */ + env_put("AUTHORIZED=1"); + if (request->credentials_scheme != NULL) { + /* passdb credentials lookup */ + env_put("CREDENTIALS_LOOKUP=1"); + env_put(t_strdup_printf("SCHEME=%s", + request->credentials_scheme)); + } + } + checkpassword_setup_env(request); + cmd = checkpassword_get_cmd(request, db->checkpassword_path, + db->checkpassword_reply_path); + auth_request_log_debug(request, "checkpassword", "execute: %s", cmd); + + /* very simple argument splitting. */ + args = t_strsplit(cmd, " "); + execv_const(args[0], args); +} + +static void sigchld_handler(const struct child_wait_status *status, + struct db_checkpassword *db) +{ + struct chkpw_auth_request *request = + hash_table_lookup(db->clients, POINTER_CAST(status->pid)); + + i_assert(request != NULL); + + hash_table_remove(db->clients, POINTER_CAST(status->pid)); + request->exited = TRUE; + + if (WIFSIGNALED(status->status)) { + auth_request_log_error(request->request, "checkpassword", + "Child %s died with signal %d", + dec2str(status->pid), WTERMSIG(status->status)); + checkpassword_internal_failure(&request); + } else if (WIFEXITED(status->status)) { + request->exit_status = WEXITSTATUS(status->status); + + auth_request_log_debug(request->request, + "checkpassword", "exit_status=%d", + request->exit_status); + checkpassword_request_half_finish(request); + } else { + /* shouldn't happen */ + auth_request_log_debug(request->request, "checkpassword", + "Child %s exited with status=%d", + dec2str(status->pid), status->status); + checkpassword_internal_failure(&request); + } +} + +void db_checkpassword_call(struct db_checkpassword *db, + struct auth_request *request, + const char *auth_password, + db_checkpassword_callback_t *callback, + void (*request_callback)()) +{ + struct chkpw_auth_request *chkpw_auth_request; + unsigned int output_len; + int fd_in[2], fd_out[2]; + pid_t pid; + + /* <username> \0 <password> \0 timestamp \0 */ + output_len = strlen(request->user) + 3; + if (auth_password != NULL) + output_len += strlen(auth_password); + if (output_len > CHECKPASSWORD_MAX_REQUEST_LEN) { + auth_request_log_info(request, "checkpassword", + "Username+password combination too long (%u bytes)", + output_len); + callback(request, DB_CHECKPASSWORD_STATUS_FAILURE, + NULL, request_callback); + return; + } + + fd_in[0] = -1; + if (pipe(fd_in) < 0 || pipe(fd_out) < 0) { + auth_request_log_error(request, "checkpassword", + "pipe() failed: %m"); + if (fd_in[0] != -1) { + (void)close(fd_in[0]); + (void)close(fd_in[1]); + } + callback(request, DB_CHECKPASSWORD_STATUS_INTERNAL_FAILURE, + NULL, request_callback); + return; + } + + pid = fork(); + if (pid == -1) { + auth_request_log_error(request, "checkpassword", + "fork() failed: %m"); + (void)close(fd_in[0]); + (void)close(fd_in[1]); + (void)close(fd_out[0]); + (void)close(fd_out[1]); + callback(request, DB_CHECKPASSWORD_STATUS_INTERNAL_FAILURE, + NULL, request_callback); + return; + } + + if (pid == 0) { + /* child */ + (void)close(fd_in[0]); + (void)close(fd_out[1]); + checkpassword_exec(db, request, fd_in[1], fd_out[0], + auth_password != NULL); + /* not reached */ + } + + if (close(fd_in[1]) < 0) { + auth_request_log_error(request, "checkpassword", + "close(fd_in[1]) failed: %m"); + } + if (close(fd_out[0]) < 0) { + auth_request_log_error(request, "checkpassword", + "close(fd_out[0]) failed: %m"); + } + + auth_request_ref(request); + chkpw_auth_request = i_new(struct chkpw_auth_request, 1); + chkpw_auth_request->db = db; + chkpw_auth_request->pid = pid; + chkpw_auth_request->fd_in = fd_in[0]; + chkpw_auth_request->fd_out = fd_out[1]; + chkpw_auth_request->auth_password = i_strdup(auth_password); + chkpw_auth_request->request = request; + chkpw_auth_request->output_len = output_len; + chkpw_auth_request->input_buf = str_new(default_pool, 256); + chkpw_auth_request->callback = callback; + chkpw_auth_request->request_callback = request_callback; + + chkpw_auth_request->io_in = + io_add(fd_in[0], IO_READ, checkpassword_child_input, + chkpw_auth_request); + chkpw_auth_request->io_out = + io_add(fd_out[1], IO_WRITE, checkpassword_child_output, + chkpw_auth_request); + + hash_table_insert(db->clients, POINTER_CAST(pid), chkpw_auth_request); + child_wait_add_pid(db->child_wait, pid); +} + +struct db_checkpassword * +db_checkpassword_init(const char *checkpassword_path, + const char *checkpassword_reply_path) +{ + struct db_checkpassword *db; + + db = i_new(struct db_checkpassword, 1); + db->checkpassword_path = i_strdup(checkpassword_path); + db->checkpassword_reply_path = i_strdup(checkpassword_reply_path); + db->clients = hash_table_create(default_pool, default_pool, 0, + NULL, NULL); + db->child_wait = + child_wait_new_with_pid((pid_t)-1, sigchld_handler, db); + return db; +} + +void db_checkpassword_deinit(struct db_checkpassword **_db) +{ + struct db_checkpassword *db = *_db; + struct hash_iterate_context *iter; + void *key, *value; + + *_db = NULL; + + iter = hash_table_iterate_init(db->clients); + while (hash_table_iterate(iter, &key, &value)) { + struct chkpw_auth_request *request = value; + checkpassword_internal_failure(&request); + } + hash_table_iterate_deinit(&iter); + + child_wait_free(&db->child_wait); + hash_table_destroy(&db->clients); + i_free(db->checkpassword_reply_path); + i_free(db->checkpassword_path); + i_free(db); +} + #endif
--- a/src/auth/db-checkpassword.h Sat May 19 22:40:08 2012 +0300 +++ b/src/auth/db-checkpassword.h Sun May 20 03:25:04 2012 +0300 @@ -2,57 +2,28 @@ #define CHECKPASSWORD_COMMON_H #include "auth-request.h" -#include "lib-signals.h" -#include "buffer.h" -#include "str.h" -#include "ioloop.h" -#include "hash.h" -#include "env-util.h" -#include "safe-memset.h" -#include "child-wait.h" -#include <stdlib.h> -#include <unistd.h> -#include <sys/wait.h> - - -struct chkpw_auth_request { - int fd_out, fd_in; - struct io *io_out, *io_in; - pid_t pid; - - string_t *input_buf; - char *password; - unsigned int write_pos; - - struct auth_request *request; - void *callback; - void (*half_finish_callback)(); - void (*finish_callback)(); - int internal_failure_code; - - int exit_status; - unsigned int exited:1; +enum db_checkpassword_status { + DB_CHECKPASSWORD_STATUS_INTERNAL_FAILURE = -1, + /* auth unsuccessful / user not found */ + DB_CHECKPASSWORD_STATUS_FAILURE = 0, + DB_CHECKPASSWORD_STATUS_OK = 1 }; -enum checkpassword_sigchld_handler_result { - SIGCHLD_RESULT_UNKNOWN_CHILD = -1, - SIGCHLD_RESULT_DEAD_CHILD = -2, - SIGCHLD_RESULT_UNKNOWN_ERROR = -3, - SIGCHLD_RESULT_OK = 1, -}; - +typedef void db_checkpassword_callback_t(struct auth_request *request, + enum db_checkpassword_status status, + const char *const *extra_fields, + void (*request_callback)()); -void checkpassword_request_free(struct chkpw_auth_request *request); -enum checkpassword_sigchld_handler_result -checkpassword_sigchld_handler(const struct child_wait_status *child_wait_status, - struct chkpw_auth_request *request); -void checkpassword_setup_env(struct auth_request *request); -const char * -checkpassword_get_cmd(struct auth_request *request, const char *args, +struct db_checkpassword * +db_checkpassword_init(const char *checkpassword_path, const char *checkpassword_reply_path); +void db_checkpassword_deinit(struct db_checkpassword **db); -void checkpassword_child_input(struct chkpw_auth_request *request); -void checkpassword_child_output(struct chkpw_auth_request *request); +void db_checkpassword_call(struct db_checkpassword *db, + struct auth_request *request, + const char *auth_password, + db_checkpassword_callback_t *callback, + void (*request_callback)()); #endif
--- a/src/auth/db-ldap.c Sat May 19 22:40:08 2012 +0300 +++ b/src/auth/db-ldap.c Sun May 20 03:25:04 2012 +0300 @@ -1032,9 +1032,12 @@ } templ = strchr(name, '='); - if (templ == NULL) - templ = ""; - else { + if (templ == NULL) { + if (*ldap_attr == '\0') { + /* =foo static value */ + templ = ""; + } + } else { *templ++ = '\0'; str_truncate(tmp_str, 0); var_expand_with_funcs(tmp_str, templ, NULL, @@ -1245,10 +1248,15 @@ values = ctx->val_1_arr; } - if (*field->value == '\0') { + if (field->value == NULL) { /* use the LDAP attribute's value */ } else { /* template */ + if (values[0] == NULL && *field->ldap_attr_name != '\0') { + /* ldapAttr=key=template%$, but ldapAttr doesn't + exist. */ + return values; + } if (values[0] != NULL && values[1] != NULL) { auth_request_log_warning(ctx->auth_request, "ldap", "Multiple values found for '%s', " @@ -1461,33 +1469,6 @@ pool_unref(&conn->pool); } -void db_ldap_check_userdb_warning(struct ldap_connection *conn) -{ - const struct ldap_settings *def = &default_ldap_settings; - const char *set_name; - - if (worker || conn->userdb_used || conn->set.userdb_warning_disable) - return; - - if (strcmp(conn->set.user_attrs, def->user_attrs) != 0) - set_name = "user_attrs"; - else if (strcmp(conn->set.user_filter, def->user_filter) != 0) - set_name = "user_filter"; - else if (strcmp(conn->set.iterate_attrs, def->iterate_attrs) != 0) - set_name = "iterate_attrs"; - else if (strcmp(conn->set.iterate_filter, def->iterate_filter) != 0) - set_name = "iterate_filter"; - else - set_name = NULL; - - if (set_name != NULL) { - i_warning("ldap: Ignoring changed %s in %s, " - "because userdb ldap not used. " - "(If this is intentional, set userdb_warning_disable=yes)", - set_name, conn->config_path); - } -} - #ifndef BUILTIN_LDAP /* Building a plugin */ extern struct passdb_module_interface passdb_ldap_plugin;
--- a/src/auth/db-ldap.h Sat May 19 22:40:08 2012 +0300 +++ b/src/auth/db-ldap.h Sun May 20 03:25:04 2012 +0300 @@ -63,7 +63,7 @@ const char *iterate_filter; const char *default_pass_scheme; - bool userdb_warning_disable; + bool userdb_warning_disable; /* deprecated for now at least */ /* ... */ int ldap_deref, ldap_scope; @@ -122,7 +122,7 @@ struct ldap_field { /* Dovecot field name. */ const char *name; - /* Field value template with %vars. "" = same as LDAP value. */ + /* Field value template with %vars. NULL = same as LDAP value. */ const char *value; /* LDAP attribute name, or "" if this is a static field. */ const char *ldap_attr_name; @@ -172,7 +172,6 @@ struct ldap_connection *db_ldap_init(const char *config_path, bool userdb); void db_ldap_unref(struct ldap_connection **conn); -void db_ldap_check_userdb_warning(struct ldap_connection *conn); int db_ldap_connect(struct ldap_connection *conn); void db_ldap_enable_input(struct ldap_connection *conn, bool enable);
--- a/src/auth/passdb-blocking.c Sat May 19 22:40:08 2012 +0300 +++ b/src/auth/passdb-blocking.c Sun May 20 03:25:04 2012 +0300 @@ -30,7 +30,7 @@ enum passdb_result ret; const char *const *args; - args = t_strsplit(reply, "\t"); + args = t_strsplit_tab(reply); if (strcmp(*args, "OK") == 0 && args[1] != NULL && args[2] != NULL) { /* OK \t user \t password [\t extra] */
--- a/src/auth/passdb-cache.c Sat May 19 22:40:08 2012 +0300 +++ b/src/auth/passdb-cache.c Sun May 20 03:25:04 2012 +0300 @@ -53,7 +53,7 @@ return TRUE; } - list = t_strsplit(value, "\t"); + list = t_strsplit_tab(value); cached_pw = list[0]; if (*cached_pw == '\0') { @@ -117,7 +117,7 @@ return TRUE; } - list = t_strsplit(value, "\t"); + list = t_strsplit_tab(value); auth_request_set_fields(request, list + 1, NULL); *result_r = PASSDB_RESULT_OK;
--- a/src/auth/passdb-checkpassword.c Sat May 19 22:40:08 2012 +0300 +++ b/src/auth/passdb-checkpassword.c Sun May 20 03:25:04 2012 +0300 @@ -1,154 +1,58 @@ /* Copyright (c) 2004-2012 Dovecot authors, see the included COPYING file */ #include "auth-common.h" -#include "execv-const.h" #include "passdb.h" #ifdef PASSDB_CHECKPASSWORD +#include "password-scheme.h" #include "db-checkpassword.h" struct checkpassword_passdb_module { struct passdb_module module; - - const char *checkpassword_path, *checkpassword_reply_path; - struct hash_table *clients; + struct db_checkpassword *db; }; -static struct child_wait *checkpassword_passdb_children = NULL; - -static void checkpassword_request_finish(struct chkpw_auth_request *request, - enum passdb_result result) -{ - struct passdb_module *_module = request->request->passdb->passdb; - struct checkpassword_passdb_module *module = - (struct checkpassword_passdb_module *)_module; - verify_plain_callback_t *callback = - (verify_plain_callback_t *)request->callback; - - hash_table_remove(module->clients, POINTER_CAST(request->pid)); - - if (result == PASSDB_RESULT_OK) { - if (strchr(str_c(request->input_buf), '\n') != NULL) { - auth_request_log_error(request->request, - "checkpassword", - "LF characters in checkpassword reply"); - result = PASSDB_RESULT_INTERNAL_FAILURE; - } else { - auth_request_log_debug(request->request, - "checkpassword", "input: %s", - str_c(request->input_buf)); - auth_request_set_fields(request->request, - t_strsplit(str_c(request->input_buf), "\t"), - NULL); - } - } - - callback(result, request->request); - - auth_request_unref(&request->request); - checkpassword_request_free(request); -} - static void -checkpassword_request_half_finish(struct chkpw_auth_request *request) +auth_checkpassword_callback(struct auth_request *request, + enum db_checkpassword_status status, + const char *const *extra_fields, + void (*request_callback)()) { - if (!request->exited || request->fd_in != -1) - return; - - switch (request->exit_status) { - /* vpopmail exit codes: */ - case 3: /* password fail / vpopmail user not found */ - case 12: /* null user name given */ - case 13: /* null password given */ - case 15: /* user has no password */ - case 20: /* invalid user/domain characters */ - case 21: /* system user not found */ - case 22: /* system user shadow entry not found */ - case 23: /* system password fail */ + verify_plain_callback_t *callback = request_callback; + const char *scheme, *crypted_pass = NULL; + unsigned int i; - /* standard checkpassword exit codes: */ - case 1: - /* (1 is additionally defined in vpopmail for - "pop/smtp/webmal/ imap/access denied") */ - auth_request_log_info(request->request, "checkpassword", - "Login failed (status=%d)", - request->exit_status); - checkpassword_request_finish(request, - PASSDB_RESULT_PASSWORD_MISMATCH); - break; - case 0: - if (request->input_buf != NULL) { - checkpassword_request_finish(request, PASSDB_RESULT_OK); - break; - } - /* missing input - fall through */ - case 2: - /* checkpassword is called with wrong - parameters? unlikely */ - auth_request_log_error(request->request, "checkpassword", - "Child %s exited with status 2 (tried to use " - "userdb-only checkpassword program for passdb?)", - dec2str(request->pid)); - checkpassword_request_finish(request, - PASSDB_RESULT_INTERNAL_FAILURE); - break; - case 111: - /* temporary problem, treat as internal error */ - default: - /* whatever error.. */ - auth_request_log_error(request->request, "checkpassword", - "Child %s exited with status %d", - dec2str(request->pid), request->exit_status); - checkpassword_request_finish(request, - PASSDB_RESULT_INTERNAL_FAILURE); + switch (status) { + case DB_CHECKPASSWORD_STATUS_INTERNAL_FAILURE: + callback(PASSDB_RESULT_INTERNAL_FAILURE, request); + return; + case DB_CHECKPASSWORD_STATUS_FAILURE: + callback(PASSDB_RESULT_PASSWORD_MISMATCH, request); + return; + case DB_CHECKPASSWORD_STATUS_OK: break; } -} - -static void sigchld_handler(const struct child_wait_status *status, - struct checkpassword_passdb_module *module) -{ - struct chkpw_auth_request *request = - hash_table_lookup(module->clients, POINTER_CAST(status->pid)); - - switch (checkpassword_sigchld_handler(status, request)) { - case SIGCHLD_RESULT_UNKNOWN_CHILD: - case SIGCHLD_RESULT_DEAD_CHILD: - break; - case SIGCHLD_RESULT_UNKNOWN_ERROR: - checkpassword_request_finish(request, - PASSDB_RESULT_INTERNAL_FAILURE); - break; - case SIGCHLD_RESULT_OK: - checkpassword_request_half_finish(request); - request = NULL; - break; + for (i = 0; extra_fields[i] != NULL; i++) { + if (strncmp(extra_fields[i], "password=", 9) == 0) + crypted_pass = extra_fields[i]+9; + else if (extra_fields[i][0] != '\0') { + auth_request_set_field_keyvalue(request, + extra_fields[i], NULL); + } } -} - -static void ATTR_NORETURN -checkpassword_verify_plain_child(struct auth_request *request, - struct checkpassword_passdb_module *module, - int fd_in, int fd_out) -{ - const char *cmd, *const *args; - - if (dup2(fd_out, 3) < 0 || dup2(fd_in, 4) < 0) { - auth_request_log_error(request, "checkpassword", - "dup2() failed: %m"); - } else { - checkpassword_setup_env(request); - cmd = checkpassword_get_cmd(request, module->checkpassword_path, - module->checkpassword_reply_path); - auth_request_log_debug(request, "checkpassword", - "execute: %s", cmd); - - /* very simple argument splitting. */ - args = t_strsplit(cmd, " "); - execv_const(args[0], args); + if (crypted_pass != NULL) { + /* for cache */ + scheme = password_get_scheme(&crypted_pass); + if (scheme != NULL) { + auth_request_set_field(request, "password", + crypted_pass, scheme); + } else { + auth_request_log_error(request, "checkpassword", + "password field returned without {scheme} prefix"); + } } - exit(2); + callback(PASSDB_RESULT_OK, request); } static void @@ -158,97 +62,70 @@ struct passdb_module *_module = request->passdb->passdb; struct checkpassword_passdb_module *module = (struct checkpassword_passdb_module *)_module; - struct chkpw_auth_request *chkpw_auth_request; - int fd_in[2], fd_out[2]; - pid_t pid; - fd_in[0] = -1; - if (pipe(fd_in) < 0 || pipe(fd_out) < 0) { - auth_request_log_error(request, "checkpassword", - "pipe() failed: %m"); - callback(PASSDB_RESULT_INTERNAL_FAILURE, request); - if (fd_in[0] != -1) { - (void)close(fd_in[0]); - (void)close(fd_in[1]); - } - return; - } + db_checkpassword_call(module->db, request, password, + auth_checkpassword_callback, callback); +} - pid = fork(); - if (pid == -1) { - auth_request_log_error(request, "checkpassword", - "fork() failed: %m"); - callback(PASSDB_RESULT_INTERNAL_FAILURE, request); - (void)close(fd_in[0]); - (void)close(fd_in[1]); - (void)close(fd_out[0]); - (void)close(fd_out[1]); - return; - } - - if (pid == 0) { - (void)close(fd_in[0]); - (void)close(fd_out[1]); - checkpassword_verify_plain_child(request, module, - fd_in[1], fd_out[0]); - /* not reached */ - } +static void +credentials_checkpassword_callback(struct auth_request *request, + enum db_checkpassword_status status, + const char *const *extra_fields, + void (*request_callback)()) +{ + lookup_credentials_callback_t *callback = request_callback; + const char *scheme, *crypted_pass = NULL; + unsigned int i; - if (close(fd_in[1]) < 0) { - auth_request_log_error(request, "checkpassword", - "close(fd_in[1]) failed: %m"); + switch (status) { + case DB_CHECKPASSWORD_STATUS_INTERNAL_FAILURE: + callback(PASSDB_RESULT_INTERNAL_FAILURE, NULL, 0, request); + return; + case DB_CHECKPASSWORD_STATUS_FAILURE: + callback(PASSDB_RESULT_USER_UNKNOWN, NULL, 0, request); + return; + case DB_CHECKPASSWORD_STATUS_OK: + break; } - if (close(fd_out[0]) < 0) { - auth_request_log_error(request, "checkpassword", - "close(fd_out[0]) failed: %m"); + for (i = 0; extra_fields[i] != NULL; i++) { + if (strncmp(extra_fields[i], "password=", 9) == 0) + crypted_pass = extra_fields[i]+9; + else if (extra_fields[i][0] != '\0') { + auth_request_set_field_keyvalue(request, + extra_fields[i], NULL); + } } + scheme = password_get_scheme(&crypted_pass); + if (scheme == NULL) + scheme = request->credentials_scheme; - auth_request_ref(request); - chkpw_auth_request = i_new(struct chkpw_auth_request, 1); - chkpw_auth_request->fd_in = fd_in[0]; - chkpw_auth_request->fd_out = fd_out[1]; - chkpw_auth_request->pid = pid; - chkpw_auth_request->password = i_strdup(password); - chkpw_auth_request->request = request; - chkpw_auth_request->callback = callback; - chkpw_auth_request->half_finish_callback = - checkpassword_request_half_finish; - chkpw_auth_request->finish_callback = - checkpassword_request_finish; - chkpw_auth_request->internal_failure_code = - PASSDB_RESULT_INTERNAL_FAILURE; + passdb_handle_credentials(PASSDB_RESULT_OK, crypted_pass, scheme, + callback, request); +} - chkpw_auth_request->io_in = - io_add(fd_in[0], IO_READ, checkpassword_child_input, - chkpw_auth_request); - chkpw_auth_request->io_out = - io_add(fd_out[1], IO_WRITE, checkpassword_child_output, - chkpw_auth_request); +static void +checkpassword_lookup_credentials(struct auth_request *request, + lookup_credentials_callback_t *callback) +{ + struct passdb_module *_module = request->passdb->passdb; + struct checkpassword_passdb_module *module = + (struct checkpassword_passdb_module *)_module; - hash_table_insert(module->clients, POINTER_CAST(pid), - chkpw_auth_request); - - if (checkpassword_passdb_children != NULL) - child_wait_add_pid(checkpassword_passdb_children, pid); - else { - checkpassword_passdb_children = - child_wait_new_with_pid(pid, sigchld_handler, module); - } + db_checkpassword_call(module->db, request, NULL, + credentials_checkpassword_callback, callback); } static struct passdb_module * checkpassword_preinit(pool_t pool, const char *args) { struct checkpassword_passdb_module *module; + const char *checkpassword_path = args; + const char *checkpassword_reply_path = + PKG_LIBEXECDIR"/checkpassword-reply"; module = p_new(pool, struct checkpassword_passdb_module, 1); - module->checkpassword_path = p_strdup(pool, args); - module->checkpassword_reply_path = - PKG_LIBEXECDIR"/checkpassword-reply"; - - module->clients = - hash_table_create(default_pool, default_pool, 0, NULL, NULL); - + module->db = db_checkpassword_init(checkpassword_path, + checkpassword_reply_path); return &module->module; } @@ -256,19 +133,8 @@ { struct checkpassword_passdb_module *module = (struct checkpassword_passdb_module *)_module; - struct hash_iterate_context *iter; - void *key, *value; - iter = hash_table_iterate_init(module->clients); - while (hash_table_iterate(iter, &key, &value)) { - checkpassword_request_finish(value, - PASSDB_RESULT_INTERNAL_FAILURE); - } - hash_table_iterate_deinit(&iter); - hash_table_destroy(&module->clients); - - if (checkpassword_passdb_children != NULL) - child_wait_free(&checkpassword_passdb_children); + db_checkpassword_deinit(&module->db); } struct passdb_module_interface passdb_checkpassword = { @@ -279,7 +145,7 @@ checkpassword_deinit, checkpassword_verify_plain, - NULL, + checkpassword_lookup_credentials, NULL }; #else
--- a/src/auth/passdb-ldap.c Sat May 19 22:40:08 2012 +0300 +++ b/src/auth/passdb-ldap.c Sun May 20 03:25:04 2012 +0300 @@ -432,7 +432,6 @@ /* Credential lookups can't be done with authentication binds */ _module->iface.lookup_credentials = NULL; } - db_ldap_check_userdb_warning(module->conn); } static void passdb_ldap_deinit(struct passdb_module *_module)
--- a/src/auth/passdb-static.c Sat May 19 22:40:08 2012 +0300 +++ b/src/auth/passdb-static.c Sun May 20 03:25:04 2012 +0300 @@ -25,7 +25,7 @@ passdb_template_export(module->tmpl, request); if (module->static_password_tmpl == NULL) - *password_r = NULL; + *password_r = ""; else { table = auth_request_get_var_expand_table(request, NULL); var_expand(str, module->static_password_tmpl, table);
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/auth/test-auth-cache.c Sun May 20 03:25:04 2012 +0300 @@ -0,0 +1,57 @@ +/* Copyright (c) 2012 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "auth-request.h" +#include "auth-cache.h" +#include "test-common.h" + +const struct var_expand_table auth_request_var_expand_static_tab[] = { + { 'a', NULL, NULL }, + { '\0', NULL, "longb" }, + { 'c', NULL, "longc" }, + { '\0', NULL, NULL } +}; + +const struct var_expand_table * +auth_request_get_var_expand_table(const struct auth_request *auth_request ATTR_UNUSED, + auth_request_escape_func_t *escape_func ATTR_UNUSED) +{ + return auth_request_var_expand_static_tab; +} + +static void test_auth_cache_parse_key(void) +{ + struct { + const char *in, *out; + } tests[] = { + { "foo%5.5Mabar", "%a" }, + { "foo%5.5M{longb}bar", "%{longb}" }, + { "foo%5.5Mcbar", "%c" }, + { "foo%5.5M{longc}bar", "%{longc}" }, + { "%a%b", "%a\t%b" }, + { "%a%{longb}%a", "%a\t%{longb}" }, + { "%{longc}%c", "%{longc}" }, + { "%c%a%{longc}%c", "%c\t%a" }, + { "%a%{env:foo}%{env:foo}%a", "%a\t%{env:foo}\t%{env:foo}" } + }; + const char *cache_key; + unsigned int i; + + test_begin("auth cache parse key"); + + for (i = 0; i < N_ELEMENTS(tests); i++) { + cache_key = auth_cache_parse_key(pool_datastack_create(), + tests[i].in); + test_assert(strcmp(cache_key, tests[i].out) == 0); + } + test_end(); +} + +int main(void) +{ + static void (*test_functions[])(void) = { + test_auth_cache_parse_key, + NULL + }; + return test_run(test_functions); +}
--- a/src/auth/userdb-checkpassword.c Sat May 19 22:40:08 2012 +0300 +++ b/src/auth/userdb-checkpassword.c Sun May 20 03:25:04 2012 +0300 @@ -1,7 +1,6 @@ /* Copyright (c) 2004-2012 Dovecot authors, see the included COPYING file */ #include "auth-common.h" -#include "execv-const.h" #include "userdb.h" #ifdef USERDB_CHECKPASSWORD @@ -10,225 +9,59 @@ struct checkpassword_userdb_module { struct userdb_module module; - - const char *checkpassword_path, *checkpassword_reply_path; - struct hash_table *clients; + struct db_checkpassword *db; }; -static struct child_wait *checkpassword_userdb_children = NULL; - -static void checkpassword_request_finish(struct chkpw_auth_request *request, - enum userdb_result result) -{ - struct userdb_module *_module = request->request->userdb->userdb; - struct checkpassword_userdb_module *module = - (struct checkpassword_userdb_module *)_module; - userdb_callback_t *callback = - (userdb_callback_t *)request->callback; - - hash_table_remove(module->clients, POINTER_CAST(request->pid)); - - if (result == USERDB_RESULT_OK) { - if (strchr(str_c(request->input_buf), '\n') != NULL) { - auth_request_log_error(request->request, - "userdb-checkpassword", - "LF characters in checkpassword reply"); - result = USERDB_RESULT_INTERNAL_FAILURE; - } else { - auth_request_init_userdb_reply(request->request); - auth_request_set_fields(request->request, - t_strsplit(str_c(request->input_buf), "\t"), - NULL); - } - } - - callback(result, request->request); - - auth_request_unref(&request->request); - checkpassword_request_free(request); -} - static void -checkpassword_request_half_finish(struct chkpw_auth_request *request) +userdb_checkpassword_callback(struct auth_request *request, + enum db_checkpassword_status status, + const char *const *extra_fields, + void (*request_callback)()) { - if (!request->exited || request->fd_in != -1) - return; + userdb_callback_t *callback = request_callback; + unsigned int i; - switch (request->exit_status) { - case 3: - /* User does not exist. */ - auth_request_log_info(request->request, "userdb-checkpassword", - "User unknown"); - checkpassword_request_finish(request, - USERDB_RESULT_USER_UNKNOWN); + switch (status) { + case DB_CHECKPASSWORD_STATUS_INTERNAL_FAILURE: + callback(USERDB_RESULT_INTERNAL_FAILURE, request); break; - case 2: - /* This is intentionally not 0. checkpassword-reply exits with - 2 on success when AUTHORIZED is set. */ - if (request->input_buf != NULL) { - checkpassword_request_finish(request, USERDB_RESULT_OK); - break; + case DB_CHECKPASSWORD_STATUS_FAILURE: + callback(USERDB_RESULT_USER_UNKNOWN, request); + break; + case DB_CHECKPASSWORD_STATUS_OK: + for (i = 0; extra_fields[i] != NULL; i++) { + if (strncmp(extra_fields[i], "userdb_", 7) != 0) + continue; + auth_request_set_field_keyvalue(request, + extra_fields[i], NULL); } - /* missing input - fall through */ - default: - /* whatever error... */ - auth_request_log_error(request->request, "userdb-checkpassword", - "Child %s exited with status %d", - dec2str(request->pid), request->exit_status); - checkpassword_request_finish(request, - USERDB_RESULT_INTERNAL_FAILURE); + callback(USERDB_RESULT_OK, request); break; } } -static void sigchld_handler(const struct child_wait_status *status, - struct checkpassword_userdb_module *module) -{ - struct chkpw_auth_request *request = - hash_table_lookup(module->clients, POINTER_CAST(status->pid)); - - switch (checkpassword_sigchld_handler(status, request)) { - case SIGCHLD_RESULT_UNKNOWN_CHILD: - case SIGCHLD_RESULT_DEAD_CHILD: - break; - case SIGCHLD_RESULT_UNKNOWN_ERROR: - checkpassword_request_finish(request, - USERDB_RESULT_INTERNAL_FAILURE); - break; - case SIGCHLD_RESULT_OK: - checkpassword_request_half_finish(request); - request = NULL; - break; - } -} - -static void ATTR_NORETURN -checkpassword_lookup_child(struct auth_request *request, - struct checkpassword_userdb_module *module, - int fd_in, int fd_out) -{ - const char *cmd, *const *args; - - if (dup2(fd_out, 3) < 0 || dup2(fd_in, 4) < 0) { - auth_request_log_error(request, "userdb-checkpassword", - "dup2() failed: %m"); - } else { - /* We want to retrieve user data and don't do - authorization, so we need to signalize the - checkpassword program that the password shall be - ignored by setting AUTHORIZED. This needs a - special checkpassword program which knows how to - handle this. */ - env_put("AUTHORIZED=1"); - checkpassword_setup_env(request); - cmd = checkpassword_get_cmd(request, module->checkpassword_path, - module->checkpassword_reply_path); - auth_request_log_debug(request, "userdb-checkpassword", - "execute: %s", cmd); - - /* very simple argument splitting. */ - args = t_strsplit(cmd, " "); - execv_const(args[0], args); - } - exit(2); -} - static void checkpassword_lookup(struct auth_request *request, userdb_callback_t *callback) { struct userdb_module *_module = request->userdb->userdb; struct checkpassword_userdb_module *module = (struct checkpassword_userdb_module *)_module; - struct chkpw_auth_request *chkpw_auth_request; - int fd_in[2], fd_out[2]; - pid_t pid; - fd_in[0] = -1; - if (pipe(fd_in) < 0 || pipe(fd_out) < 0) { - auth_request_log_error(request, "userdb-checkpassword", - "pipe() failed: %m"); - callback(USERDB_RESULT_INTERNAL_FAILURE, request); - if (fd_in[0] != -1) { - (void)close(fd_in[0]); - (void)close(fd_in[1]); - } - return; - } - - pid = fork(); - if (pid == -1) { - auth_request_log_error(request, "userdb-checkpassword", - "fork() failed: %m"); - callback(USERDB_RESULT_INTERNAL_FAILURE, request); - (void)close(fd_in[0]); - (void)close(fd_in[1]); - (void)close(fd_out[0]); - (void)close(fd_out[1]); - return; - } - - if (pid == 0) { - (void)close(fd_in[0]); - (void)close(fd_out[1]); - checkpassword_lookup_child(request, module, - fd_in[1], fd_out[0]); - /* not reached */ - } - - if (close(fd_in[1]) < 0) { - auth_request_log_error(request, "userdb-checkpassword", - "close(fd_in[1]) failed: %m"); - } - if (close(fd_out[0]) < 0) { - auth_request_log_error(request, "userdb-checkpassword", - "close(fd_out[0]) failed: %m"); - } - - auth_request_ref(request); - chkpw_auth_request = i_new(struct chkpw_auth_request, 1); - chkpw_auth_request->fd_in = fd_in[0]; - chkpw_auth_request->fd_out = fd_out[1]; - chkpw_auth_request->pid = pid; - chkpw_auth_request->request = request; - chkpw_auth_request->callback = callback; - chkpw_auth_request->half_finish_callback = - checkpassword_request_half_finish; - chkpw_auth_request->finish_callback = - checkpassword_request_finish; - chkpw_auth_request->internal_failure_code = - USERDB_RESULT_INTERNAL_FAILURE; - - chkpw_auth_request->io_in = - io_add(fd_in[0], IO_READ, checkpassword_child_input, - chkpw_auth_request); - chkpw_auth_request->io_out = - io_add(fd_out[1], IO_WRITE, checkpassword_child_output, - chkpw_auth_request); - - hash_table_insert(module->clients, POINTER_CAST(pid), - chkpw_auth_request); - - if (checkpassword_userdb_children != NULL) - child_wait_add_pid(checkpassword_userdb_children, pid); - else { - checkpassword_userdb_children = - child_wait_new_with_pid(pid, sigchld_handler, module); - } + db_checkpassword_call(module->db, request, NULL, + userdb_checkpassword_callback, callback); } static struct userdb_module * checkpassword_preinit(pool_t pool, const char *args) { struct checkpassword_userdb_module *module; + const char *checkpassword_path = args; + const char *checkpassword_reply_path = + PKG_LIBEXECDIR"/checkpassword-reply"; module = p_new(pool, struct checkpassword_userdb_module, 1); - module->checkpassword_path = p_strdup(pool, args); - module->checkpassword_reply_path = - PKG_LIBEXECDIR"/checkpassword-reply"; - - module->clients = - hash_table_create(default_pool, default_pool, 0, NULL, NULL); - + module->db = db_checkpassword_init(checkpassword_path, + checkpassword_reply_path); return &module->module; } @@ -236,19 +69,8 @@ { struct checkpassword_userdb_module *module = (struct checkpassword_userdb_module *)_module; - struct hash_iterate_context *iter; - void *key, *value; - iter = hash_table_iterate_init(module->clients); - while (hash_table_iterate(iter, &key, &value)) { - checkpassword_request_finish(value, - USERDB_RESULT_INTERNAL_FAILURE); - } - hash_table_iterate_deinit(&iter); - hash_table_destroy(&module->clients); - - if (checkpassword_userdb_children != NULL) - child_wait_free(&checkpassword_userdb_children); + db_checkpassword_deinit(&module->db); } struct userdb_module_interface userdb_checkpassword = {
--- a/src/auth/userdb-passwd.c Sat May 19 22:40:08 2012 +0300 +++ b/src/auth/userdb-passwd.c Sun May 20 03:25:04 2012 +0300 @@ -137,6 +137,25 @@ return &ctx->ctx; } +static bool +passwd_iterate_want_pw(struct passwd *pw, const struct auth_settings *set) +{ + /* skip entries not in valid UID range. + they're users for daemons and such. */ + if (pw->pw_uid < (uid_t)set->first_valid_uid) + return FALSE; + if (pw->pw_uid > (uid_t)set->last_valid_uid && set->last_valid_uid != 0) + return FALSE; + + /* skip entries that don't have a valid shell. + they're again probably not real users. */ + if (strcmp(pw->pw_shell, "/bin/false") == 0 || + strcmp(pw->pw_shell, "/sbin/nologin") == 0 || + strcmp(pw->pw_shell, "/usr/sbin/nologin") == 0) + return FALSE; + return TRUE; +} + static void passwd_iterate_next(struct userdb_iterate_context *_ctx) { struct passwd_userdb_iterate_context *ctx = @@ -154,11 +173,7 @@ errno = 0; while ((pw = getpwent()) != NULL) { - /* skip entries not in valid UID range. - they're users for daemons and such. */ - if (pw->pw_uid >= (uid_t)set->first_valid_uid && - (set->last_valid_uid == 0 || - pw->pw_uid <= (uid_t)set->last_valid_uid)) { + if (passwd_iterate_want_pw(pw, set)) { _ctx->callback(pw->pw_name, _ctx->context); return; }
--- a/src/config/config-connection.c Sat May 19 22:40:08 2012 +0300 +++ b/src/config/config-connection.c Sun May 20 03:25:04 2012 +0300 @@ -42,7 +42,7 @@ if (line == NULL) return NULL; - return t_strsplit(line, "\t"); + return t_strsplit_tab(line); } static void
--- a/src/config/config-filter.c Sat May 19 22:40:08 2012 +0300 +++ b/src/config/config-filter.c Sun May 20 03:25:04 2012 +0300 @@ -287,14 +287,17 @@ const struct config_filter_parser *src, pool_t pool, const char **error_r) { + const char *conflict_key; unsigned int i; for (i = 0; dest[i].root != NULL; i++) { if (settings_parser_apply_changes(dest[i].parser, src->parsers[i].parser, pool, - error_r) < 0) { + error_r == NULL ? NULL : + &conflict_key) < 0) { + i_assert(error_r != NULL); *error_r = t_strdup_printf("Conflict in setting %s " - "found from filter at %s", *error_r, + "found from filter at %s", conflict_key, src->file_and_line); return -1; } @@ -314,6 +317,11 @@ const char *error = NULL, **error_p; unsigned int i, count; + /* get the matching filters. the most specific ones are handled first, + so that if more generic filters try to override settings we'll fail + with an error. Merging SET_STRLIST types requires + settings_parser_apply_changes() to work a bit unintuitively by + letting the destination settings override the source settings. */ src = config_filter_find_all(ctx, module, filter, output_r); /* all of them should have the same number of parsers.
--- a/src/config/config-request.c Sat May 19 22:40:08 2012 +0300 +++ b/src/config/config-request.c Sun May 20 03:25:04 2012 +0300 @@ -350,7 +350,7 @@ i_assert(module != NULL); - pool = pool_alloconly_create("config export", 1024*64); + pool = pool_alloconly_create(MEMPOOL_GROWING"config export", 1024*64); ctx = p_new(pool, struct config_export_context, 1); ctx->pool = pool;
--- a/src/config/doveconf.c Sat May 19 22:40:08 2012 +0300 +++ b/src/config/doveconf.c Sun May 20 03:25:04 2012 +0300 @@ -434,7 +434,7 @@ ret = config_dump_human_output(ctx, output, 0, setting_name_filter); config_dump_human_deinit(ctx); - if (ret == 0 && setting_name_filter == NULL) + if (setting_name_filter == NULL) ret = config_dump_human_sections(output, filter, module); o_stream_uncork(output);
--- a/src/dict/dict-commands.c Sat May 19 22:40:08 2012 +0300 +++ b/src/dict/dict-commands.c Sun May 20 03:25:04 2012 +0300 @@ -90,7 +90,7 @@ return -1; } - args = t_strsplit(line, "\t"); + args = t_strsplit_tab(line); if (str_array_length(args) < 2 || str_to_uint(args[0], &flags) < 0) { i_error("dict client: ITERATE: broken input"); @@ -272,7 +272,7 @@ const char *const *args; /* <id> <key> <value> */ - args = t_strsplit(line, "\t"); + args = t_strsplit_tab(line); if (str_array_length(args) != 3) { i_error("dict client: SET: broken input"); return -1; @@ -291,7 +291,7 @@ const char *const *args; /* <id> <key> */ - args = t_strsplit(line, "\t"); + args = t_strsplit_tab(line); if (str_array_length(args) != 2) { i_error("dict client: UNSET: broken input"); return -1; @@ -311,7 +311,7 @@ long long diff; /* <id> <key> <diff> */ - args = t_strsplit(line, "\t"); + args = t_strsplit_tab(line); if (str_array_length(args) != 3 || str_to_llong(args[2], &diff) < 0) { i_error("dict client: ATOMIC_INC: broken input");
--- a/src/director/Makefile.am Sat May 19 22:40:08 2012 +0300 +++ b/src/director/Makefile.am Sun May 20 03:25:04 2012 +0300 @@ -4,10 +4,12 @@ AM_CPPFLAGS = \ -I$(top_srcdir)/src/lib \ + -I$(top_srcdir)/src/lib-test \ -I$(top_srcdir)/src/lib-auth \ -I$(top_srcdir)/src/lib-imap \ -I$(top_srcdir)/src/lib-settings \ - -I$(top_srcdir)/src/lib-master + -I$(top_srcdir)/src/lib-master \ + -I$(top_srcdir)/src/lib-mail director_LDADD = $(LIBDOVECOT) director_DEPENDENCIES = $(LIBDOVECOT_DEPS) @@ -39,10 +41,27 @@ notify-connection.h \ user-directory.h -noinst_PROGRAMS = director-test +noinst_PROGRAMS = director-test $(test_programs) director_test_LDADD = $(LIBDOVECOT) director_test_DEPENDENCIES = $(LIBDOVECOT_DEPS) director_test_SOURCES = \ director-test.c + +test_programs = \ + test-user-directory + +test_libs = \ + ../lib-test/libtest.la \ + ../lib/liblib.la + +test_user_directory_SOURCES = test-user-directory.c +test_user_directory_LDADD = user-directory.o $(test_libs) +test_user_directory_DEPENDENCIES = user-directory.o $(test_libs) + +check: check-am check-test +check-test: all-am + for bin in $(test_programs); do \ + if ! $(RUN_TEST) ./$$bin; then exit 1; fi; \ + done
--- a/src/director/director-connection.c Sat May 19 22:40:08 2012 +0300 +++ b/src/director/director-connection.c Sun May 20 03:25:04 2012 +0300 @@ -1,5 +1,33 @@ /* Copyright (c) 2010-2012 Dovecot authors, see the included COPYING file */ +/* + Handshaking: + + Incoming director connections send: + + VERSION + ME + <wait for DONE from remote handshake> + DONE + <make this connection our "left" connection, potentially disconnecting + another one> + + Outgoing director connections send: + + VERSION + ME + [0..n] DIRECTOR + HOST-HAND-START + [0..n] HOST + HOST-HAND-END + [0..n] USER + <possibly other non-handshake commands between USERs> + DONE + <wait for DONE from remote> + <make this connection our "right" connection, potentially disconnecting + another one> +*/ + #include "lib.h" #include "ioloop.h" #include "array.h" @@ -7,7 +35,6 @@ #include "istream.h" #include "ostream.h" #include "str.h" -#include "llist.h" #include "master-service.h" #include "mail-host.h" #include "director.h" @@ -19,31 +46,48 @@ #include <stdlib.h> #include <unistd.h> -#define DIRECTOR_VERSION_NAME "director" -#define DIRECTOR_VERSION_MAJOR 1 -#define DIRECTOR_VERSION_MINOR 0 - #define MAX_INBUF_SIZE 1024 #define MAX_OUTBUF_SIZE (1024*1024*10) #define OUTBUF_FLUSH_THRESHOLD (1024*128) -/* Max idling time while connecting/handshaking before disconnecting */ -#define DIRECTOR_CONNECTION_INIT_TIMEOUT_MSECS (2*1000) -/* How long to wait for PONG after PING request */ -#define DIRECTOR_CONNECTION_PING_TIMEOUT_MSECS (2*1000) +/* Max idling time before "ME" command must have been received, + or we'll disconnect. */ +#define DIRECTOR_CONNECTION_ME_TIMEOUT_MSECS (10*1000) +/* Max time to wait for USERs in handshake to be sent. With a lot of users the + kernel may quickly eat up everything we send, while the receiver is busy + parsing the data. */ +#define DIRECTOR_CONNECTION_SEND_USERS_TIMEOUT_MSECS (30*1000) +/* Max idling time before "DONE" command must have been received, + or we'll disconnect. */ +#define DIRECTOR_CONNECTION_DONE_TIMEOUT_MSECS (30*1000) +/* How long to wait for PONG for an idling connection */ +#define DIRECTOR_CONNECTION_PING_IDLE_TIMEOUT_MSECS (10*1000) +/* Maximum time to wait for PONG reply */ +#define DIRECTOR_CONNECTION_PONG_TIMEOUT_MSECS (60*1000) /* How long to wait to send PING when connection is idle */ #define DIRECTOR_CONNECTION_PING_INTERVAL_MSECS (15*1000) /* How long to wait before sending PING while waiting for SYNC reply */ -#define DIRECTOR_CONNECTION_SYNC_TIMEOUT_MSECS 1000 +#define DIRECTOR_CONNECTION_PING_SYNC_INTERVAL_MSECS 1000 /* If outgoing director connection exists for less than this many seconds, mark the host as failed so we won't try to reconnect to it immediately */ -#define DIRECTOR_SUCCESS_MIN_CONNECT_SECS 10 +#define DIRECTOR_SUCCESS_MIN_CONNECT_SECS 40 +#define DIRECTOR_WAIT_DISCONNECT_SECS 10 + +#if DIRECTOR_CONNECTION_DONE_TIMEOUT_MSECS <= DIRECTOR_CONNECTION_PING_TIMEOUT_MSECS +# error DIRECTOR_CONNECTION_DONE_TIMEOUT_MSECS is too low +#endif + +#if DIRECTOR_CONNECTION_PONG_TIMEOUT_MSECS <= DIRECTOR_CONNECTION_PING_IDLE_TIMEOUT_MSECS +# error DIRECTOR_CONNECTION_PONG_TIMEOUT_MSECS is too low +#endif + +#define CMD_IS_USER_HANDHAKE(args) \ + (str_array_length(args) > 2) struct director_connection { - struct director_connection *prev, *next; - struct director *dir; char *name; time_t created; + unsigned int minor_version; /* for incoming connections the director host isn't known until ME-line is received */ @@ -53,10 +97,13 @@ struct io *io; struct istream *input; struct ostream *output; - struct timeout *to, *to_ping; + struct timeout *to_disconnect, *to_ping, *to_pong; struct user_directory_iter *user_iter; + /* set during command execution */ + const char *cur_cmd, *cur_line; + unsigned int in:1; unsigned int connected:1; unsigned int version_received:1; @@ -65,25 +112,224 @@ unsigned int ignore_host_events:1; unsigned int handshake_sending_hosts:1; unsigned int ping_waiting:1; - unsigned int sync_ping:1; + unsigned int synced:1; + unsigned int wrong_host:1; + unsigned int verifying_left:1; }; -static void director_connection_ping(struct director_connection *conn); static void director_connection_disconnected(struct director_connection **conn); +static void director_connection_reconnect(struct director_connection **conn); + +static void ATTR_FORMAT(2, 3) +director_cmd_error(struct director_connection *conn, const char *fmt, ...) +{ + va_list args; + + va_start(args, fmt); + i_error("director(%s): Command %s: %s (input: %s)", conn->name, + conn->cur_cmd, t_strdup_vprintf(fmt, args), conn->cur_line); + va_end(args); + + conn->host->last_protocol_failure = ioloop_time; +} + +static void +director_connection_init_timeout(struct director_connection *conn) +{ + unsigned int secs = ioloop_time - conn->created; + + if (!conn->connected) { + i_error("director(%s): Connect timed out (%u secs)", + conn->name, secs); + } else if (conn->io == NULL) { + i_error("director(%s): Sending handshake (%u secs)", + conn->name, secs); + } else if (!conn->me_received) { + i_error("director(%s): Handshaking ME timed out (%u secs)", + conn->name, secs); + } else { + i_error("director(%s): Handshaking DONE timed out (%u secs)", + conn->name, secs); + } + director_connection_disconnected(&conn); +} + +static void +director_connection_set_ping_timeout(struct director_connection *conn) +{ + unsigned int msecs; + + msecs = conn->synced || !conn->handshake_received ? + DIRECTOR_CONNECTION_PING_INTERVAL_MSECS : + DIRECTOR_CONNECTION_PING_SYNC_INTERVAL_MSECS; + + timeout_remove(&conn->to_ping); + conn->to_ping = timeout_add(msecs, director_connection_ping, conn); +} + +static void director_connection_wait_timeout(struct director_connection *conn) +{ + director_connection_deinit(&conn); +} + +static void director_connection_send_connect(struct director_connection *conn, + struct director_host *host) +{ + const char *connect_str; + + if (conn->to_disconnect != NULL) + return; + + connect_str = t_strdup_printf("CONNECT\t%s\t%u\n", + net_ip2addr(&host->ip), host->port); + director_connection_send(conn, connect_str); + (void)o_stream_flush(conn->output); + o_stream_uncork(conn->output); + + conn->to_disconnect = + timeout_add(DIRECTOR_WAIT_DISCONNECT_SECS*1000, + director_connection_wait_timeout, conn); +} + +static void director_connection_assigned(struct director_connection *conn) +{ + struct director *dir = conn->dir; + + if (dir->left != NULL && dir->right != NULL) { + /* we're connected to both directors. see if the ring is + finished by sending a SYNC. if we get it back, it's done. */ + dir->sync_seq++; + director_set_ring_unsynced(dir); + director_sync_send(dir, dir->self_host, dir->sync_seq, + DIRECTOR_VERSION_MINOR); + } + director_connection_set_ping_timeout(conn); +} + +static bool director_connection_assign_left(struct director_connection *conn) +{ + struct director *dir = conn->dir; + + i_assert(conn->in); + i_assert(dir->left != conn); + + /* make sure this is the correct incoming connection */ + if (conn->host->self) { + i_error("Connection from self, dropping"); + return FALSE; + } else if (dir->left == NULL) { + /* no conflicts yet */ + } else if (dir->left->host == conn->host) { + i_info("Dropping existing connection %s " + "in favor of its new connection %s", + dir->left->host->name, conn->host->name); + director_connection_deinit(&dir->left); + } else if (dir->left->verifying_left) { + /* we're waiting to verify if our current left is still + working. if we don't receive a PONG, the current left + gets disconnected and a new left gets assigned. if we do + receive a PONG, we'll wait until the current left + disconnects us and then reassign the new left. */ + return TRUE; + } else if (director_host_cmp_to_self(dir->left->host, conn->host, + dir->self_host) < 0) { + /* the old connection is the correct one. + refer the client there (FIXME: do we ever get here?) */ + i_warning("Director connection %s tried to connect to " + "us, should use %s instead", + conn->name, dir->left->host->name); + director_connection_send_connect(conn, dir->left->host); + return TRUE; + } else { + /* this new connection is the correct one, but wait until the + old connection gets disconnected before using this one. + that guarantees that the director inserting itself into + the ring has finished handshaking its left side, so the + switch will be fast. */ + return TRUE; + } + dir->left = conn; + i_free(conn->name); + conn->name = i_strdup_printf("%s/left", conn->host->name); + director_connection_assigned(conn); + return TRUE; +} + +static void director_assign_left(struct director *dir) +{ + struct director_connection *conn, *const *connp; + + array_foreach(&dir->connections, connp) { + conn = *connp; + + if (conn->in && conn->handshake_received && + conn->to_disconnect == NULL && conn != dir->left) { + /* either use this or disconnect it */ + if (!director_connection_assign_left(conn)) { + /* we don't want this */ + director_connection_deinit(&conn); + director_assign_left(dir); + break; + } + } + } +} + +static bool director_has_outgoing_connections(struct director *dir) +{ + struct director_connection *const *connp; + + array_foreach(&dir->connections, connp) { + if (!(*connp)->in && (*connp)->to_disconnect == NULL) + return TRUE; + } + return FALSE; +} + +static bool director_connection_assign_right(struct director_connection *conn) +{ + struct director *dir = conn->dir; + + i_assert(!conn->in); + + if (dir->right != NULL) { + /* see if we should disconnect or keep the existing + connection. */ + if (director_host_cmp_to_self(conn->host, dir->right->host, + dir->self_host) <= 0) { + /* the old connection is the correct one */ + i_warning("Aborting incorrect outgoing connection to %s " + "(already connected to correct one: %s)", + conn->host->name, dir->right->host->name); + conn->wrong_host = TRUE; + return FALSE; + } + i_info("Replacing right director connection %s with %s", + dir->right->host->name, conn->host->name); + director_connection_deinit(&dir->right); + } + dir->right = conn; + i_free(conn->name); + conn->name = i_strdup_printf("%s/right", conn->host->name); + director_connection_assigned(conn); + return TRUE; +} static bool director_args_parse_ip_port(struct director_connection *conn, const char *const *args, struct ip_addr *ip_r, unsigned int *port_r) { + if (args[0] == NULL || args[1] == NULL) { + director_cmd_error(conn, "Missing IP+port parameters"); + return FALSE; + } if (net_addr2ip(args[0], ip_r) < 0) { - i_error("director(%s): Command has invalid IP address: %s", - conn->name, args[0]); + director_cmd_error(conn, "Invalid IP address: %s", args[0]); return FALSE; } if (str_to_uint(args[1], port_r) < 0) { - i_error("director(%s): Command has invalid port: %s", - conn->name, args[1]); + director_cmd_error(conn, "Invalid port: %s", args[1]); return FALSE; } return TRUE; @@ -93,13 +339,17 @@ const char *const *args) { struct director *dir = conn->dir; - struct director_host *host; const char *connect_str; struct ip_addr ip; unsigned int port; + time_t next_comm_attempt; if (!director_args_parse_ip_port(conn, args, &ip, &port)) return FALSE; + if (conn->me_received) { + director_cmd_error(conn, "Duplicate ME"); + return FALSE; + } if (!conn->in && (!net_ip_compare(&conn->host->ip, &ip) || conn->host->port != port)) { @@ -109,103 +359,125 @@ net_ip2addr(&ip), port); return FALSE; } - host = director_host_get(dir, &ip, port); - /* the host is up now, make sure we can connect to it immediately - if needed */ - host->last_failed = 0; conn->me_received = TRUE; + timeout_remove(&conn->to_ping); + conn->to_ping = timeout_add(DIRECTOR_CONNECTION_DONE_TIMEOUT_MSECS, + director_connection_init_timeout, conn); + if (!conn->in) return TRUE; - i_free(conn->name); - conn->name = i_strdup_printf("%s/left", host->name); - conn->host = host; - /* make sure we don't keep old sequence values across restarts */ - host->last_seq = 0; + /* Incoming connection: + + a) we don't have an established ring yet. make sure we're connecting + to our right side (which might become our left side). + + b) it's our current "left" connection. the previous connection + is most likely dead. + + c) we have an existing ring. tell our current "left" to connect to + it with CONNECT command. - connect_str = t_strdup_printf("CONNECT\t%s\t%u\n", - net_ip2addr(&host->ip), host->port); - /* make sure this is the correct incoming connection */ - if (host->self) { - /* probably we're trying to find our own ip. it's no */ - i_error("Connection from self, dropping"); + d) the incoming connection doesn't belong to us at all, refer it + elsewhere with CONNECT. however, before disconnecting it verify + first that our left side is actually still functional. + */ + i_assert(conn->host == NULL); + conn->host = director_host_get(dir, &ip, port); + /* the host shouldn't be removed at this point, but if for some + reason it is we don't want to crash */ + conn->host->removed = FALSE; + director_host_ref(conn->host); + /* make sure we don't keep old sequence values across restarts */ + conn->host->last_seq = 0; + + next_comm_attempt = conn->host->last_protocol_failure + + DIRECTOR_PROTOCOL_FAILURE_RETRY_SECS; + if (next_comm_attempt > ioloop_time) { + /* the director recently sent invalid protocol data, + don't try retrying yet */ + i_error("director(%s): Remote sent invalid protocol data recently, " + "waiting %u secs before allowing further communication", + conn->name, (unsigned int)(next_comm_attempt-ioloop_time)); return FALSE; } else if (dir->left == NULL) { - /* no conflicts yet */ - } else if (dir->left->host == host) { - i_warning("Dropping existing connection %s " - "in favor of its new connection %s", - dir->left->host->name, host->name); + /* a) - just in case the left is also our right side reset + its failed state, so we can connect to it */ + conn->host->last_network_failure = 0; + if (!director_has_outgoing_connections(dir)) + director_connect(dir); + } else if (dir->left->host == conn->host) { + /* b) */ + i_assert(dir->left != conn); director_connection_deinit(&dir->left); + } else if (director_host_cmp_to_self(conn->host, dir->left->host, + dir->self_host) < 0) { + /* c) */ + connect_str = t_strdup_printf("CONNECT\t%s\t%u\n", + net_ip2addr(&conn->host->ip), + conn->host->port); + director_connection_send(dir->left, connect_str); } else { - if (director_host_cmp_to_self(dir->left->host, host, - dir->self_host) < 0) { - /* the old connection is the correct one. - refer the client there. */ - i_warning("Director connection %s tried to connect to " - "us, should use %s instead", - host->name, dir->left->host->name); - director_connection_send(conn, t_strdup_printf( - "CONNECT\t%s\t%u\n", - net_ip2addr(&dir->left->host->ip), - dir->left->host->port)); - /* also make sure that the connection is alive */ - director_connection_ping(dir->left); - return FALSE; - } - - /* this new connection is the correct one. disconnect the old - one, but before that tell it to connect to the new one. - that message might not reach it, so also send the same - message to right side. */ - i_warning("Replacing director connection %s with %s", - dir->left->host->name, host->name); - director_connection_send(dir->left, connect_str); - (void)o_stream_flush(dir->left->output); - director_connection_deinit(&dir->left); - } - dir->left = conn; - - /* tell the ring's right side to connect to this new director. */ - if (dir->right != NULL) { - if (dir->left->host != dir->right->host) - director_connection_send(dir->right, connect_str); - else { - /* there are only two directors, and we already have - a connection to this server. */ - } - } else { - /* there are only two directors. connect to the other one. */ - (void)director_connect_host(dir, host); + /* d) */ + dir->left->verifying_left = TRUE; + director_connection_ping(dir->left); } return TRUE; } static bool -director_user_refresh(struct director *dir, unsigned int username_hash, - struct mail_host *host, time_t timestamp, - struct user **user_r) +director_user_refresh(struct director_connection *conn, + unsigned int username_hash, struct mail_host *host, + time_t timestamp, bool weak, struct user **user_r) { + struct director *dir = conn->dir; struct user *user; - bool ret = FALSE; + bool ret = FALSE, unset_weak_user = FALSE; user = user_directory_lookup(dir->users, username_hash); if (user == NULL) { *user_r = user_directory_add(dir->users, username_hash, host, timestamp); + (*user_r)->weak = weak; return TRUE; } - if (timestamp == ioloop_time && (time_t)user->timestamp != timestamp) { - user_directory_refresh(dir->users, user); - ret = TRUE; - } - if (user->host != host) { - i_error("User hash %u is being redirected to two hosts: " - "%s and %s", username_hash, - net_ip2addr(&user->host->ip), - net_ip2addr(&host->ip)); + if (user->weak) { + if (!weak) { + /* removing user's weakness */ + unset_weak_user = TRUE; + user->weak = FALSE; + ret = TRUE; + } else { + /* weak user marked again as weak */ + } + } else if (weak && + !user_directory_user_is_recently_updated(dir->users, user)) { + /* mark the user as weak */ + user->weak = TRUE; + ret = TRUE; + } else if (user->host != host) { + /* non-weak user received a non-weak update with + conflicting host. this shouldn't happen. */ + string_t *str = t_str_new(128); + + str_printfa(str, "User hash %u " + "is being redirected to two hosts: %s and %s", + username_hash, net_ip2addr(&user->host->ip), + net_ip2addr(&host->ip)); + str_printfa(str, " (old_ts=%ld", (long)user->timestamp); + + if (!conn->handshake_received) { + str_printfa(str, ",handshaking,recv_ts=%ld", + (long)timestamp); + } + if (user->to_move != NULL) + str_append(str, ",moving"); + if (user->kill_state != USER_KILL_STATE_NONE) + str_printfa(str, ",kill_state=%d", user->kill_state); + str_append_c(str, ')'); + i_error("%s", str_c(str)); /* we want all the directors to redirect the user to same server, but we don't want two directors fighting over which @@ -214,12 +486,29 @@ /* change the host. we'll also need to remove the user from the old host's user_count, because we can't keep track of the user for more than one host */ - user->host->user_count--; - user->host = host; - user->host->user_count++; + } else { + /* keep the host */ + host = user->host; } ret = TRUE; } + if (user->host != host) { + user->host->user_count--; + user->host = host; + user->host->user_count++; + ret = TRUE; + } + if (timestamp == ioloop_time && (time_t)user->timestamp != timestamp) { + user_directory_refresh(dir->users, user); + ret = TRUE; + } + + if (unset_weak_user) { + /* user is no longer weak. handle pending requests for + this user if there are any */ + director_set_state_changed(conn->dir); + } + *user_r = user; return ret; } @@ -232,15 +521,16 @@ struct ip_addr ip; struct mail_host *host; struct user *user; + bool weak; - if (str_array_length(args) != 3 || + if (str_array_length(args) < 3 || str_to_uint(args[0], &username_hash) < 0 || net_addr2ip(args[1], &ip) < 0 || str_to_uint(args[2], ×tamp) < 0) { - i_error("director(%s): Invalid USER handshake args", - conn->name); + director_cmd_error(conn, "Invalid parameters"); return FALSE; } + weak = args[3] != NULL && args[3][0] == 'w'; host = mail_host_lookup(conn->dir->mail_hosts, &ip); if (host == NULL) { @@ -249,22 +539,25 @@ return FALSE; } - director_user_refresh(conn->dir, username_hash, host, timestamp, &user); + director_user_refresh(conn, username_hash, host, timestamp, weak, &user); return TRUE; } static bool -director_cmd_user(struct director_connection *conn, const char *const *args) +director_cmd_user(struct director_connection *conn, + const char *const *args) { unsigned int username_hash; struct ip_addr ip; struct mail_host *host; struct user *user; + /* NOTE: if more parameters are added, update also + CMD_IS_USER_HANDHAKE() macro */ if (str_array_length(args) != 2 || str_to_uint(args[0], &username_hash) < 0 || net_addr2ip(args[1], &ip) < 0) { - i_error("director(%s): Invalid USER args", conn->name); + director_cmd_error(conn, "Invalid parameters"); return FALSE; } @@ -274,9 +567,11 @@ return TRUE; } - if (director_user_refresh(conn->dir, username_hash, - host, ioloop_time, &user)) + if (director_user_refresh(conn, username_hash, + host, ioloop_time, FALSE, &user)) { + i_assert(!user->weak); director_update_user(conn->dir, conn->host, user); + } return TRUE; } @@ -286,22 +581,55 @@ struct director_host *host; struct ip_addr ip; unsigned int port; + bool forward = FALSE; if (!director_args_parse_ip_port(conn, args, &ip, &port)) return FALSE; host = director_host_lookup(conn->dir, &ip, port); if (host != NULL) { - /* already have this. just reset its last_failed timestamp, - since it might be up now. */ - host->last_failed = 0; - return TRUE; + if (host == conn->dir->self_host) { + /* ignore updates to ourself */ + return TRUE; + } + if (host->removed) { + /* ignore re-adds of removed directors */ + return TRUE; + } + + /* already have this. just reset its last_network_failure + timestamp, since it might be up now. */ + host->last_network_failure = 0; + if (host->last_seq != 0) { + /* it also may have been restarted, reset last_seq */ + host->last_seq = 0; + forward = TRUE; + } + } else { + /* save the director and forward it */ + host = director_host_add(conn->dir, &ip, port); + forward = TRUE; } + if (forward) { + director_notify_ring_added(host, + director_connection_get_host(conn)); + } + return TRUE; +} - /* save the director and forward it */ - director_host_add(conn->dir, &ip, port); - director_connection_send(conn->dir->right, - t_strdup_printf("DIRECTOR\t%s\t%u\n", net_ip2addr(&ip), port)); +static bool director_cmd_director_remove(struct director_connection *conn, + const char *const *args) +{ + struct director_host *host; + struct ip_addr ip; + unsigned int port; + + if (!director_args_parse_ip_port(conn, args, &ip, &port)) + return FALSE; + + host = director_host_lookup(conn->dir, &ip, port); + if (host != NULL && !host->removed) + director_ring_remove(host, director_connection_get_host(conn)); return TRUE; } @@ -313,9 +641,9 @@ struct mail_host *const *hostp; unsigned int remote_ring_completed; - if (args == NULL || str_to_uint(args[0], &remote_ring_completed) < 0) { - i_error("director(%s): Invalid HOST-HAND-START args", - conn->name); + if (args[0] == NULL || + str_to_uint(args[0], &remote_ring_completed) < 0) { + director_cmd_error(conn, "Invalid parameters"); return FALSE; } @@ -348,14 +676,13 @@ net_addr2ip(args[0], &ip) < 0 || str_to_uint(args[1], &port) < 0 || str_to_uint(args[2], &seq) < 0) { - i_error("director(%s): Command is missing parameters: %s", - conn->name, t_strarray_join(args, " ")); + director_cmd_error(conn, "Invalid parameters"); return -1; } *_args = args + 3; host = director_host_lookup(conn->dir, &ip, port); - if (host == NULL) { + if (host == NULL || host->removed) { /* director is already gone, but we can't be sure if this command was sent everywhere. re-send it as if it was from ourself. */ @@ -372,6 +699,54 @@ } static bool +director_cmd_user_weak(struct director_connection *conn, + const char *const *args) +{ + struct director_host *dir_host; + struct ip_addr ip; + unsigned int username_hash; + struct mail_host *host; + struct user *user; + struct director_host *src_host = conn->host; + bool weak = TRUE; + int ret; + + if ((ret = director_cmd_is_seen(conn, &args, &dir_host)) < 0) + return FALSE; + + if (str_array_length(args) != 2 || + str_to_uint(args[0], &username_hash) < 0 || + net_addr2ip(args[1], &ip) < 0) { + director_cmd_error(conn, "Invalid parameters"); + return FALSE; + } + + host = mail_host_lookup(conn->dir->mail_hosts, &ip); + if (host == NULL) { + /* we probably just removed this host. */ + return TRUE; + } + + if (ret > 0) { + /* The entire ring has seen this USER-WEAK. + make it non-weak now. */ + weak = FALSE; + src_host = conn->dir->self_host; + } + + if (director_user_refresh(conn, username_hash, + host, ioloop_time, weak, &user)) { + if (!user->weak) + director_update_user(conn->dir, src_host, user); + else { + director_update_user_weak(conn->dir, src_host, + dir_host, user); + } + } + return TRUE; +} + +static bool director_cmd_host_int(struct director_connection *conn, const char *const *args, struct director_host *dir_host) { @@ -383,7 +758,7 @@ if (str_array_length(args) != 2 || net_addr2ip(args[0], &ip) < 0 || str_to_uint(args[1], &vhost_count) < 0) { - i_error("director(%s): Invalid HOST args", conn->name); + director_cmd_error(conn, "Invalid parameters"); return FALSE; } if (conn->ignore_host_events) { @@ -441,7 +816,7 @@ if (str_array_length(args) != 1 || net_addr2ip(args[0], &ip) < 0) { - i_error("director(%s): Invalid HOST-REMOVE args", conn->name); + director_cmd_error(conn, "Invalid parameters"); return FALSE; } @@ -465,7 +840,7 @@ if (str_array_length(args) != 1 || net_addr2ip(args[0], &ip) < 0) { - i_error("director(%s): Invalid HOST-FLUSH args", conn->name); + director_cmd_error(conn, "Invalid parameters"); return FALSE; } @@ -491,7 +866,7 @@ if (str_array_length(args) != 2 || str_to_uint(args[0], &username_hash) < 0 || net_addr2ip(args[1], &ip) < 0) { - i_error("director(%s): Invalid USER-MOVE args", conn->name); + director_cmd_error(conn, "Invalid parameters"); return FALSE; } @@ -511,7 +886,7 @@ if (str_array_length(args) != 1 || str_to_uint(args[0], &username_hash) < 0) { - i_error("director(%s): Invalid USER-KILLED args", conn->name); + director_cmd_error(conn, "Invalid parameters"); return FALSE; } @@ -532,8 +907,7 @@ if (str_array_length(args) != 1 || str_to_uint(args[0], &username_hash) < 0) { - i_error("director(%s): Invalid USER-KILLED-EVERYWHERE args", - conn->name); + director_cmd_error(conn, "Invalid parameters"); return FALSE; } @@ -542,152 +916,127 @@ return TRUE; } -static void director_handshake_cmd_done(struct director_connection *conn) +static bool director_handshake_cmd_done(struct director_connection *conn) { struct director *dir = conn->dir; - if (dir->debug) - i_debug("Handshaked to %s", conn->host->name); + if (dir->debug) { + unsigned int secs = time(NULL)-conn->created; - conn->host->last_failed = 0; + i_debug("director(%s): Handshake took %u secs, " + "bytes in=%"PRIuUOFF_T" out=%"PRIuUOFF_T, + conn->name, secs, conn->input->v_offset, + conn->output->offset); + } + + /* the host is up now, make sure we can connect to it immediately + if needed */ + conn->host->last_network_failure = 0; + conn->handshake_received = TRUE; if (conn->in) { /* handshaked to left side. tell it we've received the whole handshake. */ director_connection_send(conn, "DONE\n"); - /* tell the right director about the left one */ - if (dir->right != NULL) { - director_connection_send(dir->right, - t_strdup_printf("DIRECTOR\t%s\t%u\n", - net_ip2addr(&conn->host->ip), - conn->host->port)); - } + /* tell the "right" director about the "left" one */ + director_update_send(dir, director_connection_get_host(conn), + t_strdup_printf("DIRECTOR\t%s\t%u\n", + net_ip2addr(&conn->host->ip), + conn->host->port)); + /* this is our "left" side. */ + return director_connection_assign_left(conn); + } else { + /* handshaked to "right" side. */ + return director_connection_assign_right(conn); } - - if (dir->left != NULL && dir->right != NULL && - dir->left->handshake_received && dir->right->handshake_received) { - /* we're connected to both directors. see if the ring is - finished by sending a SYNC. if we get it back, it's done. */ - dir->sync_seq++; - director_set_ring_unsynced(dir); - director_connection_send(dir->right, - t_strdup_printf("SYNC\t%s\t%u\t%u\n", - net_ip2addr(&dir->self_ip), - dir->self_port, dir->sync_seq)); - } - if (conn->to_ping != NULL) - timeout_remove(&conn->to_ping); - conn->to_ping = timeout_add(DIRECTOR_CONNECTION_PING_INTERVAL_MSECS, - director_connection_ping, conn); } -static bool +static int director_connection_handle_handshake(struct director_connection *conn, const char *cmd, const char *const *args) { - struct director_host *host; - struct ip_addr ip; - unsigned int port; - /* both incoming and outgoing connections get VERSION and ME */ if (strcmp(cmd, "VERSION") == 0 && str_array_length(args) >= 3) { if (strcmp(args[0], DIRECTOR_VERSION_NAME) != 0) { i_error("director(%s): Wrong protocol in socket " "(%s vs %s)", conn->name, args[0], DIRECTOR_VERSION_NAME); - return FALSE; + return -1; } else if (atoi(args[1]) != DIRECTOR_VERSION_MAJOR) { i_error("director(%s): Incompatible protocol version: " "%u vs %u", conn->name, atoi(args[1]), DIRECTOR_VERSION_MAJOR); - return FALSE; + return -1; } + conn->minor_version = atoi(args[2]); conn->version_received = TRUE; - return TRUE; + return 1; } if (!conn->version_received) { - i_error("director(%s): Incompatible protocol", conn->name); - return FALSE; + director_cmd_error(conn, "Incompatible protocol"); + return -1; + } + + if (strcmp(cmd, "ME") == 0) + return director_cmd_me(conn, args) ? 1 : -1; + if (!conn->me_received) { + director_cmd_error(conn, "Expecting ME command first"); + return -1; } - if (strcmp(cmd, "ME") == 0 && !conn->me_received && - str_array_length(args) == 2) - return director_cmd_me(conn, args); - - /* only outgoing connections get a CONNECT reference */ - if (!conn->in && strcmp(cmd, "CONNECT") == 0 && - str_array_length(args) == 2) { - /* remote wants us to connect elsewhere */ - if (!director_args_parse_ip_port(conn, args, &ip, &port)) - return FALSE; - - conn->dir->right = NULL; - host = director_host_get(conn->dir, &ip, port); - /* reset failure timestamp so we'll actually try to - connect there. */ - host->last_failed = 0; - if (conn->dir->debug) - i_debug("Received CONNECT reference to %s", host->name); - (void)director_connect_host(conn->dir, host); - return FALSE; + /* incoming connections get a HOST list */ + if (conn->handshake_sending_hosts) { + if (strcmp(cmd, "HOST") == 0) + return director_cmd_host_handshake(conn, args) ? 1 : -1; + if (strcmp(cmd, "HOST-HAND-END") == 0) { + conn->ignore_host_events = FALSE; + conn->handshake_sending_hosts = FALSE; + return 1; + } + director_cmd_error(conn, "Unexpected command during host list"); + return -1; } - /* only incoming connections get DIRECTOR and HOST lists */ - if (conn->in && strcmp(cmd, "DIRECTOR") == 0 && conn->me_received) - return director_cmd_director(conn, args); + if (strcmp(cmd, "HOST-HAND-START") == 0) { + if (!conn->in) { + director_cmd_error(conn, + "Host list is only for incoming connections"); + return -1; + } + return director_cmd_host_hand_start(conn, args) ? 1 : -1; + } - if (strcmp(cmd, "HOST") == 0) { - /* allow hosts from all connections always, - this could be an host update */ - if (conn->handshake_sending_hosts) - return director_cmd_host_handshake(conn, args); - else - return director_cmd_host(conn, args); - } - if (conn->handshake_sending_hosts && - strcmp(cmd, "HOST-HAND-END") == 0) { - conn->ignore_host_events = FALSE; - conn->handshake_sending_hosts = FALSE; - return TRUE; - } - if (conn->in && strcmp(cmd, "HOST-HAND-START") == 0 && - conn->me_received) - return director_cmd_host_hand_start(conn, args); + if (conn->in && strcmp(cmd, "USER") == 0 && CMD_IS_USER_HANDHAKE(args)) + return director_handshake_cmd_user(conn, args) ? 1 : -1; - /* only incoming connections get a full USER list, but outgoing - connections can also receive USER updates during handshake and - it wouldn't be safe to ignore them. */ - if (strcmp(cmd, "USER") == 0 && conn->me_received) { - if (conn->in) - return director_handshake_cmd_user(conn, args); - else - return director_cmd_user(conn, args); - } /* both get DONE */ - if (strcmp(cmd, "DONE") == 0 && !conn->handshake_received && - !conn->handshake_sending_hosts) { - director_handshake_cmd_done(conn); - return TRUE; - } - i_error("director(%s): Invalid handshake command: %s " - "(in=%d me_received=%d)", conn->name, cmd, - conn->in, conn->me_received); - return FALSE; + if (strcmp(cmd, "DONE") == 0) + return director_handshake_cmd_done(conn) ? 1 : -1; + return 0; } static void director_connection_sync_host(struct director_connection *conn, struct director_host *host, - uint32_t seq, const char *line) + uint32_t seq, unsigned int minor_version) { struct director *dir = conn->dir; + if (minor_version > DIRECTOR_VERSION_MINOR) { + /* we're not up to date */ + minor_version = DIRECTOR_VERSION_MINOR; + } + if (host->self) { if (dir->sync_seq != seq) { /* stale SYNC event */ return; } + /* sync_seq increases when we get disconnected, so we must be + successfully connected to both directions */ + i_assert(dir->left != NULL && dir->right != NULL); + dir->ring_min_version = minor_version; if (!dir->ring_handshaked) { /* the ring is handshaked */ director_set_ring_handshaked(dir); @@ -701,35 +1050,36 @@ } director_set_ring_synced(dir); } - } else { + } else if (dir->right != NULL) { /* forward it to the connection on right */ - if (dir->right != NULL) { - director_connection_send(dir->right, - t_strconcat(line, "\n", NULL)); - } + director_sync_send(dir, host, seq, minor_version); } } static bool director_connection_sync(struct director_connection *conn, - const char *const *args, const char *line) + const char *const *args) { struct director *dir = conn->dir; struct director_host *host; struct ip_addr ip; - unsigned int port, seq; + unsigned int port, seq, minor_version = 0; - if (str_array_length(args) != 3 || + if (str_array_length(args) < 3 || !director_args_parse_ip_port(conn, args, &ip, &port) || str_to_uint(args[2], &seq) < 0) { - i_error("director(%s): Invalid SYNC args", conn->name); + director_cmd_error(conn, "Invalid parameters"); return FALSE; } + if (args[3] != NULL) + minor_version = atoi(args[3]); /* find the originating director. if we don't see it, it was already removed and we can ignore this sync. */ host = director_host_lookup(dir, &ip, port); - if (host != NULL) - director_connection_sync_host(conn, host, seq, line); + if (host != NULL) { + director_connection_sync_host(conn, host, seq, + minor_version); + } if (host == NULL || !host->self) director_resend_sync(dir); @@ -746,16 +1096,13 @@ if (str_array_length(args) != 2 || !director_args_parse_ip_port(conn, args, &ip, &port)) { - i_error("director(%s): Invalid CONNECT args", conn->name); + director_cmd_error(conn, "Invalid parameters"); return FALSE; } - host = director_host_lookup(dir, &ip, port); - if (host == NULL) { - i_error("Received CONNECT request to unknown host %s:%u", - net_ip2addr(&ip), port); - return TRUE; - } + host = director_host_get(conn->dir, &ip, port); + /* reset failure timestamp so we'll actually try to connect there. */ + host->last_network_failure = 0; /* remote suggests us to connect elsewhere */ if (dir->right != NULL && @@ -786,56 +1133,73 @@ return TRUE; } +static void director_disconnect_wrong_lefts(struct director *dir) +{ + struct director_connection *const *connp, *conn; + + array_foreach(&dir->connections, connp) { + conn = *connp; + + if (conn->in && conn != dir->left && conn->me_received && + conn->to_disconnect == NULL && + director_host_cmp_to_self(dir->left->host, conn->host, + dir->self_host) < 0) { + i_warning("Director connection %s tried to connect to " + "us, should use %s instead", + conn->name, dir->left->host->name); + director_connection_send_connect(conn, dir->left->host); + } + } +} + static bool director_cmd_pong(struct director_connection *conn) { if (!conn->ping_waiting) return TRUE; + conn->ping_waiting = FALSE; + timeout_remove(&conn->to_pong); - conn->ping_waiting = FALSE; - timeout_remove(&conn->to_ping); - conn->to_ping = timeout_add(DIRECTOR_CONNECTION_PING_INTERVAL_MSECS, - director_connection_ping, conn); + if (conn->verifying_left) { + conn->verifying_left = FALSE; + if (conn == conn->dir->left) { + /* our left side is functional. tell all the wrong + incoming connections to connect to it instead. */ + director_disconnect_wrong_lefts(conn->dir); + } + } + + director_connection_set_ping_timeout(conn); return TRUE; } static bool -director_connection_handle_line(struct director_connection *conn, - const char *line) +director_connection_handle_cmd(struct director_connection *conn, + const char *cmd, const char *const *args) { - const char *cmd, *const *args; + int ret; - args = t_strsplit(line, "\t"); - cmd = args[0]; args++; - if (cmd == NULL) { - i_error("director(%s): Received empty line", conn->name); - return FALSE; + if (!conn->handshake_received) { + ret = director_connection_handle_handshake(conn, cmd, args); + if (ret > 0) + return TRUE; + if (ret < 0) { + /* invalid commands during handshake, + we probably don't want to reconnect here */ + return FALSE; + } + /* allow also other commands during handshake */ } - /* ping/pong is always handled */ if (strcmp(cmd, "PING") == 0) { director_connection_send(conn, "PONG\n"); return TRUE; } if (strcmp(cmd, "PONG") == 0) return director_cmd_pong(conn); - - if (!conn->handshake_received) { - if (!director_connection_handle_handshake(conn, cmd, args)) { - /* invalid commands during handshake, - we probably don't want to reconnect here */ - if (conn->dir->debug) { - i_debug("director(%s): Handshaking failed", - conn->host->name); - } - if (conn->host != NULL) - conn->host->last_failed = ioloop_time; - return FALSE; - } - return TRUE; - } - if (strcmp(cmd, "USER") == 0) return director_cmd_user(conn, args); + if (strcmp(cmd, "USER-WEAK") == 0) + return director_cmd_user_weak(conn, args); if (strcmp(cmd, "HOST") == 0) return director_cmd_host(conn, args); if (strcmp(cmd, "HOST-REMOVE") == 0) @@ -850,24 +1214,45 @@ return director_cmd_user_killed_everywhere(conn, args); if (strcmp(cmd, "DIRECTOR") == 0) return director_cmd_director(conn, args); + if (strcmp(cmd, "DIRECTOR-REMOVE") == 0) + return director_cmd_director_remove(conn, args); if (strcmp(cmd, "SYNC") == 0) - return director_connection_sync(conn, args, line); + return director_connection_sync(conn, args); if (strcmp(cmd, "CONNECT") == 0) return director_cmd_connect(conn, args); - i_error("director(%s): Unknown command (in this state): %s", - conn->name, cmd); + director_cmd_error(conn, "Unknown command %s", cmd); return FALSE; } +static bool +director_connection_handle_line(struct director_connection *conn, + const char *line) +{ + const char *cmd, *const *args; + bool ret; + + args = t_strsplit_tab(line); + cmd = args[0]; args++; + if (cmd == NULL) { + director_cmd_error(conn, "Received empty line"); + return FALSE; + } + + conn->cur_cmd = cmd; + conn->cur_line = line; + ret = director_connection_handle_cmd(conn, cmd, args); + conn->cur_cmd = NULL; + conn->cur_line = NULL; + return ret; +} + static void director_connection_input(struct director_connection *conn) { struct director *dir = conn->dir; char *line; bool ret; - if (conn->to_ping != NULL) - timeout_reset(conn->to_ping); switch (i_stream_read(conn->input)) { case 0: return; @@ -880,9 +1265,20 @@ return; case -2: /* buffer full */ - i_error("BUG: Director %s sent us more than %d bytes", - conn->name, MAX_INBUF_SIZE); - director_connection_disconnected(&conn); + director_cmd_error(conn, "Director sent us more than %d bytes", + MAX_INBUF_SIZE); + director_connection_reconnect(&conn); + return; + } + + if (conn->to_disconnect != NULL) { + /* just read everything the remote sends, and wait for it + to disconnect. we mainly just want the remote to read the + CONNECT we sent it. */ + size_t size; + + (void)i_stream_get_data(conn->input, &size); + i_stream_skip(conn->input, size); return; } @@ -893,15 +1289,13 @@ } T_END; if (!ret) { - if (dir->debug) { - i_debug("director(%s): Invalid input, disconnecting", - conn->name); - } - director_connection_disconnected(&conn); + director_connection_reconnect(&conn); break; } } director_sync_thaw(dir); + if (conn != NULL) + timeout_reset(conn->to_ping); } static void director_connection_send_directors(struct director_connection *conn, @@ -910,6 +1304,8 @@ struct director_host *const *hostp; array_foreach(&conn->dir->dir_hosts, hostp) { + if ((*hostp)->removed) + continue; str_printfa(str, "DIRECTOR\t%s\t%u\n", net_ip2addr(&(*hostp)->ip), (*hostp)->port); } @@ -933,27 +1329,24 @@ struct user *user; int ret; - o_stream_cork(conn->output); while ((user = user_directory_iter_next(conn->user_iter)) != NULL) { - if (!user_directory_user_has_connections(conn->dir->users, - user)) { - /* user is already expired */ - continue; - } + T_BEGIN { + string_t *str = t_str_new(128); - T_BEGIN { - const char *line; - - line = t_strdup_printf("USER\t%u\t%s\t%u\n", - user->username_hash, - net_ip2addr(&user->host->ip), - user->timestamp); - director_connection_send(conn, line); + str_printfa(str, "USER\t%u\t%s\t%u", + user->username_hash, + net_ip2addr(&user->host->ip), + user->timestamp); + if (user->weak) + str_append(str, "\tw"); + str_append_c(str, '\n'); + director_connection_send(conn, str_c(str)); } T_END; if (o_stream_get_buffer_used_size(conn->output) >= OUTBUF_FLUSH_THRESHOLD) { if ((ret = o_stream_flush(conn->output)) <= 0) { /* continue later */ + timeout_reset(conn->to_ping); return ret; } } @@ -961,32 +1354,27 @@ user_directory_iter_deinit(&conn->user_iter); director_connection_send(conn, "DONE\n"); - i_assert(conn->io == NULL); - conn->io = io_add(conn->fd, IO_READ, director_connection_input, conn); - ret = o_stream_flush(conn->output); - o_stream_uncork(conn->output); + timeout_reset(conn->to_ping); return ret; } static int director_connection_output(struct director_connection *conn) { - if (conn->user_iter != NULL) - return director_connection_send_users(conn); - else - return o_stream_flush(conn->output); -} + int ret; -static void -director_connection_init_timeout(struct director_connection *conn) -{ - if (conn->host != NULL) - conn->host->last_failed = ioloop_time; - if (!conn->connected) - i_error("director(%s): Connect timed out", conn->name); - else - i_error("director(%s): Handshaking timed out", conn->name); - director_connection_disconnected(&conn); + if (conn->user_iter != NULL) { + /* still handshaking USER list */ + o_stream_cork(conn->output); + ret = director_connection_send_users(conn); + o_stream_uncork(conn->output); + if (ret < 0) + director_connection_disconnected(&conn); + else + o_stream_set_flush_pending(conn->output, TRUE); + return ret; + } + return o_stream_flush(conn->output); } static struct director_connection * @@ -1000,11 +1388,9 @@ conn->dir = dir; conn->input = i_stream_create_fd(conn->fd, MAX_INBUF_SIZE, FALSE); conn->output = o_stream_create_fd(conn->fd, MAX_OUTBUF_SIZE, FALSE); - o_stream_set_flush_callback(conn->output, - director_connection_output, conn); - conn->to_ping = timeout_add(DIRECTOR_CONNECTION_INIT_TIMEOUT_MSECS, + conn->to_ping = timeout_add(DIRECTOR_CONNECTION_ME_TIMEOUT_MSECS, director_connection_init_timeout, conn); - DLLIST_PREPEND(&dir->connections, conn); + array_append(&dir->connections, &conn, 1); return conn; } @@ -1040,43 +1426,32 @@ int err; if ((err = net_geterror(conn->fd)) != 0) { - conn->host->last_failed = ioloop_time; i_error("director(%s): connect() failed: %s", conn->name, strerror(err)); director_connection_disconnected(&conn); return; } conn->connected = TRUE; - - if (dir->right != NULL) { - /* see if we should disconnect or keep the existing - connection. */ - if (director_host_cmp_to_self(conn->host, dir->right->host, - dir->self_host) <= 0) { - /* the old connection is the correct one */ - i_warning("Aborting incorrect outgoing connection to %s " - "(already connected to correct one: %s)", - conn->host->name, dir->right->host->name); - director_connection_deinit(&conn); - return; - } - i_warning("Replacing director connection %s with %s", - dir->right->host->name, conn->host->name); - director_connection_deinit(&dir->right); - } - dir->right = conn; - i_free(conn->name); - conn->name = i_strdup_printf("%s/right", conn->host->name); + o_stream_set_flush_callback(conn->output, + director_connection_output, conn); io_remove(&conn->io); + conn->io = io_add(conn->fd, IO_READ, director_connection_input, conn); + timeout_remove(&conn->to_ping); + conn->to_ping = timeout_add(DIRECTOR_CONNECTION_SEND_USERS_TIMEOUT_MSECS, + director_connection_init_timeout, conn); + + o_stream_cork(conn->output); director_connection_send_handshake(conn); director_connection_send_directors(conn, str); director_connection_send_hosts(conn, str); director_connection_send(conn, str_c(str)); conn->user_iter = user_directory_iter_init(dir->users); - (void)director_connection_send_users(conn); + if (director_connection_send_users(conn) == 0) + o_stream_set_flush_pending(conn->output, TRUE); + o_stream_uncork(conn->output); } struct director_connection * @@ -1085,45 +1460,57 @@ { struct director_connection *conn; + i_assert(!host->removed); + /* make sure we don't keep old sequence values across restarts */ host->last_seq = 0; conn = director_connection_init_common(dir, fd); conn->name = i_strdup_printf("%s/out", host->name); conn->host = host; - /* use IO_READ instead of IO_WRITE, so that we don't assign - dir->right until remote has actually sent some data */ - conn->io = io_add(conn->fd, IO_READ, + director_host_ref(host); + conn->io = io_add(conn->fd, IO_WRITE, director_connection_connected, conn); return conn; } void director_connection_deinit(struct director_connection **_conn) { - struct director_connection *conn = *_conn; + struct director_connection *const *conns, *conn = *_conn; struct director *dir = conn->dir; + unsigned int i, count; *_conn = NULL; if (dir->debug && conn->host != NULL) i_debug("Disconnecting from %s", conn->host->name); - if (conn->host != NULL && !conn->in && - conn->created + DIRECTOR_SUCCESS_MIN_CONNECT_SECS > ioloop_time) - conn->host->last_failed = ioloop_time; - - DLLIST_REMOVE(&dir->connections, conn); - if (dir->left == conn) + conns = array_get(&dir->connections, &count); + for (i = 0; i < count; i++) { + if (conns[i] == conn) { + array_delete(&dir->connections, i, 1); + break; + } + } + i_assert(i < count); + if (dir->left == conn) { dir->left = NULL; + /* if there is already another handshaked incoming connection, + use it as the new "left" */ + director_assign_left(dir); + } if (dir->right == conn) dir->right = NULL; + if (conn->host != NULL) + director_host_unref(conn->host); if (conn->user_iter != NULL) user_directory_iter_deinit(&conn->user_iter); - if (conn->to != NULL) - timeout_remove(&conn->to); - if (conn->to_ping != NULL) - timeout_remove(&conn->to_ping); + if (conn->to_disconnect != NULL) + timeout_remove(&conn->to_disconnect); + if (conn->to_pong != NULL) + timeout_remove(&conn->to_pong); + timeout_remove(&conn->to_ping); if (conn->io != NULL) io_remove(&conn->io); i_stream_unref(&conn->input); @@ -1148,14 +1535,26 @@ struct director_connection *conn = *_conn; struct director *dir = conn->dir; + if (conn->created + DIRECTOR_SUCCESS_MIN_CONNECT_SECS > ioloop_time && + conn->host != NULL) { + /* connection didn't exist for very long, assume it has a + network problem */ + conn->host->last_network_failure = ioloop_time; + } + director_connection_deinit(_conn); if (dir->right == NULL) director_connect(dir); } -static void director_connection_timeout(struct director_connection *conn) +void director_connection_reconnect(struct director_connection **_conn) { - director_connection_disconnected(&conn); + struct director_connection *conn = *_conn; + struct director *dir = conn->dir; + + director_connection_deinit(_conn); + if (dir->right == NULL) + director_connect(dir); } void director_connection_send(struct director_connection *conn, @@ -1176,34 +1575,33 @@ "disconnecting", conn->name); } o_stream_close(conn->output); - conn->to = timeout_add(0, director_connection_timeout, conn); } } -void director_connection_send_except(struct director_connection *conn, - struct director_host *skip_host, - const char *data) -{ - if (conn->host != skip_host) - director_connection_send(conn, data); -} - -static void director_connection_ping_timeout(struct director_connection *conn) +static void +director_connection_ping_idle_timeout(struct director_connection *conn) { i_error("director(%s): Ping timed out, disconnecting", conn->name); director_connection_disconnected(&conn); } -static void director_connection_ping(struct director_connection *conn) +static void director_connection_pong_timeout(struct director_connection *conn) { - conn->sync_ping = FALSE; + i_error("director(%s): PONG reply not received although other " + "input keeps coming, disconnecting", conn->name); + director_connection_disconnected(&conn); +} + +void director_connection_ping(struct director_connection *conn) +{ if (conn->ping_waiting) return; - if (conn->to_ping != NULL) - timeout_remove(&conn->to_ping); - conn->to_ping = timeout_add(DIRECTOR_CONNECTION_PING_TIMEOUT_MSECS, - director_connection_ping_timeout, conn); + timeout_remove(&conn->to_ping); + conn->to_ping = timeout_add(DIRECTOR_CONNECTION_PING_IDLE_TIMEOUT_MSECS, + director_connection_ping_idle_timeout, conn); + conn->to_pong = timeout_add(DIRECTOR_CONNECTION_PONG_TIMEOUT_MSECS, + director_connection_pong_timeout, conn); director_connection_send(conn, "PING\n"); conn->ping_waiting = TRUE; } @@ -1219,17 +1617,20 @@ return conn->host; } -struct director_connection * -director_connection_find_outgoing(struct director *dir, - struct director_host *host) +bool director_connection_is_handshaked(struct director_connection *conn) { - struct director_connection *conn; + return conn->handshake_received; +} - for (conn = dir->connections; conn != NULL; conn = conn->next) { - if (conn->host == host && !conn->in) - return conn; - } - return NULL; +bool director_connection_is_incoming(struct director_connection *conn) +{ + return conn->in; +} + +unsigned int +director_connection_get_minor_version(struct director_connection *conn) +{ + return conn->minor_version; } void director_connection_cork(struct director_connection *conn) @@ -1242,27 +1643,16 @@ o_stream_uncork(conn->output); } -void director_connection_wait_sync(struct director_connection *conn) +void director_connection_set_synced(struct director_connection *conn, + bool synced) { - /* switch to faster ping timeout. avoid reseting the timeout if it's - already fast. */ - if (conn->ping_waiting || conn->sync_ping) + if (conn->synced == synced) + return; + conn->synced = synced; + + /* switch ping timeout, unless we're already waiting for PONG */ + if (conn->ping_waiting) return; - if (conn->to_ping != NULL) - timeout_remove(&conn->to_ping); - conn->to_ping = timeout_add(DIRECTOR_CONNECTION_SYNC_TIMEOUT_MSECS, - director_connection_ping, conn); - conn->sync_ping = TRUE; + director_connection_set_ping_timeout(conn); } - -void director_connections_deinit(struct director *dir) -{ - struct director_connection *conn; - - while (dir->connections != NULL) { - conn = dir->connections; - dir->connections = conn->next; - director_connection_deinit(&conn); - } -}
--- a/src/director/director-connection.h Sat May 19 22:40:08 2012 +0300 +++ b/src/director/director-connection.h Sun May 20 03:25:04 2012 +0300 @@ -14,21 +14,19 @@ void director_connection_send(struct director_connection *conn, const char *data); -void director_connection_wait_sync(struct director_connection *conn); -void director_connection_send_except(struct director_connection *conn, - struct director_host *skip_host, - const char *data); +void director_connection_set_synced(struct director_connection *conn, + bool synced); +void director_connection_ping(struct director_connection *conn); const char *director_connection_get_name(struct director_connection *conn); struct director_host * director_connection_get_host(struct director_connection *conn); -struct director_connection * -director_connection_find_outgoing(struct director *dir, - struct director_host *host); +bool director_connection_is_handshaked(struct director_connection *conn); +bool director_connection_is_incoming(struct director_connection *conn); +unsigned int +director_connection_get_minor_version(struct director_connection *conn); void director_connection_cork(struct director_connection *conn); void director_connection_uncork(struct director_connection *conn); -void director_connections_deinit(struct director *dir); - #endif
--- a/src/director/director-host.c Sat May 19 22:40:08 2012 +0300 +++ b/src/director/director-host.c Sun May 20 03:25:04 2012 +0300 @@ -29,6 +29,8 @@ struct director_host *host; host = i_new(struct director_host, 1); + host->dir = dir; + host->refcount = 1; host->ip = *ip; host->port = port; host->name = i_strdup_printf("%s:%u", net_ip2addr(ip), port); @@ -41,8 +43,39 @@ return host; } -void director_host_free(struct director_host *host) +void director_host_free(struct director_host **_host) +{ + struct director_host *host = *_host; + + i_assert(host->refcount == 1); + + *_host = NULL; + director_host_unref(host); +} + +void director_host_ref(struct director_host *host) { + i_assert(host->refcount > 0); + host->refcount++; +} + +void director_host_unref(struct director_host *host) +{ + struct director_host *const *hosts; + unsigned int i, count; + + i_assert(host->refcount > 0); + + if (--host->refcount > 0) + return; + + hosts = array_get(&host->dir->dir_hosts, &count); + for (i = 0; i < count; i++) { + if (hosts[i] == host) { + array_delete(&host->dir->dir_hosts, i, 1); + break; + } + } i_free(host->name); i_free(host); }
--- a/src/director/director-host.h Sat May 19 22:40:08 2012 +0300 +++ b/src/director/director-host.h Sun May 20 03:25:04 2012 +0300 @@ -6,6 +6,9 @@ struct director; struct director_host { + struct director *dir; + int refcount; + struct ip_addr ip; unsigned int port; @@ -17,16 +20,21 @@ it can be ignored (or: it must be ignored to avoid potential command loops) */ unsigned int last_seq; - /* Last time host was detected to be down/broken */ - time_t last_failed; + /* Last time host was detected to be down */ + time_t last_network_failure; + time_t last_protocol_failure; /* we are this director */ unsigned int self:1; + unsigned int removed:1; }; struct director_host * director_host_add(struct director *dir, const struct ip_addr *ip, unsigned int port); -void director_host_free(struct director_host *host); +void director_host_free(struct director_host **host); + +void director_host_ref(struct director_host *host); +void director_host_unref(struct director_host *host); struct director_host * director_host_get(struct director *dir, const struct ip_addr *ip,
--- a/src/director/director-request.c Sat May 19 22:40:08 2012 +0300 +++ b/src/director/director-request.c Sun May 20 03:25:04 2012 +0300 @@ -26,6 +26,7 @@ director_request_get_timeout_error(struct director_request *request) { string_t *str = t_str_new(128); + struct user *user; unsigned int secs; str_printfa(str, "Timeout - queued for %u secs (", @@ -34,12 +35,21 @@ if (request->dir->ring_last_sync_time == 0) str_append(str, "Ring has never been synced"); else { - secs =ioloop_time - request->dir->ring_last_sync_time; + secs = ioloop_time - request->dir->ring_last_sync_time; if (request->dir->ring_synced) str_printfa(str, "Ring synced for %u secs", secs); else str_printfa(str, "Ring not synced for %u secs", secs); } + + user = user_directory_lookup(request->dir->users, + request->username_hash); + if (user != NULL) { + if (user->weak) + str_append(str, ", weak user"); + str_printfa(str, ", user refreshed %u secs ago", + (unsigned int)(ioloop_time - user->timestamp)); + } str_append_c(str, ')'); return str_c(str); } @@ -73,7 +83,8 @@ director_request_callback *callback, void *context) { struct director_request *request; - unsigned int username_hash = user_directory_get_username_hash(username); + unsigned int username_hash = + user_directory_get_username_hash(dir->users, username); request = i_new(struct director_request, 1); request->dir = dir; @@ -116,6 +127,74 @@ ring_noconn_warning, dir); } +static bool director_request_existing(struct director *dir, struct user *user) +{ + struct mail_host *host; + + if (user->kill_state != USER_KILL_STATE_NONE) { + /* delay processing this user's connections until + its existing connections have been killed */ + return FALSE; + } + if (user->weak) { + /* wait for user to become non-weak */ + return FALSE; + } + if (!user_directory_user_is_near_expiring(dir->users, user)) + return TRUE; + + /* user is close to being expired. another director may have + already expired it. */ + host = mail_host_get_by_hash(dir->mail_hosts, user->username_hash); + if (!dir->ring_synced) { + /* try again later once ring is synced */ + return FALSE; + } + if (user->host == host) { + /* doesn't matter, other directors would + assign the user the same way regardless */ + return TRUE; + } + + /* We have to worry about two separate timepoints in here: + + a) some directors think the user isn't expiring, and + others think the user is near expiring + + b) some directors think the user is near expiring, and + others think the user has already expired + + What we don't have to worry about is: + + !c) some directors think the user isn't expiring, and + others think the user has already expired + + If !c) happens, the user might get redirected to different backends. + We'll use a large enough timeout between a) and b) states, so that + !c) should never happen. + + So what we'll do here is: + + 1. Send a USER-WEAK notification to all directors with the new host. + 2. Each director receiving USER-WEAK refreshes the user's timestamp + and host, but marks the user as being weak. + 3. Once USER-WEAK has reached all directors, a real USER update is + sent, which removes the weak-flag. + 4. If a director ever receives a USER update for a weak user, the + USER update overrides the host and removes the weak-flag. + 5. Director doesn't let any weak user log in, until the weak-flag + gets removed. + */ + if (dir->ring_min_version < DIRECTOR_VERSION_WEAK_USERS) { + /* weak users not supported by ring currently */ + return TRUE; + } else { + user->weak = TRUE; + director_update_user_weak(dir, dir->self_host, NULL, user); + return FALSE; + } +} + bool director_request_continue(struct director_request *request) { struct director *dir = request->dir; @@ -130,11 +209,8 @@ user = user_directory_lookup(dir->users, request->username_hash); if (user != NULL) { - if (user->kill_state != USER_KILL_STATE_NONE) { - /* delay processing this user's connections until - its existing connections have been killed */ + if (!director_request_existing(dir, user)) return FALSE; - } user_directory_refresh(dir->users, user); } else { if (!dir->ring_synced) { @@ -152,6 +228,7 @@ host, ioloop_time); } + i_assert(!user->weak); director_update_user(dir, dir->self_host, user); T_BEGIN { request->callback(&user->host->ip, NULL, request->context);
--- a/src/director/director-settings.c Sat May 19 22:40:08 2012 +0300 +++ b/src/director/director-settings.c Sun May 20 03:25:04 2012 +0300 @@ -68,6 +68,7 @@ DEF(SET_STR, director_servers), DEF(SET_STR, director_mail_servers), + DEF(SET_STR, director_username_hash), DEF(SET_TIME, director_user_expire), DEF(SET_UINT, director_doveadm_port), @@ -79,6 +80,7 @@ .director_servers = "", .director_mail_servers = "", + .director_username_hash = "%u", .director_user_expire = 60*15, .director_doveadm_port = 0 };
--- a/src/director/director-settings.h Sat May 19 22:40:08 2012 +0300 +++ b/src/director/director-settings.h Sun May 20 03:25:04 2012 +0300 @@ -6,6 +6,7 @@ const char *director_servers; const char *director_mail_servers; + const char *director_username_hash; unsigned int director_user_expire; unsigned int director_doveadm_port; };
--- a/src/director/director-test.c Sat May 19 22:40:08 2012 +0300 +++ b/src/director/director-test.c Sun May 20 03:25:04 2012 +0300 @@ -492,7 +492,7 @@ break; /* ip vhost-count user-count */ T_BEGIN { - const char *const *args = t_strsplit(line, "\t"); + const char *const *args = t_strsplit_tab(line); struct host *host; host = i_new(struct host, 1);
--- a/src/director/director.c Sat May 19 22:40:08 2012 +0300 +++ b/src/director/director.c Sun May 20 03:25:04 2012 +0300 @@ -17,7 +17,10 @@ #define DIRECTOR_RECONNECT_TIMEOUT_MSECS (30*1000) #define DIRECTOR_USER_MOVE_TIMEOUT_MSECS (30*1000) #define DIRECTOR_USER_MOVE_FINISH_DELAY_MSECS (2*1000) -#define DIRECTOR_SYNC_TIMEOUT_MSECS (15*1000) +#define DIRECTOR_SYNC_TIMEOUT_MSECS (5*1000) +#define DIRECTOR_RING_MIN_WAIT_SECS 20 +#define DIRECTOR_QUICK_RECONNECT_TIMEOUT_MSECS 1000 +#define DIRECTOR_DELAYED_DIR_REMOVE_MSECS (1000*30) static bool director_is_self_ip_set(struct director *dir) { @@ -81,12 +84,26 @@ i_unreached(); } +static bool +director_has_outgoing_connection(struct director *dir, + struct director_host *host) +{ + struct director_connection *const *connp; + + array_foreach(&dir->connections, connp) { + if (director_connection_get_host(*connp) == host && + !director_connection_is_incoming(*connp)) + return TRUE; + } + return FALSE; +} + int director_connect_host(struct director *dir, struct director_host *host) { unsigned int port; int fd; - if (director_connection_find_outgoing(dir, host) != NULL) + if (director_has_outgoing_connection(dir, host)) return 0; if (dir->debug) { @@ -96,13 +113,13 @@ port = dir->test_port != 0 ? dir->test_port : host->port; fd = net_connect_ip(&host->ip, port, &dir->self_ip); if (fd == -1) { - host->last_failed = ioloop_time; + host->last_network_failure = ioloop_time; i_error("connect(%s) failed: %m", host->name); return -1; } /* Reset timestamp so that director_connect() won't skip this host while we're still trying to connect to it */ - host->last_failed = 0; + host->last_network_failure = 0; director_connection_init_out(dir, fd, host); return 0; @@ -111,15 +128,47 @@ static struct director_host * director_get_preferred_right_host(struct director *dir) { - struct director_host *const *hosts; - unsigned int count, self_idx; + struct director_host *const *hosts, *host; + unsigned int i, count, self_idx; hosts = array_get(&dir->dir_hosts, &count); - if (count == 1) + if (count == 1) { + /* self */ return NULL; + } self_idx = director_find_self_idx(dir); - return hosts[(self_idx + 1) % count]; + for (i = 0; i < count; i++) { + host = hosts[(self_idx + i + 1) % count]; + if (!host->removed) + return host; + } + /* self, with some removed hosts */ + return NULL; +} + +static bool director_wait_for_others(struct director *dir) +{ + struct director_host *const *hostp; + + /* don't assume we're alone until we've attempted to connect + to others for a while */ + if (dir->ring_first_alone != 0 && + ioloop_time - dir->ring_first_alone > DIRECTOR_RING_MIN_WAIT_SECS) + return FALSE; + + if (dir->ring_first_alone == 0) + dir->ring_first_alone = ioloop_time; + /* reset all failures and try again */ + array_foreach(&dir->dir_hosts, hostp) { + (*hostp)->last_network_failure = 0; + (*hostp)->last_protocol_failure = 0; + } + if (dir->to_reconnect != NULL) + timeout_remove(&dir->to_reconnect); + dir->to_reconnect = timeout_add(DIRECTOR_QUICK_RECONNECT_TIMEOUT_MSECS, + director_connect, dir); + return TRUE; } void director_connect(struct director *dir) @@ -135,31 +184,45 @@ for (i = 1; i < count; i++) { unsigned int idx = (self_idx + i) % count; - if (hosts[idx]->last_failed + + if (hosts[idx]->removed) + continue; + + if (hosts[idx]->last_network_failure + DIRECTOR_RECONNECT_RETRY_SECS > ioloop_time) { - /* failed recently, don't try retrying here */ + /* connection failed recently, don't try retrying here */ + continue; + } + if (hosts[idx]->last_protocol_failure + + DIRECTOR_PROTOCOL_FAILURE_RETRY_SECS > ioloop_time) { + /* the director recently sent invalid protocol data, + don't try retrying yet */ continue; } - if (director_connect_host(dir, hosts[idx]) == 0) - break; - } - if (i == count) { - /* we're the only one */ - if (dir->debug) { - i_debug("director: Couldn't connect to right side, " - "we must be the only director left"); + if (director_connect_host(dir, hosts[idx]) == 0) { + /* success */ + return; } - if (dir->left != NULL) { - /* since we couldn't connect to it, - it must have failed recently */ - director_connection_deinit(&dir->left); - } - if (!dir->ring_handshaked) - director_set_ring_handshaked(dir); - else - director_set_ring_synced(dir); + } + + if (count > 1 && director_wait_for_others(dir)) + return; + + /* we're the only one */ + if (count > 1) { + i_warning("director: Couldn't connect to right side, " + "we must be the only director left"); } + if (dir->left != NULL) { + /* since we couldn't connect to it, + it must have failed recently */ + director_connection_deinit(&dir->left); + } + dir->ring_min_version = DIRECTOR_VERSION_MINOR; + if (!dir->ring_handshaked) + director_set_ring_handshaked(dir); + else + director_set_ring_synced(dir); } void director_set_ring_handshaked(struct director *dir) @@ -188,7 +251,9 @@ cur_host = dir->right == NULL ? NULL : director_connection_get_host(dir->right); - if (cur_host != preferred_host) + if (preferred_host == NULL) { + /* all directors have been removed, try again later */ + } else if (cur_host != preferred_host) (void)director_connect_host(dir, preferred_host); else { /* the connection hasn't finished sync yet. @@ -213,18 +278,20 @@ host = dir->right == NULL ? NULL : director_connection_get_host(dir->right); + + if (dir->to_reconnect != NULL) + timeout_remove(&dir->to_reconnect); if (host != director_get_preferred_right_host(dir)) { /* try to reconnect to preferred host later */ - if (dir->to_reconnect == NULL) { - dir->to_reconnect = - timeout_add(DIRECTOR_RECONNECT_TIMEOUT_MSECS, - director_reconnect_timeout, dir); - } - } else { - if (dir->to_reconnect != NULL) - timeout_remove(&dir->to_reconnect); + dir->to_reconnect = + timeout_add(DIRECTOR_RECONNECT_TIMEOUT_MSECS, + director_reconnect_timeout, dir); } + if (dir->left != NULL) + director_connection_set_synced(dir->left, TRUE); + if (dir->right != NULL) + director_connection_set_synced(dir->right, TRUE); if (dir->to_sync != NULL) timeout_remove(&dir->to_sync); dir->ring_synced = TRUE; @@ -232,14 +299,36 @@ director_set_state_changed(dir); } +void director_sync_send(struct director *dir, struct director_host *host, + uint32_t seq, unsigned int minor_version) +{ + string_t *str; + + str = t_str_new(128); + str_printfa(str, "SYNC\t%s\t%u\t%u", + net_ip2addr(&host->ip), host->port, seq); + if (minor_version > 0 && + director_connection_get_minor_version(dir->right) > 0) { + /* only minor_version>0 supports this parameter */ + str_printfa(str, "\t%u", minor_version); + } + str_append_c(str, '\n'); + director_connection_send(dir->right, str_c(str)); + + /* ping our connections in case either of them are hanging. + if they are, we want to know it fast. */ + if (dir->left != NULL) + director_connection_ping(dir->left); + if (dir->right != NULL) + director_connection_ping(dir->right); +} + bool director_resend_sync(struct director *dir) { if (!dir->ring_synced && dir->left != NULL && dir->right != NULL) { /* send a new SYNC in case the previous one got dropped */ - director_connection_send(dir->right, - t_strdup_printf("SYNC\t%s\t%u\t%u\n", - net_ip2addr(&dir->self_ip), - dir->self_port, dir->sync_seq)); + director_sync_send(dir, dir->self_host, dir->sync_seq, + DIRECTOR_VERSION_MINOR); if (dir->to_sync != NULL) timeout_reset(dir->to_sync); return TRUE; @@ -272,6 +361,10 @@ static void director_sync(struct director *dir) { + /* we're synced again when we receive this SYNC back */ + dir->sync_seq++; + director_set_ring_unsynced(dir); + if (dir->sync_frozen) { dir->sync_pending = TRUE; return; @@ -282,38 +375,38 @@ return; } - /* we're synced again when we receive this SYNC back */ - dir->sync_seq++; - director_set_ring_unsynced(dir); - if (dir->debug) { i_debug("Ring is desynced (seq=%u, sending SYNC to %s)", dir->sync_seq, dir->right == NULL ? "(nowhere)" : director_connection_get_name(dir->right)); } + /* send PINGs to our connections more rapidly until we've synced again. + if the connection has actually died, we don't need to wait (and + delay requests) for as long to detect it */ if (dir->left != NULL) - director_connection_wait_sync(dir->left); - director_connection_wait_sync(dir->right); - director_connection_send(dir->right, t_strdup_printf( - "SYNC\t%s\t%u\t%u\n", net_ip2addr(&dir->self_ip), - dir->self_port, dir->sync_seq)); + director_connection_set_synced(dir->left, FALSE); + director_connection_set_synced(dir->right, FALSE); + director_sync_send(dir, dir->self_host, dir->sync_seq, + DIRECTOR_VERSION_MINOR); } void director_sync_freeze(struct director *dir) { + struct director_connection *const *connp; + i_assert(!dir->sync_frozen); i_assert(!dir->sync_pending); - if (dir->left != NULL) - director_connection_cork(dir->left); - if (dir->right != NULL) - director_connection_cork(dir->right); + array_foreach(&dir->connections, connp) + director_connection_cork(*connp); dir->sync_frozen = TRUE; } void director_sync_thaw(struct director *dir) { + struct director_connection *const *connp; + i_assert(dir->sync_frozen); dir->sync_frozen = FALSE; @@ -321,10 +414,81 @@ dir->sync_pending = FALSE; director_sync(dir); } - if (dir->left != NULL) - director_connection_uncork(dir->left); - if (dir->right != NULL) - director_connection_uncork(dir->right); + array_foreach(&dir->connections, connp) + director_connection_uncork(*connp); +} + +void director_notify_ring_added(struct director_host *added_host, + struct director_host *src) +{ + const char *cmd; + + cmd = t_strdup_printf("DIRECTOR\t%s\t%u\n", + net_ip2addr(&added_host->ip), added_host->port); + director_update_send(added_host->dir, src, cmd); +} + +static void director_delayed_dir_remove_timeout(struct director *dir) +{ + struct director_host *const *hosts, *host; + unsigned int i, count; + + timeout_remove(&dir->to_remove_dirs); + + hosts = array_get(&dir->dir_hosts, &count); + for (i = 0; i < count; ) { + if (hosts[i]->removed) { + host = hosts[i]; + director_host_free(&host); + hosts = array_get(&dir->dir_hosts, &count); + } else { + i++; + } + } +} + +void director_ring_remove(struct director_host *removed_host, + struct director_host *src) +{ + struct director *dir = removed_host->dir; + struct director_connection *const *conns, *conn; + unsigned int i, count; + const char *cmd; + + if (removed_host->self) { + /* others will just disconnect us */ + return; + } + + /* mark the host as removed and fully remove it later. this delay is + needed, because the removal may trigger director reconnections, + which may send the director back and we don't want to re-add it */ + removed_host->removed = TRUE; + if (dir->to_remove_dirs == NULL) { + dir->to_remove_dirs = + timeout_add(DIRECTOR_DELAYED_DIR_REMOVE_MSECS, + director_delayed_dir_remove_timeout, dir); + } + + /* disconnect any connections to the host */ + conns = array_get(&dir->connections, &count); + for (i = 0; i < count; ) { + conn = conns[i]; + if (director_connection_get_host(conn) != removed_host) + i++; + else { + director_connection_deinit(&conn); + conns = array_get(&dir->connections, &count); + } + } + if (dir->right == NULL) + director_connect(dir); + + cmd = t_strdup_printf("DIRECTOR-REMOVE\t%s\t%u\n", + net_ip2addr(&removed_host->ip), + removed_host->port); + director_update_send_version(dir, src, + DIRECTOR_VERSION_RING_REMOVE, cmd); } void director_update_host(struct director *dir, struct director_host *src, @@ -387,9 +551,29 @@ void director_update_user(struct director *dir, struct director_host *src, struct user *user) { + i_assert(src != NULL); + + i_assert(!user->weak); + director_update_send(dir, src, t_strdup_printf("USER\t%u\t%s\n", + user->username_hash, net_ip2addr(&user->host->ip))); +} + +void director_update_user_weak(struct director *dir, struct director_host *src, + struct director_host *orig_src, + struct user *user) +{ + i_assert(src != NULL); + i_assert(user->weak); + + if (orig_src == NULL) { + orig_src = dir->self_host; + orig_src->last_seq++; + } + director_update_send(dir, src, t_strdup_printf( - "USER\t%u\t%s\n", user->username_hash, - net_ip2addr(&user->host->ip))); + "USER-WEAK\t%s\t%u\t%u\t%u\t%s\n", + net_ip2addr(&orig_src->ip), orig_src->port, orig_src->last_seq, + user->username_hash, net_ip2addr(&user->host->ip))); } struct director_user_kill_finish_ctx { @@ -434,7 +618,7 @@ static void director_finish_user_kill(struct director *dir, struct user *user, bool self) { - if (dir->right == NULL || dir->right == dir->left) { + if (dir->right == NULL) { /* we're alone */ director_user_kill_finish_delayed(dir, user); } else if (self || @@ -603,12 +787,22 @@ void director_update_send(struct director *dir, struct director_host *src, const char *cmd) { + director_update_send_version(dir, src, 0, cmd); +} + +void director_update_send_version(struct director *dir, + struct director_host *src, + unsigned int min_version, const char *cmd) +{ + struct director_connection *const *connp; + i_assert(src != NULL); - if (dir->left != NULL) - director_connection_send_except(dir->left, src, cmd); - if (dir->right != NULL && dir->right != dir->left) - director_connection_send_except(dir->right, src, cmd); + array_foreach(&dir->connections, connp) { + if (director_connection_get_host(*connp) != src && + director_connection_get_minor_version(*connp) >= min_version) + director_connection_send(*connp, cmd); + } } struct director * @@ -625,21 +819,30 @@ dir->state_change_callback = callback; i_array_init(&dir->dir_hosts, 16); i_array_init(&dir->pending_requests, 16); - dir->users = user_directory_init(set->director_user_expire); + i_array_init(&dir->connections, 8); + dir->users = user_directory_init(set->director_user_expire, + set->director_username_hash); dir->mail_hosts = mail_hosts_init(); dir->ipc_proxy = ipc_client_init(DIRECTOR_IPC_PROXY_PATH); + dir->ring_min_version = DIRECTOR_VERSION_MINOR; return dir; } void director_deinit(struct director **_dir) { struct director *dir = *_dir; - struct director_host *const *hostp; + struct director_host *const *hostp, *host; + struct director_connection *conn, *const *connp; *_dir = NULL; - director_connections_deinit(dir); + while (array_count(&dir->connections) > 0) { + connp = array_idx(&dir->connections, 0); + conn = *connp; + director_connection_deinit(&conn); + } + user_directory_deinit(&dir->users); mail_hosts_deinit(&dir->mail_hosts); mail_hosts_deinit(&dir->orig_config_hosts); @@ -653,9 +856,15 @@ timeout_remove(&dir->to_request); if (dir->to_sync != NULL) timeout_remove(&dir->to_sync); - array_foreach(&dir->dir_hosts, hostp) - director_host_free(*hostp); + if (dir->to_remove_dirs != NULL) + timeout_remove(&dir->to_remove_dirs); + while (array_count(&dir->dir_hosts) > 0) { + hostp = array_idx(&dir->dir_hosts, 0); + host = *hostp; + director_host_free(&host); + } array_free(&dir->pending_requests); array_free(&dir->dir_hosts); + array_free(&dir->connections); i_free(dir); }
--- a/src/director/director.h Sat May 19 22:40:08 2012 +0300 +++ b/src/director/director.h Sun May 20 03:25:04 2012 +0300 @@ -4,6 +4,19 @@ #include "network.h" #include "director-settings.h" +#define DIRECTOR_VERSION_NAME "director" +#define DIRECTOR_VERSION_MAJOR 1 +#define DIRECTOR_VERSION_MINOR 2 + +/* weak users supported in protocol v1.1+ */ +#define DIRECTOR_VERSION_WEAK_USERS 1 +/* director removes supported in v1.2+ */ +#define DIRECTOR_VERSION_RING_REMOVE 2 + +/* Minimum time between even attempting to communicate with a director that + failed due to a protocol error. */ +#define DIRECTOR_PROTOCOL_FAILURE_RETRY_SECS 60 + struct director; struct mail_host; struct user; @@ -20,9 +33,13 @@ unsigned int test_port; struct director_host *self_host; + /* left and right connections are set only after they have finished + handshaking. until then they're in the connections list, although + updates are still sent to them during handshaking if the USER list + is long. */ struct director_connection *left, *right; /* all director connections */ - struct director_connection *connections; + ARRAY_DEFINE(connections, struct director_connection *); struct timeout *to_reconnect; struct timeout *to_sync; @@ -43,11 +60,16 @@ /* director hosts are sorted by IP (and port) */ ARRAY_DEFINE(dir_hosts, struct director_host *); + struct timeout *to_remove_dirs; struct ipc_client *ipc_proxy; unsigned int sync_seq; + /* the lowest minor version supported by the ring */ + unsigned int ring_min_version; time_t ring_last_sync_time; + time_t ring_first_alone; + /* director ring handshaking is complete. director can start serving clients. */ unsigned int ring_handshaked:1; @@ -76,8 +98,15 @@ void director_set_ring_synced(struct director *dir); void director_set_ring_unsynced(struct director *dir); void director_set_state_changed(struct director *dir); +void director_sync_send(struct director *dir, struct director_host *host, + uint32_t seq, unsigned int minor_version); bool director_resend_sync(struct director *dir); +void director_notify_ring_added(struct director_host *added_host, + struct director_host *src); +void director_ring_remove(struct director_host *removed_host, + struct director_host *src); + void director_update_host(struct director *dir, struct director_host *src, struct director_host *orig_src, struct mail_host *host); @@ -89,6 +118,9 @@ struct mail_host *host); void director_update_user(struct director *dir, struct director_host *src, struct user *user); +void director_update_user_weak(struct director *dir, struct director_host *src, + struct director_host *orig_src, + struct user *user); void director_move_user(struct director *dir, struct director_host *src, struct director_host *orig_src, unsigned int username_hash, struct mail_host *host); @@ -97,6 +129,7 @@ struct director_host *src, struct director_host *orig_src, unsigned int username_hash); +void director_user_weak(struct director *dir, struct user *user); void director_sync_freeze(struct director *dir); void director_sync_thaw(struct director *dir); @@ -104,7 +137,10 @@ /* Send data to all directors using both left and right connections (unless they're the same). */ void director_update_send(struct director *dir, struct director_host *src, - const char *data); + const char *cmd); +void director_update_send_version(struct director *dir, + struct director_host *src, + unsigned int min_version, const char *cmd); int director_connect_host(struct director *dir, struct director_host *host);
--- a/src/director/doveadm-connection.c Sat May 19 22:40:08 2012 +0300 +++ b/src/director/doveadm-connection.c Sun May 20 03:25:04 2012 +0300 @@ -99,6 +99,7 @@ string_t *str = t_str_new(1024); const char *type; bool left, right; + time_t last_failed; array_foreach(&dir->dir_hosts, hostp) { const struct director_host *host = *hostp; @@ -108,7 +109,9 @@ right = dir->right != NULL && director_connection_get_host(dir->right) == host; - if (dir->self_host == host) + if (host->removed) + type = "removed"; + else if (dir->self_host == host) type = "self"; else if (left) type = right ? "l+r" : "left"; @@ -116,15 +119,70 @@ type = "right"; else type = ""; + + last_failed = I_MAX(host->last_network_failure, + host->last_protocol_failure); str_printfa(str, "%s\t%u\t%s\t%lu\n", net_ip2addr(&host->ip), host->port, type, - (unsigned long)host->last_failed); + (unsigned long)last_failed); } str_append_c(str, '\n'); o_stream_send(conn->output, str_data(str), str_len(str)); } static bool +doveadm_cmd_director_add(struct doveadm_connection *conn, const char *line) +{ + const char *const *args; + struct director_host *host; + struct ip_addr ip; + unsigned int port = conn->dir->self_port; + + args = t_strsplit_tab(line); + if (args[0] == NULL || + net_addr2ip(line, &ip) < 0 || + (args[1] != NULL && str_to_uint(args[1], &port) < 0)) { + i_error("doveadm sent invalid DIRECTOR-ADD parameters"); + return FALSE; + } + + if (director_host_lookup(conn->dir, &ip, port) == NULL) { + host = director_host_add(conn->dir, &ip, port); + director_notify_ring_added(host, conn->dir->self_host); + } + o_stream_send(conn->output, "OK\n", 3); + return TRUE; +} + +static bool +doveadm_cmd_director_remove(struct doveadm_connection *conn, const char *line) +{ + const char *const *args; + struct director_host *host; + struct ip_addr ip; + unsigned int port = 0; + + args = t_strsplit_tab(line); + if (args[0] == NULL || + net_addr2ip(line, &ip) < 0 || + (args[1] != NULL && str_to_uint(args[1], &port) < 0)) { + i_error("doveadm sent invalid DIRECTOR-REMOVE parameters"); + return FALSE; + } + + host = port != 0 ? + director_host_lookup(conn->dir, &ip, port) : + director_host_lookup_ip(conn->dir, &ip); + if (host == NULL) + o_stream_send_str(conn->output, "NOTFOUND\n"); + else { + director_ring_remove(host, conn->dir->self_host); + o_stream_send(conn->output, "OK\n", 3); + } + return TRUE; +} + +static bool doveadm_cmd_host_set(struct doveadm_connection *conn, const char *line) { struct director *dir = conn->dir; @@ -133,7 +191,7 @@ struct ip_addr ip; unsigned int vhost_count = -1U; - args = t_strsplit(line, "\t"); + args = t_strsplit_tab(line); if (args[0] == NULL || net_addr2ip(args[0], &ip) < 0 || (args[1] != NULL && str_to_uint(args[1], &vhost_count) < 0)) { @@ -223,7 +281,7 @@ string_t *str = t_str_new(256); if (str_to_uint(line, &username_hash) < 0) - username_hash = user_directory_get_username_hash(line); + username_hash = user_directory_get_username_hash(conn->dir->users, line); /* get user's current host */ user = user_directory_lookup(conn->dir->users, username_hash); @@ -296,7 +354,7 @@ struct mail_host *host; struct ip_addr ip; - args = t_strsplit(line, "\t"); + args = t_strsplit_tab(line); if (args[0] == NULL || args[1] == NULL || net_addr2ip(args[1], &ip) < 0) { i_error("doveadm sent invalid USER-MOVE parameters: %s", line); @@ -309,7 +367,7 @@ } if (str_to_uint(args[0], &username_hash) < 0) - username_hash = user_directory_get_username_hash(line); + username_hash = user_directory_get_username_hash(conn->dir->users, line); user = user_directory_lookup(conn->dir->users, username_hash); if (user != NULL && user->kill_state != USER_KILL_STATE_NONE) { o_stream_send_str(conn->output, "TRYAGAIN\n"); @@ -360,6 +418,10 @@ doveadm_cmd_host_list_removed(conn); else if (strcmp(cmd, "DIRECTOR-LIST") == 0) doveadm_cmd_director_list(conn); + else if (strcmp(cmd, "DIRECTOR-ADD") == 0) + doveadm_cmd_director_add(conn, args); + else if (strcmp(cmd, "DIRECTOR-REMOVE") == 0) + doveadm_cmd_director_remove(conn, args); else if (strcmp(cmd, "HOST-SET") == 0) ret = doveadm_cmd_host_set(conn, args); else if (strcmp(cmd, "HOST-REMOVE") == 0)
--- a/src/director/login-connection.c Sat May 19 22:40:08 2012 +0300 +++ b/src/director/login-connection.c Sun May 20 03:25:04 2012 +0300 @@ -121,7 +121,7 @@ } /* OK <id> [<parameters>] */ - args = t_strsplit(line_params, "\t"); + args = t_strsplit_tab(line_params); if (*args != NULL) { /* we should always get here, but in case we don't just forward as-is and let login process handle the error. */
--- a/src/director/main.c Sat May 19 22:40:08 2012 +0300 +++ b/src/director/main.c Sun May 20 03:25:04 2012 +0300 @@ -28,7 +28,10 @@ static int director_client_connected(int fd, const struct ip_addr *ip) { - if (director_host_lookup_ip(director, ip) == NULL) { + struct director_host *host; + + host = director_host_lookup_ip(director, ip); + if (host == NULL || host->removed) { i_warning("Connection from %s: Server not listed in " "director_servers, dropping", net_ip2addr(ip)); return -1; @@ -125,7 +128,8 @@ array_foreach(&dir->pending_requests, requestp) { ret = director_request_continue(*requestp); if (!ret) { - /* request for a user being killed */ + /* a) request for a user being killed + b) user is weak */ array_append(&new_requests, requestp, 1); } }
--- a/src/director/notify-connection.c Sat May 19 22:40:08 2012 +0300 +++ b/src/director/notify-connection.c Sun May 20 03:25:04 2012 +0300 @@ -22,11 +22,18 @@ struct user *user; const char *line; unsigned int hash; + int diff; while ((line = i_stream_read_next_line(conn->input)) != NULL) { - hash = user_directory_get_username_hash(line); + hash = user_directory_get_username_hash(conn->dir->users, line); user = user_directory_lookup(conn->dir->users, hash); if (user != NULL) { + diff = ioloop_time - user->timestamp; + if (diff >= (int)conn->dir->set->director_user_expire) { + i_warning("notify: User %s refreshed too late " + "(%d secs)", line, diff); + } + user->weak = FALSE; user_directory_refresh(conn->dir->users, user); director_update_user(conn->dir, conn->dir->self_host, user);
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/director/test-user-directory.c Sun May 20 03:25:04 2012 +0300 @@ -0,0 +1,104 @@ +/* Copyright (c) 2012 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "ioloop.h" +#include "mail-user-hash.h" +#include "mail-host.h" +#include "user-directory.h" +#include "test-common.h" + +#include <stdlib.h> + +#define USER_DIR_TIMEOUT 1000000 + +unsigned int mail_user_hash(const char *username ATTR_UNUSED, + const char *format ATTR_UNUSED) { return 0; } + +static void +verify_user_directory(struct user_directory *dir, unsigned int user_count) +{ + struct user_directory_iter *iter; + struct user *user, *prev = NULL; + unsigned int prev_stamp = 0, iter_count = 0; + + iter = user_directory_iter_init(dir); + while ((user = user_directory_iter_next(iter)) != NULL) { + test_assert(prev_stamp <= user->timestamp); + test_assert(user->prev == prev); + test_assert(prev == NULL || user->prev->next == user); + + iter_count++; + prev = user; + } + test_assert(prev == NULL || prev->next == NULL); + user_directory_iter_deinit(&iter); + test_assert(iter_count == user_count); +} + +static void test_user_directory_ascending(void) +{ + const unsigned int count = 100000; + struct user_directory *dir; + struct mail_host *host = t_new(struct mail_host, 1); + unsigned int i; + + test_begin("user directory ascending"); + dir = user_directory_init(USER_DIR_TIMEOUT, "%u"); + user_directory_add(dir, 1, host, ioloop_time + count+1); + + for (i = 0; i < count; i++) + user_directory_add(dir, i+2, host, ioloop_time + i); + verify_user_directory(dir, count+1); + user_directory_deinit(&dir); + test_end(); +} + +static void test_user_directory_descending(void) +{ + const unsigned int count = 1000; + struct user_directory *dir; + struct mail_host *host = t_new(struct mail_host, 1); + unsigned int i; + + test_begin("user directory descending"); + dir = user_directory_init(USER_DIR_TIMEOUT, "%u"); + + for (i = 0; i < count; i++) + user_directory_add(dir, i+1, host, ioloop_time - i); + verify_user_directory(dir, count); + user_directory_deinit(&dir); + test_end(); +} + +static void test_user_directory_random(void) +{ + struct user_directory *dir; + struct mail_host *host = t_new(struct mail_host, 1); + time_t timestamp; + unsigned int i, count = 10000 + rand()%10000; + + test_begin("user directory random"); + dir = user_directory_init(USER_DIR_TIMEOUT, "%u"); + for (i = 0; i < count; i++) { + if (rand() % 10 == 0) + timestamp = ioloop_time; + else + timestamp = ioloop_time-rand()%100; + user_directory_add(dir, i+1, host, timestamp); + } + verify_user_directory(dir, count); + user_directory_deinit(&dir); + test_end(); +} + +int main(void) +{ + static void (*test_functions[])(void) = { + test_user_directory_ascending, + test_user_directory_descending, + test_user_directory_random, + NULL + }; + ioloop_time = 1234567890; + return test_run(test_functions); +}
--- a/src/director/user-directory.c Sat May 19 22:40:08 2012 +0300 +++ b/src/director/user-directory.c Sun May 20 03:25:04 2012 +0300 @@ -3,13 +3,16 @@ #include "lib.h" #include "ioloop.h" #include "array.h" -#include "md5.h" #include "hash.h" #include "llist.h" +#include "mail-user-hash.h" #include "mail-host.h" #include "user-directory.h" -#define MAX_CLOCK_DRIFT_SECS 2 +/* n% of timeout_secs */ +#define USER_NEAR_EXPIRING_PERCENTAGE 10 +/* but max. of this many secs */ +#define USER_NEAR_EXPIRING_MAX 30 struct user_directory_iter { struct user_directory *dir; @@ -21,10 +24,15 @@ struct hash_table *hash; /* sorted by time */ struct user *head, *tail; + struct user *prev_insert_pos; ARRAY_DEFINE(iters, struct user_directory_iter *); + char *username_hash_fmt; unsigned int timeout_secs; + /* If user's expire time is less than this many seconds away, + don't assume that other directors haven't yet expired it */ + unsigned int user_near_expiring_secs; }; static void user_move_iters(struct user_directory *dir, struct user *user) @@ -35,6 +43,9 @@ if ((*iterp)->pos == user) (*iterp)->pos = user->next; } + + if (dir->prev_insert_pos == user) + dir->prev_insert_pos = user->next; } static void user_free(struct user_directory *dir, struct user *user) @@ -49,6 +60,14 @@ i_free(user); } +static bool user_directory_user_has_connections(struct user_directory *dir, + struct user *user) +{ + time_t expire_timestamp = user->timestamp + dir->timeout_secs; + + return expire_timestamp >= ioloop_time; +} + static void user_directory_drop_expired(struct user_directory *dir) { while (dir->head != NULL && @@ -64,11 +83,57 @@ return hash_table_lookup(dir->hash, POINTER_CAST(username_hash)); } +static void +user_directory_insert_backwards(struct user_directory *dir, + struct user *pos, struct user *user) +{ + for (; pos != NULL; pos = pos->prev) { + if ((time_t)pos->timestamp <= user->timestamp) + break; + } + if (pos == NULL) + DLLIST2_PREPEND(&dir->head, &dir->tail, user); + else { + user->prev = pos; + user->next = pos->next; + user->prev->next = user; + if (user->next != NULL) + user->next->prev = user; + else + dir->tail = user; + } +} + +static void +user_directory_insert_forwards(struct user_directory *dir, + struct user *pos, struct user *user) +{ + for (; pos != NULL; pos = pos->next) { + if ((time_t)pos->timestamp >= user->timestamp) + break; + } + if (pos == NULL) + DLLIST2_APPEND(&dir->head, &dir->tail, user); + else { + user->prev = pos->prev; + user->next = pos; + if (user->prev != NULL) + user->prev->next = user; + else + dir->head = user; + user->next->prev = user; + } +} + struct user * user_directory_add(struct user_directory *dir, unsigned int username_hash, struct mail_host *host, time_t timestamp) { - struct user *user, *pos; + struct user *user; + + /* make sure we don't add timestamps higher than ioloop time */ + if (timestamp > ioloop_time) + timestamp = ioloop_time; user = i_new(struct user, 1); user->username_hash = username_hash; @@ -79,21 +144,24 @@ if (dir->tail == NULL || (time_t)dir->tail->timestamp <= timestamp) DLLIST2_APPEND(&dir->head, &dir->tail, user); else { - /* need to insert to correct position */ - for (pos = dir->tail; pos != NULL; pos = pos->prev) { - if ((time_t)pos->timestamp <= timestamp) - break; - } - if (pos == NULL) - DLLIST2_PREPEND(&dir->head, &dir->tail, user); - else { - user->prev = pos; - user->next = pos->next; - user->prev->next = user; - user->next->prev = user; + /* need to insert to correct position. we should get here + only when handshaking. the handshaking USER requests should + come sorted by timestamp. so keep track of the previous + insert position, the next USER should be inserted after + it. */ + if (dir->prev_insert_pos == NULL) { + /* find the position starting from tail */ + user_directory_insert_backwards(dir, dir->tail, user); + } else if (timestamp < dir->prev_insert_pos->timestamp) { + user_directory_insert_backwards(dir, dir->prev_insert_pos, + user); + } else { + user_directory_insert_forwards(dir, dir->prev_insert_pos, + user); } } + dir->prev_insert_pos = user; hash_table_insert(dir->hash, POINTER_CAST(user->username_hash), user); return user; } @@ -120,33 +188,43 @@ } } -unsigned int user_directory_get_username_hash(const char *username) +unsigned int user_directory_get_username_hash(struct user_directory *dir, + const char *username) { - /* NOTE: If you modify this, modify also - director_username_hash() in login-common/login-proxy.c */ - unsigned char md5[MD5_RESULTLEN]; - unsigned int i, hash = 0; + return mail_user_hash(username, dir->username_hash_fmt); +} - md5_get_digest(username, strlen(username), md5); - for (i = 0; i < sizeof(hash); i++) - hash = (hash << CHAR_BIT) | md5[i]; - return hash; +bool user_directory_user_is_recently_updated(struct user_directory *dir, + struct user *user) +{ + return (time_t)(user->timestamp + dir->timeout_secs/2) >= ioloop_time; } -bool user_directory_user_has_connections(struct user_directory *dir, - struct user *user) +bool user_directory_user_is_near_expiring(struct user_directory *dir, + struct user *user) { - time_t expire_timestamp = user->timestamp + dir->timeout_secs; + time_t expire_timestamp; - return expire_timestamp - MAX_CLOCK_DRIFT_SECS >= ioloop_time; + expire_timestamp = user->timestamp + + (dir->timeout_secs - dir->user_near_expiring_secs); + return expire_timestamp < ioloop_time; } -struct user_directory *user_directory_init(unsigned int timeout_secs) +struct user_directory * +user_directory_init(unsigned int timeout_secs, const char *username_hash_fmt) { struct user_directory *dir; dir = i_new(struct user_directory, 1); dir->timeout_secs = timeout_secs; + dir->user_near_expiring_secs = + timeout_secs * USER_NEAR_EXPIRING_PERCENTAGE / 100; + dir->user_near_expiring_secs = + I_MIN(dir->user_near_expiring_secs, USER_NEAR_EXPIRING_MAX); + dir->user_near_expiring_secs = + I_MAX(dir->user_near_expiring_secs, 1); + + dir->username_hash_fmt = i_strdup(username_hash_fmt); dir->hash = hash_table_create(default_pool, default_pool, 0, NULL, NULL); i_array_init(&dir->iters, 8); @@ -165,6 +243,7 @@ user_free(dir, dir->head); hash_table_destroy(&dir->hash); array_free(&dir->iters); + i_free(dir->username_hash_fmt); i_free(dir); } @@ -177,6 +256,7 @@ iter->dir = dir; iter->pos = dir->head; array_append(&dir->iters, &iter, 1); + user_directory_drop_expired(dir); return iter; }
--- a/src/director/user-directory.h Sat May 19 22:40:08 2012 +0300 +++ b/src/director/user-directory.h Sun May 20 03:25:04 2012 +0300 @@ -38,11 +38,17 @@ /* If not USER_KILL_STATE_NONE, don't allow new connections until all directors have killed the user's connections. */ enum user_kill_state kill_state; + + /* TRUE, if the user's timestamp was close to being expired and we're + now doing a ring-wide sync for this user to make sure we don't + assign conflicting hosts to it */ + unsigned int weak:1; }; /* Create a new directory. Users are dropped if their time gets older than timeout_secs. */ -struct user_directory *user_directory_init(unsigned int timeout_secs); +struct user_directory * +user_directory_init(unsigned int timeout_secs, const char *username_hash_fmt); void user_directory_deinit(struct user_directory **dir); /* Look up username from directory. Returns NULL if not found. */ @@ -59,11 +65,13 @@ void user_directory_remove_host(struct user_directory *dir, struct mail_host *host); -unsigned int user_directory_get_username_hash(const char *username); +unsigned int user_directory_get_username_hash(struct user_directory *dir, + const char *username); -/* Returns TRUE if user still potentially has connections. */ -bool user_directory_user_has_connections(struct user_directory *dir, - struct user *user); +bool user_directory_user_is_recently_updated(struct user_directory *dir, + struct user *user); +bool user_directory_user_is_near_expiring(struct user_directory *dir, + struct user *user); struct user_directory_iter * user_directory_iter_init(struct user_directory *dir);
--- a/src/doveadm/Makefile.am Sat May 19 22:40:08 2012 +0300 +++ b/src/doveadm/Makefile.am Sun May 20 03:25:04 2012 +0300 @@ -39,26 +39,29 @@ libs = \ dsync/libdsync.a \ - $(LIBDOVECOT_STORAGE) \ $(unused_objects) doveadm_LDADD = \ $(libs) \ $(cmd_pw_libs) \ $(CRYPT_LIBS) \ + $(LIBDOVECOT_STORAGE) \ $(LIBDOVECOT) \ $(MODULE_LIBS) doveadm_DEPENDENCIES = \ $(libs) \ $(cmd_pw_libs) \ + $(LIBDOVECOT_STORAGE_DEPS) \ $(LIBDOVECOT_DEPS) doveadm_server_LDADD = \ $(libs) \ + $(LIBDOVECOT_STORAGE) \ $(LIBDOVECOT) \ $(MODULE_LIBS) doveadm_server_DEPENDENCIES = \ $(libs) \ + $(LIBDOVECOT_STORAGE_DEPS) \ $(LIBDOVECOT_DEPS) common = \
--- a/src/doveadm/client-connection.c Sat May 19 22:40:08 2012 +0300 +++ b/src/doveadm/client-connection.c Sun May 20 03:25:04 2012 +0300 @@ -35,22 +35,22 @@ unsigned int authenticated:1; }; -static bool -doveadm_mail_cmd_server(const char *cmd_name, - const struct doveadm_settings *set, - const struct mail_storage_service_input *input, - int argc, char *argv[]) +static struct doveadm_mail_cmd_context * +doveadm_mail_cmd_server_parse(const char *cmd_name, + const struct doveadm_settings *set, + const struct mail_storage_service_input *input, + int argc, char *argv[]) { struct doveadm_mail_cmd_context *ctx; const struct doveadm_mail_cmd *cmd; const char *getopt_args; - bool ret, add_username_header = FALSE; + bool add_username_header = FALSE; int c; cmd = doveadm_mail_cmd_find(cmd_name); if (cmd == NULL) { i_error("doveadm: Client sent unknown command: %s", cmd_name); - return FALSE; + return NULL; } ctx = doveadm_mail_cmd_init(cmd, set); @@ -83,7 +83,7 @@ "Client sent unknown parameter: %c", cmd->name, c); ctx->v.deinit(ctx); - return FALSE; + return NULL; } } } @@ -95,8 +95,9 @@ i_error("doveadm %s: Client sent unknown parameter: %s", cmd->name, argv[0]); ctx->v.deinit(ctx); - return FALSE; + return NULL; } + ctx->args = (const void *)argv; if (doveadm_print_is_initialized() && add_username_header) { doveadm_print_header("username", "Username", @@ -104,20 +105,38 @@ DOVEADM_PRINT_HEADER_FLAG_HIDE_TITLE); doveadm_print_sticky("username", input->username); } + return ctx; +} - ctx->args = (const void *)argv; +static void +doveadm_mail_cmd_server_run(struct client_connection *conn, + struct doveadm_mail_cmd_context *ctx, + const struct mail_storage_service_input *input) +{ + const char *error; + int ret; + if (ctx->v.preinit != NULL) ctx->v.preinit(ctx); - doveadm_mail_single_user(ctx, input); + ret = doveadm_mail_single_user(ctx, input, &error); doveadm_mail_server_flush(); ctx->v.deinit(ctx); doveadm_print_flush(); mail_storage_service_deinit(&ctx->storage_service); - ret = ctx->exit_code == 0; + + if (ret < 0) { + i_error("%s: %s", ctx->cmd->name, error); + o_stream_send(conn->output, "\n-\n", 3); + } else if (ret == 0) { + o_stream_send_str(conn->output, "\n-NOUSER\n"); + } else if (ctx->exit_code != 0) { + /* maybe not an error, but not a full success either */ + o_stream_send(conn->output, "\n-\n", 3); + } else { + o_stream_send(conn->output, "\n+\n", 3); + } pool_unref(&ctx->pool); - - return ret; } static bool client_is_allowed_command(const struct doveadm_settings *set, @@ -144,9 +163,9 @@ static bool client_handle_command(struct client_connection *conn, char **args) { struct mail_storage_service_input input; + struct doveadm_mail_cmd_context *ctx; const char *flags, *cmd_name; unsigned int argc; - bool ret; memset(&input, 0, sizeof(input)); input.service = "doveadm"; @@ -190,11 +209,11 @@ } o_stream_cork(conn->output); - ret = doveadm_mail_cmd_server(cmd_name, conn->set, &input, argc, args); - if (ret) - o_stream_send(conn->output, "\n+\n", 3); + ctx = doveadm_mail_cmd_server_parse(cmd_name, conn->set, &input, argc, args); + if (ctx == NULL) + o_stream_send(conn->output, "\n-\n", 3); else - o_stream_send(conn->output, "\n-\n", 3); + doveadm_mail_cmd_server_run(conn, ctx, &input); o_stream_uncork(conn->output); /* flush the output and disconnect */ @@ -345,7 +364,7 @@ fstat() always returns mode as 0777 */ if (net_getunixname(listen_fd, &listen_path) == 0 && stat(listen_path, &st) == 0 && S_ISSOCK(st.st_mode) && - (st.st_mode & 0777) == 0600 && st.st_uid == geteuid()) { + (st.st_mode & 0777) == 0600) { /* no need for client to authenticate */ conn->authenticated = TRUE; o_stream_send(conn->output, "+\n", 2);
--- a/src/doveadm/doveadm-auth.c Sat May 19 22:40:08 2012 +0300 +++ b/src/doveadm/doveadm-auth.c Sun May 20 03:25:04 2012 +0300 @@ -7,10 +7,14 @@ #include "base64.h" #include "str.h" #include "wildcard-match.h" +#include "master-service.h" #include "auth-client.h" #include "auth-master.h" #include "auth-server-connection.h" +#include "mail-storage-service.h" +#include "mail-user.h" #include "doveadm.h" +#include "doveadm-print.h" #include <stdio.h> #include <stdlib.h> @@ -245,19 +249,74 @@ doveadm_exit_code = EX_NOPERM; } +static void cmd_user_mail_input_field(const char *key, const char *value, + const char *show_field) +{ + if (show_field == NULL) { + doveadm_print(key); + doveadm_print(value); + } else if (strcmp(show_field, key) == 0) { + printf("%s\n", value); + } +} + +static int cmd_user_mail_input(struct mail_storage_service_ctx *storage_service, + const struct authtest_input *input, + const char *show_field) +{ + struct mail_storage_service_input service_input; + struct mail_storage_service_user *service_user; + struct mail_user *user; + const struct mail_storage_settings *mail_set; + const char *error; + int ret; + + memset(&service_input, 0, sizeof(service_input)); + service_input.module = "mail"; + service_input.service = input->info.service; + service_input.username = input->username; + service_input.local_ip = input->info.local_ip; + service_input.local_port = input->info.local_port; + service_input.remote_ip = input->info.remote_ip; + service_input.remote_port = input->info.remote_port; + + if ((ret = mail_storage_service_lookup_next(storage_service, &service_input, + &service_user, &user, + &error)) <= 0) + return ret == 0 ? 0 : -1; + + if (show_field == NULL) { + doveadm_print_init(DOVEADM_PRINT_TYPE_TAB); + doveadm_print_header_simple("field"); + doveadm_print_header_simple("value"); + } + + cmd_user_mail_input_field("uid", user->set->mail_uid, show_field); + cmd_user_mail_input_field("gid", user->set->mail_gid, show_field); + cmd_user_mail_input_field("home", user->set->mail_home, show_field); + + mail_set = mail_user_set_get_storage_set(user); + cmd_user_mail_input_field("mail", mail_set->mail_location, show_field); + + mail_user_unref(&user); + mail_storage_service_user_free(&service_user); + return 1; +} + static void cmd_user(int argc, char *argv[]) { const char *auth_socket_path = NULL; struct authtest_input input; const char *show_field = NULL; + struct mail_storage_service_ctx *storage_service = NULL; unsigned int i; - bool have_wildcards; - int c; + bool have_wildcards, mail_fields = FALSE, first = TRUE; + int c, ret; memset(&input, 0, sizeof(input)); input.info.service = "doveadm"; - while ((c = getopt(argc, argv, "a:f:x:")) > 0) { + while ((c = getopt(argc, argv, "a:f:mx:")) > 0) { switch (c) { case 'a': auth_socket_path = optarg; @@ -265,6 +324,9 @@ case 'f': show_field = optarg; break; + case 'm': + mail_fields = TRUE; + break; case 'x': auth_user_info_parse(&input.info, optarg); break; @@ -285,27 +347,37 @@ } } - if (have_wildcards) + if (have_wildcards) { cmd_user_list(auth_socket_path, &input, argv + optind); - else { - bool first = TRUE; + return; + } + + if (mail_fields) { + storage_service = mail_storage_service_init(master_service, NULL, + MAIL_STORAGE_SERVICE_FLAG_NO_LOG_INIT | + MAIL_STORAGE_SERVICE_FLAG_USERDB_LOOKUP); + } - while ((input.username = argv[optind++]) != NULL) { - if (first) - first = FALSE; - else - putchar('\n'); - switch (cmd_user_input(auth_socket_path, &input, - show_field)) { - case -1: - doveadm_exit_code = EX_TEMPFAIL; - break; - case 0: - doveadm_exit_code = EX_NOUSER; - break; - } + while ((input.username = argv[optind++]) != NULL) { + if (first) + first = FALSE; + else + putchar('\n'); + + ret = mail_fields ? + cmd_user_mail_input(storage_service, &input, show_field) : + cmd_user_input(auth_socket_path, &input, show_field); + switch (ret) { + case -1: + doveadm_exit_code = EX_TEMPFAIL; + break; + case 0: + doveadm_exit_code = EX_NOUSER; + break; } } + if (storage_service != NULL) + mail_storage_service_deinit(&storage_service); } struct doveadm_cmd doveadm_cmd_auth = { @@ -315,5 +387,5 @@ struct doveadm_cmd doveadm_cmd_user = { cmd_user, "user", - "[-a <userdb socket path>] [-x <auth info>] [-f field] <user mask> [...]" + "[-a <userdb socket path>] [-x <auth info>] [-f field] [-m] <user mask> [...]" };
--- a/src/doveadm/doveadm-director.c Sat May 19 22:40:08 2012 +0300 +++ b/src/doveadm/doveadm-director.c Sun May 20 03:25:04 2012 +0300 @@ -3,6 +3,7 @@ #include "lib.h" #include "md5.h" #include "hash.h" +#include "str.h" #include "network.h" #include "istream.h" #include "write-full.h" @@ -120,7 +121,7 @@ return; } - args = t_strsplit(line, "\t"); + args = t_strsplit_tab(line); if (str_array_length(args) != 4 || str_to_uint(args[1], &expires) < 0) { i_error("Invalid reply from director"); @@ -162,7 +163,7 @@ if (*line == '\0') break; T_BEGIN { - args = t_strsplit(line, "\t"); + args = t_strsplit_tab(line); if (str_array_length(args) >= 3) { doveadm_print(args[0]); doveadm_print(args[1]); @@ -315,7 +316,7 @@ if (*line == '\0') break; T_BEGIN { - args = t_strsplit(line, "\t"); + args = t_strsplit_tab(line); if (str_array_length(args) < 3 || str_to_uint(args[0], &user_hash) < 0 || str_to_uint(args[1], &expires) < 0 || @@ -559,7 +560,7 @@ if (*line == '\0') break; T_BEGIN { - args = t_strsplit(line, "\t"); + args = t_strsplit_tab(line); if (str_array_length(args) >= 2) { director_dump_cmd(ctx, "add", "%s %s", args[0], args[1]); @@ -580,6 +581,68 @@ director_disconnect(ctx); } + +static void director_read_ok_reply(struct director_context *ctx) +{ + const char *line; + + line = i_stream_read_next_line(ctx->input); + if (line == NULL) { + i_error("Director disconnected unexpectedly"); + doveadm_exit_code = EX_TEMPFAIL; + } else if (strcmp(line, "NOTFOUND") == 0) { + i_error("Not found"); + doveadm_exit_code = DOVEADM_EX_NOTFOUND; + } else if (strcmp(line, "OK") != 0) { + i_error("Failed: %s", line); + doveadm_exit_code = EX_TEMPFAIL; + } +} + +static void cmd_director_ring_add(int argc, char *argv[]) +{ + struct director_context *ctx; + struct ip_addr ip; + string_t *str = t_str_new(64); + unsigned int port = 0; + + ctx = cmd_director_init(argc, argv, "a:", cmd_director_ring_add); + if (argv[optind] == NULL || + net_addr2ip(argv[optind], &ip) < 0 || + (argv[optind+1] != NULL && str_to_uint(argv[optind+1], &port) < 0)) + director_cmd_help(cmd_director_ring_add); + + str_printfa(str, "DIRECTOR-ADD\t%s", net_ip2addr(&ip)); + if (port != 0) + str_printfa(str, "\t%u", port); + str_append_c(str, '\n'); + director_send(ctx, str_c(str)); + director_read_ok_reply(ctx); + director_disconnect(ctx); +} + +static void cmd_director_ring_remove(int argc, char *argv[]) +{ + struct director_context *ctx; + struct ip_addr ip; + string_t *str = t_str_new(64); + unsigned int port = 0; + + ctx = cmd_director_init(argc, argv, "a:", cmd_director_ring_remove); + if (argv[optind] == NULL || + net_addr2ip(argv[optind], &ip) < 0 || + (argv[optind+1] != NULL && str_to_uint(argv[optind+1], &port) < 0)) + director_cmd_help(cmd_director_ring_remove); + + str_printfa(str, "DIRECTOR-REMOVE\t%s", net_ip2addr(&ip)); + if (port != 0) + str_printfa(str, "\t%u", port); + str_append_c(str, '\n'); + director_send(ctx, str_c(str)); + director_read_ok_reply(ctx); + director_disconnect(ctx); +} + static void cmd_director_ring_status(int argc, char *argv[]) { struct director_context *ctx; @@ -599,7 +662,7 @@ if (*line == '\0') break; T_BEGIN { - args = t_strsplit(line, "\t"); + args = t_strsplit_tab(line); if (str_array_length(args) >= 4 && str_to_ulong(args[3], &l) == 0) { doveadm_print(args[0]); @@ -634,6 +697,10 @@ "[-a <director socket path>] <host>|all" }, { cmd_director_dump, "director dump", "[-a <director socket path>]" }, + { cmd_director_ring_add, "director ring add", + "[-a <director socket path>] <ip> [<port>]" }, + { cmd_director_ring_remove, "director ring remove", + "[-a <director socket path>] <ip> [<port>]" }, { cmd_director_ring_status, "director ring status", "[-a <director socket path>]" } };
--- a/src/doveadm/doveadm-mail-expunge.c Sat May 19 22:40:08 2012 +0300 +++ b/src/doveadm/doveadm-mail-expunge.c Sun May 20 03:25:04 2012 +0300 @@ -9,16 +9,25 @@ #include "doveadm-mail-iter.h" #include "doveadm-mail.h" +struct expunge_cmd_context { + struct doveadm_mail_cmd_context ctx; + bool delete_empty_mailbox; +}; + static int -cmd_expunge_box(struct doveadm_mail_cmd_context *ctx, +cmd_expunge_box(struct doveadm_mail_cmd_context *_ctx, const struct mailbox_info *info, struct mail_search_args *search_args) { + struct expunge_cmd_context *ctx = (struct expunge_cmd_context *)_ctx; struct doveadm_mail_iter *iter; + struct mailbox *box; struct mailbox_transaction_context *trans; struct mail *mail; + enum mail_error error; + int ret = 0; - if (doveadm_mail_iter_init(ctx, info, search_args, 0, NULL, + if (doveadm_mail_iter_init(_ctx, info, search_args, 0, NULL, &trans, &iter) < 0) return -1; @@ -29,7 +38,30 @@ } mail_expunge(mail); } - return doveadm_mail_iter_deinit_sync(&iter); + + if (doveadm_mail_iter_deinit_keep_box(&iter, &box) < 0) + ret = -1; + else if (mailbox_sync(box, 0) < 0) { + doveadm_mail_failed_mailbox(_ctx, box); + ret = -1; + } + + if (ctx->delete_empty_mailbox && ret == 0) { + if (mailbox_delete_empty(box) < 0) { + (void)mailbox_get_last_error(box, &error); + if (error != MAIL_ERROR_EXISTS) { + doveadm_mail_failed_mailbox(_ctx, box); + ret = -1; + } + } else { + if (mailbox_set_subscribed(box, FALSE) < 0) { + doveadm_mail_failed_mailbox(_ctx, box); + ret = -1; + } + } + } + mailbox_free(&box); + return ret; } static bool @@ -208,16 +240,32 @@ expunge_search_args_check(ctx->search_args, "expunge"); } +static bool cmd_expunge_parse_arg(struct doveadm_mail_cmd_context *_ctx, int c) +{ + struct expunge_cmd_context *ctx = (struct expunge_cmd_context *)_ctx; + + switch (c) { + case 'd': + ctx->delete_empty_mailbox = TRUE; + break; + default: + return FALSE; + } + return TRUE; +} + static struct doveadm_mail_cmd_context *cmd_expunge_alloc(void) { - struct doveadm_mail_cmd_context *ctx; + struct expunge_cmd_context *ctx; - ctx = doveadm_mail_cmd_alloc(struct doveadm_mail_cmd_context); - ctx->v.init = cmd_expunge_init; - ctx->v.run = cmd_expunge_run; - return ctx; + ctx = doveadm_mail_cmd_alloc(struct expunge_cmd_context); + ctx->ctx.getopt_args = "d"; + ctx->ctx.v.parse_arg = cmd_expunge_parse_arg; + ctx->ctx.v.init = cmd_expunge_init; + ctx->ctx.v.run = cmd_expunge_run; + return &ctx->ctx; } struct doveadm_mail_cmd cmd_expunge = { - cmd_expunge_alloc, "expunge", "<search query>" + cmd_expunge_alloc, "expunge", "[-d] <search query>" };
--- a/src/doveadm/doveadm-mail-import.c Sat May 19 22:40:08 2012 +0300 +++ b/src/doveadm/doveadm-mail-import.c Sun May 20 03:25:04 2012 +0300 @@ -91,6 +91,7 @@ mailbox, src_mail->uid); } save_ctx = mailbox_save_alloc(dest_trans); + mailbox_save_copy_flags(save_ctx, src_mail); if (mailbox_copy(&save_ctx, src_mail) < 0) { i_error("Copying box=%s uid=%u failed: %s", mailbox, src_mail->uid,
--- a/src/doveadm/doveadm-mail-iter.c Sat May 19 22:40:08 2012 +0300 +++ b/src/doveadm/doveadm-mail-iter.c Sun May 20 03:25:04 2012 +0300 @@ -83,7 +83,7 @@ static int doveadm_mail_iter_deinit_full(struct doveadm_mail_iter **_iter, - bool sync, bool commit) + bool sync, bool commit, bool keep_box) { struct doveadm_mail_iter *iter = *_iter; int ret; @@ -95,24 +95,32 @@ ret = mailbox_sync(iter->box, 0); if (ret < 0) doveadm_mail_failed_mailbox(iter->ctx, iter->box); - mailbox_free(&iter->box); + if (!keep_box) + mailbox_free(&iter->box); i_free(iter); return ret; } int doveadm_mail_iter_deinit(struct doveadm_mail_iter **_iter) { - return doveadm_mail_iter_deinit_full(_iter, FALSE, TRUE); + return doveadm_mail_iter_deinit_full(_iter, FALSE, TRUE, FALSE); } int doveadm_mail_iter_deinit_sync(struct doveadm_mail_iter **_iter) { - return doveadm_mail_iter_deinit_full(_iter, TRUE, TRUE); + return doveadm_mail_iter_deinit_full(_iter, TRUE, TRUE, FALSE); +} + +int doveadm_mail_iter_deinit_keep_box(struct doveadm_mail_iter **iter, + struct mailbox **box_r) +{ + *box_r = (*iter)->box; + return doveadm_mail_iter_deinit_full(iter, FALSE, TRUE, TRUE); } void doveadm_mail_iter_deinit_rollback(struct doveadm_mail_iter **_iter) { - (void)doveadm_mail_iter_deinit_full(_iter, FALSE, FALSE); + (void)doveadm_mail_iter_deinit_full(_iter, FALSE, FALSE, FALSE); } bool doveadm_mail_iter_next(struct doveadm_mail_iter *iter,
--- a/src/doveadm/doveadm-mail-iter.h Sat May 19 22:40:08 2012 +0300 +++ b/src/doveadm/doveadm-mail-iter.h Sun May 20 03:25:04 2012 +0300 @@ -13,6 +13,8 @@ struct doveadm_mail_iter **iter_r); int doveadm_mail_iter_deinit(struct doveadm_mail_iter **iter); int doveadm_mail_iter_deinit_sync(struct doveadm_mail_iter **iter); +int doveadm_mail_iter_deinit_keep_box(struct doveadm_mail_iter **iter, + struct mailbox **box_r); void doveadm_mail_iter_deinit_rollback(struct doveadm_mail_iter **iter); bool doveadm_mail_iter_next(struct doveadm_mail_iter *iter,
--- a/src/doveadm/doveadm-mail-mailbox-status.c Sat May 19 22:40:08 2012 +0300 +++ b/src/doveadm/doveadm-mail-mailbox-status.c Sun May 20 03:25:04 2012 +0300 @@ -182,7 +182,7 @@ doveadm_mail_help_name("mailbox status"); status_parse_fields(ctx, t_strsplit_spaces(fields, " ")); - ctx->search_args = doveadm_mail_mailbox_search_args_build(args); + ctx->search_args = doveadm_mail_mailbox_search_args_build(args+1); if (!ctx->total_sum) { doveadm_print_header("mailbox", "mailbox",
--- a/src/doveadm/doveadm-mail-mailbox.c Sat May 19 22:40:08 2012 +0300 +++ b/src/doveadm/doveadm-mail-mailbox.c Sun May 20 03:25:04 2012 +0300 @@ -139,12 +139,18 @@ { struct mail_search_args *search_args; struct mail_search_arg *arg; + enum mail_search_arg_type type; unsigned int i; doveadm_mailbox_args_check(args); search_args = mail_search_build_init(); for (i = 0; args[i] != NULL; i++) { - arg = mail_search_build_add(search_args, SEARCH_MAILBOX_GLOB); + if (strchr(args[i], '*') != NULL || + strchr(args[i], '%') != NULL) + type = SEARCH_MAILBOX_GLOB; + else + type = SEARCH_MAILBOX; + arg = mail_search_build_add(search_args, type); arg->value.str = p_strdup(search_args->pool, args[i]); } if (i > 1) { @@ -371,6 +377,9 @@ case 'r': ctx->recursive = TRUE; break; + case 's': + ctx->ctx.subscriptions = TRUE; + break; default: return FALSE; } @@ -385,7 +394,7 @@ ctx->ctx.ctx.v.init = cmd_mailbox_delete_init; ctx->ctx.ctx.v.run = cmd_mailbox_delete_run; ctx->ctx.ctx.v.parse_arg = cmd_mailbox_delete_parse_arg; - ctx->ctx.ctx.getopt_args = "r"; + ctx->ctx.ctx.getopt_args = "rs"; p_array_init(&ctx->mailboxes, ctx->ctx.ctx.pool, 16); return &ctx->ctx.ctx; }
--- a/src/doveadm/doveadm-mail-server.c Sat May 19 22:40:08 2012 +0300 +++ b/src/doveadm/doveadm-mail-server.c Sun May 20 03:25:04 2012 +0300 @@ -83,15 +83,23 @@ struct server_connection *conn = context; struct doveadm_server *server; - if (reply == SERVER_CMD_REPLY_INTERNAL_FAILURE) { + switch (reply) { + case SERVER_CMD_REPLY_INTERNAL_FAILURE: internal_failure = TRUE; master_service_stop(master_service); return; + case SERVER_CMD_REPLY_UNKNOWN_USER: + i_error("No such user"); + if (cmd_ctx->exit_code == 0) + cmd_ctx->exit_code = EX_NOUSER; + break; + case SERVER_CMD_REPLY_FAIL: + doveadm_mail_failed_error(cmd_ctx, MAIL_ERROR_TEMP); + break; + case SERVER_CMD_REPLY_OK: + break; } - if (reply != SERVER_CMD_REPLY_OK) - doveadm_mail_failed_error(cmd_ctx, MAIL_ERROR_TEMP); - server = server_connection_get_server(conn); if (array_count(&server->queue) > 0) { char *const *usernamep = array_idx(&server->queue, 0);
--- a/src/doveadm/doveadm-mail.c Sat May 19 22:40:08 2012 +0300 +++ b/src/doveadm/doveadm-mail.c Sun May 20 03:25:04 2012 +0300 @@ -292,6 +292,13 @@ return ret; } + if (ctx->v.prerun != NULL) { + if (ctx->v.prerun(ctx, ctx->cur_service_user, error_r) < 0) { + mail_storage_service_user_free(&ctx->cur_service_user); + return -1; + } + } + ret = mail_storage_service_next(ctx->storage_service, ctx->cur_service_user, &ctx->cur_mail_user); @@ -309,12 +316,10 @@ return 1; } -void doveadm_mail_single_user(struct doveadm_mail_cmd_context *ctx, - const struct mail_storage_service_input *input) +int doveadm_mail_single_user(struct doveadm_mail_cmd_context *ctx, + const struct mail_storage_service_input *input, + const char **error_r) { - const char *error; - int ret; - i_assert(input->username != NULL); ctx->cur_username = input->username; @@ -324,11 +329,7 @@ if (hook_doveadm_mail_init != NULL) hook_doveadm_mail_init(ctx); - ret = doveadm_mail_next_user(ctx, input, &error); - if (ret < 0) - i_fatal("%s", error); - else if (ret == 0) - i_fatal_status(EX_NOUSER, "User doesn't exist"); + return doveadm_mail_next_user(ctx, input, error_r); } static void sig_die(const siginfo_t *si, void *context ATTR_UNUSED) @@ -372,6 +373,7 @@ continue; } input.username = user; + ctx->cur_username = user; doveadm_print_sticky("username", user); T_BEGIN { ret = doveadm_mail_next_user(ctx, &input, &error); @@ -445,8 +447,8 @@ doveadm_mail_cmd(const struct doveadm_mail_cmd *cmd, int argc, char *argv[]) { struct doveadm_mail_cmd_context *ctx; - const char *getopt_args, *wildcard_user; - int c; + const char *getopt_args, *wildcard_user, *error; + int ret, c; ctx = doveadm_mail_cmd_init(cmd, doveadm_settings); ctx->full_args = (const void *)(argv + 1); @@ -455,7 +457,10 @@ if (doveadm_debug) ctx->service_flags |= MAIL_STORAGE_SERVICE_FLAG_DEBUG; - getopt_args = t_strconcat("AS:u:", ctx->getopt_args, NULL); + getopt_args = "AS:u:"; + /* keep context's getopt_args first in case it contains '+' */ + if (ctx->getopt_args != NULL) + getopt_args = t_strconcat(ctx->getopt_args, getopt_args, NULL); ctx->cur_username = getenv("USER"); wildcard_user = NULL; while ((c = getopt(argc, argv, getopt_args)) > 0) { @@ -510,7 +515,11 @@ memset(&input, 0, sizeof(input)); input.service = "doveadm"; input.username = ctx->cur_username; - doveadm_mail_single_user(ctx, &input); + ret = doveadm_mail_single_user(ctx, &input, &error); + if (ret < 0) + i_fatal("%s", error); + else if (ret == 0) + i_fatal_status(EX_NOUSER, "User doesn't exist"); } else { ctx->service_flags |= MAIL_STORAGE_SERVICE_FLAG_TEMP_PRIV_DROP; doveadm_mail_all_users(ctx, argv, wildcard_user);
--- a/src/doveadm/doveadm-mail.h Sat May 19 22:40:08 2012 +0300 +++ b/src/doveadm/doveadm-mail.h Sun May 20 03:25:04 2012 +0300 @@ -20,6 +20,9 @@ const char *const args[]); int (*get_next_user)(struct doveadm_mail_cmd_context *ctx, const char **username_r); + int (*prerun)(struct doveadm_mail_cmd_context *ctx, + struct mail_storage_service_user *service_user, + const char **error_r); int (*run)(struct doveadm_mail_cmd_context *ctx, struct mail_user *mail_user); void (*deinit)(struct doveadm_mail_cmd_context *ctx); @@ -92,8 +95,9 @@ struct doveadm_mail_cmd_context * doveadm_mail_cmd_init(const struct doveadm_mail_cmd *cmd, const struct doveadm_settings *set); -void doveadm_mail_single_user(struct doveadm_mail_cmd_context *ctx, - const struct mail_storage_service_input *input); +int doveadm_mail_single_user(struct doveadm_mail_cmd_context *ctx, + const struct mail_storage_service_input *input, + const char **error_r); int doveadm_mail_server_user(struct doveadm_mail_cmd_context *ctx, const struct mail_storage_service_input *input, const char **error_r);
--- a/src/doveadm/doveadm-mailbox-list-iter.c Sat May 19 22:40:08 2012 +0300 +++ b/src/doveadm/doveadm-mailbox-list-iter.c Sun May 20 03:25:04 2012 +0300 @@ -10,18 +10,24 @@ #include "doveadm-mailbox-list-iter.h" struct doveadm_mailbox_list_iter { + struct mail_user *user; struct doveadm_mail_cmd_context *ctx; struct mail_search_args *search_args; enum mailbox_list_iter_flags iter_flags; struct mailbox_list_iterate_context *iter; + + struct mailbox_info info; + ARRAY_TYPE(const_string) patterns; + unsigned int pattern_idx; + bool only_selectable; }; static int search_args_get_mailbox_patterns(const struct mail_search_arg *args, ARRAY_TYPE(const_string) *patterns, - bool *have_guid_r) + bool *have_guid, bool *have_wildcards) { const struct mail_search_arg *subargs; @@ -35,12 +41,15 @@ subargs = args->value.subargs; for (; subargs != NULL; subargs = subargs->next) { if (!search_args_get_mailbox_patterns(subargs, - patterns, have_guid_r)) + patterns, have_guid, + have_wildcards)) return 0; } break; + case SEARCH_MAILBOX_GLOB: + *have_wildcards = TRUE; + /* fall through */ case SEARCH_MAILBOX: - case SEARCH_MAILBOX_GLOB: if (args->match_not) { array_clear(patterns); return 0; @@ -48,7 +57,7 @@ array_append(patterns, &args->value.str, 1); break; case SEARCH_MAILBOX_GUID: - *have_guid_r = TRUE; + *have_guid = TRUE; break; default: break; @@ -66,33 +75,36 @@ { static const char *all_pattern = "*"; struct doveadm_mailbox_list_iter *iter; - ARRAY_TYPE(const_string) patterns; - bool have_guid = FALSE; - - iter_flags |= MAILBOX_LIST_ITER_LIST_PREFIXES; + bool have_guid = FALSE, have_wildcards = FALSE; iter = i_new(struct doveadm_mailbox_list_iter, 1); iter->ctx = ctx; iter->search_args = search_args; + iter->user = user; + i_array_init(&iter->patterns, 16); + search_args_get_mailbox_patterns(search_args->args, &iter->patterns, + &have_guid, &have_wildcards); - t_array_init(&patterns, 16); - search_args_get_mailbox_patterns(search_args->args, &patterns, - &have_guid); - if (array_count(&patterns) == 0) { + if (array_count(&iter->patterns) == 0) { iter_flags |= MAILBOX_LIST_ITER_SKIP_ALIASES; if (have_guid) ns_mask |= NAMESPACE_SHARED | NAMESPACE_PUBLIC; - array_append(&patterns, &all_pattern, 1); - } else { + array_append(&iter->patterns, &all_pattern, 1); + } else if (have_wildcards) { iter_flags |= MAILBOX_LIST_ITER_STAR_WITHIN_NS; ns_mask |= NAMESPACE_SHARED | NAMESPACE_PUBLIC; + } else { + /* just return the listed mailboxes without actually + iterating through. this also allows accessing mailboxes + without lookup ACL right */ + return iter; } - (void)array_append_space(&patterns); + (void)array_append_space(&iter->patterns); iter->only_selectable = TRUE; iter->iter_flags = iter_flags; iter->iter = mailbox_list_iter_init_namespaces(user->namespaces, - array_idx(&patterns, 0), + array_idx(&iter->patterns, 0), ns_mask, iter_flags); return iter; } @@ -132,10 +144,13 @@ *_iter = NULL; - if ((ret = mailbox_list_iter_deinit(&iter->iter)) < 0) { + if (iter->iter == NULL) + ret = 0; + else if ((ret = mailbox_list_iter_deinit(&iter->iter)) < 0) { i_error("Listing mailboxes failed"); doveadm_mail_failed_error(iter->ctx, MAIL_ERROR_TEMP); } + array_free(&iter->patterns); i_free(iter); return ret; } @@ -144,6 +159,22 @@ doveadm_mailbox_list_iter_next(struct doveadm_mailbox_list_iter *iter) { const struct mailbox_info *info; + const char *const *patterns; + unsigned int count; + + while (iter->iter == NULL) { + patterns = array_get(&iter->patterns, &count); + if (iter->pattern_idx == count) + return NULL; + + iter->info.name = patterns[iter->pattern_idx++]; + iter->info.ns = mail_namespace_find(iter->user->namespaces, + iter->info.name); + if (iter->info.ns != NULL) + return &iter->info; + /* FIXME: maybe fail?.. or just wait for v2.2 to get rid of + this error condition */ + } while ((info = mailbox_list_iter_next(iter->iter)) != NULL) { char sep = mail_namespace_get_sep(info->ns);
--- a/src/doveadm/doveadm-mount.c Sat May 19 22:40:08 2012 +0300 +++ b/src/doveadm/doveadm-mount.c Sun May 20 03:25:04 2012 +0300 @@ -77,8 +77,10 @@ mountpoints = mountpoint_list_get(); if (argv[1] == NULL) { - mountpoint_list_add_missing(mountpoints, MOUNTPOINT_STATE_DEFAULT, - mountpoint_list_default_ignore_types); + mountpoint_list_add_missing(mountpoints, + MOUNTPOINT_STATE_DEFAULT, + mountpoint_list_default_ignore_prefixes, + mountpoint_list_default_ignore_types); } else { memset(&rec, 0, sizeof(rec)); rec.mount_path = argv[1];
--- a/src/doveadm/doveadm-mutf7.c Sat May 19 22:40:08 2012 +0300 +++ b/src/doveadm/doveadm-mutf7.c Sun May 20 03:25:04 2012 +0300 @@ -40,11 +40,13 @@ if (imap_utf8_to_utf7(argv[i], str) < 0) { i_error("Mailbox name not valid UTF-8: %s", argv[i]); + doveadm_exit_code = EX_DATAERR; } } else { if (imap_utf7_to_utf8(argv[i], str) < 0) { i_error("Mailbox name not valid mUTF-7: %s", argv[i]); + doveadm_exit_code = EX_DATAERR; } } printf("%s\n", str_c(str));
--- a/src/doveadm/doveadm-penalty.c Sat May 19 22:40:08 2012 +0300 +++ b/src/doveadm/doveadm-penalty.c Sun May 20 03:25:04 2012 +0300 @@ -27,7 +27,7 @@ static void penalty_parse_line(const char *line, struct penalty_line *line_r) { - const char *const *args = t_strsplit(line, "\t"); + const char *const *args = t_strsplit_tab(line); const char *ident = args[0]; const char *penalty_str = args[1]; const char *last_penalty_str = args[2];
--- a/src/doveadm/doveadm-proxy.c Sat May 19 22:40:08 2012 +0300 +++ b/src/doveadm/doveadm-proxy.c Sun May 20 03:25:04 2012 +0300 @@ -47,7 +47,7 @@ switch (state) { case IPC_CLIENT_CMD_STATE_REPLY: T_BEGIN { - const char *const *args = t_strsplit(data, "\t"); + const char *const *args = t_strsplit_tab(data); for (; *args != NULL; args++) doveadm_print(*args); } T_END;
--- a/src/doveadm/doveadm-settings.c Sat May 19 22:40:08 2012 +0300 +++ b/src/doveadm/doveadm-settings.c Sun May 20 03:25:04 2012 +0300 @@ -61,6 +61,7 @@ DEF(SET_STR, doveadm_password), DEF(SET_STR, doveadm_allowed_commands), DEF(SET_STR, dsync_alt_char), + DEF(SET_STR, dsync_remote_cmd), { SET_STRLIST, "plugin", offsetof(struct doveadm_settings, plugin_envs), NULL }, @@ -77,6 +78,7 @@ .doveadm_password = "", .doveadm_allowed_commands = "", .dsync_alt_char = "_", + .dsync_remote_cmd = "ssh -l%{login} %{host} doveadm dsync-server -u%u -l%{lock_timeout} -n%{namespace}", .plugin_envs = ARRAY_INIT };
--- a/src/doveadm/doveadm-settings.h Sat May 19 22:40:08 2012 +0300 +++ b/src/doveadm/doveadm-settings.h Sun May 20 03:25:04 2012 +0300 @@ -11,6 +11,7 @@ const char *doveadm_password; const char *doveadm_allowed_commands; const char *dsync_alt_char; + const char *dsync_remote_cmd; ARRAY_DEFINE(plugin_envs, const char *); };
--- a/src/doveadm/doveadm-stats.c Sat May 19 22:40:08 2012 +0300 +++ b/src/doveadm/doveadm-stats.c Sun May 20 03:25:04 2012 +0300 @@ -102,7 +102,7 @@ do { T_BEGIN { args = read_next_line(input); - if (args[0] == NULL) + if (args != NULL && args[0] == NULL) args = NULL; if (args != NULL) { for (i = 0; args[i] != NULL; i++)
--- a/src/doveadm/doveadm-who.c Sat May 19 22:40:08 2012 +0300 +++ b/src/doveadm/doveadm-who.c Sun May 20 03:25:04 2012 +0300 @@ -55,7 +55,7 @@ static void who_parse_line(const char *line, struct who_line *line_r) { - const char *const *args = t_strsplit(line, "\t"); + const char *const *args = t_strsplit_tab(line); const char *ident = args[0]; const char *pid_str = args[1]; const char *refcount_str = args[2];
--- a/src/doveadm/doveadm.c Sat May 19 22:40:08 2012 +0300 +++ b/src/doveadm/doveadm.c Sun May 20 03:25:04 2012 +0300 @@ -331,6 +331,7 @@ doveadm_register_cmd(doveadm_commands[i]); if (cmd_name != NULL && (quick_init || + strcmp(cmd_name, "config") == 0 || strcmp(cmd_name, "stop") == 0 || strcmp(cmd_name, "reload") == 0)) { /* special case commands: even if there is something wrong
--- a/src/doveadm/dsync/Makefile.am Sat May 19 22:40:08 2012 +0300 +++ b/src/doveadm/dsync/Makefile.am Sun May 20 03:25:04 2012 +0300 @@ -11,9 +11,6 @@ -I$(top_srcdir)/src/lib-storage \ -I$(top_srcdir)/src/doveadm -libs = \ - $(LIBDOVECOT_STORAGE) - libdsync_a_SOURCES = \ doveadm-dsync.c \ dsync-brain.c \
--- a/src/doveadm/dsync/doveadm-dsync.c Sat May 19 22:40:08 2012 +0300 +++ b/src/doveadm/dsync/doveadm-dsync.c Sun May 20 03:25:04 2012 +0300 @@ -4,6 +4,8 @@ #include "lib-signals.h" #include "array.h" #include "execv-const.h" +#include "str.h" +#include "var-expand.h" #include "settings-parser.h" #include "master-service.h" #include "mail-storage-service.h" @@ -21,54 +23,83 @@ #include <unistd.h> #include <ctype.h> +#define DSYNC_LOCK_FILENAME ".dovecot-sync.lock" + struct dsync_cmd_context { struct doveadm_mail_cmd_context ctx; enum dsync_brain_flags brain_flags; - const char *mailbox; + const char *mailbox, *namespace_prefix; - const char *const *remote_cmd_args; const char *local_location; - int fd_in, fd_out; + int fd_in, fd_out, fd_err; + struct io *io_err; + + unsigned int lock_timeout; + unsigned int lock:1; + unsigned int default_replica_location:1; unsigned int reverse_workers:1; + unsigned int remote:1; }; -static const char *ssh_cmd = "ssh"; +static bool legacy_dsync = FALSE; -static void run_cmd(const char *const *args, int *fd_in_r, int *fd_out_r) +static void remote_error_input(struct dsync_cmd_context *ctx) { - int fd_in[2], fd_out[2]; + char buf[1024]; + ssize_t ret; - if (pipe(fd_in) < 0 || pipe(fd_out) < 0) + ret = read(ctx->fd_err, buf, sizeof(buf)-1); + if (ret == -1) { + io_remove(&ctx->io_err); + return; + } + if (ret > 0) { + buf[ret-1] = '\0'; + i_error("remote: %s", buf); + } +} + +static void +run_cmd(struct dsync_cmd_context *ctx, const char *const *args) +{ + int fd_in[2], fd_out[2], fd_err[2]; + + if (pipe(fd_in) < 0 || pipe(fd_out) < 0 || pipe(fd_err) < 0) i_fatal("pipe() failed: %m"); switch (fork()) { case -1: i_fatal("fork() failed: %m"); - break; case 0: /* child, which will execute the proxy server. stdin/stdout goes to pipes which we'll pass to proxy client. */ if (dup2(fd_in[0], STDIN_FILENO) < 0 || - dup2(fd_out[1], STDOUT_FILENO) < 0) + dup2(fd_out[1], STDOUT_FILENO) < 0 || + dup2(fd_err[1], STDERR_FILENO) < 0) i_fatal("dup2() failed: %m"); (void)close(fd_in[0]); (void)close(fd_in[1]); (void)close(fd_out[0]); (void)close(fd_out[1]); + (void)close(fd_err[0]); + (void)close(fd_err[1]); execvp_const(args[0], args); - break; default: /* parent */ - (void)close(fd_in[0]); - (void)close(fd_out[1]); - *fd_in_r = fd_out[0]; - *fd_out_r = fd_in[1]; break; } + + (void)close(fd_in[0]); + (void)close(fd_out[1]); + (void)close(fd_err[1]); + ctx->fd_in = fd_out[0]; + ctx->fd_out = fd_in[1]; + ctx->fd_err = fd_err[0]; + ctx->io_err = io_add(ctx->fd_err, IO_READ, remote_error_input, ctx); } static void @@ -87,9 +118,7 @@ array_append(&cmd_args, &p, 1); } - p = strchr(argv[0], '/'); - if (p == NULL) p = argv[0]; - if (strstr(p, "dsync") != NULL) { + if (legacy_dsync) { /* we're executing dsync */ p = "server"; } else { @@ -101,11 +130,63 @@ *cmd_args_r = array_idx(&cmd_args, 0); } -static bool mirror_get_remote_cmd(const char *const *argv, const char *user, +static const char *const * +get_ssh_cmd_args(struct dsync_cmd_context *ctx, + const char *host, const char *login, const char *mail_user) +{ + static struct var_expand_table static_tab[] = { + { 'u', NULL, "user" }, + { '\0', NULL, "login" }, + { '\0', NULL, "host" }, + { '\0', NULL, "lock_timeout" }, + { '\0', NULL, "namespace" }, + { '\0', NULL, NULL } + }; + struct var_expand_table *tab; + ARRAY_TYPE(const_string) cmd_args; + string_t *str, *str2; + const char *value, *const *args; + + tab = t_malloc(sizeof(static_tab)); + memcpy(tab, static_tab, sizeof(static_tab)); + + tab[0].value = mail_user; + tab[1].value = login; + tab[2].value = host; + tab[3].value = dec2str(ctx->lock_timeout); + tab[4].value = ctx->namespace_prefix; + + t_array_init(&cmd_args, 8); + str = t_str_new(128); + str2 = t_str_new(128); + args = t_strsplit(doveadm_settings->dsync_remote_cmd, " "); + for (; *args != NULL; args++) { + if (strchr(*args, '%') == NULL) + value = *args; + else { + /* some automation: if parameter's all %variables + expand to empty, but the %variable isn't the only + text in the parameter, skip it. */ + str_truncate(str, 0); + str_truncate(str2, 0); + var_expand(str, *args, tab); + var_expand(str2, *args, static_tab); + if (strcmp(str_c(str), str_c(str2)) == 0 && + str_len(str) > 0) + continue; + value = t_strdup(str_c(str)); + } + array_append(&cmd_args, &value, 1); + } + (void)array_append_space(&cmd_args); + return array_idx(&cmd_args, 0); +} + +static bool mirror_get_remote_cmd(struct dsync_cmd_context *ctx, + const char *user, const char *const **cmd_args_r) { - ARRAY_TYPE(const_string) cmd_args; - const char *p, *host; + const char *p, *host, *const *argv = ctx->ctx.args; if (argv[1] != NULL) { /* more than one parameter, so it contains a full command @@ -142,17 +223,7 @@ /* we'll assume virtual users, so in user@host it really means not to give ssh a username, but to give dsync -u user parameter. */ - t_array_init(&cmd_args, 8); - array_append(&cmd_args, &ssh_cmd, 1); - array_append(&cmd_args, &host, 1); - p = "doveadm"; array_append(&cmd_args, &p, 1); - p = "dsync-server"; array_append(&cmd_args, &p, 1); - if (*user != '\0') { - p = "-u"; array_append(&cmd_args, &p, 1); - array_append(&cmd_args, &user, 1); - } - (void)array_append_space(&cmd_args); - *cmd_args_r = array_idx(&cmd_args, 0); + *cmd_args_r = get_ssh_cmd_args(ctx, host, "", user); return TRUE; } @@ -196,7 +267,8 @@ "points to same directory: %s", path1); } - worker2 = dsync_worker_init_local(user2, *ctx->ctx.set->dsync_alt_char); + worker2 = dsync_worker_init_local(user2, ctx->namespace_prefix, + *ctx->ctx.set->dsync_alt_char); mail_user_unref(&user2); return worker2; } @@ -209,32 +281,64 @@ return dsync_worker_init_proxy_client(ctx->fd_in, ctx->fd_out); } -static int -cmd_dsync_run(struct doveadm_mail_cmd_context *_ctx, struct mail_user *user) +static const char *const * +parse_ssh_location(struct dsync_cmd_context *ctx, + const char *location, const char *username) { - struct dsync_cmd_context *ctx = (struct dsync_cmd_context *)_ctx; - struct dsync_worker *worker1, *worker2, *workertmp; - struct dsync_brain *brain; - int ret = 0; + const char *host, *login; - user->admin = TRUE; + host = strchr(location, '@'); + if (host != NULL) + login = t_strdup_until(location, host++); + else { + host = location; + login = ""; + } + return get_ssh_cmd_args(ctx, host, login, username); +} + +static int dsync_lock(struct mail_user *user, unsigned int lock_timeout, + const char **path_r, struct file_lock **lock_r) +{ + const char *home, *path; + int ret, fd; - /* create workers */ - worker1 = dsync_worker_init_local(user, *_ctx->set->dsync_alt_char); - if (ctx->remote_cmd_args == NULL) - worker2 = cmd_dsync_run_local(ctx, user); - else - worker2 = cmd_dsync_run_remote(ctx, user); - if (ctx->reverse_workers) { - workertmp = worker1; - worker1 = worker2; - worker2 = workertmp; + if ((ret = mail_user_get_home(user, &home)) < 0) { + i_error("Couldn't look up user's home dir"); + return -1; + } + if (ret == 0) { + i_error("User has no home directory"); + return -1; + } + + path = t_strconcat(home, "/"DSYNC_LOCK_FILENAME, NULL); + fd = creat(path, 0600); + if (fd == -1) { + i_error("Couldn't create lock %s: %m", path); + return -1; } + if (file_wait_lock(fd, path, F_WRLCK, FILE_LOCK_METHOD_FCNTL, + lock_timeout, lock_r) <= 0) { + i_error("Couldn't lock %s: %m", path); + (void)close(fd); + return -1; + } + *path_r = path; + return fd; +} + +static int +cmd_dsync_start(struct dsync_cmd_context *ctx, struct dsync_worker *worker1, + struct dsync_worker *worker2) +{ + struct dsync_brain *brain; + /* create and run the brain */ brain = dsync_brain_init(worker1, worker2, ctx->mailbox, ctx->brain_flags); - if (ctx->remote_cmd_args == NULL) + if (!ctx->remote) dsync_brain_sync_all(brain); else { dsync_brain_sync(brain); @@ -245,53 +349,137 @@ if (dsync_brain_has_unexpected_changes(brain)) { i_warning("Mailbox changes caused a desync. " "You may want to run dsync again."); - _ctx->exit_code = 2; + ctx->ctx.exit_code = 2; } if (dsync_brain_deinit(&brain) < 0) { - _ctx->exit_code = EX_TEMPFAIL; - ret = -1; + ctx->ctx.exit_code = EX_TEMPFAIL; + return -1; + } + return 0; +} + +static int +cmd_dsync_run(struct doveadm_mail_cmd_context *_ctx, struct mail_user *user) +{ + struct dsync_cmd_context *ctx = (struct dsync_cmd_context *)_ctx; + struct dsync_worker *worker1, *worker2, *workertmp; + const char *lock_path; + struct file_lock *lock; + int lock_fd, ret = 0; + + user->admin = TRUE; + user->dsyncing = TRUE; + + /* create workers */ + worker1 = dsync_worker_init_local(user, ctx->namespace_prefix, + *_ctx->set->dsync_alt_char); + if (!ctx->remote) + worker2 = cmd_dsync_run_local(ctx, user); + else + worker2 = cmd_dsync_run_remote(ctx, user); + if (ctx->reverse_workers) { + workertmp = worker1; + worker1 = worker2; + worker2 = workertmp; } + if (!ctx->lock) + ret = cmd_dsync_start(ctx, worker1, worker2); + else { + lock_fd = dsync_lock(user, ctx->lock_timeout, &lock_path, &lock); + if (lock_fd == -1) { + _ctx->exit_code = EX_TEMPFAIL; + ret = -1; + } else { + ret = cmd_dsync_start(ctx, worker1, worker2); + file_lock_free(&lock); + if (close(lock_fd) < 0) + i_error("close(%s) failed: %m", lock_path); + } + } dsync_worker_deinit(&worker1); dsync_worker_deinit(&worker2); + if (ctx->io_err != NULL) + io_remove(&ctx->io_err); + if (ctx->fd_err != -1) { + (void)close(ctx->fd_err); + ctx->fd_err = -1; + } return ret; } +static int cmd_dsync_prerun(struct doveadm_mail_cmd_context *_ctx, + struct mail_storage_service_user *service_user, + const char **error_r) +{ + struct dsync_cmd_context *ctx = (struct dsync_cmd_context *)_ctx; + const char *const *remote_cmd_args = NULL; + const struct mail_user_settings *user_set; + const char *username = ""; + + user_set = mail_storage_service_user_get_set(service_user)[0]; + + ctx->fd_in = STDIN_FILENO; + ctx->fd_out = STDOUT_FILENO; + ctx->fd_err = -1; + ctx->remote = FALSE; + + if (ctx->default_replica_location) { + ctx->local_location = + mail_user_set_plugin_getenv(user_set, "mail_replica"); + if (ctx->local_location == NULL || + *ctx->local_location == '\0') { + *error_r = "User has no mail_replica in userdb"; + _ctx->exit_code = DOVEADM_EX_NOTFOUND; + return -1; + } + } else { + /* if we're executing remotely, give -u parameter if we also + did a userdb lookup. */ + if ((_ctx->service_flags & MAIL_STORAGE_SERVICE_FLAG_USERDB_LOOKUP) != 0) + username = _ctx->cur_username; + + if (!mirror_get_remote_cmd(ctx, username, &remote_cmd_args)) { + /* it's a mail_location */ + if (_ctx->args[1] != NULL) + doveadm_mail_help_name(_ctx->cmd->name); + ctx->local_location = _ctx->args[0]; + } + } + + if (remote_cmd_args == NULL && ctx->local_location != NULL && + strncmp(ctx->local_location, "remote:", 7) == 0) { + /* this is a remote (ssh) command */ + remote_cmd_args = parse_ssh_location(ctx, ctx->local_location+7, + _ctx->cur_username); + } + + if (remote_cmd_args != NULL) { + /* do this before mail_storage_service_next() in case it + drops process privileges */ + run_cmd(ctx, remote_cmd_args); + ctx->remote = TRUE; + } + return 0; +} + static void cmd_dsync_init(struct doveadm_mail_cmd_context *_ctx, const char *const args[]) { struct dsync_cmd_context *ctx = (struct dsync_cmd_context *)_ctx; - const char *username = ""; - if (args[0] == NULL) - doveadm_mail_help_name(_ctx->cmd->name); + if (ctx->default_replica_location) { + if (args[0] != NULL) + i_error("Don't give mail location with -d parameter"); + } else { + if (args[0] == NULL) + doveadm_mail_help_name(_ctx->cmd->name); + } lib_signals_ignore(SIGHUP, TRUE); if (doveadm_debug || doveadm_verbose) ctx->brain_flags |= DSYNC_BRAIN_FLAG_VERBOSE; - - /* if we're executing remotely, give -u parameter if we also - did a userdb lookup. this works only when we're handling a - single user */ - if ((_ctx->service_flags & MAIL_STORAGE_SERVICE_FLAG_USERDB_LOOKUP) != 0 && - _ctx->cur_username != NULL) - username = _ctx->cur_username; - if (!mirror_get_remote_cmd(args, username, &ctx->remote_cmd_args)) { - /* it's a mail_location */ - if (args[1] != NULL) - doveadm_mail_help_name(_ctx->cmd->name); - ctx->local_location = args[0]; - } - - if (ctx->remote_cmd_args != NULL) { - /* do this before mail_storage_service_next() in case it - drops process privileges */ - run_cmd(ctx->remote_cmd_args, &ctx->fd_in, &ctx->fd_out); - } else { - ctx->fd_in = STDIN_FILENO; - ctx->fd_out = STDOUT_FILENO; - } } static void cmd_dsync_preinit(struct doveadm_mail_cmd_context *ctx) @@ -306,15 +494,27 @@ struct dsync_cmd_context *ctx = (struct dsync_cmd_context *)_ctx; switch (c) { + case 'd': + ctx->default_replica_location = TRUE; + break; case 'E': - /* dsync backup wrapper detection flag */ + /* dsync wrapper detection flag */ + legacy_dsync = TRUE; break; case 'f': ctx->brain_flags |= DSYNC_BRAIN_FLAG_FULL_SYNC; break; + case 'l': + ctx->lock = TRUE; + if (str_to_uint(optarg, &ctx->lock_timeout) < 0) + i_error("Invalid -l parameter: %s", optarg); + break; case 'm': ctx->mailbox = optarg; break; + case 'n': + ctx->namespace_prefix = optarg; + break; case 'R': ctx->reverse_workers = TRUE; break; @@ -329,10 +529,11 @@ struct dsync_cmd_context *ctx; ctx = doveadm_mail_cmd_alloc(struct dsync_cmd_context); - ctx->ctx.getopt_args = "EfRm:"; + ctx->ctx.getopt_args = "+dEfl:m:n:R"; ctx->ctx.v.parse_arg = cmd_mailbox_dsync_parse_arg; ctx->ctx.v.preinit = cmd_dsync_preinit; ctx->ctx.v.init = cmd_dsync_init; + ctx->ctx.v.prerun = cmd_dsync_prerun; ctx->ctx.v.run = cmd_dsync_run; return &ctx->ctx; } @@ -349,41 +550,87 @@ } static int -cmd_dsync_server_run(struct doveadm_mail_cmd_context *ctx, +cmd_dsync_server_run(struct doveadm_mail_cmd_context *_ctx, struct mail_user *user) { + struct dsync_cmd_context *ctx = (struct dsync_cmd_context *)_ctx; struct dsync_proxy_server *server; struct dsync_worker *worker; + struct file_lock *lock; + const char *lock_path; + int lock_fd, ret = 0; user->admin = TRUE; + user->dsyncing = TRUE; i_set_failure_prefix(t_strdup_printf("dsync-remote(%s): ", user->username)); - worker = dsync_worker_init_local(user, *ctx->set->dsync_alt_char); + worker = dsync_worker_init_local(user, ctx->namespace_prefix, + *_ctx->set->dsync_alt_char); server = dsync_proxy_server_init(STDIN_FILENO, STDOUT_FILENO, worker); - io_loop_run(current_ioloop); + if (!ctx->lock) + io_loop_run(current_ioloop); + else { + lock_fd = dsync_lock(user, ctx->lock_timeout, &lock_path, &lock); + if (lock_fd == -1) { + _ctx->exit_code = EX_TEMPFAIL; + ret = -1; + } else { + io_loop_run(current_ioloop); + file_lock_free(&lock); + if (close(lock_fd) < 0) + i_error("close(%s) failed: %m", lock_path); + } + } dsync_proxy_server_deinit(&server); dsync_worker_deinit(&worker); - return 0; + return ret; +} + +static bool +cmd_mailbox_dsync_server_parse_arg(struct doveadm_mail_cmd_context *_ctx, int c) +{ + struct dsync_cmd_context *ctx = (struct dsync_cmd_context *)_ctx; + + switch (c) { + case 'E': + /* dsync wrapper detection flag */ + legacy_dsync = TRUE; + break; + case 'l': + ctx->lock = TRUE; + if (str_to_uint(optarg, &ctx->lock_timeout) < 0) + i_error("Invalid -l parameter: %s", optarg); + break; + case 'n': + ctx->namespace_prefix = optarg; + break; + default: + return FALSE; + } + return TRUE; } static struct doveadm_mail_cmd_context *cmd_dsync_server_alloc(void) { - struct doveadm_mail_cmd_context *ctx; + struct dsync_cmd_context *ctx; - ctx = doveadm_mail_cmd_alloc(struct doveadm_mail_cmd_context); - ctx->v.run = cmd_dsync_server_run; - return ctx; + ctx = doveadm_mail_cmd_alloc(struct dsync_cmd_context); + ctx->ctx.getopt_args = "El:n:"; + ctx->ctx.v.parse_arg = cmd_mailbox_dsync_server_parse_arg; + ctx->ctx.v.run = cmd_dsync_server_run; + return &ctx->ctx; } struct doveadm_mail_cmd cmd_dsync_mirror = { - cmd_dsync_alloc, "sync", "[-fR] [-m <mailbox>] <dest>" + cmd_dsync_alloc, "sync", + "[-dfR] [-l <secs>] [-m <mailbox>] [-n <namespace>] <dest>" }; struct doveadm_mail_cmd cmd_dsync_backup = { cmd_dsync_backup_alloc, "backup", - "[-fR] [-m <mailbox>] <dest>" + "[-dfR] [-l <secs>] [-m <mailbox>] [-n <namespace>] <dest>" }; struct doveadm_mail_cmd cmd_dsync_server = { cmd_dsync_server_alloc, "dsync-server", &doveadm_mail_cmd_hide @@ -398,7 +645,7 @@ char *p, *dup, new_flags[6]; int max_argc, src, dest, i, j; bool flag_f = FALSE, flag_R = FALSE, flag_m, flag_u, flag_C, has_arg; - bool backup_flag = FALSE; + bool dsync_server = FALSE; p = strrchr(argv[0], '/'); if (p == NULL) p = argv[0]; @@ -408,7 +655,7 @@ /* @UNSAFE: this is called when the "doveadm" binary is called as "dsync" (for backwards compatibility) */ max_argc = argc + 7; - new_argv = calloc(sizeof(char *), max_argc); + new_argv = t_new(char *, max_argc); new_argv[0] = argv[0]; dest = 1; getopt_str = master_service_getopt_string(); @@ -419,7 +666,7 @@ break; flag_m = FALSE; flag_C = FALSE; has_arg = FALSE; flag_u = FALSE; - dup = strdup(argv[src]); + dup = t_strdup_noconst(argv[src]); for (i = j = 1; argv[src][i] != '\0'; i++) { switch (argv[src][i]) { case 'C': @@ -485,12 +732,12 @@ } if (strcmp(argv[src], "mirror") == 0) new_argv[dest] = "sync"; - else if (strcmp(argv[src], "backup") == 0) { - backup_flag = TRUE; + else if (strcmp(argv[src], "backup") == 0) new_argv[dest] = "backup"; - } else if (strcmp(argv[src], "server") == 0) + else if (strcmp(argv[src], "server") == 0) { new_argv[dest] = "dsync-server"; - else + dsync_server = TRUE; + } else i_fatal("Invalid parameter: %s", argv[src]); src++; dest++; @@ -500,15 +747,16 @@ } /* dsync flags */ - new_flags[0] = '-'; i = 1; - if (backup_flag) - new_flags[i++] = 'E'; - if (flag_f) - new_flags[i++] = 'f'; - if (flag_R) - new_flags[i++] = 'R'; - if (mailbox != NULL) - new_flags[i++] = 'm'; + new_flags[0] = '-'; + new_flags[1] = 'E'; i = 2; + if (!dsync_server) { + if (flag_f) + new_flags[i++] = 'f'; + if (flag_R) + new_flags[i++] = 'R'; + if (mailbox != NULL) + new_flags[i++] = 'm'; + } i_assert((unsigned int)i < sizeof(new_flags)); new_flags[i] = '\0'; @@ -528,6 +776,7 @@ i_assert(dest < max_argc); new_argv[dest] = NULL; + legacy_dsync = TRUE; *_argc = dest; *_argv = new_argv; optind = 1;
--- a/src/doveadm/dsync/dsync-brain-msgs.c Sat May 19 22:40:08 2012 +0300 +++ b/src/doveadm/dsync/dsync-brain-msgs.c Sun May 20 03:25:04 2012 +0300 @@ -58,6 +58,8 @@ if ((msg->flags & DSYNC_MAIL_FLAG_EXPUNGED) != 0) return; + if (*msg->guid == '\0') + return; inst = p_new(iter->sync->pool, struct dsync_brain_guid_instance, 1); inst->mailbox_idx = mailbox_idx;
--- a/src/doveadm/dsync/dsync-brain.c Sat May 19 22:40:08 2012 +0300 +++ b/src/doveadm/dsync/dsync-brain.c Sun May 20 03:25:04 2012 +0300 @@ -230,7 +230,8 @@ struct dsync_brain_subs_list *list; pool_t pool; - pool = pool_alloconly_create("dsync brain subs list", 1024*4); + pool = pool_alloconly_create(MEMPOOL_GROWING"dsync brain subs list", + 1024*4); list = p_new(pool, struct dsync_brain_subs_list, 1); list->pool = pool; list->brain = brain; @@ -724,7 +725,8 @@ pool_t pool; bool ret; - pool = pool_alloconly_create("dsync changed mailboxes", 10240); + pool = pool_alloconly_create(MEMPOOL_GROWING"dsync changed mailboxes", + 10240); p_array_init(&mailboxes, pool, 128); dsync_brain_get_changed_mailboxes(brain, &mailboxes, (brain->flags & DSYNC_BRAIN_FLAG_FULL_SYNC) != 0);
--- a/src/doveadm/dsync/dsync-data.h Sat May 19 22:40:08 2012 +0300 +++ b/src/doveadm/dsync/dsync-data.h Sun May 20 03:25:04 2012 +0300 @@ -51,6 +51,7 @@ struct dsync_msg_static_data { const char *pop3_uidl; + unsigned int pop3_order; time_t received_date; struct istream *input; };
--- a/src/doveadm/dsync/dsync-proxy-server.c Sat May 19 22:40:08 2012 +0300 +++ b/src/doveadm/dsync/dsync-proxy-server.c Sun May 20 03:25:04 2012 +0300 @@ -167,7 +167,7 @@ server->fd_in = fd_in; server->fd_out = fd_out; - server->cmd_pool = pool_alloconly_create("worker server cmd", 1024); + server->cmd_pool = pool_alloconly_create("worker server cmd", 2048); server->io = io_add(fd_in, IO_READ, proxy_server_input, server); server->input = i_stream_create_fd(fd_in, (size_t)-1, FALSE); server->output = o_stream_create_fd(fd_out, (size_t)-1, FALSE);
--- a/src/doveadm/dsync/dsync-worker-local.c Sat May 19 22:40:08 2012 +0300 +++ b/src/doveadm/dsync/dsync-worker-local.c Sun May 20 03:25:04 2012 +0300 @@ -94,7 +94,7 @@ struct hash_table *dir_changes_hash; char alt_char; - ARRAY_DEFINE(wanted_namespaces, struct mail_namespace *); + const char *namespace_prefix; mailbox_guid_t selected_box_guid; struct mailbox *selected_box; @@ -151,18 +151,33 @@ return h; } -static bool local_worker_want_namespace(struct mail_namespace *ns) +static bool local_worker_want_namespace(struct local_dsync_worker *worker, + struct mail_namespace *ns) { - return strcmp(ns->unexpanded_set->location, - SETTING_STRVAR_UNEXPANDED) == 0; + if (worker->namespace_prefix == NULL) { + return strcmp(ns->unexpanded_set->location, + SETTING_STRVAR_UNEXPANDED) == 0; + } else { + return strcmp(ns->prefix, worker->namespace_prefix) == 0; + } } static void dsync_check_namespaces(struct local_dsync_worker *worker) { struct mail_namespace *ns; + if (worker->namespace_prefix != NULL) { + ns = mail_namespace_find_prefix(worker->user->namespaces, + worker->namespace_prefix); + if (ns == NULL) { + i_fatal("Namespace prefix '%s' not found", + worker->namespace_prefix); + } + return; + } + for (ns = worker->user->namespaces; ns != NULL; ns = ns->next) { - if (local_worker_want_namespace(ns)) + if (local_worker_want_namespace(worker, ns)) return; } i_fatal("All your namespaces have a location setting. " @@ -171,7 +186,8 @@ } struct dsync_worker * -dsync_worker_init_local(struct mail_user *user, char alt_char) +dsync_worker_init_local(struct mail_user *user, const char *namespace_prefix, + char alt_char) { struct local_dsync_worker *worker; pool_t pool; @@ -181,13 +197,13 @@ worker->worker.v = local_dsync_worker; worker->user = user; worker->pool = pool; + worker->namespace_prefix = p_strdup(pool, namespace_prefix); worker->alt_char = alt_char; worker->mailbox_hash = hash_table_create(default_pool, pool, 0, mailbox_guid_hash, mailbox_guid_cmp); i_array_init(&worker->saved_uids, 128); i_array_init(&worker->msg_get_queue, 32); - p_array_init(&worker->wanted_namespaces, pool, 8); dsync_check_namespaces(worker); mail_user_ref(worker->user); @@ -382,7 +398,8 @@ hash_table_create(default_pool, worker->pool, 0, dir_change_hash, dir_change_cmp); for (ns = worker->user->namespaces; ns != NULL; ns = ns->next) { - if (ns->alias_for != NULL || !local_worker_want_namespace(ns)) + if (ns->alias_for != NULL || + !local_worker_want_namespace(worker, ns)) continue; if (dsync_worker_get_list_mailbox_log(worker, ns->list) < 0) @@ -503,7 +520,7 @@ memset(dsync_box_r, 0, sizeof(*dsync_box_r)); while ((info = mailbox_list_iter_next(iter->list_iter)) != NULL) { - if (local_worker_want_namespace(info->ns)) + if (local_worker_want_namespace(worker, info->ns)) break; } if (info == NULL) @@ -526,7 +543,7 @@ dsync_box_r->last_change = dir_change->last_rename; } - if ((info->flags & MAILBOX_NOSELECT) != 0) { + if ((info->flags & (MAILBOX_NOSELECT | MAILBOX_NONEXISTENT)) != 0) { dsync_box_r->flags |= DSYNC_MAILBOX_FLAG_NOSELECT; local_dsync_worker_add_mailbox(worker, info->ns, info->name, &dsync_box_r->name_sha1); @@ -646,7 +663,7 @@ memset(rec_r, 0, sizeof(*rec_r)); while ((info = mailbox_list_iter_next(iter->list_iter)) != NULL) { - if (local_worker_want_namespace(info->ns) || + if (local_worker_want_namespace(worker, info->ns) || (info->ns->flags & NAMESPACE_FLAG_SUBSCRIPTIONS) == 0) break; } @@ -833,6 +850,7 @@ static void iter_local_mailbox_close(struct local_dsync_worker_msg_iter *iter) { + iter->prev_uid = 0; iter->expunges_set = FALSE; if (mailbox_search_deinit(&iter->search_ctx) < 0) { i_error("msg search failed: %s", @@ -1699,6 +1717,8 @@ save_ctx, msg); if (*data->pop3_uidl != '\0') mailbox_save_set_pop3_uidl(save_ctx, data->pop3_uidl); + if (data->pop3_order > 0) + mailbox_save_set_pop3_order(save_ctx, data->pop3_order); mailbox_save_set_received_date(save_ctx, data->received_date, 0); @@ -1775,6 +1795,7 @@ struct dsync_msg_static_data data; struct mailbox_transaction_context *trans; struct mailbox *box; + const char *value; i_assert(!worker->reading_mail); @@ -1806,6 +1827,9 @@ data.pop3_uidl = ""; else data.pop3_uidl = t_strdup(data.pop3_uidl); + if (mail_get_special(worker->get_mail, MAIL_FETCH_POP3_ORDER, &value) < 0 || + str_to_uint(value, &data.pop3_order) < 0) + data.pop3_order = 0; if (mail_get_received_date(worker->get_mail, &data.received_date) < 0 || mail_get_stream(worker->get_mail, NULL, NULL, &data.input) < 0) { get->callback(worker->get_mail->expunged ?
--- a/src/doveadm/dsync/dsync-worker.h Sat May 19 22:40:08 2012 +0300 +++ b/src/doveadm/dsync/dsync-worker.h Sun May 20 03:25:04 2012 +0300 @@ -30,7 +30,8 @@ typedef void dsync_worker_finish_callback_t(bool success, void *context); struct dsync_worker * -dsync_worker_init_local(struct mail_user *user, char alt_char); +dsync_worker_init_local(struct mail_user *user, const char *namespace_prefix, + char alt_char); struct dsync_worker *dsync_worker_init_proxy_client(int fd_in, int fd_out); void dsync_worker_deinit(struct dsync_worker **worker);
--- a/src/doveadm/dsync/test-dsync-proxy-server-cmd.c Sat May 19 22:40:08 2012 +0300 +++ b/src/doveadm/dsync/test-dsync-proxy-server-cmd.c Sun May 20 03:25:04 2012 +0300 @@ -30,6 +30,8 @@ { int ret; + i_assert(cur_cmd != NULL); + ret = cur_cmd->func(server, cur_cmd_args); if (ret == 0) return 0;
--- a/src/doveadm/server-connection.c Sat May 19 22:40:08 2012 +0300 +++ b/src/doveadm/server-connection.c Sun May 20 03:25:04 2012 +0300 @@ -197,6 +197,7 @@ const unsigned char *data; size_t size; const char *line; + enum server_cmd_reply reply; if (!conn->handshaked) { if ((line = i_stream_read_next_line(conn->input)) == NULL) { @@ -254,15 +255,18 @@ if (conn->state != SERVER_REPLY_STATE_RET) break; /* fall through */ - data = i_stream_get_data(conn->input, &size); case SERVER_REPLY_STATE_RET: - if (size < 2) + line = i_stream_next_line(conn->input); + if (line == NULL) return; - if (data[0] == '+' && data[1] == '\n') + if (line[0] == '+') server_connection_callback(conn, SERVER_CMD_REPLY_OK); - else if (data[0] == '-' && data[1] == '\n') - server_connection_callback(conn, SERVER_CMD_REPLY_FAIL); - else + else if (line[0] == '-') { + reply = strcmp(line+1, "NOUSER") == 0 ? + SERVER_CMD_REPLY_UNKNOWN_USER : + SERVER_CMD_REPLY_FAIL; + server_connection_callback(conn, reply); + } else i_error("doveadm server sent broken input"); /* we're finished, close the connection */ server_connection_destroy(&conn);
--- a/src/doveadm/server-connection.h Sat May 19 22:40:08 2012 +0300 +++ b/src/doveadm/server-connection.h Sun May 20 03:25:04 2012 +0300 @@ -3,6 +3,7 @@ enum server_cmd_reply { SERVER_CMD_REPLY_INTERNAL_FAILURE, + SERVER_CMD_REPLY_UNKNOWN_USER, SERVER_CMD_REPLY_FAIL, SERVER_CMD_REPLY_OK };
--- a/src/imap-login/client.c Sat May 19 22:40:08 2012 +0300 +++ b/src/imap-login/client.c Sun May 20 03:25:04 2012 +0300 @@ -123,6 +123,10 @@ client->common.local_port = atoi(value); else if (strcasecmp(key, "x-proxy-ttl") == 0) client->common.proxy_ttl = atoi(value); + else if (strcasecmp(key, "x-session-id") == 0) { + client->common.session_id = + p_strdup(client->common.pool, value); + } args += 2; } }
--- a/src/imap-login/imap-proxy.c Sat May 19 22:40:08 2012 +0300 +++ b/src/imap-login/imap-proxy.c Sun May 20 03:25:04 2012 +0300 @@ -32,11 +32,13 @@ i_assert(client->common.proxy_ttl > 0); str_printfa(str, "I ID (" + "\"x-session-id\" \"%s\" " "\"x-originating-ip\" \"%s\" " "\"x-originating-port\" \"%u\" " "\"x-connected-ip\" \"%s\" " "\"x-connected-port\" \"%u\" " "\"x-proxy-ttl\" \"%u\")\r\n", + client_get_session_id(&client->common), net_ip2addr(&client->common.ip), client->common.remote_port, net_ip2addr(&client->common.local_ip),
--- a/src/imap/Makefile.am Sat May 19 22:40:08 2012 +0300 +++ b/src/imap/Makefile.am Sun May 20 03:25:04 2012 +0300 @@ -19,12 +19,15 @@ ../lib/mountpoint.o endif -libs = \ +imap_LDADD = \ + $(unused_objects) \ $(LIBDOVECOT_STORAGE) \ - $(unused_objects) - -imap_LDADD = $(libs) $(LIBDOVECOT) $(MODULE_LIBS) -imap_DEPENDENCIES = $(libs) $(LIBDOVECOT_DEPS) + $(LIBDOVECOT) \ + $(MODULE_LIBS) +imap_DEPENDENCIES = \ + $(unused_objects) \ + $(LIBDOVECOT_STORAGE_DEPS) \ + $(LIBDOVECOT_DEPS) cmds = \ cmd-append.c \
--- a/src/imap/cmd-append.c Sat May 19 22:40:08 2012 +0300 +++ b/src/imap/cmd-append.c Sun May 20 03:25:04 2012 +0300 @@ -227,11 +227,11 @@ enum mail_flags flags; const char *const *keywords_list; struct mail_keywords *keywords; - const char *internal_date_str; + const char *internal_date_str, *msg; time_t internal_date; int ret, timezone_offset; unsigned int save_count; - bool nonsync; + bool nonsync, fatal; if (cmd->cancel) { cmd_append_finish(ctx); @@ -245,8 +245,13 @@ ret = imap_parser_read_args(ctx->save_parser, 0, IMAP_PARSE_FLAG_LITERAL_SIZE, &args); if (ret == -1) { - if (!ctx->failed) - client_send_command_error(cmd, NULL); + if (!ctx->failed) { + msg = imap_parser_get_error(ctx->save_parser, &fatal); + if (fatal) + client_disconnect_with_error(client, msg); + else + client_send_command_error(cmd, msg); + } cmd_append_finish(ctx); return TRUE; }
--- a/src/imap/cmd-fetch.c Sat May 19 22:40:08 2012 +0300 +++ b/src/imap/cmd-fetch.c Sun May 20 03:25:04 2012 +0300 @@ -58,6 +58,11 @@ } else { *next_arg_r = arg + 1; arg = imap_arg_as_list(arg); + if (IMAP_ARG_IS_EOL(arg)) { + client_send_command_error(cmd, + "FETCH list is empty."); + return FALSE; + } while (imap_arg_get_atom(arg, &str)) { str = t_str_ucase(str); arg++;
--- a/src/imap/cmd-list.c Sat May 19 22:40:08 2012 +0300 +++ b/src/imap/cmd-list.c Sun May 20 03:25:04 2012 +0300 @@ -115,9 +115,10 @@ MAILBOX_LIST_ITER_RETURN_SUBSCRIBED; } else if (strcasecmp(str, "RECURSIVEMATCH") == 0) list_flags |= MAILBOX_LIST_ITER_SELECT_RECURSIVEMATCH; - else if (strcasecmp(str, "SPECIAL-USE") == 0) - list_flags |= MAILBOX_LIST_ITER_SELECT_SPECIALUSE; - else if (strcasecmp(str, "REMOTE") == 0) { + else if (strcasecmp(str, "SPECIAL-USE") == 0) { + list_flags |= MAILBOX_LIST_ITER_SELECT_SPECIALUSE | + MAILBOX_LIST_ITER_RETURN_SPECIALUSE; + } else if (strcasecmp(str, "REMOTE") == 0) { /* not supported, ignore */ } else { /* skip also optional list value */ @@ -273,6 +274,8 @@ { struct mail_namespace *const *listed; const struct mailbox_settings *mailbox_set; + struct mailbox *box; + enum mailbox_existence existence; unsigned int len; enum mailbox_info_flags flags; const char *name; @@ -307,11 +310,18 @@ ctx->inbox_found = TRUE; flags = list_get_inbox_flags(ctx); - } else if (same_ns && - mailbox_list_mailbox(ctx->ns->list, "", &flags) > 0) { - /* mailbox with namespace prefix exists */ + } else if (!same_ns) { + /* parent */ + flags = MAILBOX_NONEXISTENT; } else { - flags = MAILBOX_NONEXISTENT; + /* see if namespace prefix is selectable */ + box = mailbox_alloc(ctx->ns->list, name, 0); + if (mailbox_exists(box, TRUE, &existence) == 0 && + existence == MAILBOX_EXISTENCE_SELECT) + flags = MAILBOX_SELECT; + else + flags = MAILBOX_NONEXISTENT; + mailbox_free(&box); } if ((flags & MAILBOX_CHILDREN) == 0) {
--- a/src/imap/imap-client.c Sat May 19 22:40:08 2012 +0300 +++ b/src/imap/imap-client.c Sun May 20 03:25:04 2012 +0300 @@ -34,7 +34,8 @@ client_destroy(client, "Disconnected for inactivity"); } -struct client *client_create(int fd_in, int fd_out, struct mail_user *user, +struct client *client_create(int fd_in, int fd_out, const char *session_id, + struct mail_user *user, struct mail_storage_service_user *service_user, const struct imap_settings *set) { @@ -52,6 +53,7 @@ client->v = imap_client_vfuncs; client->set = set; client->service_user = service_user; + client->session_id = p_strdup(pool, session_id); client->fd_in = fd_in; client->fd_out = fd_out; client->input = i_stream_create_fd(fd_in, @@ -85,6 +87,11 @@ str_append_c(client->capability_string, ' '); str_append(client->capability_string, set->imap_capability + 1); } + if (user->fuzzy_search) { + /* Enable FUZZY capability only when it actually has + a chance of working */ + str_append(client->capability_string, " SEARCH=FUZZY"); + } ident = mail_user_get_anvil_userip_ident(client->user); if (ident != NULL) { @@ -141,6 +148,7 @@ static struct var_expand_table static_tab[] = { { 'i', NULL, "input" }, { 'o', NULL, "output" }, + { '\0', NULL, "session" }, { '\0', NULL, NULL } }; struct var_expand_table *tab; @@ -151,6 +159,7 @@ tab[0].value = dec2str(i_stream_get_absolute_offset(client->input)); tab[1].value = dec2str(client->output->offset); + tab[2].value = client->session_id; str = t_str_new(128); var_expand(str, client->set->imap_logout_format, tab);
--- a/src/imap/imap-client.h Sat May 19 22:40:08 2012 +0300 +++ b/src/imap/imap-client.h Sun May 20 03:25:04 2012 +0300 @@ -95,6 +95,7 @@ struct client *prev, *next; struct imap_client_vfuncs v; + const char *session_id; int fd_in, fd_out; struct io *io; @@ -177,7 +178,8 @@ /* Create new client with specified input/output handles. socket specifies if the handle is a socket. */ -struct client *client_create(int fd_in, int fd_out, struct mail_user *user, +struct client *client_create(int fd_in, int fd_out, const char *session_id, + struct mail_user *user, struct mail_storage_service_user *service_user, const struct imap_settings *set); void client_destroy(struct client *client, const char *reason);
--- a/src/imap/imap-commands.c Sat May 19 22:40:08 2012 +0300 +++ b/src/imap/imap-commands.c Sun May 20 03:25:04 2012 +0300 @@ -59,7 +59,8 @@ { "UID SORT", cmd_sort, COMMAND_FLAG_BREAKS_SEQS }, { "UID THREAD", cmd_thread, COMMAND_FLAG_BREAKS_SEQS }, { "UNSELECT", cmd_unselect, COMMAND_FLAG_BREAKS_MAILBOX }, - { "X-CANCEL", cmd_x_cancel, 0 } + { "X-CANCEL", cmd_x_cancel, 0 }, + { "XLIST", cmd_list, 0 } }; #define IMAP_EXT_COMMANDS_COUNT N_ELEMENTS(imap_ext_commands)
--- a/src/imap/imap-sync.c Sat May 19 22:40:08 2012 +0300 +++ b/src/imap/imap-sync.c Sun May 20 03:25:04 2012 +0300 @@ -669,8 +669,11 @@ for (; cmd != NULL; cmd = prev) { prev = cmd->next; - if (cmd->state == CLIENT_COMMAND_STATE_WAIT_SYNC && - (cmd->sync->flags & MAILBOX_SYNC_FLAG_FAST) != 0) { + if (cmd->state != CLIENT_COMMAND_STATE_WAIT_SYNC) + continue; + + i_assert(cmd->sync != NULL); + if ((cmd->sync->flags & MAILBOX_SYNC_FLAG_FAST) != 0) { if (cmd_finish_sync(cmd)) { client_command_free(&cmd); ret = TRUE;
--- a/src/imap/main.c Sat May 19 22:40:08 2012 +0300 +++ b/src/imap/main.c Sun May 20 03:25:04 2012 +0300 @@ -208,12 +208,13 @@ if (set->verbose_proctitle) verbose_proctitle = TRUE; - client = client_create(fd_in, fd_out, mail_user, user, set); + client = client_create(fd_in, fd_out, input->session_id, + mail_user, user, set); T_BEGIN { client_add_input(client, input_buf); } T_END; - flags = login_client == NULL ? 0 : login_client->auth_req.flags; + flags = login_client->auth_req.flags; if ((flags & MAIL_AUTH_REQUEST_FLAG_TLS_COMPRESSION) != 0) client->tls_compression = TRUE; return 0; @@ -221,6 +222,7 @@ static void main_stdio_run(const char *username) { + struct master_login_client login_client; struct mail_storage_service_input input; const char *value, *error, *input_base64; buffer_t *input_buf; @@ -241,7 +243,9 @@ input_buf = input_base64 == NULL ? NULL : t_base64_decode_str(input_base64); - if (client_create_from_input(&input, NULL, STDIN_FILENO, STDOUT_FILENO, + memset(&login_client, 0, sizeof(login_client)); + if (client_create_from_input(&input, &login_client, + STDIN_FILENO, STDOUT_FILENO, input_buf, &error) < 0) i_fatal("%s", error); } @@ -261,6 +265,7 @@ input.remote_ip = client->auth_req.remote_ip; input.username = username; input.userdb_fields = extra_fields; + input.session_id = client->session_id; buffer_create_const_data(&input_buf, client->data, client->auth_req.data_size);
--- a/src/indexer/Makefile.am Sat May 19 22:40:08 2012 +0300 +++ b/src/indexer/Makefile.am Sun May 20 03:25:04 2012 +0300 @@ -27,12 +27,15 @@ ../lib-storage/mail-search-parser-imap.o endif -libs = \ +indexer_worker_LDADD = \ + $(unused_objects) \ $(LIBDOVECOT_STORAGE) \ - $(unused_objects) - -indexer_worker_LDADD = $(libs) $(LIBDOVECOT) $(MODULE_LIBS) -indexer_worker_DEPENDENCIES = $(libs) $(LIBDOVECOT_DEPS) + $(LIBDOVECOT) \ + $(MODULE_LIBS) +indexer_worker_DEPENDENCIES = \ + $(unused_objects) \ + $(LIBDOVECOT_STORAGE_DEPS) \ + $(LIBDOVECOT_DEPS) indexer_worker_SOURCES = \ indexer-worker.c \ indexer-worker-settings.c \
--- a/src/ipc/ipc-connection.c Sat May 19 22:40:08 2012 +0300 +++ b/src/ipc/ipc-connection.c Sun May 20 03:25:04 2012 +0300 @@ -127,7 +127,7 @@ if ((line = i_stream_next_line(conn->input)) == NULL) return; - args = t_strsplit(line, "\t"); + args = t_strsplit_tab(line); if (str_array_length(args) < 3 || strcmp(args[0], "HANDSHAKE") != 0) { i_error("IPC server sent invalid handshake");
--- a/src/lda/Makefile.am Sat May 19 22:40:08 2012 +0300 +++ b/src/lda/Makefile.am Sun May 20 03:25:04 2012 +0300 @@ -26,11 +26,17 @@ libs = \ $(unused_objects) \ - $(LIBDOVECOT_STORAGE) \ $(LIBDOVECOT_LDA) -dovecot_lda_LDADD = $(libs) $(LIBDOVECOT) $(MODULE_LIBS) -dovecot_lda_DEPENDENCIES = $(libs) $(LIBDOVECOT_DEPS) +dovecot_lda_LDADD = \ + $(libs) \ + $(LIBDOVECOT_STORAGE) \ + $(LIBDOVECOT) \ + $(MODULE_LIBS) +dovecot_lda_DEPENDENCIES = \ + $(libs) \ + $(LIBDOVECOT_STORAGE_DEPS) \ + $(LIBDOVECOT_DEPS) dovecot_lda_SOURCES = \ main.c
--- a/src/lib-auth/auth-client-request.c Sat May 19 22:40:08 2012 +0300 +++ b/src/lib-auth/auth-client-request.c Sun May 20 03:25:04 2012 +0300 @@ -44,6 +44,10 @@ if ((info->flags & AUTH_REQUEST_FLAG_VALID_CLIENT_CERT) != 0) str_append(str, "\tvalid-client-cert"); + if (info->session_id != NULL) { + str_append(str, "\tsession="); + str_tabescape_write(str, info->session_id); + } if (info->cert_username != NULL) { str_append(str, "\tcert_username="); str_tabescape_write(str, info->cert_username); @@ -82,6 +86,8 @@ request->request_info = *request_info; request->request_info.mech = p_strdup(pool, request_info->mech); request->request_info.service = p_strdup(pool, request_info->service); + request->request_info.session_id = + p_strdup_empty(pool, request_info->session_id); request->request_info.cert_username = p_strdup_empty(pool, request_info->cert_username); request->request_info.initial_resp_base64 =
--- a/src/lib-auth/auth-client.h Sat May 19 22:40:08 2012 +0300 +++ b/src/lib-auth/auth-client.h Sun May 20 03:25:04 2012 +0300 @@ -37,6 +37,7 @@ struct auth_request_info { const char *mech; const char *service; + const char *session_id; const char *cert_username; enum auth_request_flags flags;
--- a/src/lib-auth/auth-master.c Sat May 19 22:40:08 2012 +0300 +++ b/src/lib-auth/auth-master.c Sun May 20 03:25:04 2012 +0300 @@ -117,7 +117,7 @@ const char *line, *const *tmp; while ((line = i_stream_next_line(conn->input)) != NULL) { - tmp = t_strsplit(line, "\t"); + tmp = t_strsplit_tab(line); if (strcmp(tmp[0], "VERSION") == 0 && tmp[1] != NULL && tmp[2] != NULL) { if (strcmp(tmp[1], dec2str(AUTH_PROTOCOL_MAJOR)) != 0) { @@ -230,7 +230,7 @@ { const char *cmd, *const *args, *id, *wanted_id; - args = t_strsplit(line, "\t"); + args = t_strsplit_tab(line); cmd = *args; args++; if (*args == NULL) id = "";
--- a/src/lib-auth/auth-server-connection.c Sat May 19 22:40:08 2012 +0300 +++ b/src/lib-auth/auth-server-connection.c Sun May 20 03:25:04 2012 +0300 @@ -207,7 +207,7 @@ if (conn->client->debug) i_debug("auth input: %s", line); - args = t_strsplit(line, "\t"); + args = t_strsplit_tab(line); if (args[0] == NULL) { i_error("Auth server sent empty line"); return -1;
--- a/src/lib-dict/dict-file.c Sat May 19 22:40:08 2012 +0300 +++ b/src/lib-dict/dict-file.c Sun May 20 03:25:04 2012 +0300 @@ -3,12 +3,14 @@ #include "lib.h" #include "array.h" #include "hash.h" +#include "file-lock.h" #include "file-dotlock.h" #include "nfs-workarounds.h" #include "istream.h" #include "ostream.h" #include "dict-private.h" +#include <stdio.h> #include <stdlib.h> #include <unistd.h> #include <fcntl.h> @@ -17,10 +19,13 @@ struct file_dict { struct dict dict; pool_t hash_pool; + enum file_lock_method lock_method; char *path; struct hash_table *hash; int fd; + + bool refreshed; }; struct file_dict_iterate_path { @@ -75,10 +80,25 @@ const char *base_dir ATTR_UNUSED) { struct file_dict *dict; - + const char *p; + dict = i_new(struct file_dict, 1); + dict->lock_method = FILE_LOCK_METHOD_DOTLOCK; + + p = strchr(uri, ':'); + if (p == NULL) { + /* no parameters */ + dict->path = i_strdup(uri); + } else { + dict->path = i_strdup_until(uri, p++); + if (strcmp(p, "lock=fcntl") == 0) + dict->lock_method = FILE_LOCK_METHOD_FCNTL; + else if (strcmp(p, "lock=flock") == 0) + dict->lock_method = FILE_LOCK_METHOD_FLOCK; + else + i_error("dict file: Invalid parameter: %s", p+1); + } dict->dict = *driver; - dict->path = i_strdup(uri); dict->hash_pool = pool_alloconly_create("file dict", 1024); dict->hash = hash_table_create(default_pool, dict->hash_pool, 0, str_hash, (hash_cmp_callback_t *)strcmp); @@ -126,10 +146,9 @@ return FALSE; } -static int file_dict_refresh(struct file_dict *dict) +static int file_dict_open_latest(struct file_dict *dict) { - struct istream *input; - char *key, *value; + int open_type; if (!file_dict_need_refresh(dict)) return 0; @@ -138,25 +157,44 @@ if (close(dict->fd) < 0) i_error("close(%s) failed: %m", dict->path); } - dict->fd = open(dict->path, O_RDONLY); + + open_type = dict->lock_method == FILE_LOCK_METHOD_DOTLOCK ? + O_RDONLY : O_RDWR; + dict->fd = open(dict->path, open_type); if (dict->fd == -1) { if (errno == ENOENT) return 0; i_error("open(%s) failed: %m", dict->path); return -1; } + dict->refreshed = FALSE; + return 1; +} + +static int file_dict_refresh(struct file_dict *dict) +{ + struct istream *input; + char *key, *value; + + if (file_dict_open_latest(dict) < 0) + return -1; + if (dict->refreshed) + return 0; hash_table_clear(dict->hash, TRUE); p_clear(dict->hash_pool); - input = i_stream_create_fd(dict->fd, (size_t)-1, FALSE); - while ((key = i_stream_read_next_line(input)) != NULL && - (value = i_stream_read_next_line(input)) != NULL) { - key = p_strdup(dict->hash_pool, key); - value = p_strdup(dict->hash_pool, value); - hash_table_insert(dict->hash, key, value); + if (dict->fd != -1) { + input = i_stream_create_fd(dict->fd, (size_t)-1, FALSE); + while ((key = i_stream_read_next_line(input)) != NULL && + (value = i_stream_read_next_line(input)) != NULL) { + key = p_strdup(dict->hash_pool, key); + value = p_strdup(dict->hash_pool, value); + hash_table_insert(dict->hash, key, value); + } + i_stream_destroy(&input); } - i_stream_destroy(&input); + dict->refreshed = TRUE; return 0; } @@ -254,7 +292,7 @@ struct file_dict_transaction_context *ctx; pool_t pool; - pool = pool_alloconly_create("file dict transaction", 1024); + pool = pool_alloconly_create("file dict transaction", 2048); ctx = p_new(pool, struct file_dict_transaction_context, 1); ctx->ctx.dict = _dict; ctx->pool = pool; @@ -381,35 +419,92 @@ return fd_copy_stat_permissions(&src_st, dest_fd, dest_path); } +static int +file_dict_lock(struct file_dict *dict, struct file_lock **lock_r) +{ + int ret; + + if (file_dict_open_latest(dict) < 0) + return -1; + + if (dict->fd == -1) { + /* quota file doesn't exist yet, we need to create it */ + dict->fd = open(dict->path, O_CREAT | O_RDWR, 0600); + if (dict->fd == -1) { + i_error("creat(%s) failed: %m", dict->path); + return -1; + } + (void)fd_copy_parent_dir_permissions(dict->path, dict->fd, + dict->path); + } + + do { + if (file_wait_lock(dict->fd, dict->path, F_WRLCK, + dict->lock_method, + file_dict_dotlock_settings.timeout, + lock_r) <= 0) { + i_error("file_wait_lock(%s) failed: %m", dict->path); + return -1; + } + /* check again if we need to reopen the file because it was + just replaced */ + } while ((ret = file_dict_open_latest(dict)) > 0); + + return ret < 0 ? -1 : 0; +} + static int file_dict_write_changes(struct file_dict_transaction_context *ctx) { struct file_dict *dict = (struct file_dict *)ctx->ctx.dict; - struct dotlock *dotlock; + struct dotlock *dotlock = NULL; + struct file_lock *lock = NULL; + const char *temp_path = NULL; struct hash_iterate_context *iter; struct ostream *output; void *key, *value; - int fd; + int fd = -1; - fd = file_dotlock_open(&file_dict_dotlock_settings, dict->path, 0, - &dotlock); - if (fd == -1) { - i_error("file dict commit: file_dotlock_open(%s) failed: %m", - dict->path); - return -1; + switch (dict->lock_method) { + case FILE_LOCK_METHOD_FCNTL: + case FILE_LOCK_METHOD_FLOCK: + if (file_dict_lock(dict, &lock) < 0) + return -1; + temp_path = t_strdup_printf("%s.tmp", dict->path); + fd = creat(temp_path, 0600); + if (fd == -1) { + i_error("file dict commit: creat(%s) failed: %m", + temp_path); + return -1; + } + break; + case FILE_LOCK_METHOD_DOTLOCK: + fd = file_dotlock_open(&file_dict_dotlock_settings, dict->path, 0, + &dotlock); + if (fd == -1) { + i_error("file dict commit: file_dotlock_open(%s) failed: %m", + dict->path); + return -1; + } + temp_path = file_dotlock_get_lock_path(dotlock); + break; } + /* refresh once more now that we're locked */ if (file_dict_refresh(dict) < 0) { - file_dotlock_delete(&dotlock); + if (dotlock != NULL) + file_dotlock_delete(&dotlock); + else { + (void)close(fd); + file_unlock(&lock); + } return -1; } if (dict->fd != -1) { /* preserve the permissions */ - (void)fd_copy_permissions(dict->fd, dict->path, fd, - file_dotlock_get_lock_path(dotlock)); + (void)fd_copy_permissions(dict->fd, dict->path, fd, temp_path); } else { /* get initial permissions from parent directory */ - (void)fd_copy_parent_dir_permissions(dict->path, fd, - file_dotlock_get_lock_path(dotlock)); + (void)fd_copy_parent_dir_permissions(dict->path, fd, temp_path); } file_dict_apply_changes(ctx); @@ -425,10 +520,21 @@ hash_table_iterate_deinit(&iter); o_stream_destroy(&output); - if (file_dotlock_replace(&dotlock, - DOTLOCK_REPLACE_FLAG_DONT_CLOSE_FD) < 0) { - (void)close(fd); - return -1; + if (dotlock != NULL) { + if (file_dotlock_replace(&dotlock, + DOTLOCK_REPLACE_FLAG_DONT_CLOSE_FD) < 0) { + (void)close(fd); + return -1; + } + } else { + if (rename(temp_path, dict->path) < 0) { + i_error("rename(%s, %s) failed: %m", + temp_path, dict->path); + file_unlock(&lock); + (void)close(fd); + return -1; + } + file_lock_free(&lock); } if (dict->fd != -1)
--- a/src/lib-dict/dict-sql.c Sat May 19 22:40:08 2012 +0300 +++ b/src/lib-dict/dict-sql.c Sun May 20 03:25:04 2012 +0300 @@ -164,14 +164,16 @@ return FALSE; } } + + *path_len_r = path - path_start; + *pat_len_r = pat - map->pattern; + if (*pat == '\0') return *path == '\0'; else if (!partial_ok) return FALSE; else { /* partial matches must end with '/' */ - *path_len_r = path - path_start; - *pat_len_r = pat - map->pattern; return pat == map->pattern || pat[-1] == '/'; } }
--- a/src/lib-imap/test-imap-utf7.c Sat May 19 22:40:08 2012 +0300 +++ b/src/lib-imap/test-imap-utf7.c Sun May 20 03:25:04 2012 +0300 @@ -12,7 +12,7 @@ "&&x&&", "&-&-x&-&-", "~peter/mail/å°åŒ—/日本語", "~peter/mail/&U,BTFw-/&ZeVnLIqe-", "tietäjä", "tiet&AOQ-j&AOQ-", - "pää", NULL, + "p\xe4\xe4", NULL, NULL }; static const char *invalid_utf7[] = {
--- a/src/lib-index/mail-cache-fields.c Sat May 19 22:40:08 2012 +0300 +++ b/src/lib-index/mail-cache-fields.c Sun May 20 03:25:04 2012 +0300 @@ -403,7 +403,7 @@ cache->file_field_map[i] = fidx; /* update last_used if it's newer than ours */ - if (last_used[i] > cache->fields[fidx].field.last_used) + if ((time_t)last_used[i] > cache->fields[fidx].field.last_used) cache->fields[fidx].field.last_used = last_used[i]; dec = cache->fields[fidx].field.decision;
--- a/src/lib-index/mail-index-map.c Sat May 19 22:40:08 2012 +0300 +++ b/src/lib-index/mail-index-map.c Sun May 20 03:25:04 2012 +0300 @@ -174,6 +174,12 @@ return -1; } + if (ext_hdr->record_offset == 0) { + /* if we get here from extension introduction, record_offset=0 + and hdr->record_size hasn't been updated yet */ + return 0; + } + if (ext_hdr->record_offset + ext_hdr->record_size > hdr->record_size) { *error_r = t_strdup_printf("Record field points " "outside record size (%u+%u > %u)", @@ -210,9 +216,7 @@ return -1; } - /* if we get here from extension introduction, record_offset=0 and - hdr->record_size hasn't been updated yet */ - if (ext_hdr->record_offset != 0 && ext_hdr->record_size != 0) { + if (ext_hdr->record_size != 0) { if (mail_index_map_ext_hdr_check_record(hdr, ext_hdr, error_r) < 0) return -1;
--- a/src/lib-index/mail-index-modseq.c Sat May 19 22:40:08 2012 +0300 +++ b/src/lib-index/mail-index-modseq.c Sun May 20 03:25:04 2012 +0300 @@ -88,6 +88,11 @@ index->modseqs_enabled = TRUE; } +bool mail_index_have_modseq_tracking(struct mail_index *index) +{ + return mail_index_map_get_modseq_header(index->map) != NULL; +} + const struct mail_index_modseq_header * mail_index_map_get_modseq_header(struct mail_index_map *map) {
--- a/src/lib-index/mail-index-modseq.h Sat May 19 22:40:08 2012 +0300 +++ b/src/lib-index/mail-index-modseq.h Sun May 20 03:25:04 2012 +0300 @@ -26,6 +26,7 @@ mail_index_map_get_modseq_header(struct mail_index_map *map); uint64_t mail_index_map_modseq_get_highest(struct mail_index_map *map); void mail_index_modseq_enable(struct mail_index *index); +bool mail_index_have_modseq_tracking(struct mail_index *index); uint64_t mail_index_modseq_get_highest(struct mail_index_view *view); uint64_t mail_index_modseq_lookup(struct mail_index_view *view, uint32_t seq);
--- a/src/lib-index/mail-index-transaction-private.h Sat May 19 22:40:08 2012 +0300 +++ b/src/lib-index/mail-index-transaction-private.h Sun May 20 03:25:04 2012 +0300 @@ -35,6 +35,7 @@ enum mail_index_transaction_flags flags; struct mail_index_transaction_vfuncs v; struct mail_index_view *view; + struct mail_index_view *latest_view; /* NOTE: If you add anything new, remember to update mail_index_transaction_reset_v() to reset it. */ @@ -121,6 +122,9 @@ unsigned int left_idx, unsigned int right_idx, uint32_t seq); +void mail_index_transaction_lookup_latest_keywords(struct mail_index_transaction *t, + uint32_t seq, + ARRAY_TYPE(keyword_indexes) *keywords); bool mail_index_cancel_flag_updates(struct mail_index_transaction *t, uint32_t seq);
--- a/src/lib-index/mail-index-transaction-update.c Sat May 19 22:40:08 2012 +0300 +++ b/src/lib-index/mail-index-transaction-update.c Sun May 20 03:25:04 2012 +0300 @@ -977,7 +977,7 @@ t_array_init(&existing, 32); if (seq < t->first_new_seq) - mail_index_lookup_keywords(t->view, seq, &existing); + mail_index_transaction_lookup_latest_keywords(t, seq, &existing); existing_idx = array_get(&existing, &existing_count); if (modify_type == MODIFY_REPLACE && existing_count != keywords->count)
--- a/src/lib-index/mail-index-transaction.c Sat May 19 22:40:08 2012 +0300 +++ b/src/lib-index/mail-index-transaction.c Sun May 20 03:25:04 2012 +0300 @@ -52,6 +52,8 @@ array_free(&t->module_contexts); mail_index_view_transaction_unref(t->view); + if (t->latest_view != NULL) + mail_index_view_close(&t->latest_view); mail_index_view_close(&t->view); i_free(t); } @@ -89,6 +91,21 @@ return next_uid; } +void mail_index_transaction_lookup_latest_keywords(struct mail_index_transaction *t, + uint32_t seq, + ARRAY_TYPE(keyword_indexes) *keywords) +{ + uint32_t uid, latest_seq; + + if (t->latest_view == NULL) { + (void)mail_index_refresh(t->view->index); + t->latest_view = mail_index_view_open(t->view->index); + } + mail_index_lookup_uid(t->view, seq, &uid); + if (mail_index_lookup_seq(t->view, uid, &latest_seq)) + mail_index_lookup_keywords(t->view, latest_seq, keywords); +} + static int mail_transaction_log_file_refresh(struct mail_index_transaction *t, struct mail_transaction_log_append_ctx *ctx)
--- a/src/lib-index/mail-transaction-log-file.c Sat May 19 22:40:08 2012 +0300 +++ b/src/lib-index/mail-transaction-log-file.c Sun May 20 03:25:04 2012 +0300 @@ -945,8 +945,8 @@ modseq_ext_len) == 0) { /* modseq tracking started */ *cur_modseq += 1; - return; } + return; } else { /* not tracking modseqs */ return;
--- a/src/lib-index/test-mail-index-transaction-update.c Sat May 19 22:40:08 2012 +0300 +++ b/src/lib-index/test-mail-index-transaction-update.c Sun May 20 03:25:04 2012 +0300 @@ -43,6 +43,12 @@ return hdr.messages_count; } +void mail_index_transaction_lookup_latest_keywords(struct mail_index_transaction *t ATTR_UNUSED, + uint32_t seq ATTR_UNUSED, + ARRAY_TYPE(keyword_indexes) *keywords ATTR_UNUSED) +{ +} + static struct mail_index_transaction * mail_index_transaction_new(void) {
--- a/src/lib-lda/mail-deliver.c Sat May 19 22:40:08 2012 +0300 +++ b/src/lib-lda/mail-deliver.c Sun May 20 03:25:04 2012 +0300 @@ -268,7 +268,7 @@ struct mailbox_header_lookup_ctx *headers_ctx; struct mail_keywords *kw; enum mail_error error; - const char *mailbox_name, *errstr; + const char *mailbox_name, *errstr, *guid; struct mail_transaction_commit_changes changes; const struct seq_range *range; bool default_save; @@ -338,7 +338,11 @@ t = mailbox_transaction_begin(box, 0); ctx->dest_mail = mail_alloc(t, MAIL_FETCH_STREAM_BODY, NULL); - if (!mail_set_uid(ctx->dest_mail, range[0].seq1)) { + /* copying needs the message body. with maildir we also + need to get the GUID in case the message gets + expunged */ + if (!mail_set_uid(ctx->dest_mail, range[0].seq1) || + mail_get_special(ctx->dest_mail, MAIL_FETCH_GUID, &guid) < 0) { mail_free(&ctx->dest_mail); mailbox_transaction_rollback(&t); }
--- a/src/lib-mail/Makefile.am Sat May 19 22:40:08 2012 +0300 +++ b/src/lib-mail/Makefile.am Sun May 20 03:25:04 2012 +0300 @@ -8,6 +8,7 @@ libmail_la_SOURCES = \ istream-dot.c \ istream-header-filter.c \ + mail-user-hash.c \ mbox-from.c \ message-address.c \ message-date.c \ @@ -28,6 +29,7 @@ headers = \ istream-dot.h \ istream-header-filter.h \ + mail-user-hash.h \ mbox-from.h \ mail-types.h \ message-address.h \
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-mail/mail-user-hash.c Sun May 20 03:25:04 2012 +0300 @@ -0,0 +1,41 @@ +/* Copyright (c) 2012 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "md5.h" +#include "str.h" +#include "var-expand.h" +#include "mail-user-hash.h" + +unsigned int mail_user_hash(const char *username, const char *format) +{ + static struct var_expand_table static_tab[] = { + { 'u', NULL, "user" }, + { 'n', NULL, "username" }, + { 'd', NULL, "domain" }, + { '\0', NULL, NULL } + }; + struct var_expand_table *tab; + unsigned char md5[MD5_RESULTLEN]; + unsigned int i, hash = 0; + + if (strcmp(format, "%u") == 0) { + /* fast path */ + md5_get_digest(username, strlen(username), md5); + } else T_BEGIN { + string_t *str = t_str_new(128); + + tab = t_malloc(sizeof(static_tab)); + memcpy(tab, static_tab, sizeof(static_tab)); + tab[0].value = username; + tab[1].value = t_strcut(username, '@'); + tab[2].value = strchr(username, '@'); + if (tab[2].value != NULL) tab[2].value++; + + var_expand(str, format, tab); + md5_get_digest(str_data(str), str_len(str), md5); + } T_END; + for (i = 0; i < sizeof(hash); i++) + hash = (hash << CHAR_BIT) | md5[i]; + return hash; +} +
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-mail/mail-user-hash.h Sun May 20 03:25:04 2012 +0300 @@ -0,0 +1,8 @@ +#ifndef MAIL_USER_HASH +#define MAIL_USER_HASH + +/* Return a hash for username, based on given format. The format can use + %n, %d and %u variables. */ +unsigned int mail_user_hash(const char *username, const char *format); + +#endif
--- a/src/lib-mail/message-date.c Sat May 19 22:40:08 2012 +0300 +++ b/src/lib-mail/message-date.c Sun May 20 03:25:04 2012 +0300 @@ -9,6 +9,11 @@ #include <ctype.h> +/* RFC specifies ':' as the only allowed separator, + but be forgiving also for some broken ones */ +#define IS_TIME_SEP(c) \ + ((c) == ':' || (c) == '.') + struct message_date_parser_context { struct rfc822_parser_context parser; string_t *str; @@ -189,7 +194,7 @@ } /* :mm (may be the last token) */ - if (*ctx->parser.data != ':') + if (!IS_TIME_SEP(*ctx->parser.data)) return FALSE; ctx->parser.data++; (void)rfc822_skip_lwsp(&ctx->parser); @@ -200,7 +205,8 @@ tm.tm_min = (value[0]-'0') * 10 + (value[1]-'0'); /* [:ss] */ - if (ctx->parser.data != ctx->parser.end && *ctx->parser.data == ':') { + if (ctx->parser.data != ctx->parser.end && + IS_TIME_SEP(*ctx->parser.data)) { ctx->parser.data++; (void)rfc822_skip_lwsp(&ctx->parser);
--- a/src/lib-mail/message-parser.c Sat May 19 22:40:08 2012 +0300 +++ b/src/lib-mail/message-parser.c Sun May 20 03:25:04 2012 +0300 @@ -913,10 +913,10 @@ return preparsed_parse_next_header(ctx, block_r); } -struct message_parser_ctx * -message_parser_init(pool_t part_pool, struct istream *input, - enum message_header_parser_flags hdr_flags, - enum message_parser_flags flags) +static struct message_parser_ctx * +message_parser_init_int(struct istream *input, + enum message_header_parser_flags hdr_flags, + enum message_parser_flags flags) { struct message_parser_ctx *ctx; pool_t pool; @@ -924,14 +924,24 @@ pool = pool_alloconly_create("Message Parser", 1024); ctx = p_new(pool, struct message_parser_ctx, 1); ctx->parser_pool = pool; - ctx->part_pool = part_pool; ctx->hdr_flags = hdr_flags; ctx->flags = flags; ctx->input = input; - ctx->parts = ctx->part = part_pool == NULL ? NULL : - p_new(part_pool, struct message_part, 1); + i_stream_ref(input); + return ctx; +} + +struct message_parser_ctx * +message_parser_init(pool_t part_pool, struct istream *input, + enum message_header_parser_flags hdr_flags, + enum message_parser_flags flags) +{ + struct message_parser_ctx *ctx; + + ctx = message_parser_init_int(input, hdr_flags, flags); + ctx->part_pool = part_pool; + ctx->parts = ctx->part = p_new(part_pool, struct message_part, 1); ctx->parse_next_block = parse_next_header_init; - i_stream_ref(input); return ctx; } @@ -943,7 +953,7 @@ { struct message_parser_ctx *ctx; - ctx = message_parser_init(NULL, input, hdr_flags, flags); + ctx = message_parser_init_int(input, hdr_flags, flags); ctx->parts = ctx->part = parts; ctx->parse_next_block = preparsed_parse_next_header_init; return ctx; @@ -1017,6 +1027,7 @@ break; } i_assert(ret != 0); + i_assert(ctx->part != NULL); if (ret < 0) { /* well, can't return error so fake end of headers */
--- a/src/lib-mail/rfc822-parser.c Sat May 19 22:40:08 2012 +0300 +++ b/src/lib-mail/rfc822-parser.c Sun May 20 03:25:04 2012 +0300 @@ -231,7 +231,7 @@ if (ctx->data == ctx->end) return -1; - str_append_n(str, start, ctx->data - start); + str_append_n(str, start, ctx->data - start - 1); start = ctx->data; break; }
--- a/src/lib-master/anvil-client.c Sat May 19 22:40:08 2012 +0300 +++ b/src/lib-master/anvil-client.c Sun May 20 03:25:04 2012 +0300 @@ -80,7 +80,7 @@ if (ioloop_time - client->last_reconnect < ANVIL_RECONNECT_MIN_SECS) { if (client->to_reconnect == NULL) { client->to_reconnect = - timeout_add(ANVIL_RECONNECT_MIN_SECS, + timeout_add(ANVIL_RECONNECT_MIN_SECS*1000, anvil_reconnect, client); } } else {
--- a/src/lib-master/master-login-auth.c Sat May 19 22:40:08 2012 +0300 +++ b/src/lib-master/master-login-auth.c Sun May 20 03:25:04 2012 +0300 @@ -227,7 +227,7 @@ /* <id> <userid> [..] */ - list = t_strsplit(args, "\t"); + list = t_strsplit_tab(args); if (list[0] == NULL || list[1] == NULL || str_to_uint(list[0], &id) < 0) { i_error("Auth server sent corrupted USER line"); @@ -273,7 +273,7 @@ const char *const *args, *error = NULL; unsigned int i, id; - args = t_strsplit(args_line, "\t"); + args = t_strsplit_tab(args_line); if (args[0] == NULL || str_to_uint(args[0], &id) < 0) { i_error("Auth server sent broken FAIL line"); return FALSE;
--- a/src/lib-master/master-login.c Sat May 19 22:40:08 2012 +0300 +++ b/src/lib-master/master-login.c Sun May 20 03:25:04 2012 +0300 @@ -187,8 +187,10 @@ /* FIXME: currently we create a separate connection for each request, so close the connection after we're done with this client */ - if (!master_login_conn_is_closed(client->conn)) - master_login_conn_unref(&client->conn); + if (!master_login_conn_is_closed(client->conn)) { + i_assert(client->conn->refcount > 1); + client->conn->refcount--; + } master_login_conn_unref(&client->conn); i_free(client); } @@ -274,7 +276,7 @@ return; } - auth_args = t_strsplit(str_c(pl->input), "\t"); + auth_args = t_strsplit_tab(str_c(pl->input)); for (p = auth_args; *p != NULL; p++) *p = str_tabunescape(t_strdup_noconst(*p)); @@ -392,6 +394,7 @@ struct master_login_client *client; struct master_login *login = conn->login; unsigned char data[MASTER_AUTH_MAX_DATA_SIZE]; + unsigned int i, session_len = 0; int ret, client_fd; ret = master_login_conn_read_request(conn, &req, data, &client_fd); @@ -408,12 +411,26 @@ } fd_close_on_exec(client_fd, TRUE); + /* extract the session ID from the request data */ + for (i = 0; i < req.data_size; i++) { + if (data[i] == '\0') { + session_len = i++; + break; + } + } + if (session_len >= sizeof(client->session_id)) { + i_error("login client: Session ID too long"); + session_len = 0; + } + /* @UNSAFE: we have a request. do userdb lookup for it. */ + req.data_size -= i; client = i_malloc(sizeof(struct master_login_client) + req.data_size); client->conn = conn; client->fd = client_fd; client->auth_req = req; - memcpy(client->data, data, req.data_size); + memcpy(client->session_id, data, session_len); + memcpy(client->data, data+i, req.data_size); conn->refcount++; master_login_auth_request(login->auth, &req,
--- a/src/lib-master/master-login.h Sat May 19 22:40:08 2012 +0300 +++ b/src/lib-master/master-login.h Sun May 20 03:25:04 2012 +0300 @@ -4,12 +4,15 @@ #include "master-auth.h" #define MASTER_POSTLOGIN_TIMEOUT_DEFAULT 60 +/* base64(<IPv6><port><48bit timestamp>) + NUL */ +#define LOGIN_MAX_SESSION_ID_LEN 33 struct master_login_client { struct master_login_connection *conn; int fd; struct master_auth_request auth_req; + char session_id[LOGIN_MAX_SESSION_ID_LEN]; unsigned char data[FLEXIBLE_ARRAY_MEMBER]; };
--- a/src/lib-master/master-service-settings.c Sat May 19 22:40:08 2012 +0300 +++ b/src/lib-master/master-service-settings.c Sun May 20 03:25:04 2012 +0300 @@ -297,7 +297,7 @@ } T_BEGIN { - const char *const *arg = t_strsplit(line, "\t"); + const char *const *arg = t_strsplit_tab(line); ARRAY_TYPE(const_string) services; p_array_init(&services, pool, 8);
--- a/src/lib-master/master-service.c Sat May 19 22:40:08 2012 +0300 +++ b/src/lib-master/master-service.c Sun May 20 03:25:04 2012 +0300 @@ -734,6 +734,8 @@ lib_signals_deinit(); io_loop_destroy(&service->ioloop); + if (service->listener_names != NULL) + p_strsplit_free(default_pool, service->listener_names); i_free(service->listeners); i_free(service->getopt_str); i_free(service->name);
--- a/src/lib-master/mountpoint-list.c Sat May 19 22:40:08 2012 +0300 +++ b/src/lib-master/mountpoint-list.c Sun May 20 03:25:04 2012 +0300 @@ -50,6 +50,14 @@ NULL }; +const char *const mountpoint_list_default_ignore_prefixes[] = { + "/cdrom", + "/media", + "/sys", + "/proc", + NULL +}; + struct mountpoint_list * mountpoint_list_init(const char *perm_path, const char *state_path) { @@ -251,8 +259,20 @@ return FALSE; } +static bool str_array_find_prefix(const char *const *prefixes, const char *str) +{ + if (prefixes == NULL) + return FALSE; + for (; *prefixes != NULL; prefixes++) { + if (strncmp(*prefixes, str, strlen(*prefixes)) == 0) + return TRUE; + } + return FALSE; +} + int mountpoint_list_add_missing(struct mountpoint_list *list, const char *default_state, + const char *const *ignore_prefixes, const char *const *ignore_types) { struct mountpoint_list_rec new_rec, *rec, *const *recp; @@ -269,15 +289,15 @@ /* get a sorted list of all current mountpoints */ iter = mountpoint_iter_init(); while ((mnt = mountpoint_iter_next(iter)) != NULL) { - if (str_array_find(ignore_types, mnt->type)) - continue; - rec = mountpoint_list_find(list, mnt->mount_path); - if (rec == NULL) { + if (rec != NULL) { + if (!rec->wildcard) + rec->mounted = TRUE; + } else if (!str_array_find(ignore_types, mnt->type) && + !str_array_find_prefix(ignore_prefixes, + mnt->mount_path)) { new_rec.mount_path = mnt->mount_path; mountpoint_list_add(list, &new_rec); - } else if (!rec->wildcard) { - rec->mounted = TRUE; } } return mountpoint_iter_deinit(&iter);
--- a/src/lib-master/mountpoint-list.h Sat May 19 22:40:08 2012 +0300 +++ b/src/lib-master/mountpoint-list.h Sun May 20 03:25:04 2012 +0300 @@ -21,6 +21,9 @@ /* A default known good list of mountpoint types that don't contain emails (e.g. proc, tmpfs, etc.) */ extern const char *const mountpoint_list_default_ignore_types[]; +/* A default known good list of directories which shouldn't contain emails + (e.g. /media) */ +extern const char *const mountpoint_list_default_ignore_prefixes[]; struct mountpoint_list * mountpoint_list_init(const char *perm_path, const char *state_path); @@ -43,10 +46,12 @@ const char *mount_path); /* Add all currently mounted missing mountpoints to the list and update all mountpoints' mounted state. The mountpoints that match existing wildcards - aren't added. Mountpoints with type in ignore_types list also aren't added. + aren't added. Mountpoints with paths under ignore_prefixes aren't added. + Mountpoints with type in ignore_types list also aren't added. Returns 0 if we successfully iterated through all mountpoints, -1 if not. */ int mountpoint_list_add_missing(struct mountpoint_list *list, const char *default_state, + const char *const *ignore_prefixes, const char *const *ignore_types); /* Update "mounted" status for all mountpoints. */ int mountpoint_list_update_mounted(struct mountpoint_list *list);
--- a/src/lib-settings/settings-parser.c Sat May 19 22:40:08 2012 +0300 +++ b/src/lib-settings/settings-parser.c Sun May 20 03:25:04 2012 +0300 @@ -200,7 +200,8 @@ i_assert(count > 0); - parser_pool = pool_alloconly_create("settings parser", 16384); + parser_pool = pool_alloconly_create(MEMPOOL_GROWING"settings parser", + 8192); ctx = p_new(parser_pool, struct setting_parser_context, 1); ctx->set_pool = set_pool; ctx->parser_pool = parser_pool; @@ -772,7 +773,8 @@ } do { - if (link->info == &strlist_info) { + if (def == NULL) { + i_assert(link->info == &strlist_info); settings_parse_strlist(ctx, link, key, value); return 1; } @@ -803,6 +805,7 @@ return NULL; if (def == NULL) { /* strlist */ + i_assert(link->info == &strlist_info); return key; } @@ -822,7 +825,7 @@ if (!settings_find_key(ctx, key, &def, &link)) return NULL; - if (link->set_struct == NULL) + if (link->set_struct == NULL || def == NULL) return NULL; *type_r = def->type; @@ -838,7 +841,7 @@ if (!settings_find_key(ctx, key, &def, &link)) return FALSE; - if (link->change_struct == NULL) + if (link->change_struct == NULL || def == NULL) return FALSE; p = STRUCT_MEMBER_P(link->change_struct, def->offset); @@ -1158,8 +1161,9 @@ if (!settings_find_key(ctx, key, &def, &link)) return; - if (link->info == &strlist_info) { + if (def == NULL) { /* parent is strlist, no expansion needed */ + i_assert(link->info == &strlist_info); return; } @@ -1378,18 +1382,32 @@ case SET_STRLIST: { const ARRAY_TYPE(const_string) *src_arr = src; ARRAY_TYPE(const_string) *dest_arr = dest; - const char *const *strings, *dup; - unsigned int i, count; + const char *const *strings, *const *dest_strings, *dup; + unsigned int i, j, count, dest_count; if (!array_is_created(src_arr)) break; strings = array_get(src_arr, &count); + i_assert(count % 2 == 0); if (!array_is_created(dest_arr)) p_array_init(dest_arr, pool, count); - for (i = 0; i < count; i++) { + dest_count = array_count(dest_arr); + i_assert(dest_count % 2 == 0); + for (i = 0; i < count; i += 2) { + if (dest_count > 0) { + dest_strings = array_idx(dest_arr, 0); + for (j = 0; j < dest_count; j += 2) { + if (strcmp(strings[i], dest_strings[j]) == 0) + break; + } + if (j < dest_count) + continue; + } dup = p_strdup(pool, strings[i]); array_append(dest_arr, &dup, 1); + dup = p_strdup(pool, strings[i+1]); + array_append(dest_arr, &dup, 1); } break; } @@ -1720,7 +1738,8 @@ pool_t parser_pool; pool_ref(new_pool); - parser_pool = pool_alloconly_create("dup settings parser", 8192); + parser_pool = pool_alloconly_create(MEMPOOL_GROWING"dup settings parser", + 8192); new_ctx = p_new(parser_pool, struct setting_parser_context, 1); new_ctx->set_pool = new_pool; new_ctx->parser_pool = parser_pool;
--- a/src/lib-settings/settings-parser.h Sat May 19 22:40:08 2012 +0300 +++ b/src/lib-settings/settings-parser.h Sun May 20 03:25:04 2012 +0300 @@ -208,7 +208,11 @@ /* Copy changed settings from src to dest. If conflict_key_r is not NULL and both src and dest have changed the same setting, return -1 and set the - key name. If it's NULL, the old setting is kept. */ + key name. If it's NULL, the old setting is kept. + + KLUDGE: For SET_STRLIST types if both source and destination have identical + keys, the duplicates in the source side are ignored. This is required to + make the current config code work correctly. */ int settings_parser_apply_changes(struct setting_parser_context *dest, const struct setting_parser_context *src, pool_t pool, const char **conflict_key_r);
--- a/src/lib-sql/driver-mysql.c Sat May 19 22:40:08 2012 +0300 +++ b/src/lib-sql/driver-mysql.c Sun May 20 03:25:04 2012 +0300 @@ -464,9 +464,11 @@ struct mysql_db *db = (struct mysql_db *)_result->db; const char *errstr; unsigned int idle_time; + int err; + err = mysql_errno(db->mysql); errstr = mysql_error(db->mysql); - if (mysql_errno(db->mysql) == CR_SERVER_GONE_ERROR && + if ((err == CR_SERVER_GONE_ERROR || err == CR_SERVER_LOST) && db->last_success != 0) { idle_time = ioloop_time - db->last_success; errstr = t_strdup_printf("%s (idled for %u secs)",
--- a/src/lib-sql/driver-pgsql.c Sat May 19 22:40:08 2012 +0300 +++ b/src/lib-sql/driver-pgsql.c Sun May 20 03:25:04 2012 +0300 @@ -603,6 +603,9 @@ /* we don't end up in pgsql's free function, so sync_result won't be set to NULL if we don't do it here. */ db->sync_result = NULL; + } else if (result == NULL) { + result = &sql_not_connected_result; + result->refcount++; } i_assert(db->io == NULL);
--- a/src/lib-sql/driver-sqlpool.c Sat May 19 22:40:08 2012 +0300 +++ b/src/lib-sql/driver-sqlpool.c Sun May 20 03:25:04 2012 +0300 @@ -614,8 +614,8 @@ if (result->failed_try_retry && request->retry_count < array_count(&db->hosts)) { - i_error("%s: Query failed, retrying: %s", - db->driver->name, sql_result_get_error(result)); + i_warning("%s: Query failed, retrying: %s", + db->driver->name, sql_result_get_error(result)); request->retry_count++; driver_sqlpool_prepend_request(db, request);
--- a/src/lib-ssl-iostream/Makefile.am Sat May 19 22:40:08 2012 +0300 +++ b/src/lib-ssl-iostream/Makefile.am Sun May 20 03:25:04 2012 +0300 @@ -30,6 +30,6 @@ pkglib_LTLIBRARIES = libdovecot-ssl.la libdovecot_ssl_la_SOURCES = -libdovecot_ssl_la_LIBADD = libssl_iostream.la ../lib/liblib.la $(MODULE_LIBS) +libdovecot_ssl_la_LIBADD = libssl_iostream.la ../lib/liblib.la $(MODULE_LIBS) $(SSL_LIBS) libdovecot_ssl_la_DEPENDENCIES = libssl_iostream.la libdovecot_ssl_la_LDFLAGS = -export-dynamic
--- a/src/lib-ssl-iostream/iostream-openssl.c Sat May 19 22:40:08 2012 +0300 +++ b/src/lib-ssl-iostream/iostream-openssl.c Sun May 20 03:25:04 2012 +0300 @@ -517,6 +517,8 @@ } } sk_GENERAL_NAME_pop_free(gnames, GENERAL_NAME_free); + X509_free(cert); + /* verify against CommonName only when there wasn't any DNS SubjectAltNames */ if (dns_names)
--- a/src/lib-storage/Makefile.am Sat May 19 22:40:08 2012 +0300 +++ b/src/lib-storage/Makefile.am Sun May 20 03:25:04 2012 +0300 @@ -75,6 +75,7 @@ mailbox-uidvalidity.h shlibs = \ + @LINKED_STORAGE_LIBS@ \ libstorage.la \ libstorage_service.la \ list/libstorage_list.la \ @@ -85,7 +86,7 @@ pkglib_LTLIBRARIES = libdovecot-storage.la libdovecot_storage_la_SOURCES = -libdovecot_storage_la_LIBADD = $(shlibs) $(MODULE_LIBS) +libdovecot_storage_la_LIBADD = $(shlibs) $(LINKED_STORAGE_LDADD) $(MODULE_LIBS) libdovecot_storage_la_DEPENDENCIES = $(shlibs) libdovecot_storage_la_LDFLAGS = -export-dynamic
--- a/src/lib-storage/fail-mailbox.c Sat May 19 22:40:08 2012 +0300 +++ b/src/lib-storage/fail-mailbox.c Sun May 20 03:25:04 2012 +0300 @@ -299,7 +299,7 @@ struct mailbox *box; pool_t pool; - pool = pool_alloconly_create("fail mailbox", 1024); + pool = pool_alloconly_create("fail mailbox", 1024+512); box = p_new(pool, struct mailbox, 1); *box = fail_mailbox; box->vname = p_strdup(pool, vname);
--- a/src/lib-storage/index/Makefile.am Sat May 19 22:40:08 2012 +0300 +++ b/src/lib-storage/index/Makefile.am Sun May 20 03:25:04 2012 +0300 @@ -32,9 +32,6 @@ index-thread-links.c \ index-transaction.c -libstorage_index_la_LIBADD = @LINKED_STORAGE_LIBS@ -libstorage_index_la_DEPENDENCIES = @LINKED_STORAGE_LIBS@ - headers = \ istream-attachment.h \ istream-mail.h \
--- a/src/lib-storage/index/cydir/cydir-mail.c Sat May 19 22:40:08 2012 +0300 +++ b/src/lib-storage/index/cydir/cydir-mail.c Sun May 20 03:25:04 2012 +0300 @@ -22,8 +22,10 @@ { const char *path; - if (mail->lookup_abort == MAIL_LOOKUP_ABORT_NOT_IN_CACHE) - return mail_set_aborted(mail); + if (mail->lookup_abort == MAIL_LOOKUP_ABORT_NOT_IN_CACHE) { + mail_set_aborted(mail); + return -1; + } mail->transaction->stats.stat_lookup_count++; path = cydir_mail_get_path(mail);
--- a/src/lib-storage/index/dbox-common/dbox-file.c Sat May 19 22:40:08 2012 +0300 +++ b/src/lib-storage/index/dbox-common/dbox-file.c Sun May 20 03:25:04 2012 +0300 @@ -407,7 +407,7 @@ /* skip over the actual metadata */ buf_size = i_stream_get_max_buffer_size(file->input); - i_stream_set_max_buffer_size(file->input, 0); + i_stream_set_max_buffer_size(file->input, (size_t)-1); while ((line = i_stream_read_next_line(file->input)) != NULL) { if (*line == DBOX_METADATA_OLDV1_SPACE || *line == '\0') { /* end of metadata */ @@ -657,7 +657,8 @@ ret = 0; buf_size = i_stream_get_max_buffer_size(file->input); - i_stream_set_max_buffer_size(file->input, 0); + /* use unlimited line length for metadata */ + i_stream_set_max_buffer_size(file->input, (size_t)-1); while ((line = i_stream_read_next_line(file->input)) != NULL) { if (*line == DBOX_METADATA_OLDV1_SPACE || *line == '\0') { /* end of metadata */
--- a/src/lib-storage/index/dbox-common/dbox-storage.c Sat May 19 22:40:08 2012 +0300 +++ b/src/lib-storage/index/dbox-common/dbox-storage.c Sun May 20 03:25:04 2012 +0300 @@ -147,15 +147,18 @@ dbox_cleanup_if_exists(struct mailbox_list *list, const char *path) { struct stat st; + unsigned int interval = list->mail_set->mail_temp_scan_interval; if (stat(path, &st) < 0) return FALSE; /* check once in a while if there are temp files to clean up */ - if (st.st_atime > st.st_ctime + DBOX_TMP_DELETE_SECS) { + if (interval == 0) { + /* disabled */ + } else if (st.st_atime > st.st_ctime + DBOX_TMP_DELETE_SECS) { /* there haven't been any changes to this directory since we last checked it. */ - } else if (st.st_atime < ioloop_time - DBOX_TMP_SCAN_SECS) { + } else if (st.st_atime < ioloop_time - interval) { /* time to scan */ const char *prefix = mailbox_list_get_global_temp_prefix(list); @@ -172,7 +175,7 @@ if (dbox_cleanup_if_exists(box->list, box_path)) ; - else if (errno == ENOENT) { + else if (errno == ENOENT || errno == ENAMETOOLONG) { mail_storage_set_error(box->storage, MAIL_ERROR_NOTFOUND, T_MAIL_ERR_MAILBOX_NOT_FOUND(box->name)); return -1;
--- a/src/lib-storage/index/dbox-common/dbox-storage.h Sat May 19 22:40:08 2012 +0300 +++ b/src/lib-storage/index/dbox-common/dbox-storage.h Sun May 20 03:25:04 2012 +0300 @@ -18,8 +18,6 @@ #define DBOX_TRASH_DIR_NAME "trash" #define DBOX_MAILDIR_NAME "dbox-Mails" -/* How often to scan for stale temp files (based on dir's atime) */ -#define DBOX_TMP_SCAN_SECS (8*60*60) /* Delete temp files having ctime older than this. */ #define DBOX_TMP_DELETE_SECS (36*60*60)
--- a/src/lib-storage/index/dbox-multi/mdbox-map.c Sat May 19 22:40:08 2012 +0300 +++ b/src/lib-storage/index/dbox-multi/mdbox-map.c Sun May 20 03:25:04 2012 +0300 @@ -133,16 +133,20 @@ static void mdbox_map_cleanup(struct mdbox_map *map) { + unsigned int interval = + MAP_STORAGE(map)->set->mail_temp_scan_interval; struct stat st; if (stat(map->path, &st) < 0) return; /* check once in a while if there are temp files to clean up */ - if (st.st_atime > st.st_ctime + DBOX_TMP_DELETE_SECS) { + if (interval == 0) { + /* disabled */ + } else if (st.st_atime > st.st_ctime + DBOX_TMP_DELETE_SECS) { /* there haven't been any changes to this directory since we last checked it. */ - } else if (st.st_atime < ioloop_time - DBOX_TMP_SCAN_SECS) { + } else if (st.st_atime < ioloop_time - interval) { /* time to scan */ (void)unlink_old_files(map->path, DBOX_TEMP_FILE_PREFIX, ioloop_time - DBOX_TMP_DELETE_SECS);
--- a/src/lib-storage/index/dbox-multi/mdbox-purge.c Sat May 19 22:40:08 2012 +0300 +++ b/src/lib-storage/index/dbox-multi/mdbox-purge.c Sun May 20 03:25:04 2012 +0300 @@ -107,7 +107,8 @@ o_stream_send(output, &meta_hdr, sizeof(meta_hdr)); buf_size = i_stream_get_max_buffer_size(file->input); - i_stream_set_max_buffer_size(file->input, 0); + /* use unlimited line length for metadata */ + i_stream_set_max_buffer_size(file->input, (size_t)-1); while ((line = i_stream_read_next_line(file->input)) != NULL) { if (*line == '\0') { /* end of metadata */ @@ -140,7 +141,8 @@ return ret; buf_size = i_stream_get_max_buffer_size(file->input); - i_stream_set_max_buffer_size(file->input, 0); + /* use unlimited line length for metadata */ + i_stream_set_max_buffer_size(file->input, (size_t)-1); while ((line = i_stream_read_next_line(file->input)) != NULL) { if (*line == '\0') { /* end of metadata */
--- a/src/lib-storage/index/dbox-multi/mdbox-storage.c Sat May 19 22:40:08 2012 +0300 +++ b/src/lib-storage/index/dbox-multi/mdbox-storage.c Sun May 20 03:25:04 2012 +0300 @@ -249,6 +249,7 @@ { struct mdbox_mailbox *mbox = (struct mdbox_mailbox *)box; struct mail_index_transaction *new_trans = NULL; + struct mail_index_view *view; const struct mail_index_header *hdr; uint32_t uid_validity, uid_next; @@ -260,7 +261,8 @@ trans = new_trans; } - hdr = mail_index_get_header(box->view); + view = mail_index_view_open(box->index); + hdr = mail_index_get_header(view); uid_validity = hdr->uid_validity; if (update != NULL && update->uid_validity != 0) uid_validity = update->uid_validity; @@ -293,12 +295,12 @@ &first_recent_uid, sizeof(first_recent_uid), FALSE); } if (update != NULL && update->min_highest_modseq != 0 && - mail_index_modseq_get_highest(box->view) < - update->min_highest_modseq) { + mail_index_modseq_get_highest(view) < update->min_highest_modseq) { mail_index_modseq_enable(box->index); mail_index_update_highest_modseq(trans, update->min_highest_modseq); } + mail_index_view_close(&view); mdbox_update_header(mbox, trans, update); if (new_trans != NULL) {
--- a/src/lib-storage/index/dbox-multi/mdbox-sync.c Sat May 19 22:40:08 2012 +0300 +++ b/src/lib-storage/index/dbox-multi/mdbox-sync.c Sun May 20 03:25:04 2012 +0300 @@ -115,11 +115,13 @@ if (box->v.sync_notify != NULL) { /* do notifications after commit finished successfully */ + box->tmp_sync_view = ctx->sync_view; seq_range_array_iter_init(&iter, &ctx->expunged_seqs); n = 0; while (seq_range_array_iter_nth(&iter, n++, &seq)) { mail_index_lookup_uid(ctx->sync_view, seq, &uid); box->v.sync_notify(box, uid, MAILBOX_SYNC_TYPE_EXPUNGE); } + box->tmp_sync_view = NULL; } return 0; } @@ -274,6 +276,12 @@ /* we'll need to rebuild storage. try again from the beginning. */ mdbox_storage_set_corrupted(mbox->storage); + if ((flags & MDBOX_SYNC_FLAG_NO_REBUILD) != 0) { + mail_storage_set_critical(storage, + "mdbox %s: Can't rebuild storage", + mailbox_get_path(&mbox->box)); + return -1; + } return mdbox_sync_begin(mbox, flags, atomic, ctx_r); }
--- a/src/lib-storage/index/dbox-single/sdbox-copy.c Sat May 19 22:40:08 2012 +0300 +++ b/src/lib-storage/index/dbox-single/sdbox-copy.c Sun May 20 03:25:04 2012 +0300 @@ -104,9 +104,12 @@ if (ret < 0) { if (ECANTLINK(errno)) ret = 0; - else if (errno == ENOENT) - mail_set_expunged(mail); - else { + else if (errno == ENOENT) { + /* try if the fallback copying code can still + read the file (the mail could still have the + stream open) */ + ret = 0; + } else { mail_storage_set_critical( _ctx->transaction->box->storage, "link(%s, %s) failed: %m",
--- a/src/lib-storage/index/dbox-single/sdbox-save.c Sat May 19 22:40:08 2012 +0300 +++ b/src/lib-storage/index/dbox-single/sdbox-save.c Sun May 20 03:25:04 2012 +0300 @@ -78,9 +78,18 @@ void sdbox_save_add_file(struct mail_save_context *_ctx, struct dbox_file *file) { struct sdbox_save_context *ctx = (struct sdbox_save_context *)_ctx; + struct dbox_file *const *files; + unsigned int count; if (ctx->first_saved_seq == 0) ctx->first_saved_seq = ctx->ctx.seq; + + files = array_get(&ctx->files, &count); + if (count > 0) { + /* a plugin may leave a previously saved file open. + we'll close it here to avoid eating too many fds. */ + dbox_file_close(files[count-1]); + } array_append(&ctx->files, &file, 1); }
--- a/src/lib-storage/index/dbox-single/sdbox-sync-rebuild.c Sat May 19 22:40:08 2012 +0300 +++ b/src/lib-storage/index/dbox-single/sdbox-sync-rebuild.c Sun May 20 03:25:04 2012 +0300 @@ -195,6 +195,7 @@ return 0; } } + i_warning("sdbox %s: Rebuilding index", mailbox_get_path(&mbox->box)); if (dbox_sync_rebuild_verify_alt_storage(mbox->box.list) < 0) { mail_storage_set_critical(mbox->box.storage,
--- a/src/lib-storage/index/dbox-single/sdbox-sync.c Sat May 19 22:40:08 2012 +0300 +++ b/src/lib-storage/index/dbox-single/sdbox-sync.c Sun May 20 03:25:04 2012 +0300 @@ -150,10 +150,12 @@ /* NOTE: Index is no longer locked. Multiple processes may be unlinking the files at the same time. */ + ctx->mbox->box.tmp_sync_view = ctx->sync_view; array_foreach(&ctx->expunged_uids, uidp) dbox_sync_file_expunge(ctx, *uidp); if (ctx->mbox->box.v.sync_notify != NULL) ctx->mbox->box.v.sync_notify(&ctx->mbox->box, 0, 0); + ctx->mbox->box.tmp_sync_view = NULL; } static int @@ -235,8 +237,6 @@ ret = -1; } else { /* do a full resync and try again. */ - i_warning("sdbox %s: Rebuilding index", - mailbox_get_path(&ctx->mbox->box)); rebuild = FALSE; ret = sdbox_sync_index_rebuild(mbox, force_rebuild); @@ -262,11 +262,13 @@ *_ctx = NULL; if (success) { + mail_index_view_ref(ctx->sync_view); if (mail_index_sync_commit(&ctx->index_sync_ctx) < 0) { mail_storage_set_index_error(&ctx->mbox->box); ret = -1; } else { dbox_sync_expunge_files(ctx); + mail_index_view_close(&ctx->sync_view); } } else { mail_index_sync_rollback(&ctx->index_sync_ctx);
--- a/src/lib-storage/index/imapc/imapc-list.c Sat May 19 22:40:08 2012 +0300 +++ b/src/lib-storage/index/imapc/imapc-list.c Sun May 20 03:25:04 2012 +0300 @@ -33,6 +33,7 @@ list->list.pool = pool; /* separator is set when storage is created */ list->mailboxes = mailbox_tree_init('\0'); + mailbox_tree_set_parents_nonexistent(list->mailboxes); return &list->list; } @@ -92,7 +93,7 @@ T_BEGIN { const char *vname = - mailbox_list_default_get_vname(&list->list, name); + mailbox_list_get_vname(&list->list, name); if ((info_flags & MAILBOX_NONEXISTENT) != 0) node = mailbox_tree_lookup(tree, vname); @@ -191,6 +192,37 @@ return list->sep; } +static const char * +imapc_list_get_storage_name(struct mailbox_list *_list, const char *vname) +{ + struct imapc_mailbox_list *list = (struct imapc_mailbox_list *)_list; + const char *prefix = list->storage->set->imapc_list_prefix; + const char *storage_name; + + storage_name = mailbox_list_default_get_storage_name(_list, vname); + if (*prefix != '\0' && strcasecmp(storage_name, "INBOX") != 0) { + storage_name = t_strdup_printf("%s%c%s", prefix, list->sep, + storage_name); + } + return storage_name; +} + +static const char * +imapc_list_get_vname(struct mailbox_list *_list, const char *storage_name) +{ + struct imapc_mailbox_list *list = (struct imapc_mailbox_list *)_list; + const char *prefix = list->storage->set->imapc_list_prefix; + unsigned int prefix_len; + + if (*prefix != '\0' && strcasecmp(storage_name, "INBOX") != 0) { + prefix_len = strlen(prefix); + i_assert(strncmp(prefix, storage_name, prefix_len) == 0 && + storage_name[prefix_len] == list->sep); + storage_name += prefix_len+1; + } + return mailbox_list_default_get_vname(_list, storage_name); +} + static struct mailbox_list *imapc_list_get_fs(struct imapc_mailbox_list *list) { struct mailbox_list_settings list_set; @@ -313,16 +345,33 @@ { struct imapc_command *cmd; struct imapc_simple_context ctx; + const char *pattern; i_assert(list->sep != '\0'); if (list->refreshed_mailboxes) return 0; + if (*list->storage->set->imapc_list_prefix == '\0') + pattern = "*"; + else { + pattern = t_strdup_printf("%s%c*", + list->storage->set->imapc_list_prefix, list->sep); + } + cmd = imapc_list_simple_context_init(&ctx, list); - imapc_command_send(cmd, "LIST \"\" *"); + imapc_command_sendf(cmd, "LIST \"\" %s", pattern); mailbox_tree_deinit(&list->mailboxes); list->mailboxes = mailbox_tree_init(list->sep); + mailbox_tree_set_parents_nonexistent(list->mailboxes); + + if ((list->list.ns->flags & NAMESPACE_FLAG_INBOX_USER) != 0) { + /* INBOX always exists in IMAP server. since this namespace is + marked with inbox=yes, show the INBOX even if + imapc_list_prefix doesn't match it */ + bool created; + (void)mailbox_tree_get(list->mailboxes, "INBOX", &created); + } imapc_simple_run(&ctx); if (ctx.ret == 0) { @@ -397,6 +446,7 @@ ctx->info.ns = _list->ns; ctx->tree = mailbox_tree_init(sep); + mailbox_tree_set_parents_nonexistent(ctx->tree); imapc_list_build_match_tree(ctx); if (list->list.ns->prefix_len > 0) { @@ -429,7 +479,7 @@ node = mailbox_tree_iterate_next(ctx->iter, &name); if (node == NULL) return NULL; - } while (node == ctx->ns_root); + } while ((node->flags & MAILBOX_MATCHED) == 0); ctx->info.name = name; ctx->info.flags = node->flags; @@ -468,6 +518,7 @@ (struct imapc_mailbox_list *)_src_list; struct imapc_simple_context ctx; struct imapc_command *cmd; + const char *pattern; char sep; i_assert(src_list->tmp_subscriptions == NULL); @@ -487,7 +538,14 @@ src_list->tmp_subscriptions = mailbox_tree_init(src_list->sep); cmd = imapc_list_simple_context_init(&ctx, src_list); - imapc_command_send(cmd, "LSUB \"\" *"); + if (*src_list->storage->set->imapc_list_prefix == '\0') + pattern = "*"; + else { + pattern = t_strdup_printf("%s%c*", + src_list->storage->set->imapc_list_prefix, + src_list->sep); + } + imapc_command_sendf(cmd, "LSUB \"\" %s", pattern); imapc_simple_run(&ctx); /* replace subscriptions tree in destination */ @@ -611,7 +669,7 @@ i_assert(list->sep != '\0'); - vname = mailbox_list_default_get_vname(_list, name); + vname = mailbox_list_get_vname(_list, name); if (!list->refreshed_mailboxes) { node = mailbox_tree_lookup(list->mailboxes, vname); if (node != NULL) @@ -646,8 +704,8 @@ imapc_is_valid_existing_name, imapc_is_valid_create_name, imapc_list_get_hierarchy_sep, - mailbox_list_default_get_vname, - mailbox_list_default_get_storage_name, + imapc_list_get_vname, + imapc_list_get_storage_name, imapc_list_get_path, imapc_list_get_temp_prefix, imapc_list_join_refpattern,
--- a/src/lib-storage/index/imapc/imapc-mail-fetch.c Sat May 19 22:40:08 2012 +0300 +++ b/src/lib-storage/index/imapc/imapc-mail-fetch.c Sun May 20 03:25:04 2012 +0300 @@ -95,6 +95,8 @@ str_printfa(str, "UID FETCH %u (", _mail->uid); if ((fields & MAIL_FETCH_RECEIVED_DATE) != 0) str_append(str, "INTERNALDATE "); + if ((fields & MAIL_FETCH_PHYSICAL_SIZE) != 0) + str_append(str, "RFC822.SIZE "); if ((fields & MAIL_FETCH_GUID) != 0) { str_append(str, mbox->guid_fetch_field_name); str_append_c(str, ' '); @@ -157,6 +159,10 @@ if ((data->wanted_fields & MAIL_FETCH_RECEIVED_DATE) != 0 && data->received_date == (time_t)-1) fields |= MAIL_FETCH_RECEIVED_DATE; + if ((data->wanted_fields & MAIL_FETCH_PHYSICAL_SIZE) != 0 && + data->physical_size == (uoff_t)-1 && + IMAPC_BOX_HAS_FEATURE(mbox, IMAPC_FEATURE_RFC822_SIZE)) + fields |= MAIL_FETCH_PHYSICAL_SIZE; if ((data->wanted_fields & MAIL_FETCH_GUID) != 0 && data->guid == NULL && mbox->guid_fetch_field_name != NULL) fields |= MAIL_FETCH_GUID; @@ -181,6 +187,11 @@ return FALSE; fields &= ~MAIL_FETCH_RECEIVED_DATE; } + if ((fields & MAIL_FETCH_PHYSICAL_SIZE) != 0) { + if (imail->imail.data.physical_size == (uoff_t)-1) + return FALSE; + fields &= ~MAIL_FETCH_PHYSICAL_SIZE; + } if ((fields & MAIL_FETCH_GUID) != 0) { if (imail->imail.data.guid == NULL) return FALSE; @@ -357,6 +368,7 @@ (struct imapc_mailbox *)mail->imail.mail.mail.box; const char *key, *value; unsigned int i; + uoff_t size; time_t t; int tz; bool match = FALSE; @@ -377,6 +389,12 @@ imap_parse_datetime(value, &t, &tz)) mail->imail.data.received_date = t; match = TRUE; + } else if (strcasecmp(key, "RFC822.SIZE") == 0) { + if (imap_arg_get_atom(&args[i+1], &value) && + str_to_uoff(value, &size) == 0 && + IMAPC_BOX_HAS_FEATURE(mbox, IMAPC_FEATURE_RFC822_SIZE)) + mail->imail.data.physical_size = size; + match = TRUE; } else if (strcasecmp(key, "X-GM-MSGID") == 0 || strcasecmp(key, "X-GUID") == 0) { if (imap_arg_get_astring(&args[i+1], &value)) {
--- a/src/lib-storage/index/imapc/imapc-mail.c Sat May 19 22:40:08 2012 +0300 +++ b/src/lib-storage/index/imapc/imapc-mail.c Sun May 20 03:25:04 2012 +0300 @@ -2,6 +2,8 @@ #include "lib.h" #include "str.h" +#include "hex-binary.h" +#include "sha1.h" #include "istream.h" #include "imap-envelope.h" #include "imapc-msgmap.h" @@ -107,29 +109,50 @@ static int imapc_mail_get_physical_size(struct mail *_mail, uoff_t *size_r) { + struct imapc_mailbox *mbox = (struct imapc_mailbox *)_mail->box; struct index_mail *mail = (struct index_mail *)_mail; struct index_mail_data *data = &mail->data; struct istream *input; uoff_t old_offset; int ret; - if (data->physical_size == (uoff_t)-1) + if (data->physical_size == (uoff_t)-1) { (void)index_mail_get_physical_size(_mail, size_r); - if (data->physical_size == (uoff_t)-1) { - old_offset = data->stream == NULL ? 0 : data->stream->v_offset; - if (mail_get_stream(_mail, NULL, NULL, &input) < 0) - return -1; - i_stream_seek(data->stream, old_offset); + if (data->physical_size != (uoff_t)-1) { + *size_r = data->physical_size; + return 0; + } + } - ret = i_stream_get_size(data->stream, TRUE, - &data->physical_size); - if (ret <= 0) { - i_assert(ret != 0); - mail_storage_set_critical(_mail->box->storage, - "imapc: stat(%s) failed: %m", - i_stream_get_name(data->stream)); + if (IMAPC_BOX_HAS_FEATURE(mbox, IMAPC_FEATURE_RFC822_SIZE) && + data->stream == NULL) { + /* trust RFC822.SIZE to be correct */ + if (imapc_mail_fetch(_mail, MAIL_FETCH_PHYSICAL_SIZE) < 0) return -1; + if (data->physical_size == (uoff_t)-1) { + if (imapc_mail_failed(_mail, "RFC822.SIZE") < 0) + return -1; + /* assume that the server never returns RFC822.SIZE + for this mail (see BODY[] failure handling) */ + data->physical_size = 0; } + *size_r = data->physical_size; + return 0; + } + + old_offset = data->stream == NULL ? 0 : data->stream->v_offset; + if (mail_get_stream(_mail, NULL, NULL, &input) < 0) + return -1; + i_stream_seek(data->stream, old_offset); + + ret = i_stream_get_size(data->stream, TRUE, + &data->physical_size); + if (ret <= 0) { + i_assert(ret != 0); + mail_storage_set_critical(_mail->box->storage, + "imapc: stat(%s) failed: %m", + i_stream_get_name(data->stream)); + return -1; } *size_r = data->physical_size; return 0; @@ -200,6 +223,7 @@ static void index_mail_update_access_parts(struct index_mail *mail) { struct mail *_mail = &mail->mail.mail; + struct imapc_mailbox *mbox = (struct imapc_mailbox *)_mail->box; struct index_mail_data *data = &mail->data; struct mailbox_header_lookup_ctx *header_ctx; time_t date; @@ -208,7 +232,8 @@ if ((data->wanted_fields & MAIL_FETCH_RECEIVED_DATE) != 0) (void)index_mail_get_received_date(_mail, &date); if ((data->wanted_fields & MAIL_FETCH_PHYSICAL_SIZE) != 0) { - if (index_mail_get_physical_size(_mail, &size) < 0) + if (index_mail_get_physical_size(_mail, &size) < 0 && + !IMAPC_BOX_HAS_FEATURE(mbox, IMAPC_FEATURE_RFC822_SIZE)) data->access_part |= READ_HDR | READ_BODY; } @@ -284,6 +309,33 @@ buffer_free(&mail->body); } +static int imapc_mail_get_hdr_hash(struct index_mail *imail) +{ + struct istream *input; + const unsigned char *data; + size_t size; + uoff_t old_offset; + struct sha1_ctxt sha1_ctx; + unsigned char sha1_output[SHA1_RESULTLEN]; + const char *sha1_str; + + sha1_init(&sha1_ctx); + old_offset = imail->data.stream == NULL ? 0 : + imail->data.stream->v_offset; + if (mail_get_hdr_stream(&imail->mail.mail, NULL, &input) < 0) + return -1; + while (i_stream_read_data(input, &data, &size, 0) > 0) { + sha1_loop(&sha1_ctx, data, size); + i_stream_skip(input, size); + } + i_stream_seek(imail->data.stream, old_offset); + sha1_result(&sha1_ctx, sha1_output); + + sha1_str = binary_to_hex(sha1_output, sizeof(sha1_output)); + imail->data.guid = p_strdup(imail->data_pool, sha1_str); + return 0; +} + static int imapc_mail_get_guid(struct mail *_mail, const char **value_r) { struct index_mail *imail = (struct index_mail *)_mail; @@ -305,11 +357,17 @@ } /* GUID not in cache, fetch it */ - if (imapc_mail_fetch(_mail, MAIL_FETCH_GUID) < 0) - return -1; - if (imail->data.guid == NULL) { - (void)imapc_mail_failed(_mail, mbox->guid_fetch_field_name); - return -1; + if (mbox->guid_fetch_field_name != NULL) { + if (imapc_mail_fetch(_mail, MAIL_FETCH_GUID) < 0) + return -1; + if (imail->data.guid == NULL) { + (void)imapc_mail_failed(_mail, mbox->guid_fetch_field_name); + return -1; + } + } else { + /* use hash of message headers as the GUID */ + if (imapc_mail_get_hdr_hash(imail) < 0) + return -1; } index_mail_cache_add_idx(imail, cache_idx, @@ -326,7 +384,8 @@ switch (field) { case MAIL_FETCH_GUID: - if (mbox->guid_fetch_field_name == NULL) { + if (!IMAPC_BOX_HAS_FEATURE(mbox, IMAPC_FEATURE_GUID_FORCED) && + mbox->guid_fetch_field_name == NULL) { /* GUIDs not supported by server */ break; }
--- a/src/lib-storage/index/imapc/imapc-settings.c Sat May 19 22:40:08 2012 +0300 +++ b/src/lib-storage/index/imapc/imapc-settings.c Sun May 20 03:25:04 2012 +0300 @@ -25,7 +25,9 @@ DEF(SET_STR, imapc_ssl_ca_dir), DEF(SET_BOOL, imapc_ssl_verify), + DEF(SET_STR, imapc_features), DEF(SET_STR, imapc_rawlog_dir), + DEF(SET_STR, imapc_list_prefix), DEF(SET_STR, ssl_crypto_device), SETTING_DEFINE_LIST_END @@ -43,7 +45,9 @@ .imapc_ssl_ca_dir = "", .imapc_ssl_verify = TRUE, + .imapc_features = "", .imapc_rawlog_dir = "", + .imapc_list_prefix = "", .ssl_crypto_device = "" }; @@ -67,6 +71,44 @@ } /* <settings checks> */ +struct imapc_feature_list { + const char *name; + enum imapc_features num; +}; + +static const struct imapc_feature_list imapc_feature_list[] = { + { "rfc822.size", IMAPC_FEATURE_RFC822_SIZE }, + { "guid-forced", IMAPC_FEATURE_GUID_FORCED }, + { NULL, 0 } +}; + +static int +imapc_settings_parse_features(struct imapc_settings *set, + const char **error_r) +{ + enum imapc_features features = 0; + const struct imapc_feature_list *list; + const char *const *str; + + str = t_strsplit_spaces(set->imapc_features, " ,"); + for (; *str != NULL; str++) { + list = imapc_feature_list; + for (; list->name != NULL; list++) { + if (strcasecmp(*str, list->name) == 0) { + features |= list->num; + break; + } + } + if (list->name == NULL) { + *error_r = t_strdup_printf("imapc_features: " + "Unknown feature: %s", *str); + return -1; + } + } + set->parsed_features = features; + return 0; +} + static bool imapc_settings_check(void *_set, pool_t pool ATTR_UNUSED, const char **error_r) { @@ -85,5 +127,7 @@ return FALSE; } #endif + if (imapc_settings_parse_features(set, error_r) < 0) + return FALSE; return TRUE; }
--- a/src/lib-storage/index/imapc/imapc-settings.h Sat May 19 22:40:08 2012 +0300 +++ b/src/lib-storage/index/imapc/imapc-settings.h Sun May 20 03:25:04 2012 +0300 @@ -1,6 +1,13 @@ #ifndef IMAPC_SETTINGS_H #define IMAPC_SETTINGS_H +/* <settings checks> */ +enum imapc_features { + IMAPC_FEATURE_RFC822_SIZE = 0x01, + IMAPC_FEATURE_GUID_FORCED = 0x02 +}; +/* </settings checks> */ + struct imapc_settings { const char *imapc_host; unsigned int imapc_port; @@ -13,8 +20,12 @@ const char *imapc_ssl_ca_dir; bool imapc_ssl_verify; + const char *imapc_features; const char *imapc_rawlog_dir; + const char *imapc_list_prefix; const char *ssl_crypto_device; + + enum imapc_features parsed_features; }; const struct setting_parser_info *imapc_get_setting_parser_info(void);
--- a/src/lib-storage/index/imapc/imapc-storage.h Sat May 19 22:40:08 2012 +0300 +++ b/src/lib-storage/index/imapc/imapc-storage.h Sun May 20 03:25:04 2012 +0300 @@ -2,6 +2,7 @@ #define IMAPC_STORAGE_H #include "index-storage.h" +#include "imapc-settings.h" #define IMAPC_STORAGE_NAME "imapc" #define IMAPC_INDEX_PREFIX "dovecot.index" @@ -28,6 +29,11 @@ imapc_mailbox_callback_t *callback; }; +#define IMAPC_HAS_FEATURE(mstorage, feature) \ + (((mstorage)->set->parsed_features & feature) != 0) +#define IMAPC_BOX_HAS_FEATURE(mbox, feature) \ + (((mbox)->storage->set->parsed_features & feature) != 0) + struct imapc_storage { struct mail_storage storage; const struct imapc_settings *set; @@ -124,7 +130,7 @@ bool *changes_r); void imapc_mailbox_noop(struct imapc_mailbox *mbox); void imapc_mailbox_set_corrupted(struct imapc_mailbox *mbox, - const char *reason, ...); + const char *reason, ...) ATTR_FORMAT(2, 3); void imapc_storage_register_untagged(struct imapc_storage *storage, const char *name,
--- a/src/lib-storage/index/index-mail.c Sat May 19 22:40:08 2012 +0300 +++ b/src/lib-storage/index/index-mail.c Sun May 20 03:25:04 2012 +0300 @@ -442,7 +442,7 @@ if (set->mail_cache_min_mail_count > 0) { /* First check if we've configured caching not to be used with low enough message count. */ - hdr = mail_index_get_header(_mail->box->view); + hdr = mail_index_get_header(_mail->transaction->view); if (hdr->messages_count < set->mail_cache_min_mail_count) return; } @@ -1357,7 +1357,7 @@ /* open the stream only if we didn't get here from mailbox_save_init() */ - hdr = mail_index_get_header(_mail->box->view); + hdr = mail_index_get_header(_mail->transaction->view); if (!_mail->saving && _mail->uid < hdr->next_uid) { if ((data->access_part & READ_BODY) != 0) (void)mail_get_stream(_mail, NULL, NULL, &input); @@ -1453,7 +1453,7 @@ struct index_mail *mail = (struct index_mail *)_mail; uint32_t seq; - if (mail_index_lookup_seq(_mail->box->view, uid, &seq)) { + if (mail_index_lookup_seq(_mail->transaction->view, uid, &seq)) { index_mail_set_seq(_mail, seq, FALSE); return TRUE; } else {
--- a/src/lib-storage/index/index-mail.h Sat May 19 22:40:08 2012 +0300 +++ b/src/lib-storage/index/index-mail.h Sun May 20 03:25:04 2012 +0300 @@ -77,7 +77,7 @@ uint32_t parse_line_num; struct message_part *parts; - const char *envelope, *body, *bodystructure, *uid_string, *guid; + const char *envelope, *body, *bodystructure, *guid, *filename; const char *from_envelope; struct message_part_envelope_data *envelope_data;
--- a/src/lib-storage/index/index-search-private.h Sat May 19 22:40:08 2012 +0300 +++ b/src/lib-storage/index/index-search-private.h Sun May 20 03:25:04 2012 +0300 @@ -3,6 +3,8 @@ #include "mail-storage-private.h" +#include <sys/time.h> + struct index_search_context { struct mail_search_context mail_ctx; struct mail_index_view *view;
--- a/src/lib-storage/index/index-search.c Sat May 19 22:40:08 2012 +0300 +++ b/src/lib-storage/index/index-search.c Sun May 20 03:25:04 2012 +0300 @@ -1220,17 +1220,12 @@ static int search_match_once(struct index_search_context *ctx) { - unsigned long long cost1, cost2; int ret; - cost1 = search_get_cost(ctx->cur_mail->transaction); ret = mail_search_args_foreach(ctx->mail_ctx.args->args, search_cached_arg, ctx); if (ret < 0) ret = search_arg_match_text(ctx->mail_ctx.args->args, ctx); - - cost2 = search_get_cost(ctx->cur_mail->transaction); - ctx->cost += cost2 - cost1; return ret; } @@ -1423,7 +1418,8 @@ struct mail_search_context *_ctx = &ctx->mail_ctx; struct mailbox *box = _ctx->transaction->box; struct index_mail *imail = (struct index_mail *)mail; - int match; + unsigned long long cost1, cost2; + int match, ret; if (search_would_block(ctx)) { /* this lookup is useful when a large number of @@ -1437,6 +1433,8 @@ mail_search_args_reset(_ctx->args->args, FALSE); + cost1 = search_get_cost(mail->transaction); + ret = -1; while (box->v.search_next_update_seq(_ctx)) { mail_set_seq(mail, _ctx->seq); @@ -1458,12 +1456,23 @@ mail_search_args_reset(_ctx->args->args, FALSE); - if (match != 0) - return 1; - if (search_would_block(ctx)) - return 0; + if (match != 0) { + ret = 1; + break; + } + + cost2 = search_get_cost(mail->transaction); + ctx->cost += cost2 - cost1; + cost1 = cost2; + + if (search_would_block(ctx)) { + ret = 0; + break; + } } - return -1; + cost2 = search_get_cost(mail->transaction); + ctx->cost += cost2 - cost1; + return ret; } struct mail *index_search_get_mail(struct index_search_context *ctx) @@ -1501,6 +1510,14 @@ ret = search_more_with_mail(ctx, mail); if (ret <= 0) break; + + if (ctx->mail_ctx.sort_program != NULL) { + /* don't prefetch when using a sort program, + since the mails' access order will change */ + i_assert(ctx->unused_mail_idx == 0); + *mail_r = mail; + return 1; + } if (mail_prefetch(mail) && ctx->unused_mail_idx == 0) { /* no prefetching done, return it immediately */ *mail_r = mail;
--- a/src/lib-storage/index/index-status.c Sat May 19 22:40:08 2012 +0300 +++ b/src/lib-storage/index/index-status.c Sun May 20 03:25:04 2012 +0300 @@ -58,7 +58,9 @@ status_r->uidvalidity = hdr->uid_validity; status_r->uidnext = hdr->next_uid; status_r->first_recent_uid = hdr->first_recent_uid; - status_r->nonpermanent_modseqs = mail_index_is_in_memory(box->index); + status_r->nonpermanent_modseqs = + mail_index_is_in_memory(box->index) || + !mail_index_have_modseq_tracking(box->index); if ((items & STATUS_HIGHESTMODSEQ) != 0) { status_r->highest_modseq = mail_index_modseq_get_highest(box->view); @@ -98,11 +100,17 @@ struct mailbox_cache_field *cf; unsigned int i, count; - fields = mail_cache_register_get_list(box->cache, - pool_datastack_create(), &count); + if (box->metadata_pool == NULL) { + box->metadata_pool = + pool_alloconly_create("mailbox metadata", 2048); + } - cache_fields = t_new(ARRAY_TYPE(mailbox_cache_field), 1); - t_array_init(cache_fields, count); + fields = mail_cache_register_get_list(box->cache, + box->metadata_pool, &count); + + cache_fields = p_new(box->metadata_pool, + ARRAY_TYPE(mailbox_cache_field), 1); + p_array_init(cache_fields, box->metadata_pool, count); for (i = 0; i < count; i++) { dec = fields[i].decision & ~MAIL_CACHE_DECISION_FORCED; if (dec != MAIL_CACHE_DECISION_NO) {
--- a/src/lib-storage/index/index-storage.c Sat May 19 22:40:08 2012 +0300 +++ b/src/lib-storage/index/index-storage.c Sun May 20 03:25:04 2012 +0300 @@ -272,6 +272,9 @@ box->opened = TRUE; + if ((box->enabled_features & MAILBOX_FEATURE_CONDSTORE) != 0) + mail_index_modseq_enable(box->index); + index_thread_mailbox_opened(box); hook_mailbox_opened(box); return 0; @@ -315,11 +318,8 @@ { if ((feature & MAILBOX_FEATURE_CONDSTORE) != 0) { box->enabled_features |= MAILBOX_FEATURE_CONDSTORE; - if (mailbox_open(box) < 0) - return -1; - T_BEGIN { + if (box->opened) mail_index_modseq_enable(box->index); - } T_END; } return 0; } @@ -519,10 +519,13 @@ return index_storage_mailbox_delete_dir(box, FALSE); } - /* specifically support symlinked shared mailboxes. a deletion will - simply remove the symlink, not actually expunge any mails */ - if (mailbox_list_delete_symlink(box->list, box->name) == 0) - return 0; + if ((box->list->flags & MAILBOX_LIST_FLAG_MAILBOX_FILES) == 0) { + /* specifically support symlinked shared mailboxes. a deletion + will simply remove the symlink, not actually expunge any + mails */ + if (mailbox_list_delete_symlink(box->list, box->name) == 0) + return 0; + } /* we can't easily atomically delete all mails and the mailbox. so: 1) expunge all mails @@ -532,8 +535,10 @@ no) finish deleting the mailbox */ - if (mailbox_expunge_all_mails(box) < 0) - return -1; + if (!box->deleting_must_be_empty) { + if (mailbox_expunge_all_mails(box) < 0) + return -1; + } if (mailbox_mark_index_deleted(box, TRUE) < 0) return -1;
--- a/src/lib-storage/index/index-thread.c Sat May 19 22:40:08 2012 +0300 +++ b/src/lib-storage/index/index-thread.c Sun May 20 03:25:04 2012 +0300 @@ -325,6 +325,7 @@ if (seq1 == 0) { /* nothing is missing */ mail_index_strmap_view_sync_commit(&ctx->strmap_sync); + mailbox_header_lookup_unref(&headers_ctx); return 0; } @@ -332,6 +333,8 @@ mail_search_build_add_seqset(search_args, seq1, seq2); search_ctx = mailbox_search_init(ctx->t, search_args, NULL, 0, headers_ctx); + mailbox_header_lookup_unref(&headers_ctx); + mail_search_args_unref(&search_args); while (mailbox_search_next(search_ctx, &mail)) { if (mail_thread_map_add_mail(ctx, mail) < 0) {
--- a/src/lib-storage/index/index-transaction.c Sat May 19 22:40:08 2012 +0300 +++ b/src/lib-storage/index/index-transaction.c Sun May 20 03:25:04 2012 +0300 @@ -21,10 +21,15 @@ MAIL_STORAGE_CONTEXT(index_trans); int ret = 0; + if (t->nontransactional_changes) + t->changes->changed = TRUE; + if (t->save_ctx != NULL) { if (t->box->v.transaction_save_commit_pre(t->save_ctx) < 0) { t->save_ctx = NULL; ret = -1; + } else { + t->changes->changed = TRUE; } } @@ -35,6 +40,9 @@ if (t->super.commit(index_trans, result_r) < 0) { mail_storage_set_index_error(t->box); ret = -1; + } else if (result_r->commit_size > 0) { + /* something was written to the transaction log */ + t->changes->changed = TRUE; } }
--- a/src/lib-storage/index/maildir/maildir-mail.c Sat May 19 22:40:08 2012 +0300 +++ b/src/lib-storage/index/maildir/maildir-mail.c Sun May 20 03:25:04 2012 +0300 @@ -102,7 +102,7 @@ return input; } -static int maildir_mail_stat(struct mail *mail, struct stat *st) +static int maildir_mail_stat(struct mail *mail, struct stat *st_r) { struct maildir_mailbox *mbox = (struct maildir_mailbox *)mail->box; struct index_mail *imail = (struct index_mail *)mail; @@ -110,8 +110,10 @@ const char *path; int ret; - if (mail->lookup_abort == MAIL_LOOKUP_ABORT_NOT_IN_CACHE) - return mail_set_aborted(mail); + if (mail->lookup_abort == MAIL_LOOKUP_ABORT_NOT_IN_CACHE) { + mail_set_aborted(mail); + return -1; + } if (imail->data.access_part != 0 && imail->data.stream == NULL) { @@ -126,10 +128,10 @@ stp = i_stream_stat(imail->data.stream, FALSE); if (stp == NULL) return -1; - *st = *stp; + *st_r = *stp; } else if (!mail->saving) { mail->transaction->stats.stat_lookup_count++; - ret = maildir_file_do(mbox, mail->uid, do_stat, st); + ret = maildir_file_do(mbox, mail->uid, do_stat, st_r); if (ret <= 0) { if (ret == 0) mail_set_expunged(mail); @@ -138,7 +140,7 @@ } else { mail->transaction->stats.stat_lookup_count++; path = maildir_save_file_get_path(mail->transaction, mail->seq); - if (stat(path, st) < 0) { + if (stat(path, st_r) < 0) { mail_storage_set_critical(mail->box->storage, "stat(%s) failed: %m", path); return -1; @@ -480,6 +482,11 @@ /* use GUID from uidlist if it exists */ i_assert(!_mail->saving); + if (mail->data.guid != NULL) { + *value_r = mail->data.guid; + return 0; + } + /* first make sure that we have a refreshed uidlist */ if (maildir_mail_get_fname(mbox, _mail, &fname) <= 0) return -1; @@ -488,7 +495,8 @@ MAILDIR_UIDLIST_REC_EXT_GUID); if (guid != NULL) { if (*guid != '\0') { - *value_r = p_strdup(mail->data_pool, guid); + *value_r = mail->data.guid = + p_strdup(mail->data_pool, guid); return 0; } @@ -501,9 +509,14 @@ } /* default to base filename: */ + if (maildir_mail_get_special(_mail, MAIL_FETCH_UIDL_FILE_NAME, + value_r) < 0) + return -1; + mail->data.guid = mail->data.filename; + return 0; case MAIL_FETCH_UIDL_FILE_NAME: - if (mail->data.guid != NULL) { - *value_r = mail->data.guid; + if (mail->data.filename != NULL) { + *value_r = mail->data.filename; return 0; } if (fname != NULL) { @@ -519,10 +532,10 @@ fname = fname != NULL ? fname + 1 : path; } end = strchr(fname, MAILDIR_INFO_SEP); - mail->data.guid = end == NULL ? + mail->data.filename = end == NULL ? p_strdup(mail->data_pool, fname) : p_strdup_until(mail->data_pool, fname, end); - *value_r = mail->data.guid; + *value_r = mail->data.filename; return 0; case MAIL_FETCH_UIDL_BACKEND: uidl = maildir_uidlist_lookup_ext(mbox->uidlist, _mail->uid, @@ -593,6 +606,7 @@ uidl = ""; } + _mail->transaction->nontransactional_changes = TRUE; maildir_uidlist_set_ext(mbox->uidlist, _mail->uid, MAILDIR_UIDLIST_REC_EXT_POP3_UIDL, uidl); }
--- a/src/lib-storage/index/maildir/maildir-save.c Sat May 19 22:40:08 2012 +0300 +++ b/src/lib-storage/index/maildir/maildir-save.c Sun May 20 03:25:04 2012 +0300 @@ -34,6 +34,7 @@ uoff_t size, vsize; enum mail_flags flags; + unsigned int pop3_order; unsigned int preserve_filename:1; unsigned int keywords_count; /* unsigned int keywords[]; */ @@ -187,6 +188,7 @@ } if (_ctx->pop3_uidl != NULL) mf->pop3_uidl = p_strdup(ctx->pool, _ctx->pop3_uidl); + mf->pop3_order = _ctx->pop3_order; /* insert into index */ mail_index_append(ctx->trans, _ctx->uid, &ctx->seq); @@ -516,6 +518,7 @@ struct mail_storage *storage = &ctx->mbox->storage->storage; const char *path; off_t real_size; + uoff_t size; int output_errno; ctx->last_save_finished = TRUE; @@ -575,10 +578,16 @@ if (real_size == (off_t)-1) { mail_storage_set_critical(storage, "lseek(%s) failed: %m", path); - } else if (real_size != (off_t)ctx->file_last->size) { + } else if (real_size != (off_t)ctx->file_last->size && + (!maildir_filename_get_size(ctx->file_last->dest_basename, + MAILDIR_EXTRA_FILE_SIZE, &size) || + size != ctx->file_last->size)) { /* e.g. zlib plugin was used. the "physical size" must be in the maildir filename, since stat() will return wrong size */ ctx->file_last->preserve_filename = FALSE; + /* reset the base name as well, just in case there's a + ,W=vsize */ + ctx->file_last->dest_basename = ctx->file_last->tmp_name; } if (close(ctx->fd) < 0) { if (!mail_storage_set_error_from_errno(storage)) { @@ -922,6 +931,11 @@ MAILDIR_UIDLIST_REC_EXT_POP3_UIDL, mf->pop3_uidl); } + if (mf->pop3_order > 0) { + maildir_uidlist_sync_set_ext(ctx->uidlist_sync_ctx, rec, + MAILDIR_UIDLIST_REC_EXT_POP3_ORDER, + t_strdup_printf("%u", mf->pop3_order)); + } } T_END; i_assert(!seq_range_array_iter_nth(&iter, n, &uid)); }
--- a/src/lib-storage/index/maildir/maildir-storage.c Sat May 19 22:40:08 2012 +0300 +++ b/src/lib-storage/index/maildir/maildir-storage.c Sun May 20 03:25:04 2012 +0300 @@ -201,13 +201,14 @@ static int maildir_check_tmp(struct mail_storage *storage, const char *dir) { + unsigned int interval = storage->set->mail_temp_scan_interval; const char *path; struct stat st; /* if tmp/ directory exists, we need to clean it up once in a while */ path = t_strconcat(dir, "/tmp", NULL); if (stat(path, &st) < 0) { - if (errno == ENOENT) + if (errno == ENOENT || errno == ENAMETOOLONG) return 0; if (errno == EACCES) { mail_storage_set_critical(storage, "%s", @@ -218,10 +219,12 @@ return -1; } - if (st.st_atime > st.st_ctime + MAILDIR_TMP_DELETE_SECS) { + if (interval == 0) { + /* disabled */ + } else if (st.st_atime > st.st_ctime + MAILDIR_TMP_DELETE_SECS) { /* the directory should be empty. we won't do anything until ctime changes. */ - } else if (st.st_atime < ioloop_time - MAILDIR_TMP_SCAN_SECS) { + } else if (st.st_atime < ioloop_time - interval) { /* time to scan */ (void)unlink_old_files(path, "", ioloop_time - MAILDIR_TMP_DELETE_SECS); @@ -361,7 +364,7 @@ return maildir_mailbox_open_existing(box); } - if (errno == ENOENT) { + if (errno == ENOENT || errno == ENAMETOOLONG) { mail_storage_set_error(box->storage, MAIL_ERROR_NOTFOUND, T_MAIL_ERR_MAILBOX_NOT_FOUND(box->name)); return -1;
--- a/src/lib-storage/index/maildir/maildir-storage.h Sat May 19 22:40:08 2012 +0300 +++ b/src/lib-storage/index/maildir/maildir-storage.h Sun May 20 03:25:04 2012 +0300 @@ -31,8 +31,6 @@ calculating file's virtual size (added missing CRs). */ #define MAILDIR_EXTRA_VIRTUAL_SIZE 'W' -/* How often to scan tmp/ directory for old files (based on dir's atime) */ -#define MAILDIR_TMP_SCAN_SECS (8*60*60) /* Delete files having ctime older than this from tmp/. 36h is standard. */ #define MAILDIR_TMP_DELETE_SECS (36*60*60)
--- a/src/lib-storage/index/maildir/maildir-sync-index.c Sat May 19 22:40:08 2012 +0300 +++ b/src/lib-storage/index/maildir/maildir-sync-index.c Sun May 20 03:25:04 2012 +0300 @@ -492,6 +492,7 @@ } hdr_next_uid = hdr->next_uid; + ctx->mbox->box.tmp_sync_view = view; private_flags_mask = mailbox_get_private_flags_mask(&mbox->box); time_before_sync = time(NULL); mbox->syncing_commit = TRUE; @@ -647,6 +648,7 @@ if (mbox->box.v.sync_notify != NULL) mbox->box.v.sync_notify(&mbox->box, 0, 0); + ctx->mbox->box.tmp_sync_view = NULL; /* check cur/ mtime later. if we came here from saving messages they could still be moved to cur/ directory. */
--- a/src/lib-storage/index/maildir/maildir-sync.c Sat May 19 22:40:08 2012 +0300 +++ b/src/lib-storage/index/maildir/maildir-sync.c Sun May 20 03:25:04 2012 +0300 @@ -595,6 +595,8 @@ struct stat new_st, cur_st; bool refreshed = FALSE, check_new = FALSE, check_cur = FALSE; + *why_r = 0; + if (mbox->maildir_hdr.new_mtime == 0) { maildir_sync_get_header(mbox); if (mbox->maildir_hdr.new_mtime == 0) { @@ -710,6 +712,8 @@ enum mail_index_sync_flags flags = 0; bool undirty = (ctx->flags & MAILBOX_SYNC_FLAG_FULL_READ) != 0; + *why_r = 0; + if (maildir_sync_quick_check(mbox, undirty, ctx->new_dir, ctx->cur_dir, new_changed_r, cur_changed_r, why_r) < 0) return -1; @@ -752,7 +756,7 @@ enum maildir_uidlist_rec_flag flags; bool new_changed, cur_changed, lock_failure; const char *fname; - enum maildir_scan_why why = 0; + enum maildir_scan_why why; int ret; *lost_files_r = FALSE; @@ -989,6 +993,34 @@ return ret; } +int maildir_sync_refresh_flags_view(struct maildir_mailbox *mbox) +{ + struct mail_index_view_sync_ctx *sync_ctx; + bool delayed_expunges; + + (void)mail_index_refresh(mbox->box.index); + if (mbox->flags_view == NULL) + mbox->flags_view = mail_index_view_open(mbox->box.index); + + sync_ctx = mail_index_view_sync_begin(mbox->flags_view, + MAIL_INDEX_VIEW_SYNC_FLAG_FIX_INCONSISTENT); + if (mail_index_view_sync_commit(&sync_ctx, &delayed_expunges) < 0) { + mail_storage_set_index_error(&mbox->box); + return -1; + } + /* make sure the map stays in private memory */ + if (mbox->flags_view->map->refcount > 1) { + struct mail_index_map *map; + + map = mail_index_map_clone(mbox->flags_view->map); + mail_index_unmap(&mbox->flags_view->map); + mbox->flags_view->map = map; + } + mail_index_record_map_move_to_private(mbox->flags_view->map); + mail_index_map_move_to_memory(mbox->flags_view->map); + return 0; +} + struct mailbox_sync_context * maildir_storage_sync_init(struct mailbox *box, enum mailbox_sync_flags flags) { @@ -1021,29 +1053,8 @@ } if (mbox->storage->set->maildir_very_dirty_syncs) { - struct mail_index_view_sync_ctx *sync_ctx; - bool b; - - if (mbox->flags_view == NULL) { - mbox->flags_view = - mail_index_view_open(mbox->box.index); - } - sync_ctx = mail_index_view_sync_begin(mbox->flags_view, - MAIL_INDEX_VIEW_SYNC_FLAG_FIX_INCONSISTENT); - if (mail_index_view_sync_commit(&sync_ctx, &b) < 0) { - mail_storage_set_index_error(&mbox->box); + if (maildir_sync_refresh_flags_view(mbox) < 0) ret = -1; - } - /* make sure the map stays in private memory */ - if (mbox->flags_view->map->refcount > 1) { - struct mail_index_map *map; - - map = mail_index_map_clone(mbox->flags_view->map); - mail_index_unmap(&mbox->flags_view->map); - mbox->flags_view->map = map; - } - mail_index_record_map_move_to_private(mbox->flags_view->map); - mail_index_map_move_to_memory(mbox->flags_view->map); maildir_uidlist_set_all_nonsynced(mbox->uidlist); } mbox->synced = TRUE;
--- a/src/lib-storage/index/maildir/maildir-sync.h Sat May 19 22:40:08 2012 +0300 +++ b/src/lib-storage/index/maildir/maildir-sync.h Sun May 20 03:25:04 2012 +0300 @@ -42,6 +42,7 @@ void maildir_sync_notify(struct maildir_sync_context *ctx); void maildir_sync_set_new_msgs_count(struct maildir_index_sync_context *ctx, unsigned int count); +int maildir_sync_refresh_flags_view(struct maildir_mailbox *mbox); int maildir_sync_lookup(struct maildir_mailbox *mbox, uint32_t uid, enum maildir_uidlist_rec_flag *flags_r,
--- a/src/lib-storage/index/maildir/maildir-uidlist.c Sat May 19 22:40:08 2012 +0300 +++ b/src/lib-storage/index/maildir/maildir-uidlist.c Sun May 20 03:25:04 2012 +0300 @@ -97,6 +97,7 @@ unsigned int locked_refresh:1; unsigned int unsorted:1; unsigned int have_mailbox_guid:1; + unsigned int opened_readonly:1; }; struct maildir_uidlist_sync_ctx { @@ -702,11 +703,16 @@ struct stat st; uoff_t last_read_offset; int fd, ret; + bool readonly = FALSE; *retry_r = FALSE; if (uidlist->fd == -1) { fd = nfs_safe_open(uidlist->path, O_RDWR); + if (fd == -1 && errno == EACCES) { + fd = nfs_safe_open(uidlist->path, O_RDONLY); + readonly = TRUE; + } if (fd == -1) { if (errno != ENOENT) { mail_storage_set_critical(storage, @@ -807,6 +813,8 @@ (void)unlink(uidlist->path); } else if (ret > 0) { /* success */ + if (readonly) + uidlist->recreate_on_change = TRUE; uidlist->fd = fd; uidlist->fd_dev = st.st_dev; uidlist->fd_ino = st.st_ino; @@ -917,6 +925,10 @@ } uidlist->fd = nfs_safe_open(uidlist->path, O_RDWR); + if (uidlist->fd == -1 && errno == EACCES) { + uidlist->fd = nfs_safe_open(uidlist->path, O_RDONLY); + uidlist->recreate_on_change = TRUE; + } if (uidlist->fd == -1 && errno != ENOENT) { mail_storage_set_critical(uidlist->box->storage, "open(%s) failed: %m", uidlist->path); @@ -1510,6 +1522,8 @@ i_assert(uidlist->initial_hdr_read); if (maildir_uidlist_open_latest(uidlist) < 0) return -1; + if (uidlist->recreate_on_change) + return maildir_uidlist_recreate(uidlist); } i_assert(ctx->first_unwritten_pos != (unsigned int)-1);
--- a/src/lib-storage/index/maildir/maildir-util.c Sat May 19 22:40:08 2012 +0300 +++ b/src/lib-storage/index/maildir/maildir-util.c Sun May 20 03:25:04 2012 +0300 @@ -103,6 +103,12 @@ if (ret > 0 && (flags & MAILDIR_UIDLIST_REC_FLAG_NONSYNCED) != 0) { /* file was found. make sure we remember its latest name. */ maildir_uidlist_update_fname(mbox->uidlist, fname); + } else if (ret == 0 && + (flags & MAILDIR_UIDLIST_REC_FLAG_NONSYNCED) == 0) { + /* file wasn't found. mark this message nonsynced, so we can + retry the lookup by guessing the flags */ + maildir_uidlist_add_flags(mbox->uidlist, fname, + MAILDIR_UIDLIST_REC_FLAG_NONSYNCED); } return ret; } @@ -133,6 +139,11 @@ T_BEGIN { ret = maildir_file_do_try(mbox, uid, callback, context); } T_END; + if (ret == 0 && mbox->storage->set->maildir_very_dirty_syncs) T_BEGIN { + /* try guessing again with refreshed flags */ + if (maildir_sync_refresh_flags_view(mbox) == 0) + ret = maildir_file_do_try(mbox, uid, callback, context); + } T_END; for (i = 0; i < MAILDIR_RESYNC_RETRY_COUNT && ret == 0; i++) { /* file is either renamed or deleted. sync the maildir and see which one. if file appears to be renamed constantly,
--- a/src/lib-storage/index/mbox/mbox-storage.h Sat May 19 22:40:08 2012 +0300 +++ b/src/lib-storage/index/mbox/mbox-storage.h Sun May 20 03:25:04 2012 +0300 @@ -14,6 +14,7 @@ #define MBOX_SUBSCRIPTION_FILE_NAME ".subscriptions" #define MBOX_INDEX_PREFIX "dovecot.index" #define MBOX_INDEX_DIR_NAME ".imap" +#define MBOX_UIDVALIDITY_FNAME "dovecot-uidvalidity" struct mbox_index_header { uint64_t sync_size;
--- a/src/lib-storage/index/mbox/mbox-sync.c Sat May 19 22:40:08 2012 +0300 +++ b/src/lib-storage/index/mbox/mbox-sync.c Sun May 20 03:25:04 2012 +0300 @@ -45,6 +45,7 @@ #include "istream-raw-mbox.h" #include "mbox-storage.h" #include "index-sync-changes.h" +#include "mailbox-uidvalidity.h" #include "mbox-from.h" #include "mbox-file.h" #include "mbox-lock.h" @@ -1394,12 +1395,29 @@ &data, &data_size); if (data_size != sizeof(mbox->mbox_hdr) || memcmp(data, &mbox->mbox_hdr, data_size) != 0) { + if (data_size != sizeof(mbox->mbox_hdr)) { + /* upgrading from v1.x */ + mail_index_ext_resize(trans, mbox->mbox_ext_idx, + sizeof(mbox->mbox_hdr), + sizeof(uint64_t), + sizeof(uint64_t)); + } mail_index_update_header_ext(trans, mbox->mbox_ext_idx, 0, &mbox->mbox_hdr, sizeof(mbox->mbox_hdr)); } } +static uint32_t mbox_get_uidvalidity_next(struct mailbox_list *list) +{ + const char *path; + + path = mailbox_list_get_path(list, NULL, + MAILBOX_LIST_PATH_TYPE_CONTROL); + path = t_strconcat(path, "/"MBOX_UIDVALIDITY_FNAME, NULL); + return mailbox_uidvalidity_next(list, path); +} + static int mbox_sync_update_index_header(struct mbox_sync_context *sync_ctx) { struct mail_index_view *view; @@ -1455,7 +1473,7 @@ if (sync_ctx->base_uid_validity == 0) { sync_ctx->base_uid_validity = sync_ctx->hdr->uid_validity != 0 ? sync_ctx->hdr->uid_validity : - I_MAX((unsigned int)ioloop_time, 1); + mbox_get_uidvalidity_next(sync_ctx->mbox->box.list); } if (sync_ctx->base_uid_validity != sync_ctx->hdr->uid_validity) { mail_index_update_header(sync_ctx->t,
--- a/src/lib-storage/index/pop3c/pop3c-client.c Sat May 19 22:40:08 2012 +0300 +++ b/src/lib-storage/index/pop3c/pop3c-client.c Sun May 20 03:25:04 2012 +0300 @@ -669,7 +669,7 @@ *reply_r = line + 3; ret = 0; } else if (strncasecmp(line, "-ERR", 4) == 0) { - *reply_r = line + 3; + *reply_r = line + 4; ret = -1; } else { *reply_r = line;
--- a/src/lib-storage/index/pop3c/pop3c-mail.c Sat May 19 22:40:08 2012 +0300 +++ b/src/lib-storage/index/pop3c/pop3c-mail.c Sun May 20 03:25:04 2012 +0300 @@ -113,7 +113,7 @@ if (get_body || (capa & POP3C_CAPABILITY_TOP) == 0) cmd = t_strdup_printf("RETR %u\r\n", _mail->seq); else - cmd = t_strdup_printf("TOP %u\r\n", _mail->seq); + cmd = t_strdup_printf("TOP %u 0\r\n", _mail->seq); if (pop3c_client_cmd_stream(mbox->client, cmd, &input, &error) < 0) { mail_storage_set_error(mbox->box.storage,
--- a/src/lib-storage/index/pop3c/pop3c-storage.c Sat May 19 22:40:08 2012 +0300 +++ b/src/lib-storage/index/pop3c/pop3c-storage.c Sun May 20 03:25:04 2012 +0300 @@ -162,7 +162,7 @@ { struct pop3c_mailbox *mbox = (struct pop3c_mailbox *)box; - if (!box->inbox_any) { + if (strcmp(box->name, "INBOX") != 0) { mail_storage_set_error(box->storage, MAIL_ERROR_NOTFOUND, T_MAIL_ERR_MAILBOX_NOT_FOUND(box->name)); return -1;
--- a/src/lib-storage/index/raw/raw-storage.c Sat May 19 22:40:08 2012 +0300 +++ b/src/lib-storage/index/raw/raw-storage.c Sun May 20 03:25:04 2012 +0300 @@ -28,6 +28,7 @@ i_fatal("Raw user initialization failed: %s", error); ns_set = p_new(user->pool, struct mail_namespace_settings, 1); + ns_set->name = "raw-storage"; ns_set->location = ":LAYOUT=none"; ns_set->separator = "/";
--- a/src/lib-storage/index/shared/shared-storage.c Sat May 19 22:40:08 2012 +0300 +++ b/src/lib-storage/index/shared/shared-storage.c Sun May 20 03:25:04 2012 +0300 @@ -120,6 +120,19 @@ str_append(location, username); } +static bool shared_namespace_exists(struct mail_namespace *ns) +{ + const char *path; + struct stat st; + + path = mailbox_list_get_path(ns->list, NULL, MAILBOX_LIST_PATH_TYPE_DIR); + if (path == NULL) { + /* we can't know if this exists */ + return TRUE; + } + return stat(path, &st) == 0; +} + int shared_storage_get_namespace(struct mail_namespace **_ns, const char **_name) { @@ -243,6 +256,13 @@ owner = mail_user_alloc(userdomain, user->set_info, user->unexpanded_set); owner->autocreated = TRUE; + if (mail_user_init(owner, &error) < 0) { + mailbox_list_set_critical(list, + "Couldn't create namespace '%s' for user %s: %s", + ns->prefix, userdomain, error); + mail_user_unref(&owner); + return -1; + } if (!var_has_key(storage->location, 'h', "home")) ret = 1; else { @@ -255,13 +275,6 @@ return -1; } } - if (mail_user_init(owner, &error) < 0) { - mailbox_list_set_critical(list, - "Couldn't create namespace '%s' for user %s: %s", - ns->prefix, userdomain, error); - mail_user_unref(&owner); - return -1; - } /* create the new namespace */ new_ns = i_new(struct mail_namespace, 1); @@ -310,6 +323,13 @@ mail_namespace_destroy(new_ns); return -1; } + if ((new_ns->flags & NAMESPACE_FLAG_UNUSABLE) == 0 && + !shared_namespace_exists(new_ns)) { + /* this user doesn't have a usable storage */ + new_ns->flags |= NAMESPACE_FLAG_UNUSABLE; + } + /* mark the shared namespace root as usable, since it now has + child namespaces */ ns->flags |= NAMESPACE_FLAG_USABLE; *_name = mailbox_list_get_storage_name(new_ns->list, t_strconcat(new_ns->prefix, name, NULL));
--- a/src/lib-storage/list/mailbox-list-fs-iter.c Sat May 19 22:40:08 2012 +0300 +++ b/src/lib-storage/list/mailbox-list-fs-iter.c Sun May 20 03:25:04 2012 +0300 @@ -2,12 +2,16 @@ #include "lib.h" #include "array.h" +#include "str.h" +#include "unichar.h" #include "imap-match.h" +#include "imap-utf7.h" #include "mail-storage.h" #include "mailbox-tree.h" #include "mailbox-list-subscriptions.h" #include "mailbox-list-fs.h" +#include <stdio.h> #include <ctype.h> #include <dirent.h> #include <sys/stat.h> @@ -81,6 +85,49 @@ return 0; } +static void +fs_list_rename_invalid(struct fs_list_iterate_context *ctx, + const char *storage_name) +{ + /* the storage_name is completely invalid, rename it to + something more sensible. we could do this for all names that + aren't valid mUTF-7, but that might lead to accidents in + future when UTF-8 storage names are used */ + string_t *destname = t_str_new(128); + string_t *dest = t_str_new(128); + const char *root, *src; + + root = mailbox_list_get_path(ctx->ctx.list, NULL, + MAILBOX_LIST_PATH_TYPE_MAILBOX); + src = t_strconcat(root, "/", storage_name, NULL); + + (void)uni_utf8_get_valid_data((const void *)storage_name, + strlen(storage_name), destname); + + str_append(dest, root); + str_append_c(dest, '/'); + (void)imap_utf8_to_utf7(str_c(destname), dest); + + if (rename(src, str_c(dest)) < 0 && errno != ENOENT) + i_error("rename(%s, %s) failed: %m", src, str_c(dest)); +} + +static const char * +dir_get_storage_name(struct list_dir_context *dir, const char *fname) +{ + if (*dir->storage_name == '\0') { + /* regular root */ + return fname; + } else if (strcmp(dir->storage_name, "/") == 0) { + /* full_filesystem_access=yes "/" root */ + return t_strconcat("/", fname, NULL); + } else { + /* child */ + return *fname == '\0' ? dir->storage_name : + t_strconcat(dir->storage_name, "/", fname, NULL); + } +} + static int dir_entry_get(struct fs_list_iterate_context *ctx, const char *dir_path, struct list_dir_context *dir, const struct dirent *d) @@ -113,9 +160,15 @@ } /* check the pattern */ - storage_name = *dir->storage_name == '\0' ? d->d_name : - t_strconcat(dir->storage_name, "/", d->d_name, NULL); + storage_name = dir_get_storage_name(dir, d->d_name); vname = mailbox_list_get_vname(ctx->ctx.list, storage_name); + if (!uni_utf8_str_is_valid(vname)) { + fs_list_rename_invalid(ctx, storage_name); + /* just skip this in this iteration, we'll see it on the + next list */ + return 0; + } + match = imap_match(ctx->ctx.glob, vname); if ((dir->info_flags & (MAILBOX_CHILDREN | MAILBOX_NOCHILDREN | @@ -167,8 +220,9 @@ if (*path == '~') { if (!mailbox_list_try_get_absolute_path(ctx->ctx.list, &path)) { - /* couldn't expand ~user/ */ - return FALSE; + /* a) couldn't expand ~user/ + b) mailbox is under our mail root, we changed + path to storage_name */ } /* NOTE: the path may have been translated to a storage_name instead of path */ @@ -196,6 +250,10 @@ if (!fs_list_get_storage_path(ctx, dir->storage_name, &path)) return 0; + if (path == NULL) { + /* no mailbox root dir */ + return 0; + } fsdir = opendir(path); if (fsdir == NULL) { @@ -266,13 +324,14 @@ dir->info_flags = info_flags; p_array_init(&dir->entries, pool, 16); - if ((dir->info_flags & MAILBOX_CHILDREN) == 0) { - /* start with the assumption of not having children */ + if (fs_list_dir_read(ctx, dir) < 0) + ctx->ctx.failed = TRUE; + + if ((dir->info_flags & (MAILBOX_CHILDREN | MAILBOX_NOCHILDREN | + MAILBOX_NOINFERIORS)) == 0) { + /* assume this directory has no children */ dir->info_flags |= MAILBOX_NOCHILDREN; } - - if (fs_list_dir_read(ctx, dir) < 0) - ctx->ctx.failed = TRUE; return dir; } @@ -313,10 +372,12 @@ static void fs_list_get_roots(struct fs_list_iterate_context *ctx) { + struct mail_namespace *ns = ctx->ctx.list->ns; + char ns_sep = mail_namespace_get_sep(ns); bool full_fs_access = ctx->ctx.list->mail_set->mail_full_filesystem_access; const char *const *patterns, *pattern, *const *parentp, *const *childp; - const char *p, *last, *root; + const char *p, *last, *root, *prefix_vname; unsigned int i, parentlen; i_assert(*ctx->valid_patterns != NULL); @@ -329,14 +390,28 @@ for (p = last = pattern; *p != '\0'; p++) { if (*p == '%' || *p == '*') break; - if (*p == '/') + if (*p == ns_sep) last = p; } - if (p == last && *pattern == '/') + prefix_vname = t_strdup_until(pattern, last); + + if (p == last+1 && *pattern == ns_sep) root = "/"; - else { + else if ((ns->flags & NAMESPACE_FLAG_INBOX_USER) != 0 && + strcasecmp(prefix_vname, "INBOX") == 0 && + strncasecmp(ns->prefix, pattern, ns->prefix_len) == 0) { + /* special case: Namespace prefix is INBOX/ and + we just want to see its contents (not the + INBOX's children). */ + root = ""; + } else if (*prefix_vname == '\0') { + /* we need to handle "" explicitly here, because getting + storage name with mail_shared_explicit_inbox=no + would return root=INBOX. */ + root = ""; + } else { root = mailbox_list_get_storage_name(ctx->ctx.list, - t_strdup_until(pattern, last)); + prefix_vname); } if (*root == '/') { @@ -360,7 +435,8 @@ childp = array_idx(&ctx->roots, i); parentlen = strlen(*parentp); if (strncmp(*parentp, *childp, parentlen) == 0 && - ((*childp)[parentlen] == ctx->sep || + (parentlen == 0 || + (*childp)[parentlen] == ctx->sep || (*childp)[parentlen] == '\0')) array_delete(&ctx->roots, i, 1); } @@ -514,14 +590,13 @@ struct mail_namespace *ns = ctx->ctx.list->ns; struct list_dir_context *dir, *subdir = NULL; enum imap_match_result match, child_dir_match; - const char *storage_name, *child_dir_name; + const char *storage_name, *vname, *child_dir_name; dir = ctx->dir; - storage_name = *dir->storage_name == '\0' ? entry->fname : - t_strconcat(dir->storage_name, "/", entry->fname, NULL); + storage_name = dir_get_storage_name(dir, entry->fname); - ctx->info.name = mailbox_list_get_vname(ctx->ctx.list, storage_name); - ctx->info.name = p_strdup(ctx->info_pool, ctx->info.name); + vname = mailbox_list_get_vname(ctx->ctx.list, storage_name); + ctx->info.name = p_strdup(ctx->info_pool, vname); ctx->info.flags = entry->info_flags; match = imap_match(ctx->ctx.glob, ctx->info.name);
--- a/src/lib-storage/list/mailbox-list-index-iter.c Sat May 19 22:40:08 2012 +0300 +++ b/src/lib-storage/list/mailbox-list-index-iter.c Sun May 20 03:25:04 2012 +0300 @@ -25,6 +25,7 @@ ctx->ctx.glob = imap_match_init_multiple(pool, patterns, TRUE, ns_sep); array_create(&ctx->ctx.module_contexts, pool, sizeof(void *), 5); ctx->sep = ns_sep; + ctx->info_pool = pool_alloconly_create("mailbox list index iter info", 128); if (mailbox_list_index_refresh(ctx->ctx.list) < 0) { /* no indexing */ @@ -46,12 +47,15 @@ struct mailbox_list_index_node *node = ctx->next_node; struct mailbox *box; + p_clear(ctx->info_pool); + str_truncate(ctx->path, ctx->parent_len); if (str_len(ctx->path) > 0) str_append_c(ctx->path, ctx->sep); str_append(ctx->path, node->name); - ctx->info.name = str_c(ctx->path); + ctx->info.name = mailbox_list_get_vname(ctx->ctx.list, str_c(ctx->path)); + ctx->info.name = p_strdup(ctx->info_pool, ctx->info.name); ctx->info.flags = 0; if ((node->flags & MAILBOX_LIST_INDEX_FLAG_NONEXISTENT) != 0) ctx->info.flags |= MAILBOX_NONEXISTENT; @@ -166,6 +170,7 @@ ilist->iter_refcount--; } + pool_unref(&ctx->info_pool); pool_unref(&_ctx->pool); return ret; }
--- a/src/lib-storage/list/mailbox-list-index-status.c Sat May 19 22:40:08 2012 +0300 +++ b/src/lib-storage/list/mailbox-list-index-status.c Sun May 20 03:25:04 2012 +0300 @@ -289,6 +289,8 @@ struct mailbox_status status; uint32_t seq, seq1, seq2; + (void)mailbox_list_index_refresh(box->list); + node = mailbox_list_index_lookup(box->list, box->name); if (node == NULL) { mailbox_list_index_refresh_later(box->list);
--- a/src/lib-storage/list/mailbox-list-index-sync.c Sat May 19 22:40:08 2012 +0300 +++ b/src/lib-storage/list/mailbox-list-index-sync.c Sun May 20 03:25:04 2012 +0300 @@ -267,8 +267,11 @@ /* clear EXISTS-flags, so after sync we know what can be expunged */ mailbox_list_index_node_clear_exists(ilist->mailbox_tree); + /* don't include autocreated mailboxes in index until they're + actually created. */ patterns[0] = "*"; patterns[1] = NULL; - iter = ilist->module_ctx.super.iter_init(list, patterns, 0); + iter = ilist->module_ctx.super. + iter_init(list, patterns, MAILBOX_LIST_ITER_NO_AUTO_BOXES); while ((info = ilist->module_ctx.super.iter_next(iter)) != NULL) { flags = 0; if ((info->flags & MAILBOX_NONEXISTENT) != 0) @@ -301,7 +304,7 @@ T_BEGIN { mailbox_list_index_sync_names(&sync_ctx); } T_END; - } else { + } else if (mailbox_list_index_need_refresh(ilist, sync_ctx.view)) { /* we're synced, reset refresh flag */ struct mailbox_list_index_header new_hdr;
--- a/src/lib-storage/list/mailbox-list-index.c Sat May 19 22:40:08 2012 +0300 +++ b/src/lib-storage/list/mailbox-list-index.c Sun May 20 03:25:04 2012 +0300 @@ -30,6 +30,34 @@ ilist->sync_log_file_offset = 0; } +static void mailbox_list_index_index_open(struct mailbox_list *list) +{ + struct mailbox_list_index *ilist = INDEX_LIST_CONTEXT(list); + const struct mail_storage_settings *set = list->mail_set; + enum mail_index_open_flags index_flags; + unsigned int lock_timeout; + + if (ilist->opened) + return; + ilist->opened = TRUE; + + index_flags = mail_storage_settings_to_index_flags(set); + lock_timeout = set->mail_max_lock_timeout == 0 ? -1U : + set->mail_max_lock_timeout; + + mail_index_set_lock_method(ilist->index, set->parsed_lock_method, + lock_timeout); + if (mail_index_open_or_create(ilist->index, index_flags) < 0) { + if (mail_index_move_to_memory(ilist->index) < 0) { + /* try opening once more. it should be created + directly into memory now. */ + if (mail_index_open_or_create(ilist->index, + index_flags) < 0) + i_panic("in-memory index creation failed"); + } + } +} + struct mailbox_list_index_node * mailbox_list_index_node_find_sibling(struct mailbox_list_index_node *node, const char *name) @@ -197,9 +225,8 @@ return 0; } -static bool -mailbox_list_index_need_refresh(struct mailbox_list_index *ilist, - struct mail_index_view *view) +bool mailbox_list_index_need_refresh(struct mailbox_list_index *ilist, + struct mail_index_view *view) { const struct mailbox_list_index_header *hdr; const void *data; @@ -221,6 +248,7 @@ return 0; } + mailbox_list_index_index_open(list); if (mail_index_refresh(ilist->index) < 0) { mailbox_list_index_set_index_error(list); return -1; @@ -245,6 +273,8 @@ struct mail_index_view *view; struct mail_index_transaction *trans; + mailbox_list_index_index_open(list); + view = mail_index_view_open(ilist->index); if (!mailbox_list_index_need_refresh(ilist, view)) { new_hdr.refresh_flag = 1; @@ -268,36 +298,12 @@ hash_table_destroy(&ilist->mailbox_hash); hash_table_destroy(&ilist->mailbox_names); pool_unref(&ilist->mailbox_pool); - mail_index_close(ilist->index); + if (ilist->opened) + mail_index_close(ilist->index); mail_index_free(&ilist->index); ilist->module_ctx.super.deinit(list); } -static int mailbox_list_index_index_open(struct mailbox_list *list) -{ - struct mailbox_list_index *ilist = INDEX_LIST_CONTEXT(list); - const struct mail_storage_settings *set = list->mail_set; - enum mail_index_open_flags index_flags; - unsigned int lock_timeout; - - index_flags = mail_storage_settings_to_index_flags(set); - lock_timeout = set->mail_max_lock_timeout == 0 ? -1U : - set->mail_max_lock_timeout; - - mail_index_set_lock_method(ilist->index, set->parsed_lock_method, - lock_timeout); - if (mail_index_open_or_create(ilist->index, index_flags) < 0) { - if (mail_index_move_to_memory(ilist->index) < 0) { - /* try opening once more. it should be created - directly into memory now. */ - if (mail_index_open_or_create(ilist->index, - index_flags) < 0) - i_panic("in-memory index creation failed"); - } - } - return 0; -} - static int mailbox_list_index_create_mailbox_dir(struct mailbox_list *list, const char *name, @@ -399,11 +405,6 @@ hash_table_create(default_pool, ilist->mailbox_pool, 0, NULL, NULL); - if (mailbox_list_index_index_open(list) < 0) { - list->v = ilist->module_ctx.super; - mail_index_free(&ilist->index); - MODULE_CONTEXT_UNSET(list, mailbox_list_index_module); - } mailbox_list_index_status_init_list(list); }
--- a/src/lib-storage/list/mailbox-list-index.h Sat May 19 22:40:08 2012 +0300 +++ b/src/lib-storage/list/mailbox-list-index.h Sun May 20 03:25:04 2012 +0300 @@ -102,6 +102,8 @@ /* uint32_t uid => struct mailbox_list_index_node* */ struct hash_table *mailbox_hash; struct mailbox_list_index_node *mailbox_tree; + + unsigned int opened:1; }; struct mailbox_list_index_iterate_context { @@ -109,6 +111,8 @@ struct mailbox_list_iterate_context *backend_ctx; struct mailbox_info info; + pool_t info_pool; + unsigned int parent_len; string_t *path; struct mailbox_list_index_node *next_node; @@ -124,6 +128,8 @@ struct mailbox_list_index_node * mailbox_list_index_lookup(struct mailbox_list *list, const char *name); +bool mailbox_list_index_need_refresh(struct mailbox_list_index *ilist, + struct mail_index_view *view); int mailbox_list_index_refresh(struct mailbox_list *list); void mailbox_list_index_refresh_later(struct mailbox_list *list);
--- a/src/lib-storage/list/mailbox-list-maildir-iter.c Sat May 19 22:40:08 2012 +0300 +++ b/src/lib-storage/list/mailbox-list-maildir-iter.c Sun May 20 03:25:04 2012 +0300 @@ -135,16 +135,20 @@ node = mailbox_tree_lookup(ctx->tree_ctx, inbox_name); if (node != NULL) node->flags &= ~MAILBOX_NONEXISTENT; - } else { + return 0; + } + + /* add the INBOX only if it matches the patterns */ + match = imap_match(glob, inbox_name); + if (match == IMAP_MATCH_PARENT) + maildir_fill_parents(ctx, glob, FALSE, inbox_name); + else if (match == IMAP_MATCH_YES) { node = mailbox_tree_get(ctx->tree_ctx, inbox_name, &created); if (created) node->flags = MAILBOX_NOCHILDREN; else node->flags &= ~MAILBOX_NONEXISTENT; - - match = imap_match(glob, inbox_name); - if ((match & (IMAP_MATCH_YES | IMAP_MATCH_PARENT)) != 0) - node->flags |= MAILBOX_MATCHED; + node->flags |= MAILBOX_MATCHED; } return 0; } @@ -436,7 +440,11 @@ if ((flags & MAILBOX_LIST_ITER_SELECT_SUBSCRIBED) != 0) { /* Listing only subscribed mailboxes. Flags are set later if needed. */ - mailbox_list_subscriptions_fill(&ctx->ctx, ctx->tree_ctx); + bool default_nonexistent = + (flags & MAILBOX_LIST_ITER_RETURN_NO_FLAGS) == 0; + + mailbox_list_subscriptions_fill(&ctx->ctx, ctx->tree_ctx, + default_nonexistent); } if ((flags & MAILBOX_LIST_ITER_SELECT_SUBSCRIBED) == 0 ||
--- a/src/lib-storage/list/mailbox-list-subscriptions.c Sat May 19 22:40:08 2012 +0300 +++ b/src/lib-storage/list/mailbox-list-subscriptions.c Sun May 20 03:25:04 2012 +0300 @@ -185,7 +185,8 @@ } void mailbox_list_subscriptions_fill(struct mailbox_list_iterate_context *ctx, - struct mailbox_tree_context *tree) + struct mailbox_tree_context *tree, + bool default_nonexistent) { struct mailbox_list_iter_update_context update_ctx; struct mailbox_tree_iterate_context *iter; @@ -197,6 +198,8 @@ update_ctx.tree_ctx = tree; update_ctx.glob = ctx->glob; update_ctx.leaf_flags = MAILBOX_SUBSCRIBED; + if (default_nonexistent) + update_ctx.leaf_flags |= MAILBOX_NONEXISTENT; update_ctx.parent_flags = MAILBOX_CHILD_SUBSCRIBED; update_ctx.match_parents = (ctx->flags & MAILBOX_LIST_ITER_SELECT_RECURSIVEMATCH) != 0; @@ -226,7 +229,7 @@ array_create(&ctx->ctx.module_contexts, pool, sizeof(void *), 5); ctx->tree = mailbox_tree_init(sep); - mailbox_list_subscriptions_fill(&ctx->ctx, ctx->tree); + mailbox_list_subscriptions_fill(&ctx->ctx, ctx->tree, FALSE); ctx->info.ns = list->ns; /* the tree usually has only those entries we want to iterate through,
--- a/src/lib-storage/list/mailbox-list-subscriptions.h Sat May 19 22:40:08 2012 +0300 +++ b/src/lib-storage/list/mailbox-list-subscriptions.h Sun May 20 03:25:04 2012 +0300 @@ -17,7 +17,8 @@ /* Add subscriptions matching the iteration to the given tree */ void mailbox_list_subscriptions_fill(struct mailbox_list_iterate_context *ctx, - struct mailbox_tree_context *tree); + struct mailbox_tree_context *tree, + bool default_nonexistent); /* Iterate through subscriptions, call mailbox_list.get_mailbox_flags() if necessary for mailboxes to get their flags. */
--- a/src/lib-storage/list/subscription-file.c Sat May 19 22:40:08 2012 +0300 +++ b/src/lib-storage/list/subscription-file.c Sun May 20 03:25:04 2012 +0300 @@ -57,8 +57,6 @@ const char *line; *failed_r = FALSE; - if (input == NULL) - return NULL; while ((line = i_stream_next_line(input)) == NULL) { switch (i_stream_read(input)) { @@ -138,24 +136,26 @@ return -1; } - input = fd_in == -1 ? NULL : - i_stream_create_fd(fd_in, list->mailbox_name_max_length+1, - TRUE); + found = FALSE; output = o_stream_create_fd_file(fd_out, 0, FALSE); o_stream_cork(output); - found = FALSE; - while ((line = next_line(list, path, input, - &failed, FALSE)) != NULL) { - if (strcmp(line, name) == 0) { - found = TRUE; - if (!set) { - changed = TRUE; - continue; + if (fd_in != -1) { + input = i_stream_create_fd(fd_in, list->mailbox_name_max_length+1, + TRUE); + while ((line = next_line(list, path, input, + &failed, FALSE)) != NULL) { + if (strcmp(line, name) == 0) { + found = TRUE; + if (!set) { + changed = TRUE; + continue; + } } + + (void)o_stream_send_str(output, line); + (void)o_stream_send(output, "\n", 1); } - - (void)o_stream_send_str(output, line); - (void)o_stream_send(output, "\n", 1); + i_stream_destroy(&input); } if (!failed && set && !found) { @@ -178,8 +178,6 @@ } } - if (input != NULL) - i_stream_destroy(&input); o_stream_destroy(&output); if (failed || !changed) {
--- a/src/lib-storage/mail-namespace.c Sat May 19 22:40:08 2012 +0300 +++ b/src/lib-storage/mail-namespace.c Sun May 20 03:25:04 2012 +0300 @@ -189,6 +189,7 @@ { struct mail_namespace *ns, *inbox_ns = NULL; unsigned int subscriptions_count = 0; + bool visible_namespaces = FALSE; char ns_sep, list_sep = '\0'; for (ns = namespaces; ns != NULL; ns = ns->next) { @@ -201,6 +202,8 @@ } if (namespace_set_alias_for(ns, namespaces, error_r) < 0) return FALSE; + if ((ns->flags & NAMESPACE_FLAG_HIDDEN) == 0) + visible_namespaces = TRUE; if ((ns->flags & NAMESPACE_FLAG_INBOX_USER) != 0) { if (inbox_ns != NULL) { *error_r = "There can be only one namespace with " @@ -249,6 +252,10 @@ *error_r = "no list=yes namespaces"; return FALSE; } + if (!visible_namespaces) { + *error_r = "no hidden=no namespaces"; + return FALSE; + } if (subscriptions_count == 0) { *error_r = "no subscriptions=yes namespaces"; return FALSE;
--- a/src/lib-storage/mail-search-register-human.c Sat May 19 22:40:08 2012 +0300 +++ b/src/lib-storage/mail-search-register-human.c Sun May 20 03:25:04 2012 +0300 @@ -83,17 +83,17 @@ { \ return arg_new_human_date(ctx, _type, _date_type); \ } -CALLBACK_DATE(before, SEARCH_BEFORE, MAIL_SEARCH_DATE_TYPE_RECEIVED); -CALLBACK_DATE(on, SEARCH_ON, MAIL_SEARCH_DATE_TYPE_RECEIVED); -CALLBACK_DATE(since, SEARCH_SINCE, MAIL_SEARCH_DATE_TYPE_RECEIVED); +CALLBACK_DATE(before, SEARCH_BEFORE, MAIL_SEARCH_DATE_TYPE_RECEIVED) +CALLBACK_DATE(on, SEARCH_ON, MAIL_SEARCH_DATE_TYPE_RECEIVED) +CALLBACK_DATE(since, SEARCH_SINCE, MAIL_SEARCH_DATE_TYPE_RECEIVED) -CALLBACK_DATE(sentbefore, SEARCH_BEFORE, MAIL_SEARCH_DATE_TYPE_SENT); -CALLBACK_DATE(senton, SEARCH_ON, MAIL_SEARCH_DATE_TYPE_SENT); -CALLBACK_DATE(sentsince, SEARCH_SINCE, MAIL_SEARCH_DATE_TYPE_SENT); +CALLBACK_DATE(sentbefore, SEARCH_BEFORE, MAIL_SEARCH_DATE_TYPE_SENT) +CALLBACK_DATE(senton, SEARCH_ON, MAIL_SEARCH_DATE_TYPE_SENT) +CALLBACK_DATE(sentsince, SEARCH_SINCE, MAIL_SEARCH_DATE_TYPE_SENT) -CALLBACK_DATE(savedbefore, SEARCH_BEFORE, MAIL_SEARCH_DATE_TYPE_SAVED); -CALLBACK_DATE(savedon, SEARCH_ON, MAIL_SEARCH_DATE_TYPE_SAVED); -CALLBACK_DATE(savedsince, SEARCH_SINCE, MAIL_SEARCH_DATE_TYPE_SAVED); +CALLBACK_DATE(savedbefore, SEARCH_BEFORE, MAIL_SEARCH_DATE_TYPE_SAVED) +CALLBACK_DATE(savedon, SEARCH_ON, MAIL_SEARCH_DATE_TYPE_SAVED) +CALLBACK_DATE(savedsince, SEARCH_SINCE, MAIL_SEARCH_DATE_TYPE_SAVED) static struct mail_search_arg * arg_new_human_size(struct mail_search_build_context *ctx, @@ -136,10 +136,14 @@ { struct mail_search_arg *sarg; - sarg = mail_search_build_str(ctx, SEARCH_MAILBOX_GLOB); + sarg = mail_search_build_str(ctx, SEARCH_MAILBOX); if (sarg == NULL) return NULL; + if (strchr(sarg->value.str, '*') != NULL || + strchr(sarg->value.str, '%') != NULL) + sarg->type = SEARCH_MAILBOX_GLOB; + if (!uni_utf8_str_is_valid(sarg->value.str)) { ctx->_error = p_strconcat(ctx->pool, "Mailbox name not valid UTF-8: ",
--- a/src/lib-storage/mail-search-register-imap.c Sat May 19 22:40:08 2012 +0300 +++ b/src/lib-storage/mail-search-register-imap.c Sun May 20 03:25:04 2012 +0300 @@ -116,18 +116,18 @@ sarg->match_not = _not; \ return sarg; \ } -CALLBACK_FLAG(answered, MAIL_ANSWERED, FALSE); -CALLBACK_FLAG(unanswered, MAIL_ANSWERED, TRUE); -CALLBACK_FLAG(deleted, MAIL_DELETED, FALSE); -CALLBACK_FLAG(undeleted, MAIL_DELETED, TRUE); -CALLBACK_FLAG(draft, MAIL_DRAFT, FALSE); -CALLBACK_FLAG(undraft, MAIL_DRAFT, TRUE); -CALLBACK_FLAG(flagged, MAIL_FLAGGED, FALSE); -CALLBACK_FLAG(unflagged, MAIL_FLAGGED, TRUE); -CALLBACK_FLAG(seen, MAIL_SEEN, FALSE); -CALLBACK_FLAG(unseen, MAIL_SEEN, TRUE); -CALLBACK_FLAG(recent, MAIL_RECENT, FALSE); -CALLBACK_FLAG(old, MAIL_RECENT, TRUE); +CALLBACK_FLAG(answered, MAIL_ANSWERED, FALSE) +CALLBACK_FLAG(unanswered, MAIL_ANSWERED, TRUE) +CALLBACK_FLAG(deleted, MAIL_DELETED, FALSE) +CALLBACK_FLAG(undeleted, MAIL_DELETED, TRUE) +CALLBACK_FLAG(draft, MAIL_DRAFT, FALSE) +CALLBACK_FLAG(undraft, MAIL_DRAFT, TRUE) +CALLBACK_FLAG(flagged, MAIL_FLAGGED, FALSE) +CALLBACK_FLAG(unflagged, MAIL_FLAGGED, TRUE) +CALLBACK_FLAG(seen, MAIL_SEEN, FALSE) +CALLBACK_FLAG(unseen, MAIL_SEEN, TRUE) +CALLBACK_FLAG(recent, MAIL_RECENT, FALSE) +CALLBACK_FLAG(old, MAIL_RECENT, TRUE) static struct mail_search_arg * imap_search_new(struct mail_search_build_context *ctx) @@ -141,7 +141,7 @@ return sarg; } -CALLBACK_STR(keyword, SEARCH_KEYWORDS); +CALLBACK_STR(keyword, SEARCH_KEYWORDS) static struct mail_search_arg * imap_search_unkeyword(struct mail_search_build_context *ctx) @@ -179,17 +179,17 @@ { \ return arg_new_date(ctx, _type, _date_type); \ } -CALLBACK_DATE(before, SEARCH_BEFORE, MAIL_SEARCH_DATE_TYPE_RECEIVED); -CALLBACK_DATE(on, SEARCH_ON, MAIL_SEARCH_DATE_TYPE_RECEIVED); -CALLBACK_DATE(since, SEARCH_SINCE, MAIL_SEARCH_DATE_TYPE_RECEIVED); +CALLBACK_DATE(before, SEARCH_BEFORE, MAIL_SEARCH_DATE_TYPE_RECEIVED) +CALLBACK_DATE(on, SEARCH_ON, MAIL_SEARCH_DATE_TYPE_RECEIVED) +CALLBACK_DATE(since, SEARCH_SINCE, MAIL_SEARCH_DATE_TYPE_RECEIVED) -CALLBACK_DATE(sentbefore, SEARCH_BEFORE, MAIL_SEARCH_DATE_TYPE_SENT); -CALLBACK_DATE(senton, SEARCH_ON, MAIL_SEARCH_DATE_TYPE_SENT); -CALLBACK_DATE(sentsince, SEARCH_SINCE, MAIL_SEARCH_DATE_TYPE_SENT); +CALLBACK_DATE(sentbefore, SEARCH_BEFORE, MAIL_SEARCH_DATE_TYPE_SENT) +CALLBACK_DATE(senton, SEARCH_ON, MAIL_SEARCH_DATE_TYPE_SENT) +CALLBACK_DATE(sentsince, SEARCH_SINCE, MAIL_SEARCH_DATE_TYPE_SENT) -CALLBACK_DATE(x_savedbefore, SEARCH_BEFORE, MAIL_SEARCH_DATE_TYPE_SAVED); -CALLBACK_DATE(x_savedon, SEARCH_ON, MAIL_SEARCH_DATE_TYPE_SAVED); -CALLBACK_DATE(x_savedsince, SEARCH_SINCE, MAIL_SEARCH_DATE_TYPE_SAVED); +CALLBACK_DATE(x_savedbefore, SEARCH_BEFORE, MAIL_SEARCH_DATE_TYPE_SAVED) +CALLBACK_DATE(x_savedon, SEARCH_ON, MAIL_SEARCH_DATE_TYPE_SAVED) +CALLBACK_DATE(x_savedsince, SEARCH_SINCE, MAIL_SEARCH_DATE_TYPE_SAVED) static struct mail_search_arg * arg_new_size(struct mail_search_build_context *ctx, @@ -245,11 +245,11 @@ { \ return arg_new_header(ctx, _type, #_name); \ } -CALLBACK_HDR(bcc, SEARCH_HEADER_ADDRESS); -CALLBACK_HDR(cc, SEARCH_HEADER_ADDRESS); -CALLBACK_HDR(from, SEARCH_HEADER_ADDRESS); -CALLBACK_HDR(to, SEARCH_HEADER_ADDRESS); -CALLBACK_HDR(subject, SEARCH_HEADER_COMPRESS_LWSP); +CALLBACK_HDR(bcc, SEARCH_HEADER_ADDRESS) +CALLBACK_HDR(cc, SEARCH_HEADER_ADDRESS) +CALLBACK_HDR(from, SEARCH_HEADER_ADDRESS) +CALLBACK_HDR(to, SEARCH_HEADER_ADDRESS) +CALLBACK_HDR(subject, SEARCH_HEADER_COMPRESS_LWSP) static struct mail_search_arg * imap_search_header(struct mail_search_build_context *ctx) @@ -291,8 +291,8 @@ { \ return arg_new_body(ctx, _type); \ } -CALLBACK_BODY(body, SEARCH_BODY); -CALLBACK_BODY(text, SEARCH_TEXT); +CALLBACK_BODY(body, SEARCH_BODY) +CALLBACK_BODY(text, SEARCH_TEXT) static struct mail_search_arg * arg_new_interval(struct mail_search_build_context *ctx, @@ -478,7 +478,7 @@ return sarg; } -CALLBACK_STR(x_guid, SEARCH_GUID); +CALLBACK_STR(x_guid, SEARCH_GUID) static struct mail_search_arg * imap_search_x_mailbox(struct mail_search_build_context *ctx)
--- a/src/lib-storage/mail-storage-private.h Sat May 19 22:40:08 2012 +0300 +++ b/src/lib-storage/mail-storage-private.h Sun May 20 03:25:04 2012 +0300 @@ -212,7 +212,7 @@ struct mailbox_vfuncs v, *vlast; /* private: */ - pool_t pool; + pool_t pool, metadata_pool; /* Linked list of all mailboxes in this storage */ struct mailbox *prev, *next; @@ -240,6 +240,8 @@ unsigned int transaction_count; enum mailbox_feature enabled_features; + struct mail_index_view *tmp_sync_view; + /* Mailbox notification settings: */ mailbox_notify_callback_t *notify_callback; void *notify_context; @@ -262,6 +264,8 @@ unsigned int creating:1; /* Mailbox is being deleted */ unsigned int deleting:1; + /* Delete mailbox only if it's empty */ + unsigned int deleting_must_be_empty:1; /* Mailbox was already marked as deleted within this allocation. */ unsigned int marked_deleted:1; /* TRUE if this is an INBOX for this user */ @@ -274,6 +278,8 @@ unsigned int disable_reflink_copy_to:1; /* Don't allow creating any new keywords */ unsigned int disallow_new_keywords:1; + /* Mailbox has been synced at least once */ + unsigned int synced:1; }; struct mail_vfuncs { @@ -394,6 +400,8 @@ struct mailbox_transaction_stats stats; /* Set to TRUE to update stats_* fields */ unsigned int stats_track:1; + /* We've done some non-transactional (e.g. dovecot-uidlist updates) */ + unsigned int nontransactional_changes:1; }; union mail_search_module_context { @@ -439,6 +447,7 @@ uint32_t uid; char *guid, *pop3_uidl, *from_envelope; struct ostream *output; + unsigned int pop3_order; struct mail_save_attachment *attach;
--- a/src/lib-storage/mail-storage-service.c Sat May 19 22:40:08 2012 +0300 +++ b/src/lib-storage/mail-storage-service.c Sun May 20 03:25:04 2012 +0300 @@ -374,6 +374,7 @@ { 'p', NULL, "pid" }, { 'i', NULL, "uid" }, { '\0', NULL, "gid" }, + { '\0', NULL, "session" }, { '\0', NULL, NULL } }; struct var_expand_table *tab; @@ -391,6 +392,7 @@ tab[6].value = my_pid; tab[7].value = dec2str(priv->uid == (uid_t)-1 ? geteuid() : priv->uid); tab[8].value = dec2str(priv->gid == (gid_t)-1 ? getegid() : priv->gid); + tab[9].value = input->session_id; return tab; } @@ -602,8 +604,13 @@ mail_set = mail_user_set_get_storage_set(mail_user); if (mail_set->mail_debug) { - i_debug("Effective uid=%s, gid=%s, home=%s", - dec2str(geteuid()), dec2str(getegid()), home); + string_t *str = t_str_new(64); + + str_printfa(str, "Effective uid=%s, gid=%s, home=%s", + dec2str(geteuid()), dec2str(getegid()), home); + if (*priv->chroot != '\0') + str_printfa(str, ", chroot=%s", priv->chroot); + i_debug("%s", str_c(str)); } if ((user->flags & MAIL_STORAGE_SERVICE_FLAG_TEMP_PRIV_DROP) != 0 &&
--- a/src/lib-storage/mail-storage-service.h Sat May 19 22:40:08 2012 +0300 +++ b/src/lib-storage/mail-storage-service.h Sun May 20 03:25:04 2012 +0300 @@ -40,6 +40,7 @@ const char *module; const char *service; const char *username; + const char *session_id; struct ip_addr local_ip, remote_ip; unsigned int local_port, remote_port;
--- a/src/lib-storage/mail-storage-settings.c Sat May 19 22:40:08 2012 +0300 +++ b/src/lib-storage/mail-storage-settings.c Sun May 20 03:25:04 2012 +0300 @@ -37,6 +37,7 @@ DEF(SET_TIME, mailbox_idle_check_interval), DEF(SET_UINT, mail_max_keyword_length), DEF(SET_TIME, mail_max_lock_timeout), + DEF(SET_TIME, mail_temp_scan_interval), DEF(SET_BOOL, mail_save_crlf), DEF(SET_ENUM, mail_fsync), DEF(SET_BOOL, mmap_disable), @@ -47,6 +48,7 @@ DEF(SET_BOOL, mail_debug), DEF(SET_BOOL, mail_full_filesystem_access), DEF(SET_BOOL, maildir_stat_dirs), + DEF(SET_BOOL, mail_shared_explicit_inbox), DEF(SET_ENUM, lock_method), DEF(SET_STR, pop3_uidl_format), @@ -66,6 +68,7 @@ .mailbox_idle_check_interval = 30, .mail_max_keyword_length = 50, .mail_max_lock_timeout = 0, + .mail_temp_scan_interval = 7*24*60*60, .mail_save_crlf = FALSE, .mail_fsync = "optimized:never:always", .mmap_disable = FALSE, @@ -76,6 +79,7 @@ .mail_debug = FALSE, .mail_full_filesystem_access = FALSE, .maildir_stat_dirs = FALSE, + .mail_shared_explicit_inbox = TRUE, .lock_method = "fcntl:flock:dotlock", .pop3_uidl_format = "%08Xu%08Xv" };
--- a/src/lib-storage/mail-storage-settings.h Sat May 19 22:40:08 2012 +0300 +++ b/src/lib-storage/mail-storage-settings.h Sun May 20 03:25:04 2012 +0300 @@ -22,6 +22,7 @@ unsigned int mailbox_idle_check_interval; unsigned int mail_max_keyword_length; unsigned int mail_max_lock_timeout; + unsigned int mail_temp_scan_interval; bool mail_save_crlf; const char *mail_fsync; bool mmap_disable; @@ -32,6 +33,7 @@ bool mail_debug; bool mail_full_filesystem_access; bool maildir_stat_dirs; + bool mail_shared_explicit_inbox; const char *lock_method; const char *pop3_uidl_format;
--- a/src/lib-storage/mail-storage.c Sat May 19 22:40:08 2012 +0300 +++ b/src/lib-storage/mail-storage.c Sun May 20 03:25:04 2012 +0300 @@ -749,20 +749,27 @@ return 0; } - if (!box->inbox_user && - have_listable_namespace_prefix(box->storage->user->namespaces, - box->vname)) { - /* listable namespace prefix always exists */ - *existence_r = MAILBOX_EXISTENCE_NOSELECT; - return 0; - } - if (auto_boxes && box->set != NULL && mailbox_is_autocreated(box)) { *existence_r = MAILBOX_EXISTENCE_SELECT; return 0; } - return box->v.exists(box, auto_boxes, existence_r); + if (box->v.exists(box, auto_boxes, existence_r) < 0) + return -1; + + if (!box->inbox_user && *existence_r == MAILBOX_EXISTENCE_NOSELECT && + have_listable_namespace_prefix(box->storage->user->namespaces, + box->vname)) { + /* listable namespace prefix always exists. */ + *existence_r = MAILBOX_EXISTENCE_NOSELECT; + return 0; + } + + /* if this is a shared namespace with only INBOX and + mail_shared_explicit_inbox=no, we'll need to mark the namespace as + usable here since nothing else will. */ + box->list->ns->flags |= NAMESPACE_FLAG_USABLE; + return 0; } static int mailbox_check_mismatching_separators(struct mailbox *box) @@ -911,9 +918,37 @@ return 0; } +static bool mailbox_try_undelete(struct mailbox *box) +{ + time_t mtime; + + if (mail_index_get_modification_time(box->index, &mtime) < 0) + return FALSE; + if (mtime + MAILBOX_DELETE_RETRY_SECS > time(NULL)) + return FALSE; + + if (mailbox_mark_index_deleted(box, FALSE) < 0) + return FALSE; + box->mailbox_deleted = FALSE; + return TRUE; +} + int mailbox_open(struct mailbox *box) { - return mailbox_open_full(box, NULL); + if (mailbox_open_full(box, NULL) < 0) { + if (!box->mailbox_deleted) + return -1; + + /* mailbox has been marked as deleted. if this deletion + started (and crashed) a long time ago, it can be confusing + to user that the mailbox can't be opened. so we'll just + undelete it and reopen. */ + if(!mailbox_try_undelete(box)) + return -1; + if (mailbox_open_full(box, NULL) < 0) + return -1; + } + return 0; } int mailbox_open_stream(struct mailbox *box, struct istream *input) @@ -958,6 +993,8 @@ DLLIST_REMOVE(&box->storage->mailboxes, box); mail_storage_obj_unref(box->storage); + if (box->metadata_pool != NULL) + pool_unref(&box->metadata_pool); pool_unref(&box->pool); } @@ -1040,21 +1077,6 @@ return 0; } -static bool mailbox_try_undelete(struct mailbox *box) -{ - time_t mtime; - - if (mail_index_get_modification_time(box->index, &mtime) < 0) - return FALSE; - if (mtime + MAILBOX_DELETE_RETRY_SECS > time(NULL)) - return FALSE; - - if (mailbox_mark_index_deleted(box, FALSE) < 0) - return FALSE; - box->mailbox_deleted = FALSE; - return TRUE; -} - int mailbox_delete(struct mailbox *box) { int ret; @@ -1074,19 +1096,7 @@ if (mailbox_open(box) < 0) { if (mailbox_get_last_mail_error(box) != MAIL_ERROR_NOTFOUND) return -1; - if (!box->mailbox_deleted) { - /* \noselect mailbox */ - } else { - /* if deletion happened a long time ago, it means it - crashed while doing it. undelete the mailbox in - that case. */ - if (!mailbox_try_undelete(box)) - return -1; - - /* retry */ - if (mailbox_open(box) < 0) - return -1; - } + /* \noselect mailbox */ } ret = box->v.delete(box); @@ -1102,6 +1112,18 @@ return ret; } +int mailbox_delete_empty(struct mailbox *box) +{ + int ret; + + /* FIXME: should be a parameter to delete(), but since it changes API + don't do it for now */ + box->deleting_must_be_empty = TRUE; + ret = mailbox_delete(box); + box->deleting_must_be_empty = FALSE; + return ret; +} + static bool mail_storages_rename_compatible(struct mail_storage *storage1, struct mail_storage *storage2, @@ -1266,6 +1288,9 @@ { memset(metadata_r, 0, sizeof(*metadata_r)); + if (box->metadata_pool != NULL) + p_clear(box->metadata_pool); + if (box->v.get_metadata(box, items, metadata_r) < 0) return -1; @@ -1324,6 +1349,8 @@ i_error("Syncing INBOX failed: %s", errormsg); } } + if (ret == 0) + box->synced = TRUE; return ret; } @@ -1584,6 +1611,14 @@ ctx->pop3_uidl = i_strdup(uidl); } +void mailbox_save_set_pop3_order(struct mail_save_context *ctx, + unsigned int order) +{ + i_assert(order > 0); + + ctx->pop3_order = order; +} + void mailbox_save_set_dest_mail(struct mail_save_context *ctx, struct mail *mail) { @@ -1600,7 +1635,8 @@ return -1; } - (*ctx)->saving = TRUE; + if (!(*ctx)->copying_via_save) + (*ctx)->saving = TRUE; if (box->v.save_begin == NULL) { mail_storage_set_error(box->storage, MAIL_ERROR_NOTPOSSIBLE, "Saving messages not supported"); @@ -1663,7 +1699,7 @@ if (mail_index_is_deleted(box->index)) { mailbox_set_deleted(box); - mailbox_save_cancel(_ctx); + mailbox_save_cancel(&ctx); return -1; }
--- a/src/lib-storage/mail-storage.h Sat May 19 22:40:08 2012 +0300 +++ b/src/lib-storage/mail-storage.h Sun May 20 03:25:04 2012 +0300 @@ -265,6 +265,9 @@ /* number of modseq changes that couldn't be changed as requested */ unsigned int ignored_modseq_changes; + + /* TRUE if anything actually changed with this commit */ + bool changed; }; struct mailbox_sync_rec { @@ -414,6 +417,9 @@ int mailbox_update(struct mailbox *box, const struct mailbox_update *update); /* Delete mailbox (and its parent directory, if it has no siblings) */ int mailbox_delete(struct mailbox *box); +/* Delete mailbox, but only if it's empty. If it's not, fails with + MAIL_ERROR_EXISTS. */ +int mailbox_delete_empty(struct mailbox *box); /* Rename mailbox. Renaming across different mailbox lists is possible only between private namespaces and storages of the same type. If the rename fails, the error is set to src's storage. */ @@ -644,6 +650,10 @@ /* Set message's POP3 UIDL, if the backend supports it. */ void mailbox_save_set_pop3_uidl(struct mail_save_context *ctx, const char *uidl); +/* Specify ordering for POP3 messages. The default is to add them to the end + of the mailbox. Not all backends support this. */ +void mailbox_save_set_pop3_order(struct mail_save_context *ctx, + unsigned int order); /* If dest_mail is set, the saved message can be accessed using it. Note that setting it may require mailbox syncing, so don't set it unless you need it. Also you shouldn't try to access it before mailbox_save_finish() is
--- a/src/lib-storage/mail-user.c Sat May 19 22:40:08 2012 +0300 +++ b/src/lib-storage/mail-user.c Sun May 20 03:25:04 2012 +0300 @@ -50,6 +50,7 @@ user->set_info = set_info; user->unexpanded_set = settings_dup(set_info, set, pool); user->set = settings_dup(set_info, set, pool); + user->service = master_service_get_name(master_service); /* check settings so that the duplicated structure will again contain the parsed fields */ @@ -95,25 +96,28 @@ { const struct mail_storage_settings *mail_set; const char *home, *key, *value; + bool need_home_dir; - if (user->_home == NULL && - settings_vars_have_key(user->set_info, user->set, - 'h', "home", &key, &value) && - mail_user_get_home(user, &home) <= 0) { + need_home_dir = user->_home == NULL && + settings_vars_have_key(user->set_info, user->set, + 'h', "home", &key, &value); + + /* expand mail_home setting before calling mail_user_get_home() */ + settings_var_expand(user->set_info, user->set, + user->pool, mail_user_var_expand_table(user)); + + if (need_home_dir && mail_user_get_home(user, &home) <= 0) { *error_r = t_strdup_printf( "userdb didn't return a home directory, " "but %s used it (%%h): %s", key, value); return -1; } - settings_var_expand(user->set_info, user->set, - user->pool, mail_user_var_expand_table(user)); if (mail_user_expand_plugins_envs(user, error_r) < 0) return -1; mail_set = mail_user_set_get_storage_set(user); user->mail_debug = mail_set->mail_debug; - user->service = master_service_get_name(master_service); user->initialized = TRUE; hook_mail_user_created(user); @@ -166,6 +170,8 @@ const struct ip_addr *local_ip, const struct ip_addr *remote_ip) { + i_assert(service != NULL); + user->service = p_strdup(user->pool, service); if (local_ip != NULL && local_ip->family != 0) { user->local_ip = p_new(user->pool, struct ip_addr, 1); @@ -263,7 +269,7 @@ return path; } -int mail_user_get_home(struct mail_user *user, const char **home_r) +static int mail_user_userdb_lookup_home(struct mail_user *user) { struct auth_user_info info; struct auth_user_reply reply; @@ -271,19 +277,15 @@ const char *username, *const *fields; int ret; + i_assert(!user->home_looked_up); + memset(&info, 0, sizeof(info)); - info.service = "lib-storage"; + info.service = user->service; if (user->local_ip != NULL) info.local_ip = *user->local_ip; if (user->remote_ip != NULL) info.remote_ip = *user->remote_ip; - if (user->home_looked_up) { - *home_r = user->_home; - return user->_home != NULL ? 1 : 0; - } - *home_r = NULL; - if (mail_user_auth_master_conn == NULL) return 0; @@ -291,18 +293,37 @@ ret = auth_master_user_lookup(mail_user_auth_master_conn, user->username, &info, userdb_pool, &username, &fields); - if (ret >= 0) { + if (ret > 0) { auth_user_fields_parse(fields, userdb_pool, &reply); - user->_home = ret == 0 ? NULL : - p_strdup(user->pool, reply.home); - user->home_looked_up = TRUE; - ret = user->_home != NULL ? 1 : 0; - *home_r = user->_home; + user->_home = p_strdup(user->pool, reply.home); } pool_unref(&userdb_pool); return ret; } +int mail_user_get_home(struct mail_user *user, const char **home_r) +{ + int ret; + + if (user->home_looked_up) { + *home_r = user->_home; + return user->_home != NULL ? 1 : 0; + } + + ret = mail_user_userdb_lookup_home(user); + if (ret < 0) + return -1; + + if (ret > 0 && user->_home == NULL && *user->set->mail_home != '\0') { + /* no home in userdb, fallback to mail_home setting */ + user->_home = user->set->mail_home; + } + user->home_looked_up = TRUE; + + *home_r = user->_home; + return user->_home != NULL ? 1 : 0; +} + bool mail_user_is_plugin_loaded(struct mail_user *user, struct module *module) { const char *const *plugins; @@ -341,7 +362,12 @@ { const char *home, *path = *pathp; - if (mail_user_get_home(user, &home) < 0) + if (*path != '~') { + /* no need to expand home */ + return 0; + } + + if (mail_user_get_home(user, &home) <= 0) return -1; path = home_expand_tilde(path, home);
--- a/src/lib-storage/mail-user.h Sat May 19 22:40:08 2012 +0300 +++ b/src/lib-storage/mail-user.h Sun May 20 03:25:04 2012 +0300 @@ -54,6 +54,10 @@ unsigned int mail_debug:1; /* If INBOX can't be opened, log an error, but only once. */ unsigned int inbox_open_error_logged:1; + /* Fuzzy search works for this user (FTS enabled) */ + unsigned int fuzzy_search:1; + /* We're running dsync */ + unsigned int dsyncing:1; }; struct mail_user_module_register {
--- a/src/lib-storage/mailbox-list-iter.c Sat May 19 22:40:08 2012 +0300 +++ b/src/lib-storage/mailbox-list-iter.c Sun May 20 03:25:04 2012 +0300 @@ -334,6 +334,12 @@ unsigned int i; bool ret = FALSE; + if (strncasecmp(ns->prefix, "INBOX", ns->prefix_len-1) == 0) { + /* INBOX is going to be listed in any case, + don't duplicate it */ + return FALSE; + } + for (i = 0; ctx->patterns_ns_match[i] != NULL; i++) { T_BEGIN { ret = iter_next_try_prefix_pattern(ctx, ns, @@ -527,7 +533,6 @@ if ((ctx->flags & MAILBOX_LIST_ITER_SELECT_SUBSCRIBED) == 0) match2 = match; else { - info->flags |= MAILBOX_SUBSCRIBED; match2 = autocreate_box_match(&actx->all_ns_box_sets, ctx->list->ns, info->name, FALSE, &idx); @@ -697,10 +702,12 @@ const struct mailbox_info *info; do { - if (ctx->autocreate_ctx != NULL) - info = autocreate_iter_next(ctx); - else - info = mailbox_list_iter_next_call(ctx); + T_BEGIN { + if (ctx->autocreate_ctx != NULL) + info = autocreate_iter_next(ctx); + else + info = mailbox_list_iter_next_call(ctx); + } T_END; } while (info != NULL && !special_use_selection(ctx, info)); return info; } @@ -755,6 +762,8 @@ if (node != NULL) { if (!ctx->update_only && add_matched) node->flags |= MAILBOX_MATCHED; + if ((always_flags & MAILBOX_CHILDREN) != 0) + node->flags &= ~MAILBOX_NOCHILDREN; node->flags |= always_flags; } /* We don't want to show the parent mailboxes unless
--- a/src/lib-storage/mailbox-list-private.h Sat May 19 22:40:08 2012 +0300 +++ b/src/lib-storage/mailbox-list-private.h Sun May 20 03:25:04 2012 +0300 @@ -180,9 +180,6 @@ const char *storage_name); const char *mailbox_list_get_unexpanded_path(struct mailbox_list *list, enum mailbox_list_path_type type); -const char *mailbox_list_get_storage_name(struct mailbox_list *list, - const char *vname); -const char *mailbox_list_get_vname(struct mailbox_list *list, const char *name); const char * mailbox_list_get_root_path(const struct mailbox_list_settings *set, enum mailbox_list_path_type type);
--- a/src/lib-storage/mailbox-list.c Sat May 19 22:40:08 2012 +0300 +++ b/src/lib-storage/mailbox-list.c Sun May 20 03:25:04 2012 +0300 @@ -427,6 +427,13 @@ list_sep = mailbox_list_get_hierarchy_sep(list); ns_sep = mail_namespace_get_sep(ns); + if (*storage_name == '\0' && ns->type == NAMESPACE_SHARED && + (ns->flags & NAMESPACE_FLAG_INBOX_ANY) != 0 && + !list->mail_set->mail_shared_explicit_inbox) { + /* opening shared/$user. it's the same as INBOX. */ + storage_name = "INBOX"; + } + if (list_sep == ns_sep) return storage_name; if (ns->type == NAMESPACE_SHARED && @@ -458,6 +465,11 @@ string_t *dest = t_str_new(strlen(src)); unsigned int num; + if (strncmp(src, list->ns->prefix, list->ns->prefix_len) == 0) { + str_append_n(dest, src, list->ns->prefix_len); + src += list->ns->prefix_len; + } + for (; *src != '\0'; src++) { if (*src == list->set.escape_char && i_isxdigit(src[1]) && i_isxdigit(src[2])) { @@ -496,6 +508,13 @@ and <ns prefix>/inBox. */ return vname; } + if (strcmp(vname, "INBOX") == 0 && list->ns->type == NAMESPACE_SHARED && + (list->ns->flags & NAMESPACE_FLAG_INBOX_ANY) != 0 && + !list->mail_set->mail_shared_explicit_inbox) { + /* convert to shared/$user, we don't really care about the + INBOX suffix here. */ + vname = ""; + } if (*vname == '\0') { /* return namespace prefix without the separator */ if (list->ns->prefix_len == 0)
--- a/src/lib-storage/mailbox-list.h Sat May 19 22:40:08 2012 +0300 +++ b/src/lib-storage/mailbox-list.h Sun May 20 03:25:04 2012 +0300 @@ -73,9 +73,6 @@ namespace prefixes, if there exists a parent namespace whose children it matches. */ MAILBOX_LIST_ITER_STAR_WITHIN_NS = 0x000010, - /* For mailbox_list_iter_init_namespaces(): List also namespace - prefixes if they match */ - MAILBOX_LIST_ITER_LIST_PREFIXES = 0x000020, /* List only subscribed mailboxes */ MAILBOX_LIST_ITER_SELECT_SUBSCRIBED = 0x000100, @@ -233,6 +230,10 @@ bool mailbox_list_is_valid_create_name(struct mailbox_list *list, const char *name); +const char *mailbox_list_get_storage_name(struct mailbox_list *list, + const char *vname); +const char *mailbox_list_get_vname(struct mailbox_list *list, const char *name); + /* Return full path for the given mailbox name. The name must be a valid existing mailbox name, or NULL to get the root directory. For INDEX=MEMORY it returns "" as the path. */
--- a/src/lib-storage/mailbox-tree.c Sat May 19 22:40:08 2012 +0300 +++ b/src/lib-storage/mailbox-tree.c Sun May 20 03:25:04 2012 +0300 @@ -8,6 +8,8 @@ struct mailbox_tree_context { pool_t pool; char separator; + bool parents_nonexistent; + struct mailbox_node *nodes; }; @@ -49,6 +51,11 @@ tree->separator = separator; } +void mailbox_tree_set_parents_nonexistent(struct mailbox_tree_context *tree) +{ + tree->parents_nonexistent = TRUE; +} + void mailbox_tree_clear(struct mailbox_tree_context *tree) { p_clear(tree->pool); @@ -57,14 +64,13 @@ static struct mailbox_node * mailbox_tree_traverse(struct mailbox_tree_context *tree, const char *path, - bool create, bool *created) + bool create, bool *created_r) { struct mailbox_node **node, *parent; const char *name; string_t *str; - if (created != NULL) - *created = FALSE; + *created_r = FALSE; if (path == NULL) return tree->nodes; @@ -101,9 +107,10 @@ *node = p_new(tree->pool, struct mailbox_node, 1); (*node)->parent = parent; (*node)->name = p_strdup(tree->pool, name); + if (tree->parents_nonexistent) + (*node)->flags = MAILBOX_NONEXISTENT; - if (created != NULL) - *created = TRUE; + *created_r = TRUE; } if (*path == '\0') @@ -120,13 +127,19 @@ struct mailbox_node * mailbox_tree_get(struct mailbox_tree_context *tree, const char *path, - bool *created) + bool *created_r) { struct mailbox_node *node; + bool created; T_BEGIN { - node = mailbox_tree_traverse(tree, path, TRUE, created); + node = mailbox_tree_traverse(tree, path, TRUE, &created); } T_END; + if (created && tree->parents_nonexistent) + node->flags = 0; + + if (created_r != NULL) + *created_r = created; return node; } @@ -134,9 +147,10 @@ mailbox_tree_lookup(struct mailbox_tree_context *tree, const char *path) { struct mailbox_node *node; + bool created; T_BEGIN { - node = mailbox_tree_traverse(tree, path, FALSE, NULL); + node = mailbox_tree_traverse(tree, path, FALSE, &created); } T_END; return node; }
--- a/src/lib-storage/mailbox-tree.h Sat May 19 22:40:08 2012 +0300 +++ b/src/lib-storage/mailbox-tree.h Sun May 20 03:25:04 2012 +0300 @@ -17,11 +17,12 @@ void mailbox_tree_set_separator(struct mailbox_tree_context *tree, char separator); +void mailbox_tree_set_parents_nonexistent(struct mailbox_tree_context *tree); void mailbox_tree_clear(struct mailbox_tree_context *tree); struct mailbox_node * mailbox_tree_get(struct mailbox_tree_context *tree, const char *path, - bool *created); + bool *created_r); struct mailbox_node * mailbox_tree_lookup(struct mailbox_tree_context *tree, const char *path);
--- a/src/lib-test/test-common.c Sat May 19 22:40:08 2012 +0300 +++ b/src/lib-test/test-common.c Sun May 20 03:25:04 2012 +0300 @@ -185,7 +185,7 @@ total_count++; } -static void +static void ATTR_FORMAT(2, 0) test_error_handler(const struct failure_context *ctx, const char *format, va_list args) {
--- a/src/lib/array.c Sat May 19 22:40:08 2012 +0300 +++ b/src/lib/array.c Sun May 20 03:25:04 2012 +0300 @@ -71,7 +71,7 @@ void array_reverse_i(struct array *array) { - const unsigned int element_size = array->element_size; + const size_t element_size = array->element_size; unsigned int i, count = array_count_i(array); size_t size; void *data, *tmp; @@ -92,7 +92,7 @@ { unsigned int count; - count = array->buffer->used / array->element_size; + count = array_count_i(array); qsort(buffer_get_modifiable_data(array->buffer, NULL), count, array->element_size, cmp); } @@ -102,7 +102,7 @@ { unsigned int count; - count = array->buffer->used / array->element_size; + count = array_count_i(array); return bsearch(key, array->buffer->data, count, array->element_size, cmp); }
--- a/src/lib/array.h Sat May 19 22:40:08 2012 +0300 +++ b/src/lib/array.h Sun May 20 03:25:04 2012 +0300 @@ -134,6 +134,14 @@ #define array_clear(array) \ array_clear_i(&(array)->arr) +static inline unsigned int ATTR_PURE +array_count_i(const struct array *array) +{ + return array->buffer->used / array->element_size; +} +#define array_count(array) \ + array_count_i(&(array)->arr) + static inline void array_append_i(struct array *array, const void *data, unsigned int count) { @@ -177,7 +185,7 @@ static inline const void * array_get_i(const struct array *array, unsigned int *count_r) { - *count_r = array->buffer->used / array->element_size; + *count_r = array_count_i(array); return array->buffer->data; } #define array_get(array, count) \ @@ -195,7 +203,7 @@ static inline void * array_get_modifiable_i(struct array *array, unsigned int *count_r) { - *count_r = array->buffer->used / array->element_size; + *count_r = array_count_i(array); return buffer_get_modifiable_data(array->buffer, NULL); } #define array_get_modifiable(array, count) \ @@ -244,14 +252,6 @@ count * dest->element_size); } -static inline unsigned int ATTR_PURE -array_count_i(const struct array *array) -{ - return array->buffer->used / array->element_size; -} -#define array_count(array) \ - array_count_i(&(array)->arr) - bool array_cmp_i(const struct array *array1, const struct array *array2) ATTR_PURE; #define array_cmp(array1, array2) \
--- a/src/lib/compat.h Sat May 19 22:40:08 2012 +0300 +++ b/src/lib/compat.h Sun May 20 03:25:04 2012 +0300 @@ -244,7 +244,8 @@ #endif #define ENOTFOUND(errno) \ - ((errno) == ENOENT || (errno) == ENOTDIR || (errno) == ELOOP) + ((errno) == ENOENT || (errno) == ENOTDIR || \ + (errno) == ELOOP || (errno) == ENAMETOOLONG) #define ECANTLINK(errno) \ ((errno) == EXDEV || (errno) == EMLINK || (errno) == EPERM)
--- a/src/lib/data-stack.c Sat May 19 22:40:08 2012 +0300 +++ b/src/lib/data-stack.c Sun May 20 03:25:04 2012 +0300 @@ -64,7 +64,11 @@ static struct stack_block *last_buffer_block; static size_t last_buffer_size; +#ifdef DEBUG +static bool clean_after_pop = TRUE; +#else static bool clean_after_pop = FALSE; +#endif static bool outofmem = FALSE; static union { @@ -367,7 +371,7 @@ ret = STACK_BLOCK_DATA(current_block); #ifdef DEBUG - if (warn) { + if (warn && getenv("DEBUG_SILENT") == NULL) { /* warn after allocation, so if i_warning() wants to allocate more memory we don't go to infinite loop */ i_warning("Growing data stack with: %"PRIuSIZE_T, @@ -490,26 +494,26 @@ void data_stack_init(void) { -#ifdef DEBUG - clean_after_pop = TRUE; -#endif - if (data_stack_frame == 0) { - data_stack_frame = 1; + if (data_stack_frame > 0) { + /* already initialized (we did auto-initialization in + t_malloc/t_push) */ + return; + } + data_stack_frame = 1; - outofmem_area.block.size = outofmem_area.block.left = - sizeof(outofmem_area) - sizeof(outofmem_area.block); + outofmem_area.block.size = outofmem_area.block.left = + sizeof(outofmem_area) - sizeof(outofmem_area.block); - current_block = mem_block_alloc(INITIAL_STACK_SIZE); - current_block->left = current_block->size; - current_block->next = NULL; + current_block = mem_block_alloc(INITIAL_STACK_SIZE); + current_block->left = current_block->size; + current_block->next = NULL; - current_frame_block = NULL; - unused_frame_blocks = NULL; - frame_pos = BLOCK_FRAME_COUNT-1; + current_frame_block = NULL; + unused_frame_blocks = NULL; + frame_pos = BLOCK_FRAME_COUNT-1; - last_buffer_block = NULL; - last_buffer_size = 0; - } + last_buffer_block = NULL; + last_buffer_size = 0; t_push(); }
--- a/src/lib/eacces-error.c Sat May 19 22:40:08 2012 +0300 +++ b/src/lib/eacces-error.c Sun May 20 03:25:04 2012 +0300 @@ -109,14 +109,16 @@ if (getuid() == geteuid()) { if (access(path, access_mode) == 0) return 0; - if (errno == EACCES) { write_eacces_error(errmsg, path, access_mode); - (void)test_manual_access(path, access_mode, - FALSE, errmsg); + if (test_manual_access(path, access_mode, + FALSE, errmsg) == 0) { + str_append(errmsg, ", UNIX perms appear ok " + "(ACL/MAC wrong?)"); + } errno = EACCES; } else { - str_printfa(errmsg, " access(%s, %d) failed: %m", + str_printfa(errmsg, ", access(%s, %d) failed: %m", path, access_mode); } return -1; @@ -133,7 +135,7 @@ if (errno == EACCES) write_eacces_error(errmsg, path, access_mode); else - str_printfa(errmsg, " stat(%s/test) failed: %m", path); + str_printfa(errmsg, ", stat(%s/test) failed: %m", path); return -1; case R_OK: case W_OK:
--- a/src/lib/file-cache.c Sat May 19 22:40:08 2012 +0300 +++ b/src/lib/file-cache.c Sun May 20 03:25:04 2012 +0300 @@ -49,9 +49,12 @@ int file_cache_set_size(struct file_cache *cache, uoff_t size) { size_t page_size = mmap_get_page_size(); - uoff_t diff = size % page_size; + uoff_t diff; void *new_base; + i_assert(page_size > 0); + + diff = size % page_size; if (diff != 0) size += page_size - diff; @@ -95,6 +98,8 @@ unsigned char *bits, *dest; ssize_t ret; + i_assert(page_size > 0); + if (size > SSIZE_T_MAX) { /* make sure our calculations won't overflow. most likely we'll be reading less data, but allow it anyway so caller @@ -225,6 +230,7 @@ unsigned char *bits; unsigned int first_page, last_page; + i_assert(page_size > 0); i_assert((uoff_t)-1 - offset > size); if (file_cache_set_size(cache, offset + size) < 0) { @@ -270,6 +276,8 @@ if (offset >= cache->read_highwater || size == 0) return; + i_assert(page_size > 0); + if (size > cache->read_highwater - offset) { /* ignore anything after read highwater */ size = cache->read_highwater - offset;
--- a/src/lib/ioloop.c Sat May 19 22:40:08 2012 +0300 +++ b/src/lib/ioloop.c Sun May 20 03:25:04 2012 +0300 @@ -206,8 +206,7 @@ void timeout_reset(struct timeout *timeout) { - timeout_reset_timeval(timeout, timeout->ioloop->running ? NULL : - &ioloop_timeval); + timeout_reset_timeval(timeout, NULL); } static int timeout_get_wait_time(struct timeout *timeout, struct timeval *tv_r,
--- a/src/lib/mempool-alloconly.c Sat May 19 22:40:08 2012 +0300 +++ b/src/lib/mempool-alloconly.c Sun May 20 03:25:04 2012 +0300 @@ -143,7 +143,8 @@ new_apool = p_new(&apool.pool, struct alloconly_pool, 1); *new_apool = apool; #ifdef DEBUG - if (strncmp(name, MEMPOOL_GROWING, strlen(MEMPOOL_GROWING)) == 0) { + if (strncmp(name, MEMPOOL_GROWING, strlen(MEMPOOL_GROWING)) == 0 || + getenv("DEBUG_SILENT") != NULL) { name += strlen(MEMPOOL_GROWING); new_apool->disable_warning = TRUE; }
--- a/src/lib/network.c Sat May 19 22:40:08 2012 +0300 +++ b/src/lib/network.c Sun May 20 03:25:04 2012 +0300 @@ -410,7 +410,7 @@ if (ret < 0) { if (errno != EADDRINUSE) { i_error("bind(%s, %u) failed: %m", - net_ip2addr(my_ip), *port); + my_ip == NULL ? "" : net_ip2addr(my_ip), *port); } } else { /* get the actual port we started listen */ @@ -757,7 +757,7 @@ return 0; #elif defined(HAVE_GETPEERUCRED) /* Solaris */ - ucred_t *ucred; + ucred_t *ucred = NULL; if (getpeerucred(fd, &ucred) < 0) { i_error("getpeerucred() failed: %m");
--- a/src/lib/ostream.c Sat May 19 22:40:08 2012 +0300 +++ b/src/lib/ostream.c Sun May 20 03:25:04 2012 +0300 @@ -150,10 +150,11 @@ { struct const_iovec iov; + memset(&iov, 0, sizeof(iov)); iov.iov_base = data; iov.iov_len = size; - return o_stream_sendv(stream, &iov, 1); + return o_stream_sendv(stream, &iov, 1); } ssize_t o_stream_sendv(struct ostream *stream, const struct const_iovec *iov,
--- a/src/lib/process-title.c Sat May 19 22:40:08 2012 +0300 +++ b/src/lib/process-title.c Sun May 20 03:25:04 2012 +0300 @@ -27,6 +27,8 @@ unsigned int i; bool clear_env; + i_assert(argv[0] != NULL); + /* find the last argv or environment string. it should always be the last string in environ, but don't rely on it. this is what openssh does, so hopefully it's safe enough. */
--- a/src/lib/seq-range-array.c Sat May 19 22:40:08 2012 +0300 +++ b/src/lib/seq-range-array.c Sun May 20 03:25:04 2012 +0300 @@ -86,16 +86,20 @@ data[idx-1].seq2 = data[idx].seq2; array_delete(array, idx, 1); } - } else if (data[idx].seq2 == seq-1) { - i_assert(idx+1 < count); /* already handled above */ - data[idx].seq2 = seq; - if (data[idx+1].seq1 == seq+1) { - /* merge */ - data[idx+1].seq1 = data[idx].seq1; - array_delete(array, idx, 1); + } else { + if (idx > 0 && data[idx-1].seq2 == seq-1) + idx--; + if (data[idx].seq2 == seq-1) { + i_assert(idx+1 < count); /* already handled above */ + data[idx].seq2 = seq; + if (data[idx+1].seq1 == seq+1) { + /* merge */ + data[idx+1].seq1 = data[idx].seq1; + array_delete(array, idx, 1); + } + } else { + array_insert(array, idx, &value, 1); } - } else { - array_insert(array, idx, &value, 1); } return FALSE; }
--- a/src/lib/strfuncs.c Sat May 19 22:40:08 2012 +0300 +++ b/src/lib/strfuncs.c Sun May 20 03:25:04 2012 +0300 @@ -313,11 +313,15 @@ const char *t_str_lcase(const char *str) { + i_assert(str != NULL); + return str_lcase(t_strdup_noconst(str)); } const char *t_str_ucase(const char *str) { + i_assert(str != NULL); + return str_ucase(t_strdup_noconst(str)); } @@ -459,6 +463,47 @@ return split_str(pool, data, separators, TRUE); } +const char **t_strsplit_tab(const char *data) +{ + const char **array; + char *dest; + unsigned int i, array_idx, array_size, dest_size; + + if (*data == '\0') + return t_new(const char *, 1); + + array_size = 1; + dest_size = 128; + dest = t_buffer_get(dest_size+1); + for (i = 0; data[i] != '\0'; i++) { + if (i >= dest_size) { + dest_size = nearest_power(dest_size+1); + dest = t_buffer_reget(dest, dest_size+1); + } + if (data[i] != '\t') + dest[i] = data[i]; + else { + dest[i] = '\0'; + array_size++; + } + } + i_assert(i <= dest_size); + dest[i] = '\0'; + t_buffer_alloc(i+1); + dest_size = i; + + array = t_new(const char *, array_size + 1); + array[0] = dest; array_idx = 1; + + for (i = 0; i < dest_size; i++) { + if (dest[i] == '\0') + array[array_idx++] = dest+i+1; + } + i_assert(array_idx == array_size); + array[array_idx] = NULL; + return array; +} + void p_strsplit_free(pool_t pool, char **arr) { p_free(pool, arr[0]);
--- a/src/lib/strfuncs.h Sat May 19 22:40:08 2012 +0300 +++ b/src/lib/strfuncs.h Sun May 20 03:25:04 2012 +0300 @@ -66,6 +66,8 @@ const char **t_strsplit_spaces(const char *data, const char *separators) ATTR_MALLOC; void p_strsplit_free(pool_t pool, char **arr); +/* Optimized version of t_strsplit(data, "\t") */ +const char **t_strsplit_tab(const char *data); const char *dec2str(uintmax_t number); @@ -86,7 +88,7 @@ /* INTERNAL */ char *t_noalloc_strdup_vprintf(const char *format, va_list args, - unsigned int *size_r); + unsigned int *size_r) ATTR_FORMAT(1, 0); char *vstrconcat(const char *str1, va_list args, size_t *ret_len) ATTR_MALLOC; #endif
--- a/src/lib/test-array.c Sat May 19 22:40:08 2012 +0300 +++ b/src/lib/test-array.c Sun May 20 03:25:04 2012 +0300 @@ -33,7 +33,7 @@ static void test_array_reverse(void) { ARRAY_DEFINE(intarr, int); - int input[] = { -1234567890, -272585721, 2724859223U, 824725652 }; + int input[] = { -1234567890, -272585721, 272485922, 824725652 }; const int *output; unsigned int i, j;
--- a/src/lib/test-base64.c Sat May 19 22:40:08 2012 +0300 +++ b/src/lib/test-base64.c Sun May 20 03:25:04 2012 +0300 @@ -48,9 +48,9 @@ "aGVs!!!!!" }; static const struct test_base64_decode_output output[] = { - { "hello world", 0, -1 }, - { "foo barits", 0, -1 }, - { "just niin", 1, -1 }, + { "hello world", 0, -1U }, + { "foo barits", 0, -1U }, + { "just niin", 1, -1U }, { "hel", 1, 4 }, { "hel", -1, 4 }, { "hel", -1, 4 } @@ -71,7 +71,7 @@ test_assert(output[i].ret == ret && strcmp(output[i].text, str_c(str)) == 0 && (src_pos == output[i].src_pos || - (output[i].src_pos == (unsigned int)-1 && + (output[i].src_pos == -1U && src_pos == strlen(input[i])))); } test_end();
--- a/src/lib/test-bsearch-insert-pos.c Sat May 19 22:40:08 2012 +0300 +++ b/src/lib/test-bsearch-insert-pos.c Sun May 20 03:25:04 2012 +0300 @@ -13,9 +13,9 @@ void test_bsearch_insert_pos(void) { static const unsigned int input[] = { - 1, 5, 9, 15, 16, -1, - 1, 5, 9, 15, 16, 17, -1, - -1 + 1, 5, 9, 15, 16, -1U, + 1, 5, 9, 15, 16, 17, -1U, + -1U }; static const unsigned int max_key = 18; const unsigned int *cur;
--- a/src/lib/test-primes.c Sat May 19 22:40:08 2012 +0300 +++ b/src/lib/test-primes.c Sun May 20 03:25:04 2012 +0300 @@ -14,7 +14,7 @@ success = FALSE; } for (i = 10; i < 32; i++) { - num = (1 << i) - 100; + num = (1U << i) - 100; for (j = 0; j < 200; j++, num++) { if (primes_closest(num) < num) success = FALSE;
--- a/src/lib/test-seq-range-array.c Sat May 19 22:40:08 2012 +0300 +++ b/src/lib/test-seq-range-array.c Sun May 20 03:25:04 2012 +0300 @@ -6,6 +6,19 @@ #include <stdlib.h> +static void test_seq_range_array_add_merge(void) +{ + ARRAY_TYPE(seq_range) range; + + test_begin("seq_range_array_add() merging"); + t_array_init(&range, 8); + seq_range_array_add(&range, 0, 4); + seq_range_array_add(&range, 0, 1); + seq_range_array_add(&range, 0, 2); + test_assert(array_count(&range) == 2); + test_end(); +} + static void test_seq_range_array_random(void) { #define SEQ_RANGE_TEST_BUFSIZE 20 @@ -53,7 +66,7 @@ seqs = array_get(&range, &count); for (j = 0, seq1 = 0; j < count; j++) { - if (j > 0 && seqs[j-1].seq2 >= seqs[j].seq1) + if (j > 0 && seqs[j-1].seq2+1 >= seqs[j].seq1) goto fail; for (; seq1 < seqs[j].seq1; seq1++) { if (shadowbuf[seq1] != 0) @@ -156,6 +169,7 @@ void test_seq_range_array(void) { + test_seq_range_array_add_merge(); test_seq_range_array_invert(); test_seq_range_array_have_common(); test_seq_range_array_random();
--- a/src/lib/test-str-find.c Sat May 19 22:40:08 2012 +0300 +++ b/src/lib/test-str-find.c Sun May 20 03:25:04 2012 +0300 @@ -16,7 +16,8 @@ ctx = str_find_init(pool_datastack_create(), key); /* divide text into every possible block combination and test that it matches */ - max = 1 << (text_len-1); + i_assert(text_len > 0); + max = 1U << (text_len-1); for (i = 0; i < max; i++) { str_find_reset(ctx); pos = 0; offset = 0; ret = FALSE;
--- a/src/lib/test-strfuncs.c Sat May 19 22:40:08 2012 +0300 +++ b/src/lib/test-strfuncs.c Sun May 20 03:25:04 2012 +0300 @@ -2,6 +2,8 @@ #include "test-lib.h" +#include <stdlib.h> + static void test_p_strarray_dup(void) { const char *input[][3] = { @@ -27,7 +29,58 @@ test_end(); } +static void strsplit_verify(const char *str) +{ + T_BEGIN { + const char **s1, **s2; + unsigned int i; + + s1 = t_strsplit_tab(str); + s2 = t_strsplit(str, "\t"); + for (i = 0; s1[i] != NULL; i++) + test_assert(null_strcmp(s1[i], s2[i]) == 0); + test_assert(s2[i] == NULL); + } T_END; +} + +static void test_t_strsplit_tab(void) +{ + char buf[4096]; + unsigned int i, j, max; + + test_begin("t_strsplit_tab"); + strsplit_verify(""); + strsplit_verify("\t"); + strsplit_verify("\t\t"); + strsplit_verify("foo"); + strsplit_verify("foo\tbar"); + strsplit_verify("foo\tbar\tbaz"); + strsplit_verify("foo\t\tbaz"); + buf[sizeof(buf)-1] = '\0'; + for (i = 0; i < sizeof(buf)-1; i++) + buf[i] = '\t'; + strsplit_verify(buf); + for (j = 0; j < 256; j++) { + memset(buf, '\t', j); + buf[j+1] = '\0'; + strsplit_verify(buf); + } + for (j = 0; j < 100; j++) { + max = (rand() % sizeof(buf)) + 1; + buf[--max] = '\0'; + for (i = 0; i < max; i++) { + if (rand() % 10 == 0) + buf[i] = '\t'; + else + buf[i] = 'x'; + } + strsplit_verify(buf); + } + test_end(); +} + void test_strfuncs(void) { test_p_strarray_dup(); + test_t_strsplit_tab(); }
--- a/src/lib/test-var-expand.c Sat May 19 22:40:08 2012 +0300 +++ b/src/lib/test-var-expand.c Sun May 20 03:25:04 2012 +0300 @@ -11,6 +11,11 @@ const char *out; }; +struct var_get_key_range_test { + const char *in; + unsigned int idx, size; +}; + static void test_var_expand_builtin(void) { static struct var_expand_test tests[] = { @@ -37,7 +42,33 @@ test_end(); } +static void test_var_get_key_range(void) +{ + static struct var_get_key_range_test tests[] = { + { "", 0, 0 }, + { "{", 1, 0 }, + { "k", 0, 1 }, + { "{key}", 1, 3 }, + { "5.5Rk", 4, 1 }, + { "5.5R{key}", 5, 3 }, + { "{key", 1, 3 } + }; + unsigned int i, idx, size; + + test_begin("var_get_key_range"); + for (i = 0; i < N_ELEMENTS(tests); i++) { + var_get_key_range(tests[i].in, &idx, &size); + test_assert(tests[i].idx == idx); + test_assert(tests[i].size == size); + + if (tests[i].size == 1) + test_assert(tests[i].in[idx] == var_get_key(tests[i].in)); + } + test_end(); +} + void test_var_expand(void) { test_var_expand_builtin(); + test_var_get_key_range(); }
--- a/src/lib/unlink-old-files.c Sat May 19 22:40:08 2012 +0300 +++ b/src/lib/unlink-old-files.c Sun May 20 03:25:04 2012 +0300 @@ -28,6 +28,11 @@ return -1; } + /* update atime immediately, so if this scanning is done based on + atime it won't be done by multiple processes if the scan is slow */ + if (utime(dir, NULL) < 0 && errno != ENOENT) + i_error("utime(%s) failed: %m", dir); + path = t_str_new(256); str_printfa(path, "%s/", dir); dir_len = str_len(path); @@ -54,19 +59,6 @@ } } -#ifdef HAVE_DIRFD - if (fstat(dirfd(dirp), &st) < 0) - i_error("fstat(%s) failed: %m", dir); -#else - if (stat(dir, &st) < 0) - i_error("stat(%s) failed: %m", dir); -#endif - else if (st.st_atime < ioloop_time) { - /* mounted with noatime. update it ourself. */ - if (utime(dir, NULL) < 0 && errno != ENOENT) - i_error("utime(%s) failed: %m", dir); - } - if (closedir(dirp) < 0) i_error("closedir(%s) failed: %m", dir); return 0;
--- a/src/lib/unlink-old-files.h Sat May 19 22:40:08 2012 +0300 +++ b/src/lib/unlink-old-files.h Sun May 20 03:25:04 2012 +0300 @@ -2,7 +2,8 @@ #define UNLINK_OLD_FILES_H /* Unlink all files from directory beginning with given prefix and having - ctime older than min_time. Returns -1 if there were some errors. */ + ctime older than min_time. Makes sure that the directory's atime is updated. + Returns -1 if there were some errors. */ int unlink_old_files(const char *dir, const char *prefix, time_t min_time); #endif
--- a/src/lib/var-expand.c Sat May 19 22:40:08 2012 +0300 +++ b/src/lib/var-expand.c Sun May 20 03:25:04 2012 +0300 @@ -348,28 +348,50 @@ char var_get_key(const char *str) { + unsigned int idx, size; + + var_get_key_range(str, &idx, &size); + return str[idx]; +} + +void var_get_key_range(const char *str, unsigned int *idx_r, + unsigned int *size_r) +{ const struct var_expand_modifier *m; + unsigned int i = 0; /* [<offset>.]<width>[<modifiers>]<variable> */ - while ((*str >= '0' && *str <= '9') || *str == '-') - str++; + while ((str[i] >= '0' && str[i] <= '9') || str[i] == '-') + i++; - if (*str == '.') { - str++; - while (*str >= '0' && *str <= '9') - str++; + if (str[i] == '.') { + i++; + while (str[i] >= '0' && str[i] <= '9') + i++; } do { for (m = modifiers; m->key != '\0'; m++) { - if (m->key == *str) { - str++; + if (m->key == str[i]) { + i++; break; } } } while (m->key != '\0'); - return *str; + if (str[i] != '{') { + /* short key */ + *idx_r = i; + *size_r = str[i] == '\0' ? 0 : 1; + } else { + /* long key */ + *idx_r = ++i; + for (; str[i] != '\0'; i++) { + if (str[i] == '}') + break; + } + *size_r = i - *idx_r; + } } static bool var_has_long_key(const char **str, const char *long_key)
--- a/src/lib/var-expand.h Sat May 19 22:40:08 2012 +0300 +++ b/src/lib/var-expand.h Sun May 20 03:25:04 2012 +0300 @@ -27,6 +27,10 @@ /* Returns the actual key character for given string, ie. skip any modifiers that are before it. The string should be the data after the '%' character. */ char var_get_key(const char *str) ATTR_PURE; +/* Similar to var_get_key(), but works for long keys as well. For single char + keys size=1, while for e.g. %{key} size=3 and idx points to 'k'. */ +void var_get_key_range(const char *str, unsigned int *idx_r, + unsigned int *size_r); /* Returns TRUE if key variable is used in the string. long_key may be NULL. */ bool var_has_key(const char *str, char key, const char *long_key) ATTR_PURE;
--- a/src/lmtp/Makefile.am Sat May 19 22:40:08 2012 +0300 +++ b/src/lmtp/Makefile.am Sun May 20 03:25:04 2012 +0300 @@ -29,11 +29,17 @@ libs = \ $(unused_objects) \ - $(LIBDOVECOT_STORAGE) \ $(LIBDOVECOT_LDA) -lmtp_LDADD = $(libs) $(LIBDOVECOT) $(MODULE_LIBS) -lmtp_DEPENDENCIES = $(libs) $(LIBDOVECOT_DEPS) +lmtp_LDADD = \ + $(libs) \ + $(LIBDOVECOT_STORAGE) \ + $(LIBDOVECOT) \ + $(MODULE_LIBS) +lmtp_DEPENDENCIES = \ + $(libs) \ + $(LIBDOVECOT_STORAGE_DEPS) \ + $(LIBDOVECOT_DEPS) lmtp_SOURCES = \ main.c \
--- a/src/lmtp/commands.c Sat May 19 22:40:08 2012 +0300 +++ b/src/lmtp/commands.c Sun May 20 03:25:04 2012 +0300 @@ -434,6 +434,9 @@ } if (client->proxy != NULL) { + /* NOTE: if this restriction is ever removed, we'll also need + to send different message bodies to local and proxy + (with and without Return-Path: header) */ client_send_line(client, "451 4.3.0 <%s> " "Can't handle mixed proxy/non-proxy destinations", address); @@ -760,9 +763,13 @@ rcpt_to = rcpt->address; } - str_printfa(str, "Return-Path: <%s>\r\n", client->state.mail_from); - if (rcpt_to != NULL) - str_printfa(str, "Delivered-To: <%s>\r\n", rcpt_to); + /* don't set Return-Path when proxying so it won't get added twice */ + if (array_count(&client->state.rcpt_to) > 0) { + str_printfa(str, "Return-Path: <%s>\r\n", + client->state.mail_from); + if (rcpt_to != NULL) + str_printfa(str, "Delivered-To: <%s>\r\n", rcpt_to); + } str_printfa(str, "Received: from %s", client->lhlo); if ((host = net_ip2addr(&client->remote_ip)) != NULL)
--- a/src/log/log-connection.c Sat May 19 22:40:08 2012 +0300 +++ b/src/log/log-connection.c Sun May 20 03:25:04 2012 +0300 @@ -136,7 +136,7 @@ { struct log_connection *const *logs, *log; struct log_client *client; - const char *p, *p2; + const char *p, *p2, *cmd; unsigned int count; int service_fd; long pid; @@ -148,33 +148,38 @@ } service_fd = atoi(t_strcut(line, ' ')); pid = strtol(t_strcut(p, ' '), NULL, 10); + cmd = p2 + 1; logs = array_get(&logs_by_fd, &count); if (service_fd >= (int)count || logs[service_fd] == NULL) { + if (strcmp(cmd, "BYE") == 0 && service_fd < (int)count) { + /* master is probably shutting down and we already + noticed the log fd closing */ + return; + } i_error("Received master input for invalid service_fd %d: %s", service_fd, line); return; } log = logs[service_fd]; client = hash_table_lookup(log->clients, POINTER_CAST(pid)); - line = p2 + 1; - if (strcmp(line, "BYE") == 0) { + if (strcmp(cmd, "BYE") == 0) { if (client == NULL) { /* we haven't seen anything important from this client. it's not an error. */ return; } log_client_free(log, client, pid); - } else if (strncmp(line, "FATAL ", 6) == 0) { - client_log_fatal(log, client, line + 6, log_time, tm); - } else if (strncmp(line, "DEFAULT-FATAL ", 14) == 0) { + } else if (strncmp(cmd, "FATAL ", 6) == 0) { + client_log_fatal(log, client, cmd + 6, log_time, tm); + } else if (strncmp(cmd, "DEFAULT-FATAL ", 14) == 0) { /* If the client has logged a fatal/panic, don't log this message. */ if (client == NULL || !client->fatal_logged) - client_log_fatal(log, client, line + 14, log_time, tm); + client_log_fatal(log, client, cmd + 14, log_time, tm); } else { - i_error("Received unknown command from master: %s", line); + i_error("Received unknown command from master: %s", cmd); } } @@ -341,6 +346,8 @@ i_error("close(log connection fd) failed: %m"); i_free(log->default_prefix); i_free(log); + + master_service_client_connection_destroyed(master_service); } void log_connections_init(void)
--- a/src/log/main.c Sat May 19 22:40:08 2012 +0300 +++ b/src/log/main.c Sun May 20 03:25:04 2012 +0300 @@ -37,9 +37,13 @@ static void client_connected(struct master_service_connection *conn) { - if (conn->fifo) + if (conn->fifo) { log_connection_create(errorbuf, conn->fd, conn->listen_fd); - else if (strcmp(conn->name, "log-errors") == 0) + /* kludge: normally FIFOs aren't counted as connections, + but here we want log process to stay open until all writers + have closed */ + conn->fifo = FALSE; + } else if (strcmp(conn->name, "log-errors") == 0) doveadm_connection_create(errorbuf, conn->fd); else { i_error("Unknown listener name: %s", conn->name);
--- a/src/login-common/Makefile.am Sat May 19 22:40:08 2012 +0300 +++ b/src/login-common/Makefile.am Sun May 20 03:25:04 2012 +0300 @@ -6,6 +6,7 @@ -I$(top_srcdir)/src/lib-auth \ -I$(top_srcdir)/src/lib-master \ -I$(top_srcdir)/src/lib-ssl-iostream \ + -I$(top_srcdir)/src/lib-mail \ -DPKG_STATEDIR=\""$(statedir)"\" liblogin_la_SOURCES = \ @@ -39,6 +40,6 @@ pkglib_LTLIBRARIES = libdovecot-login.la libdovecot_login_la_SOURCES = -libdovecot_login_la_LIBADD = liblogin.la ../lib-ssl-iostream/libssl_iostream.la ../lib-dovecot/libdovecot.la +libdovecot_login_la_LIBADD = liblogin.la ../lib-ssl-iostream/libssl_iostream.la ../lib-dovecot/libdovecot.la $(SSL_LIBS) libdovecot_login_la_DEPENDENCIES = liblogin.la libdovecot_login_la_LDFLAGS = -export-dynamic
--- a/src/login-common/client-common-auth.c Sat May 19 22:40:08 2012 +0300 +++ b/src/login-common/client-common-auth.c Sun May 20 03:25:04 2012 +0300 @@ -82,10 +82,16 @@ reply_r->temp = TRUE; else if (strcmp(key, "authz") == 0) reply_r->authz_failure = TRUE; + else if (strcmp(key, "user_disabled") == 0) + client->auth_user_disabled = TRUE; + else if (strcmp(key, "pass_expired") == 0) + client->auth_pass_expired = TRUE; else if (strcmp(key, "reason") == 0) reply_r->reason = value; else if (strcmp(key, "host") == 0) reply_r->host = value; + else if (strcmp(key, "hostip") == 0) + reply_r->hostip = value; else if (strcmp(key, "port") == 0) reply_r->port = atoi(value); else if (strcmp(key, "destuser") == 0) @@ -295,6 +301,9 @@ memset(&proxy_set, 0, sizeof(proxy_set)); proxy_set.host = reply->host; + if (reply->hostip != NULL && + net_addr2ip(reply->hostip, &proxy_set.ip) < 0) + proxy_set.ip.family = 0; proxy_set.port = reply->port; proxy_set.connect_timeout_msecs = reply->proxy_timeout_msecs; proxy_set.notify_refresh_secs = reply->proxy_refresh_secs;
--- a/src/login-common/client-common.c Sat May 19 22:40:08 2012 +0300 +++ b/src/login-common/client-common.c Sun May 20 03:25:04 2012 +0300 @@ -7,7 +7,9 @@ #include "ostream.h" #include "iostream-rawlog.h" #include "process-title.h" +#include "buffer.h" #include "str.h" +#include "base64.h" #include "str-sanitize.h" #include "safe-memset.h" #include "var-expand.h" @@ -26,9 +28,36 @@ static void client_idle_disconnect_timeout(struct client *client) { - client_notify_disconnect(client, CLIENT_DISCONNECT_TIMEOUT, - "Disconnected for inactivity."); - client_destroy(client, "Disconnected: Inactivity"); + const char *user_reason, *destroy_reason; + unsigned int secs; + + if (client->master_tag != 0) { + secs = ioloop_time - client->auth_finished; + user_reason = "Timeout while finishing login."; + destroy_reason = t_strdup_printf( + "Timeout while finishing login (waited %u secs)", secs); + client_log_err(client, destroy_reason); + } else if (client->auth_request != NULL) { + user_reason = + "Disconnected for inactivity during authentication."; + destroy_reason = + "Disconnected: Inactivity during authentication"; + } else if (client->login_proxy != NULL) { + secs = ioloop_time - client->created; + user_reason = "Timeout while finishing login."; + destroy_reason = t_strdup_printf( + "proxy: Logging in to %s:%u timed out " + "(state=%u, duration=%us)", + login_proxy_get_host(client->login_proxy), + login_proxy_get_port(client->login_proxy), + client->proxy_state, secs); + client_log_err(client, destroy_reason); + } else { + user_reason = "Disconnected for inactivity."; + destroy_reason = "Disconnected: Inactivity"; + } + client_notify_disconnect(client, CLIENT_DISCONNECT_TIMEOUT, user_reason); + client_destroy(client, destroy_reason); } static void client_open_streams(struct client *client) @@ -177,8 +206,6 @@ if (client->login_proxy != NULL) login_proxy_free(&client->login_proxy); - if (client->ssl_proxy != NULL) - ssl_proxy_free(&client->ssl_proxy); client->v.destroy(client); if (client_unref(&client) && initial_service_count == 1) { /* as soon as this connection is done with proxying @@ -224,9 +251,10 @@ *_client = NULL; i_assert(client->destroyed); - i_assert(client->ssl_proxy == NULL); i_assert(client->login_proxy == NULL); + if (client->ssl_proxy != NULL) + ssl_proxy_free(&client->ssl_proxy); if (client->input != NULL) i_stream_unref(&client->input); if (client->output != NULL) @@ -236,6 +264,7 @@ i_free(client->proxy_master_user); i_free(client->virtual_user); i_free(client->auth_mech_name); + i_free(client->master_data_prefix); pool_unref(&client->pool); i_assert(clients_count > 0); @@ -295,7 +324,7 @@ if (!client_unref(&client) || client->destroyed) return; - fd_ssl = ssl_proxy_alloc(client->fd, &client->ip, + fd_ssl = ssl_proxy_alloc(client->fd, &client->ip, client->pool, client->set, &client->ssl_proxy); if (fd_ssl == -1) { client_notify_disconnect(client, @@ -375,31 +404,69 @@ return clients_count; } +const char *client_get_session_id(struct client *client) +{ + buffer_t *buf, *base64_buf; + struct timeval tv; + uint64_t timestamp; + unsigned int i; + + if (client->session_id != NULL) + return client->session_id; + + buf = buffer_create_dynamic(pool_datastack_create(), 24); + base64_buf = buffer_create_dynamic(pool_datastack_create(), 24*2); + + if (gettimeofday(&tv, NULL) < 0) + i_fatal("gettimeofday(): %m"); + timestamp = tv.tv_usec + (long long)tv.tv_sec * 1000ULL*1000ULL; + + /* add lowest 48 bits of the timestamp. this gives us a bit less than + 9 years until it wraps */ + for (i = 0; i < 48; i += 8) + buffer_append_c(buf, (timestamp >> i) & 0xff); + + buffer_append_c(buf, client->remote_port & 0xff); + buffer_append_c(buf, (client->remote_port >> 16) & 0xff); +#ifdef HAVE_IPV6 + if (IPADDR_IS_V6(&client->ip)) + buffer_append(buf, &client->ip.u.ip6, sizeof(client->ip.u.ip6)); + else +#endif + buffer_append(buf, &client->ip.u.ip4, sizeof(client->ip.u.ip4)); + base64_encode(buf->data, buf->used, base64_buf); + client->session_id = p_strdup(client->pool, str_c(base64_buf)); + return client->session_id; +} + +static struct var_expand_table login_var_expand_empty_tab[] = { + { 'u', NULL, "user" }, + { 'n', NULL, "username" }, + { 'd', NULL, "domain" }, + { 's', NULL, "service" }, + { 'h', NULL, "home" }, + { 'l', NULL, "lip" }, + { 'r', NULL, "rip" }, + { 'p', NULL, "pid" }, + { 'm', NULL, "mech" }, + { 'a', NULL, "lport" }, + { 'b', NULL, "rport" }, + { 'c', NULL, "secured" }, + { 'k', NULL, "ssl_security" }, + { 'e', NULL, "mail_pid" }, + { '\0', NULL, "session" }, + { '\0', NULL, NULL } +}; + static const struct var_expand_table * get_var_expand_table(struct client *client) { - static struct var_expand_table static_tab[] = { - { 'u', NULL, "user" }, - { 'n', NULL, "username" }, - { 'd', NULL, "domain" }, - { 's', NULL, "service" }, - { 'h', NULL, "home" }, - { 'l', NULL, "lip" }, - { 'r', NULL, "rip" }, - { 'p', NULL, "pid" }, - { 'm', NULL, "mech" }, - { 'a', NULL, "lport" }, - { 'b', NULL, "rport" }, - { 'c', NULL, "secured" }, - { 'k', NULL, "ssl_security" }, - { 'e', NULL, "mail_pid" }, - { '\0', NULL, NULL } - }; struct var_expand_table *tab; unsigned int i; - tab = t_malloc(sizeof(static_tab)); - memcpy(tab, static_tab, sizeof(static_tab)); + tab = t_malloc(sizeof(login_var_expand_empty_tab)); + memcpy(tab, login_var_expand_empty_tab, + sizeof(login_var_expand_empty_tab)); if (client->virtual_user != NULL) { tab[0].value = client->virtual_user; @@ -436,23 +503,20 @@ } tab[13].value = client->mail_pid == 0 ? "" : dec2str(client->mail_pid); + tab[14].value = client_get_session_id(client); return tab; } -static bool have_key(const struct var_expand_table *table, const char *str) +static bool have_username_key(const char *str) { char key; - unsigned int i; - key = var_get_key(str); - for (i = 0; table[i].key != '\0'; i++) { - if (table[i].key == key) { - if (table[i].value == NULL) - return FALSE; - if (table[i].value[0] != '\0') + for (; *str != '\0'; str++) { + if (str[0] == '%' && str[1] != '\0') { + str++; + key = var_get_key(str); + if (key == 'u' || key == 'n') return TRUE; - /* "" key - hide except in username */ - return key == 'u' || key == 'n'; } } return FALSE; @@ -468,9 +532,9 @@ }; const struct var_expand_table *var_expand_table; struct var_expand_table *tab; - const char *p; char *const *e; - string_t *str; + string_t *str, *str2; + unsigned int pos; var_expand_table = get_var_expand_table(client); @@ -478,21 +542,29 @@ memcpy(tab, static_tab, sizeof(static_tab)); str = t_str_new(256); + str2 = t_str_new(128); for (e = client->set->log_format_elements_split; *e != NULL; e++) { - for (p = *e; *p != '\0'; p++) { - if (*p != '%' || p[1] == '\0') + pos = str_len(str); + var_expand(str, *e, var_expand_table); + if (have_username_key(*e)) { + /* username is added even if it's empty */ + } else { + str_truncate(str2, 0); + var_expand(str2, *e, login_var_expand_empty_tab); + if (strcmp(str_c(str)+pos, str_c(str2)) == 0) { + /* empty %variables, don't add */ + str_truncate(str, pos); continue; - - p++; - if (have_key(var_expand_table, p)) { - if (str_len(str) > 0) - str_append(str, ", "); - var_expand(str, *e, var_expand_table); - break; } } + + if (str_len(str) > 0) + str_append(str, ", "); } + if (str_len(str) > 0) + str_truncate(str, str_len(str)-2); + tab[0].value = t_strdup(str_c(str)); tab[1].value = msg; str_truncate(str, 0); @@ -547,7 +619,7 @@ /* some auth attempts without SSL/TLS */ if (client->auth_tried_disabled_plaintext) - return "(tried to use disabled plaintext auth)"; + return "(tried to use disallowed plaintext auth)"; if (client->set->auth_ssl_require_client_cert && client->ssl_proxy == NULL) return "(cert required, client didn't start TLS)"; @@ -576,6 +648,10 @@ return t_strdup_printf("(internal failure, %u succesful auths)", client->auth_successes); } + if (client->auth_user_disabled) + return "(user disabled)"; + if (client->auth_pass_expired) + return "(password expired)"; return t_strdup_printf("(auth failed, %u attempts in %u secs)", client->auth_attempts, auth_secs); }
--- a/src/login-common/client-common.h Sat May 19 22:40:08 2012 +0300 +++ b/src/login-common/client-common.h Sun May 20 03:25:04 2012 +0300 @@ -4,6 +4,7 @@ #include "network.h" #include "login-proxy.h" #include "sasl-server.h" +#include "master-login.h" /* for LOGIN_MAX_SESSION_ID_LEN */ #define LOGIN_MAX_MASTER_PREFIX_LEN 128 @@ -14,7 +15,8 @@ POP3: Max. length of a command line (spec says 512 would be enough) */ #define LOGIN_MAX_INBUF_SIZE \ - (MASTER_AUTH_MAX_DATA_SIZE - LOGIN_MAX_MASTER_PREFIX_LEN) + (MASTER_AUTH_MAX_DATA_SIZE - LOGIN_MAX_MASTER_PREFIX_LEN - \ + LOGIN_MAX_SESSION_ID_LEN) /* max. size of output buffer. if it gets full, the client is disconnected. SASL authentication gives the largest output. */ #define LOGIN_MAX_OUTBUF_SIZE 4096 @@ -53,7 +55,7 @@ struct client_auth_reply { const char *master_user, *reason; /* for proxying */ - const char *host, *destuser, *password; + const char *host, *hostip, *destuser, *password; unsigned int port; unsigned int proxy_timeout_msecs; unsigned int proxy_refresh_secs; @@ -103,6 +105,7 @@ unsigned int local_port, remote_port; struct ssl_proxy *ssl_proxy; const struct login_settings *set; + const char *session_id; int fd; struct istream *input; @@ -122,7 +125,7 @@ char *auth_mech_name; struct auth_client_request *auth_request; string_t *auth_response; - time_t auth_first_started; + time_t auth_first_started, auth_finished; const char *sasl_final_resp; unsigned int master_auth_id; @@ -150,6 +153,8 @@ unsigned int auth_process_comm_fail:1; unsigned int proxy_auth_failed:1; unsigned int auth_waiting:1; + unsigned int auth_user_disabled:1; + unsigned int auth_pass_expired:1; unsigned int notified_auth_ready:1; unsigned int notified_disconnect:1; /* ... */ @@ -181,6 +186,7 @@ void client_auth_respond(struct client *client, const char *response); void client_auth_abort(struct client *client); void client_auth_fail(struct client *client, const char *text); +const char *client_get_session_id(struct client *client); bool client_read(struct client *client); void client_input(struct client *client);
--- a/src/login-common/login-proxy-state.c Sat May 19 22:40:08 2012 +0300 +++ b/src/login-common/login-proxy-state.c Sun May 20 03:25:04 2012 +0300 @@ -2,6 +2,7 @@ #include "lib.h" #include "network.h" +#include "ioloop.h" #include "hash.h" #include "strescape.h" #include "fd-set-nonblock.h" @@ -10,6 +11,8 @@ #include <unistd.h> #include <fcntl.h> +#define NOTIFY_RETRY_REOPEN_MSECS (60*1000) + struct login_proxy_state { struct hash_table *hash; pool_t pool; @@ -17,9 +20,11 @@ const char *notify_path; int notify_fd; - unsigned int notify_fd_broken:1; + struct timeout *to_reopen; }; +static int login_proxy_state_notify_open(struct login_proxy_state *state); + static unsigned int login_proxy_record_hash(const void *p) { const struct login_proxy_record *rec = p; @@ -51,16 +56,24 @@ return state; } +static void login_proxy_state_close(struct login_proxy_state *state) +{ + if (state->notify_fd != -1) { + if (close(state->notify_fd) < 0) + i_error("close(%s) failed: %m", state->notify_path); + state->notify_fd = -1; + } +} + void login_proxy_state_deinit(struct login_proxy_state **_state) { struct login_proxy_state *state = *_state; *_state = NULL; - if (state->notify_fd != -1) { - if (close(state->notify_fd) < 0) - i_error("close(%s) failed: %m", state->notify_path); - } + if (state->to_reopen != NULL) + timeout_remove(&state->to_reopen); + login_proxy_state_close(state); hash_table_destroy(&state->hash); pool_unref(&state->pool); i_free(state); @@ -86,31 +99,39 @@ return rec; } +static void login_proxy_state_reopen(struct login_proxy_state *state) +{ + timeout_remove(&state->to_reopen); + (void)login_proxy_state_notify_open(state); +} + static int login_proxy_state_notify_open(struct login_proxy_state *state) { - if (state->notify_fd_broken) + if (state->to_reopen != NULL) { + /* reopen later */ return -1; + } state->notify_fd = open(state->notify_path, O_WRONLY); if (state->notify_fd == -1) { - if (errno != ENOENT) - i_error("open(%s) failed: %m", state->notify_path); - state->notify_fd_broken = TRUE; + i_error("open(%s) failed: %m", state->notify_path); + state->to_reopen = timeout_add(NOTIFY_RETRY_REOPEN_MSECS, + login_proxy_state_reopen, state); return -1; } fd_set_nonblock(state->notify_fd, TRUE); return 0; } -void login_proxy_state_notify(struct login_proxy_state *state, - const char *user) +static bool login_proxy_state_try_notify(struct login_proxy_state *state, + const char *user) { unsigned int len; ssize_t ret; if (state->notify_fd == -1) { if (login_proxy_state_notify_open(state) < 0) - return; + return TRUE; } T_BEGIN { @@ -128,5 +149,16 @@ i_error("write(%s) wrote partial update", state->notify_path); } + login_proxy_state_close(state); + /* retry sending */ + return FALSE; } + return TRUE; } + +void login_proxy_state_notify(struct login_proxy_state *state, + const char *user) +{ + if (!login_proxy_state_try_notify(state, user)) + login_proxy_state_try_notify(state, user); +}
--- a/src/login-common/login-proxy.c Sat May 19 22:40:08 2012 +0300 +++ b/src/login-common/login-proxy.c Sun May 20 03:25:04 2012 +0300 @@ -5,11 +5,11 @@ #include "istream.h" #include "ostream.h" #include "llist.h" -#include "md5.h" #include "str-sanitize.h" #include "time-util.h" #include "master-service.h" #include "ipc-server.h" +#include "mail-user-hash.h" #include "client-common.h" #include "ssl-proxy.h" #include "login-proxy-state.h" @@ -21,6 +21,7 @@ #define LOGIN_PROXY_IPC_PATH "ipc-proxy" #define LOGIN_PROXY_IPC_NAME "proxy" #define KILLED_BY_ADMIN_REASON "Killed by admin" +#define PROXY_IMMEDIATE_FAILURE_SECS 30 struct login_proxy { struct login_proxy *prev, *next; @@ -238,6 +239,7 @@ rec = login_proxy_state_get(proxy_state, &proxy->ip, proxy->port); if (timeval_cmp(&rec->last_failure, &rec->last_success) > 0 && + rec->last_failure.tv_sec - rec->last_success.tv_sec > PROXY_IMMEDIATE_FAILURE_SECS && rec->num_waiting_connections != 0) { /* the server is down. fail immediately */ i_error("proxy(%s): Host %s:%u is down", @@ -289,6 +291,7 @@ proxy->client_fd = -1; proxy->server_fd = -1; proxy->created = ioloop_timeval; + proxy->ip = set->ip; proxy->host = i_strdup(set->host); proxy->port = set->port; proxy->connect_timeout_msecs = set->connect_timeout_msecs; @@ -296,7 +299,8 @@ proxy->ssl_flags = set->ssl_flags; client_ref(client); - if (net_addr2ip(set->host, &proxy->ip) < 0) { + if (set->ip.family == 0 && + net_addr2ip(set->host, &proxy->ip) < 0) { i_error("proxy(%s): BUG: host %s is not an IP " "(auth should have changed it)", client->virtual_user, set->host); @@ -347,10 +351,11 @@ DLLIST_REMOVE(&login_proxies, proxy); ipstr = net_ip2addr(&proxy->client->ip); - i_info("proxy(%s): disconnecting %s%s", - proxy->client->virtual_user, - ipstr != NULL ? ipstr : "", - reason == NULL ? "" : t_strdup_printf(" (%s)", reason)); + client_log(proxy->client, t_strdup_printf( + "proxy(%s): disconnecting %s%s", + proxy->client->virtual_user, + ipstr != NULL ? ipstr : "", + reason == NULL ? "" : t_strdup_printf(" (%s)", reason))); if (proxy->client_io != NULL) io_remove(&proxy->client_io); @@ -492,8 +497,9 @@ if (ssl_proxy_has_broken_client_cert(proxy->ssl_server_proxy)) { client_log_err(proxy->client, t_strdup_printf( - "proxy: Received invalid SSL certificate from %s:%u", - proxy->host, proxy->port)); + "proxy: Received invalid SSL certificate from %s:%u: %s", + proxy->host, proxy->port, + ssl_proxy_get_cert_error(proxy->ssl_server_proxy))); } else if (!ssl_proxy_has_valid_client_cert(proxy->ssl_server_proxy)) { client_log_err(proxy->client, t_strdup_printf( "proxy: SSL certificate not received from %s:%u", @@ -521,7 +527,7 @@ io_remove(&proxy->server_io); fd = ssl_proxy_client_alloc(proxy->server_fd, &proxy->client->ip, - proxy->client->set, + proxy->client->pool, proxy->client->set, login_proxy_ssl_handshaked, proxy, &proxy->ssl_server_proxy); if (fd < 0) { @@ -594,17 +600,10 @@ ipc_cmd_success_reply(&cmd, t_strdup_printf("%u", count)); } -static unsigned int director_username_hash(const char *username) +static unsigned int director_username_hash(struct client *client) { - /* NOTE: If you modify this, modify also - user_directory_get_username_hash() in director/user-director.c */ - unsigned char md5[MD5_RESULTLEN]; - unsigned int i, hash = 0; - - md5_get_digest(username, strlen(username), md5); - for (i = 0; i < sizeof(hash); i++) - hash = (hash << CHAR_BIT) | md5[i]; - return hash; + return mail_user_hash(client->virtual_user, + client->set->director_username_hash); } static void @@ -621,7 +620,7 @@ for (proxy = login_proxies; proxy != NULL; proxy = next) { next = proxy->next; - if (director_username_hash(proxy->client->virtual_user) == hash) { + if (director_username_hash(proxy->client) == hash) { login_proxy_free_reason(&proxy, KILLED_BY_ADMIN_REASON); count++; } @@ -629,7 +628,7 @@ for (proxy = login_proxies_pending; proxy != NULL; proxy = next) { next = proxy->next; - if (director_username_hash(proxy->client->virtual_user) == hash) { + if (director_username_hash(proxy->client) == hash) { client_destroy(proxy->client, "Connection kicked"); count++; } @@ -667,7 +666,7 @@ static void login_proxy_ipc_cmd(struct ipc_cmd *cmd, const char *line) { - const char *const *args = t_strsplit(line, "\t"); + const char *const *args = t_strsplit_tab(line); const char *name = args[0]; args++;
--- a/src/login-common/login-proxy.h Sat May 19 22:40:08 2012 +0300 +++ b/src/login-common/login-proxy.h Sun May 20 03:25:04 2012 +0300 @@ -1,6 +1,8 @@ #ifndef LOGIN_PROXY_H #define LOGIN_PROXY_H +#include "network.h" + /* Max. number of embedded proxying connections until proxying fails. This is intended to avoid an accidental configuration where two proxies keep connecting to each others, both thinking the other one is supposed to @@ -22,6 +24,7 @@ struct login_proxy_settings { const char *host; + struct ip_addr ip; unsigned int port; unsigned int connect_timeout_msecs; /* send a notification about proxy connection to proxy-notify pipe
--- a/src/login-common/login-settings.c Sat May 19 22:40:08 2012 +0300 +++ b/src/login-common/login-settings.c Sun May 20 03:25:04 2012 +0300 @@ -24,6 +24,7 @@ DEF(SET_STR, login_log_format_elements), DEF(SET_STR, login_log_format), DEF(SET_STR, login_access_sockets), + DEF(SET_STR, director_username_hash), DEF(SET_ENUM, ssl), DEF(SET_STR, ssl_ca), @@ -37,6 +38,7 @@ DEF(SET_STR, ssl_client_key), DEF(SET_STR, ssl_crypto_device), DEF(SET_BOOL, ssl_verify_client_cert), + DEF(SET_BOOL, ssl_require_crl), DEF(SET_BOOL, auth_ssl_require_client_cert), DEF(SET_BOOL, auth_ssl_username_from_cert), DEF(SET_BOOL, verbose_ssl), @@ -54,9 +56,10 @@ static const struct login_settings login_default_settings = { .login_trusted_networks = "", .login_greeting = PACKAGE_NAME" ready.", - .login_log_format_elements = "user=<%u> method=%m rip=%r lip=%l mpid=%e %c", + .login_log_format_elements = "user=<%u> method=%m rip=%r lip=%l mpid=%e %c session=<%{session}>", .login_log_format = "%$: %s", .login_access_sockets = "", + .director_username_hash = "%u", .ssl = "yes:no:required", .ssl_ca = "", @@ -70,6 +73,7 @@ .ssl_client_key = "", .ssl_crypto_device = "", .ssl_verify_client_cert = FALSE, + .ssl_require_crl = TRUE, .auth_ssl_require_client_cert = FALSE, .auth_ssl_username_from_cert = FALSE, .verbose_ssl = FALSE,
--- a/src/login-common/login-settings.h Sat May 19 22:40:08 2012 +0300 +++ b/src/login-common/login-settings.h Sun May 20 03:25:04 2012 +0300 @@ -6,6 +6,7 @@ const char *login_greeting; const char *login_log_format_elements, *login_log_format; const char *login_access_sockets; + const char *director_username_hash; const char *ssl; const char *ssl_ca; @@ -19,6 +20,7 @@ const char *ssl_client_key; const char *ssl_crypto_device; bool ssl_verify_client_cert; + bool ssl_require_crl; bool auth_ssl_require_client_cert; bool auth_ssl_username_from_cert; bool verbose_ssl;
--- a/src/login-common/main.c Sat May 19 22:40:08 2012 +0300 +++ b/src/login-common/main.c Sun May 20 03:25:04 2012 +0300 @@ -123,7 +123,7 @@ client = client_create(conn->fd, FALSE, pool, set, other_sets, &local_ip, &conn->remote_ip); } else { - fd_ssl = ssl_proxy_alloc(conn->fd, &conn->remote_ip, set, + fd_ssl = ssl_proxy_alloc(conn->fd, &conn->remote_ip, pool, set, &proxy); if (fd_ssl == -1) { net_disconnect(conn->fd);
--- a/src/login-common/sasl-server.c Sat May 19 22:40:08 2012 +0300 +++ b/src/login-common/sasl-server.c Sun May 20 03:25:04 2012 +0300 @@ -124,6 +124,7 @@ const unsigned char *data; size_t size; buffer_t *buf; + const char *session_id = client_get_session_id(client); memset(&req, 0, sizeof(req)); req.auth_pid = anvil_request->auth_pid; @@ -137,13 +138,17 @@ memcpy(req.cookie, anvil_request->cookie, sizeof(req.cookie)); buf = buffer_create_dynamic(pool_datastack_create(), 256); + /* session ID */ + buffer_append(buf, session_id, strlen(session_id)+1); + /* protocol specific data (e.g. IMAP tag) */ buffer_append(buf, client->master_data_prefix, client->master_data_prefix_len); - + /* buffered client input */ data = i_stream_get_data(client->input, &size); buffer_append(buf, data, size); req.data_size = buf->used; + client->auth_finished = ioloop_time; client->master_auth_id = req.auth_id; master_auth_request(master_auth, client->fd, &req, buf->data, master_auth_callback, client, &client->master_tag); @@ -313,6 +318,7 @@ memset(&info, 0, sizeof(info)); info.mech = mech->name; info.service = service; + info.session_id = client_get_session_id(client); info.cert_username = client->ssl_proxy == NULL ? NULL : ssl_proxy_get_peer_name(client->ssl_proxy); info.flags = client_get_auth_flags(client);
--- a/src/login-common/ssl-proxy-openssl.c Sat May 19 22:40:08 2012 +0300 +++ b/src/login-common/ssl-proxy-openssl.c Sun May 20 03:25:04 2012 +0300 @@ -52,6 +52,7 @@ struct client *client; struct ip_addr ip; const struct login_settings *set; + pool_t set_pool; int fd_ssl, fd_plain; struct io *io_ssl_read, *io_ssl_write, *io_plain_read, *io_plain_write; @@ -65,6 +66,7 @@ ssl_handshake_callback_t *handshake_callback; void *handshake_context; + const char *cert_error; char *last_error; unsigned int handshaked:1; unsigned int destroyed:1; @@ -138,6 +140,8 @@ return 1; if (strcmp(ctx1->key, ctx2->key) != 0) return 1; + if (null_strcmp(ctx1->ca, ctx2->ca) != 0) + return 1; if (null_strcmp(ctx1->cipher_list, ctx2->cipher_list) != 0) return 1; if (null_strcmp(ctx1->protocols, ctx2->protocols) != 0) @@ -543,7 +547,7 @@ static int ssl_proxy_alloc_common(SSL_CTX *ssl_ctx, int fd, const struct ip_addr *ip, - const struct login_settings *set, + pool_t set_pool, const struct login_settings *set, struct ssl_proxy **proxy_r) { struct ssl_proxy *proxy; @@ -590,7 +594,9 @@ proxy->fd_ssl = fd; proxy->fd_plain = sfd[0]; proxy->ip = *ip; - SSL_set_ex_data(ssl, extdata_index, proxy); + proxy->set_pool = set_pool; + pool_ref(set_pool); + SSL_set_ex_data(ssl, extdata_index, proxy); ssl_proxy_count++; DLLIST_PREPEND(&ssl_proxies, proxy); @@ -618,24 +624,26 @@ return ctx; } -int ssl_proxy_alloc(int fd, const struct ip_addr *ip, +int ssl_proxy_alloc(int fd, const struct ip_addr *ip, pool_t set_pool, const struct login_settings *set, struct ssl_proxy **proxy_r) { struct ssl_server_context *ctx; ctx = ssl_server_context_get(set); - return ssl_proxy_alloc_common(ctx->ctx, fd, ip, set, proxy_r); + return ssl_proxy_alloc_common(ctx->ctx, fd, ip, + set_pool, set, proxy_r); } -int ssl_proxy_client_alloc(int fd, struct ip_addr *ip, +int ssl_proxy_client_alloc(int fd, struct ip_addr *ip, pool_t set_pool, const struct login_settings *set, ssl_handshake_callback_t *callback, void *context, struct ssl_proxy **proxy_r) { int ret; - ret = ssl_proxy_alloc_common(ssl_client_ctx, fd, ip, set, proxy_r); + ret = ssl_proxy_alloc_common(ssl_client_ctx, fd, ip, + set_pool, set, proxy_r); if (ret < 0) return -1; @@ -747,6 +755,12 @@ #endif } +const char *ssl_proxy_get_cert_error(struct ssl_proxy *proxy) +{ + return proxy->cert_error != NULL ? proxy->cert_error : + "(Unknown error)"; +} + void ssl_proxy_free(struct ssl_proxy **_proxy) { struct ssl_proxy *proxy = *_proxy; @@ -763,8 +777,7 @@ SSL_free(proxy->ssl); - if (proxy->client != NULL) - client_unref(&proxy->client); + pool_unref(&proxy->set_pool); i_free(proxy->last_error); i_free(proxy); } @@ -792,6 +805,8 @@ (void)net_disconnect(proxy->fd_ssl); (void)net_disconnect(proxy->fd_plain); + if (proxy->client != NULL) + client_unref(&proxy->client); ssl_proxy_unref(proxy); } @@ -841,32 +856,42 @@ { SSL *ssl; struct ssl_proxy *proxy; + char buf[1024]; + X509_NAME *subject; ssl = X509_STORE_CTX_get_ex_data(ctx, SSL_get_ex_data_X509_STORE_CTX_idx()); proxy = SSL_get_ex_data(ssl, extdata_index); proxy->cert_received = TRUE; - if (proxy->set->verbose_ssl || - (proxy->set->auth_verbose && !preverify_ok)) { - char buf[1024]; - X509_NAME *subject; - - subject = X509_get_subject_name(ctx->current_cert); - (void)X509_NAME_oneline(subject, buf, sizeof(buf)); - buf[sizeof(buf)-1] = '\0'; /* just in case.. */ - if (!preverify_ok) - i_info("Invalid certificate: %s: %s", X509_verify_cert_error_string(ctx->error),buf); - else - i_info("Valid certificate: %s", buf); - } - if (ctx->error == X509_V_ERR_UNABLE_TO_GET_CRL && proxy->client_proxy) { + if (proxy->client_proxy && !proxy->set->ssl_require_crl && + (ctx->error == X509_V_ERR_UNABLE_TO_GET_CRL || + ctx->error == X509_V_ERR_CRL_HAS_EXPIRED)) { /* no CRL given with the CA list. don't worry about it. */ preverify_ok = 1; } if (!preverify_ok) proxy->cert_broken = TRUE; + subject = X509_get_subject_name(ctx->current_cert); + (void)X509_NAME_oneline(subject, buf, sizeof(buf)); + buf[sizeof(buf)-1] = '\0'; /* just in case.. */ + + if (proxy->cert_error == NULL) { + proxy->cert_error = p_strdup_printf(proxy->client->pool, "%s: %s", + X509_verify_cert_error_string(ctx->error), buf); + } + + if (proxy->set->verbose_ssl || + (proxy->set->auth_verbose && !preverify_ok)) { + if (preverify_ok) + i_info("Valid certificate: %s", buf); + else { + i_info("Invalid certificate: %s: %s", + X509_verify_cert_error_string(ctx->error), buf); + } + } + /* Return success anyway, because if ssl_require_client_cert=no we could still allow authentication. */ return 1; @@ -897,11 +922,11 @@ return strstr(cert, "PRIVATE KEY---") != NULL; } -static STACK_OF(X509_NAME) *load_ca(X509_STORE *store, const char *ca) +static void load_ca(X509_STORE *store, const char *ca, + STACK_OF(X509_NAME) **xnames_r) { /* mostly just copy&pasted from X509_load_cert_crl_file() */ STACK_OF(X509_INFO) *inf; - STACK_OF(X509_NAME) *xnames; X509_INFO *itmp; X509_NAME *xname; BIO *bio; @@ -915,28 +940,32 @@ i_fatal("Couldn't parse ssl_ca: %s", ssl_last_error()); BIO_free(bio); - xnames = sk_X509_NAME_new_null(); - if (xnames == NULL) - i_fatal("sk_X509_NAME_new_null() failed"); + if (xnames_r != NULL) { + *xnames_r = sk_X509_NAME_new_null(); + if (*xnames_r == NULL) + i_fatal_status(FATAL_OUTOFMEM, "sk_X509_NAME_new_null() failed"); + } for(i = 0; i < sk_X509_INFO_num(inf); i++) { itmp = sk_X509_INFO_value(inf, i); if(itmp->x509) { X509_STORE_add_cert(store, itmp->x509); xname = X509_get_subject_name(itmp->x509); - if (xname != NULL) + if (xname != NULL && xnames_r != NULL) { xname = X509_NAME_dup(xname); - if (xname != NULL) - sk_X509_NAME_push(xnames, xname); + if (xname == NULL) + i_fatal_status(FATAL_OUTOFMEM, "X509_NAME_dup() failed"); + sk_X509_NAME_push(*xnames_r, xname); + } } if(itmp->crl) X509_STORE_add_crl(store, itmp->crl); } sk_X509_INFO_pop_free(inf, X509_INFO_free); - return xnames; } static STACK_OF(X509_NAME) * -ssl_proxy_ctx_init(SSL_CTX *ssl_ctx, const struct login_settings *set) +ssl_proxy_ctx_init(SSL_CTX *ssl_ctx, const struct login_settings *set, + bool load_xnames) { X509_STORE *store; STACK_OF(X509_NAME) *xnames = NULL; @@ -953,7 +982,7 @@ if (*set->ssl_ca != '\0') { /* set trusted CA certs */ store = SSL_CTX_get_cert_store(ssl_ctx); - xnames = load_ca(store, set->ssl_ca); + load_ca(store, set->ssl_ca, load_xnames ? &xnames : NULL); } SSL_CTX_set_info_callback(ssl_ctx, ssl_info_callback); if (SSL_CTX_need_tmp_RSA(ssl_ctx)) @@ -1192,7 +1221,7 @@ ctx->ctx = ssl_ctx = SSL_CTX_new(SSLv23_server_method()); if (ssl_ctx == NULL) i_fatal("SSL_CTX_new() failed"); - xnames = ssl_proxy_ctx_init(ssl_ctx, set); + xnames = ssl_proxy_ctx_init(ssl_ctx, set, ctx->verify_client_cert); if (SSL_CTX_set_cipher_list(ssl_ctx, ctx->cipher_list) != 1) { i_fatal("Can't set cipher list to '%s': %s", @@ -1259,7 +1288,7 @@ if ((ssl_client_ctx = SSL_CTX_new(SSLv23_client_method())) == NULL) i_fatal("SSL_CTX_new() failed"); - xnames = ssl_proxy_ctx_init(ssl_client_ctx, set); + xnames = ssl_proxy_ctx_init(ssl_client_ctx, set, TRUE); ssl_proxy_ctx_verify_client(ssl_client_ctx, xnames); ssl_proxy_client_ctx_set_client_cert(ssl_client_ctx, set);
--- a/src/login-common/ssl-proxy.c Sat May 19 22:40:08 2012 +0300 +++ b/src/login-common/ssl-proxy.c Sun May 20 03:25:04 2012 +0300 @@ -10,6 +10,7 @@ /* no SSL support */ int ssl_proxy_alloc(int fd ATTR_UNUSED, const struct ip_addr *ip ATTR_UNUSED, + pool_t set_pool ATTR_UNUSED, const struct login_settings *set ATTR_UNUSED, struct ssl_proxy **proxy_r ATTR_UNUSED) { @@ -18,6 +19,7 @@ } int ssl_proxy_client_alloc(int fd ATTR_UNUSED, struct ip_addr *ip ATTR_UNUSED, + pool_t set_pool ATTR_UNUSED, const struct login_settings *set ATTR_UNUSED, ssl_handshake_callback_t *callback ATTR_UNUSED, void *context ATTR_UNUSED, @@ -77,6 +79,11 @@ return NULL; } +const char *ssl_proxy_get_cert_error(struct ssl_proxy *proxy ATTR_UNUSED) +{ + return ""; +} + void ssl_proxy_free(struct ssl_proxy **proxy ATTR_UNUSED) {} unsigned int ssl_proxy_get_count(void)
--- a/src/login-common/ssl-proxy.h Sat May 19 22:40:08 2012 +0300 +++ b/src/login-common/ssl-proxy.h Sun May 20 03:25:04 2012 +0300 @@ -13,10 +13,10 @@ /* establish SSL connection with the given fd, returns a new fd which you must use from now on, or -1 if error occurred. Unless -1 is returned, the given fd must be simply forgotten. */ -int ssl_proxy_alloc(int fd, const struct ip_addr *ip, +int ssl_proxy_alloc(int fd, const struct ip_addr *ip, pool_t set_pool, const struct login_settings *set, struct ssl_proxy **proxy_r); -int ssl_proxy_client_alloc(int fd, struct ip_addr *ip, +int ssl_proxy_client_alloc(int fd, struct ip_addr *ip, pool_t set_pool, const struct login_settings *set, ssl_handshake_callback_t *callback, void *context, struct ssl_proxy **proxy_r); @@ -30,6 +30,7 @@ const char *ssl_proxy_get_last_error(const struct ssl_proxy *proxy) ATTR_PURE; const char *ssl_proxy_get_security_string(struct ssl_proxy *proxy); const char *ssl_proxy_get_compression(struct ssl_proxy *proxy); +const char *ssl_proxy_get_cert_error(struct ssl_proxy *proxy); void ssl_proxy_free(struct ssl_proxy **proxy); /* Return number of active SSL proxies */
--- a/src/master/main.c Sat May 19 22:40:08 2012 +0300 +++ b/src/master/main.c Sun May 20 03:25:04 2012 +0300 @@ -164,7 +164,7 @@ abort(); /* just to silence the noreturn attribute warnings */ } -static void ATTR_NORETURN +static void ATTR_NORETURN ATTR_FORMAT(2, 0) startup_fatal_handler(const struct failure_context *ctx, const char *fmt, va_list args) { @@ -177,7 +177,7 @@ abort(); } -static void +static void ATTR_FORMAT(2, 0) startup_error_handler(const struct failure_context *ctx, const char *fmt, va_list args) { @@ -317,6 +317,7 @@ mountpoints = mountpoint_list_init(perm_path, state_path); if (mountpoint_list_add_missing(mountpoints, MOUNTPOINT_STATE_DEFAULT, + mountpoint_list_default_ignore_prefixes, mountpoint_list_default_ignore_types) == 0) mountpoints_warn_missing(mountpoints); (void)mountpoint_list_save(mountpoints);
--- a/src/master/master-settings.c Sat May 19 22:40:08 2012 +0300 +++ b/src/master/master-settings.c Sun May 20 03:25:04 2012 +0300 @@ -203,7 +203,7 @@ # define ENV_SYSTEMD "" #endif #ifdef DEBUG -# define ENV_GDB " GDB" +# define ENV_GDB " GDB DEBUG_SILENT" #else # define ENV_GDB "" #endif
--- a/src/master/service-listen.c Sat May 19 22:40:08 2012 +0300 +++ b/src/master/service-listen.c Sun May 20 03:25:04 2012 +0300 @@ -14,6 +14,7 @@ #include <unistd.h> #include <fcntl.h> #include <sys/stat.h> +#include <sys/socket.h> #define MIN_BACKLOG 4 #define MAX_BACKLOG 511 @@ -242,6 +243,91 @@ return ret; } +#ifdef HAVE_SYSTEMD +static int get_socket_info(int fd, unsigned int *family, unsigned int *port) +{ + union sockaddr_union { + struct sockaddr sa; + struct sockaddr_in in4; + struct sockaddr_in6 in6; + } sockaddr; + socklen_t l; + + if (port) *port = -1; + if (family) *family = -1; + + memset(&sockaddr, 0, sizeof(sockaddr)); + l = sizeof(sockaddr); + + if (getsockname(fd, &sockaddr.sa, &l) < 0) + return -errno; + + if (family) *family = sockaddr.sa.sa_family; + if (port) { + if (sockaddr.sa.sa_family == AF_INET) { + if (l < sizeof(struct sockaddr_in)) + return -EINVAL; + + *port = ntohs(sockaddr.in4.sin_port); + } else { + if (l < sizeof(struct sockaddr_in6)) + return -EINVAL; + + *port = ntohs(sockaddr.in6.sin6_port); + } + } + return 0; +} + +static int services_verify_systemd(struct service_list *service_list) +{ + struct service *const *services; + static int sd_fds = -1; + int fd, fd_max; + + if (sd_fds < 0) { + sd_fds = sd_listen_fds(0); + if (sd_fds == -1) { + i_error("sd_listen_fds() failed: %m"); + return -1; + } + } + + fd_max = SD_LISTEN_FDS_START + sd_fds - 1; + for (fd = SD_LISTEN_FDS_START; fd <= fd_max; fd++) { + if (sd_is_socket_inet(fd, 0, SOCK_STREAM, 1, 0) > 0) { + int found = FALSE; + unsigned int port, family; + get_socket_info(fd, &family, &port); + + array_foreach(&service_list->services, services) { + struct service_listener *const *listeners; + + array_foreach(&(*services)->listeners, listeners) { + struct service_listener *l = *listeners; + if (l->type != SERVICE_LISTENER_INET) + continue; + if (l->set.inetset.set->port == port && + l->set.inetset.ip.family == family) { + found = TRUE; + break; + } + } + if (found) break; + } + if (!found) { + i_error("systemd listens on port %d, but it's not configured in Dovecot. Closing.",port); + if (shutdown(fd, SHUT_RDWR) < 0 && errno != ENOTCONN) + i_error("shutdown() failed: %m"); + if (dup2(null_fd, fd) < 0) + i_error("dup2() failed: %m"); + } + } + } + return 0; +} +#endif + int services_listen(struct service_list *service_list) { struct service *const *services; @@ -252,6 +338,11 @@ if (ret2 < ret) ret = ret2; } + +#ifdef HAVE_SYSTEMD + if (ret > 0) + services_verify_systemd(service_list); +#endif return ret; }
--- a/src/plugins/Makefile.am Sat May 19 22:40:08 2012 +0300 +++ b/src/plugins/Makefile.am Sun May 20 03:25:04 2012 +0300 @@ -23,6 +23,8 @@ mail-log \ quota \ imap-quota \ + pop3-migration \ + replication \ snarf \ stats \ imap-stats \
--- a/src/plugins/acl/acl-backend-vfile-acllist.c Sat May 19 22:40:08 2012 +0300 +++ b/src/plugins/acl/acl-backend-vfile-acllist.c Sun May 20 03:25:04 2012 +0300 @@ -81,7 +81,7 @@ path = acl_list_get_path(backend); if (path == NULL) { /* we're never going to build acllist for this namespace. */ - i_array_init(&backend->acllist, 1); + acllist_clear(backend, 0); return 0; }
--- a/src/plugins/acl/acl-backend-vfile.c Sat May 19 22:40:08 2012 +0300 +++ b/src/plugins/acl/acl-backend-vfile.c Sun May 20 03:25:04 2012 +0300 @@ -184,10 +184,9 @@ static const char * get_parent_mailbox(struct acl_backend *backend, const char *name) { - struct mail_namespace *ns = mailbox_list_get_namespace(backend->list); const char *p; - p = strrchr(name, mail_namespace_get_sep(ns)); + p = strrchr(name, mailbox_list_get_hierarchy_sep(backend->list)); return p == NULL ? NULL : t_strdup_until(name, p); }
--- a/src/plugins/acl/acl-cache.c Sat May 19 22:40:08 2012 +0300 +++ b/src/plugins/acl/acl-cache.c Sun May 20 03:25:04 2012 +0300 @@ -75,6 +75,10 @@ if (obj_cache->my_current_rights != NULL && obj_cache->my_current_rights != &negative_cache_entry) acl_cache_mask_deinit(&obj_cache->my_current_rights); + if (obj_cache->my_rights != NULL) + acl_cache_mask_deinit(&obj_cache->my_rights); + if (obj_cache->my_neg_rights != NULL) + acl_cache_mask_deinit(&obj_cache->my_neg_rights); i_free(obj_cache->name); i_free(obj_cache); }
--- a/src/plugins/acl/acl-shared-storage.c Sat May 19 22:40:08 2012 +0300 +++ b/src/plugins/acl/acl-shared-storage.c Sun May 20 03:25:04 2012 +0300 @@ -12,6 +12,24 @@ #define SHARED_NS_RETRY_SECS (60*60) +static bool acl_ns_prefix_exists(struct mail_namespace *ns) +{ + struct mailbox *box; + const char *vname; + enum mailbox_existence existence; + bool ret; + + if (ns->list->mail_set->mail_shared_explicit_inbox) + return FALSE; + + vname = t_strndup(ns->prefix, ns->prefix_len-1); + box = mailbox_alloc(ns->list, vname, 0); + ret = mailbox_exists(box, FALSE, &existence) == 0 && + existence == MAILBOX_EXISTENCE_SELECT; + mailbox_free(&box); + return ret; +} + static void acl_shared_namespace_add(struct mail_namespace *ns, struct mail_storage *storage, const char *userdomain) @@ -56,7 +74,7 @@ break; (void)mailbox_list_iter_deinit(&iter); - if (info == NULL) { + if (info == NULL && !acl_ns_prefix_exists(new_ns)) { /* no visible mailboxes, remove the namespace */ mail_namespace_destroy(new_ns); }
--- a/src/plugins/expire/doveadm-expire.c Sat May 19 22:40:08 2012 +0300 +++ b/src/plugins/expire/doveadm-expire.c Sun May 20 03:25:04 2012 +0300 @@ -304,6 +304,8 @@ const struct expire_query *queries; unsigned int i, count; + i_assert(args != NULL); + /* we support two kinds of queries: 1) mailbox-pattern savedbefore <stamp> ...
--- a/src/plugins/expire/expire-plugin.c Sat May 19 22:40:08 2012 +0300 +++ b/src/plugins/expire/expire-plugin.c Sun May 20 03:25:04 2012 +0300 @@ -162,8 +162,14 @@ this is the first mail in the database */ ret = dict_lookup(euser->db, pool_datastack_create(), key, &value); - if (ret == 0) { - /* first time saving here with expire enabled */ + if (ret <= 0) { + /* first time saving here with expire enabled. + also handle lookup errors by just assuming + it didn't exist */ + if (ret < 0) { + i_warning("expire: dict lookup failed, " + "assuming update is needed"); + } first_save_timestamp(box, &new_stamp); update_dict = TRUE; } else if (strcmp(value, "0") == 0) { @@ -188,7 +194,8 @@ dctx = dict_transaction_begin(euser->db); dict_set(dctx, key, dec2str(new_stamp)); - dict_transaction_commit(&dctx); + if (dict_transaction_commit(&dctx) < 0) + i_error("expire: dict commit failed"); } } T_END; i_free(xt);
--- a/src/plugins/fts-lucene/fts-backend-lucene.c Sat May 19 22:40:08 2012 +0300 +++ b/src/plugins/fts-lucene/fts-backend-lucene.c Sun May 20 03:25:04 2012 +0300 @@ -552,7 +552,8 @@ struct fts_backend fts_backend_lucene = { .name = "lucene", - .flags = FTS_BACKEND_FLAG_BUILD_FULL_WORDS, + .flags = FTS_BACKEND_FLAG_BUILD_FULL_WORDS | + FTS_BACKEND_FLAG_FUZZY_SEARCH, { fts_backend_lucene_alloc,
--- a/src/plugins/fts-solr/fts-backend-solr.c Sat May 19 22:40:08 2012 +0300 +++ b/src/plugins/fts-solr/fts-backend-solr.c Sun May 20 03:25:04 2012 +0300 @@ -511,9 +511,11 @@ SOLR_CMDBUF_FLUSH_SIZE - str_len(ctx->cmd)); i_assert(len > 0); + i_assert(len <= size); data += len; size -= len; } + xml_encode_data(ctx->cmd, data, size); } else { xml_encode_data(ctx->cur_value, data, size); if (ctx->cur_value2 != NULL) @@ -543,6 +545,34 @@ return 0; } +static int fts_backend_solr_rescan(struct fts_backend *backend) +{ + struct mailbox_list_iterate_context *iter; + const struct mailbox_info *info; + struct mailbox *box; + int ret = 0; + + /* FIXME: proper rescan needed. for now we'll just reset the + last-uids */ + iter = mailbox_list_iter_init(backend->ns->list, "*", + MAILBOX_LIST_ITER_NO_AUTO_BOXES); + while ((info = mailbox_list_iter_next(iter)) != NULL) { + if ((info->flags & + (MAILBOX_NONEXISTENT | MAILBOX_NOSELECT)) != 0) + continue; + + box = mailbox_alloc(info->ns->list, info->name, 0); + if (mailbox_open(box) == 0) { + if (fts_index_set_last_uid(box, 0) < 0) + ret = -1; + } + mailbox_free(&box); + } + if (mailbox_list_iter_deinit(&iter) < 0) + ret = -1; + return ret; +} + static int fts_backend_solr_optimize(struct fts_backend *backend ATTR_UNUSED) { return 0; @@ -833,7 +863,7 @@ struct fts_backend fts_backend_solr = { .name = "solr", - .flags = 0, + .flags = FTS_BACKEND_FLAG_FUZZY_SEARCH, { fts_backend_solr_alloc, @@ -848,7 +878,7 @@ fts_backend_solr_update_unset_build_key, fts_backend_solr_update_build_more, fts_backend_solr_refresh, - NULL, + fts_backend_solr_rescan, fts_backend_solr_optimize, fts_backend_default_can_lookup, fts_backend_solr_lookup,
--- a/src/plugins/fts-squat/Makefile.am Sat May 19 22:40:08 2012 +0300 +++ b/src/plugins/fts-squat/Makefile.am Sun May 20 03:25:04 2012 +0300 @@ -37,9 +37,11 @@ squat-trie.lo \ squat-uidlist.lo -libs = \ +squat_test_LDADD = \ + $(common_objects) \ $(LIBDOVECOT_STORAGE) \ - $(common_objects) - -squat_test_LDADD = $(libs) $(LIBDOVECOT) -squat_test_DEPENDENCIES = $(libs) $(LIBDOVECOT_DEPS) + $(LIBDOVECOT) +squat_test_DEPENDENCIES = \ + $(common_objects) \ + $(LIBDOVECOT_STORAGE_DEPS) \ + $(LIBDOVECOT_DEPS)
--- a/src/plugins/fts-squat/fts-backend-squat.c Sat May 19 22:40:08 2012 +0300 +++ b/src/plugins/fts-squat/fts-backend-squat.c Sun May 20 03:25:04 2012 +0300 @@ -436,6 +436,7 @@ { struct squat_fts_backend *backend = (struct squat_fts_backend *)_backend; + bool first = TRUE; int ret; fts_backend_squat_set_box(backend, box); @@ -446,14 +447,14 @@ } for (; args != NULL; args = args->next) { - ret = squat_lookup_arg(backend, args, and_args, + ret = squat_lookup_arg(backend, args, first ? FALSE : and_args, &result->definite_uids, &result->maybe_uids); if (ret < 0) return -1; if (ret > 0) args->match_always = TRUE; - + first = FALSE; } return 0; }
--- a/src/plugins/fts-squat/squat-trie.c Sat May 19 22:40:08 2012 +0300 +++ b/src/plugins/fts-squat/squat-trie.c Sun May 20 03:25:04 2012 +0300 @@ -411,6 +411,7 @@ } chars = NODE_CHILDREN_CHARS(node); + i_assert(chars != NULL); chars[node->child_count - 1] = chr; return node->child_count - 1; }
--- a/src/plugins/fts-squat/squat-uidlist.c Sat May 19 22:40:08 2012 +0300 +++ b/src/plugins/fts-squat/squat-uidlist.c Sun May 20 03:25:04 2012 +0300 @@ -332,6 +332,7 @@ return 0; } + i_assert(uidlist->data != NULL); base = CONST_PTR_OFFSET(uidlist->data, hdr->block_list_offset); memcpy(&block_count, base, sizeof(block_count)); @@ -347,6 +348,8 @@ uidlist->cur_block_count = block_count; squat_uidlist_map_blocks_set_pointers(uidlist); + i_assert(uidlist->cur_block_end_indexes != NULL); + /* verify just a couple of the end indexes to make sure they look correct */ verify_count = I_MIN(block_count, 8);
--- a/src/plugins/fts/Makefile.am Sat May 19 22:40:08 2012 +0300 +++ b/src/plugins/fts/Makefile.am Sun May 20 03:25:04 2012 +0300 @@ -3,6 +3,7 @@ AM_CPPFLAGS = \ -I$(top_srcdir)/src/lib \ + -I$(top_srcdir)/src/lib-settings \ -I$(top_srcdir)/src/lib-mail \ -I$(top_srcdir)/src/lib-index \ -I$(top_srcdir)/src/lib-storage \
--- a/src/plugins/fts/decode2text.sh Sat May 19 22:40:08 2012 +0300 +++ b/src/plugins/fts/decode2text.sh Sun May 20 03:25:04 2012 +0300 @@ -52,7 +52,7 @@ # most decoders can't handle stdin directly, so write the attachment # to a temp file path=`mktemp` -trap "rm -f $path" 0 1 2 3 15 +trap "rm -f $path" 0 1 2 3 14 15 cat > $path xmlunzip() { @@ -62,23 +62,33 @@ if [ "$tempdir" = "" ]; then exit 1 fi - trap "rm -rf $tempdir" 0 1 2 3 15 + trap "rm -rf $path $tempdir" 0 1 2 3 14 15 cd $tempdir || exit 1 unzip -q "$path" 2>/dev/null || exit 0 find . -name "$name" -print0 | xargs -0 cat | /usr/local/libexec/dovecot/xml2text } +wait_timeout() { + childpid=$! + trap "kill -9 $childpid; rm -f $path" 1 2 3 14 15 + wait $childpid +} + LANG=en_US.UTF-8 export LANG if [ $fmt = "pdf" ]; then - /usr/bin/pdftotext $path - 2>/dev/null + /usr/bin/pdftotext $path - 2>/dev/null& + wait_timeout 2>/dev/null elif [ $fmt = "doc" ]; then - (/usr/bin/catdoc $path; true) 2>/dev/null + (/usr/bin/catdoc $path; true) 2>/dev/null& + wait_timeout 2>/dev/null elif [ $fmt = "ppt" ]; then - (/usr/bin/catppt $path; true) 2>/dev/null + (/usr/bin/catppt $path; true) 2>/dev/null& + wait_timeout 2>/dev/null elif [ $fmt = "xls" ]; then - (/usr/bin/xls2csv $path; true) 2>/dev/null + (/usr/bin/xls2csv $path; true) 2>/dev/null& + wait_timeout 2>/dev/null elif [ $fmt = "odt" -o $fmt = "ods" -o $fmt = "odp" ]; then xmlunzip "content.xml" elif [ $fmt = "docx" ]; then
--- a/src/plugins/fts/fts-api-private.h Sat May 19 22:40:08 2012 +0300 +++ b/src/plugins/fts/fts-api-private.h Sun May 20 03:25:04 2012 +0300 @@ -57,7 +57,9 @@ preserving original case */ FTS_BACKEND_FLAG_BUILD_DTCASE = 0x02, /* Send only fully indexable words rather than randomly sized blocks */ - FTS_BACKEND_FLAG_BUILD_FULL_WORDS = 0x04 + FTS_BACKEND_FLAG_BUILD_FULL_WORDS = 0x04, + /* Fuzzy search works */ + FTS_BACKEND_FLAG_FUZZY_SEARCH = 0x08 }; struct fts_backend {
--- a/src/plugins/fts/fts-indexer.c Sat May 19 22:40:08 2012 +0300 +++ b/src/plugins/fts/fts-indexer.c Sun May 20 03:25:04 2012 +0300 @@ -7,6 +7,7 @@ #include "write-full.h" #include "strescape.h" #include "time-util.h" +#include "settings-parser.h" #include "mail-user.h" #include "mail-storage-private.h" #include "fts-api.h" @@ -23,6 +24,7 @@ struct timeval search_start_time, last_notify; unsigned int percentage; + unsigned int timeout_secs; char *path; int fd; @@ -93,7 +95,7 @@ struct fts_indexer_context *ctx; struct mailbox_status status; uint32_t last_uid, seq1, seq2; - const char *path, *cmd; + const char *path, *cmd, *value, *error; int fd; if (fts_backend_get_last_uid(backend, box, &last_uid) < 0) @@ -126,6 +128,13 @@ ctx->input = i_stream_create_fd(fd, 128, FALSE); ctx->search_start_time = ioloop_timeval; + value = mail_user_plugin_getenv(box->storage->user, "fts_index_timeout"); + if (value != NULL) { + if (settings_get_time(value, &ctx->timeout_secs, &error) < 0) + i_error("Invalid fts_index_timeout setting: %s", error); + } + + *ctx_r = ctx; return 1; } @@ -214,12 +223,24 @@ int fts_indexer_more(struct fts_indexer_context *ctx) { - int ret; + int ret, diff; if ((ret = fts_indexer_more_int(ctx)) < 0) { + mail_storage_set_internal_error(ctx->box->storage); ctx->failed = TRUE; return -1; } + + if (ctx->timeout_secs > 0) { + diff = ioloop_time - ctx->search_start_time.tv_sec; + if (diff > (int)ctx->timeout_secs) { + mail_storage_set_error(ctx->box->storage, + MAIL_ERROR_INUSE, + "Timeout while waiting for indexing to finish"); + ctx->failed = TRUE; + return -1; + } + } if (ret == 0) fts_indexer_notify(ctx); return ret;
--- a/src/plugins/fts/fts-storage.c Sat May 19 22:40:08 2012 +0300 +++ b/src/plugins/fts/fts-storage.c Sun May 20 03:25:04 2012 +0300 @@ -203,21 +203,29 @@ static bool fts_mailbox_build_continue(struct mail_search_context *ctx) { struct fts_search_context *fctx = FTS_CONTEXT(ctx); + enum mail_error error; int ret; - if (fctx == NULL) - return TRUE; + ret = fts_indexer_more(fctx->indexer_ctx); + if (ret == 0) + return FALSE; - if (fctx->indexer_ctx != NULL) { - /* this command is still building the indexes */ - ret = fts_indexer_more(fctx->indexer_ctx); - if (ret == 0) - return FALSE; - ctx->progress_hidden = FALSE; - if (fts_indexer_deinit(&fctx->indexer_ctx) < 0) - ret = -1; - if (ret > 0) - fts_search_lookup(fctx); + /* indexing finished */ + ctx->progress_hidden = FALSE; + if (fts_indexer_deinit(&fctx->indexer_ctx) < 0) + ret = -1; + if (ret > 0) + fts_search_lookup(fctx); + if (ret < 0) { + /* if indexing timed out, it probably means that + the mailbox is still being indexed, but it's a large + mailbox and it takes a while. in this situation + we'll simply abort the search. + + if indexing failed for any other reason, just + fallback to searching the slow way. */ + (void)mailbox_get_last_error(fctx->box, &error); + fctx->indexing_timed_out = error == MAIL_ERROR_INUSE; } return TRUE; } @@ -227,10 +235,18 @@ struct mail **mail_r, bool *tryagain_r) { struct fts_mailbox *fbox = FTS_CONTEXT(ctx->transaction->box); + struct fts_search_context *fctx = FTS_CONTEXT(ctx); - if (!fts_mailbox_build_continue(ctx)) { - *tryagain_r = TRUE; - return FALSE; + if (fctx != NULL && fctx->indexer_ctx != NULL) { + /* this command is still building the indexes */ + if (!fts_mailbox_build_continue(ctx)) { + *tryagain_r = TRUE; + return FALSE; + } + if (fctx->indexing_timed_out) { + *tryagain_r = FALSE; + return FALSE; + } } return fbox->module_ctx.super. @@ -270,6 +286,8 @@ if (fctx == NULL || !fctx->fts_lookup_success) { /* fts lookup not done for this search */ + if (fctx != NULL && fctx->indexing_timed_out) + return FALSE; return fbox->module_ctx.super.search_next_update_seq(ctx); } @@ -295,12 +313,15 @@ struct fts_mailbox *fbox = FTS_CONTEXT(ctx->transaction->box); struct fts_transaction_context *ft = FTS_CONTEXT(ctx->transaction); struct fts_search_context *fctx = FTS_CONTEXT(ctx); + int ret = 0; if (fctx != NULL) { if (fctx->indexer_ctx != NULL) { if (fts_indexer_deinit(&fctx->indexer_ctx) < 0) ft->failed = TRUE; } + if (fctx->indexing_timed_out) + ret = -1; buffer_free(&fctx->orig_matches); array_free(&fctx->levels); @@ -308,7 +329,9 @@ fts_scores_unref(&fctx->scores); i_free(fctx); } - return fbox->module_ctx.super.search_deinit(ctx); + if (fbox->module_ctx.super.search_deinit(ctx) < 0) + ret = -1; + return ret; } static int fts_score_cmp(const uint32_t *uid, const struct fts_score_map *score) @@ -637,6 +660,9 @@ struct fts_mailbox_list *flist; struct mailbox_list_vfuncs *v = list->vlast; + if ((backend->flags & FTS_BACKEND_FLAG_FUZZY_SEARCH) != 0) + list->ns->user->fuzzy_search = TRUE; + flist = p_new(list->pool, struct fts_mailbox_list, 1); flist->module_ctx.super = *v; flist->backend = backend;
--- a/src/plugins/fts/fts-storage.h Sat May 19 22:40:08 2012 +0300 +++ b/src/plugins/fts/fts-storage.h Sun May 20 03:25:04 2012 +0300 @@ -36,6 +36,7 @@ unsigned int virtual_mailbox:1; unsigned int fts_lookup_success:1; + unsigned int indexing_timed_out:1; }; /* Figure out if we want to use full text search indexes and update
--- a/src/plugins/imap-stats/imap-stats-plugin.c Sat May 19 22:40:08 2012 +0300 +++ b/src/plugins/imap-stats/imap-stats-plugin.c Sun May 20 03:25:04 2012 +0300 @@ -56,7 +56,6 @@ struct mail_stats stats, pre_trans_stats, trans_stats; unsigned int args_pos = 0; string_t *str; - bool done; if (scmd == NULL) return; @@ -76,11 +75,13 @@ str_append(str, "UPDATE-CMD\t"); str_append(str, guid_128_to_string(suser->session_guid)); - done = cmd->state == CLIENT_COMMAND_STATE_DONE; - str_printfa(str, "\t%u\t%d\t", scmd->id, done); + str_printfa(str, "\t%u\t", scmd->id); + if (cmd->state == CLIENT_COMMAND_STATE_DONE) + str_append_c(str, 'd'); if (scmd->continued) + str_append_c(str, 'c'); + else { str_append_c(str, '\t'); - else { str_append(str, cmd->name); str_append_c(str, '\t'); args_pos = str_len(str);
--- a/src/plugins/mail-log/mail-log-plugin.c Sat May 19 22:40:08 2012 +0300 +++ b/src/plugins/mail-log/mail-log-plugin.c Sun May 20 03:25:04 2012 +0300 @@ -228,6 +228,8 @@ wanted_fields |= MAIL_FETCH_VIRTUAL_SIZE; mail_add_temp_wanted_fields(mail, wanted_fields, wanted_headers); + if (wanted_headers != NULL) + mailbox_header_lookup_unref(&wanted_headers); } static void @@ -360,17 +362,11 @@ (struct mail_log_mail_txn_context *)txn; const char *desc; - if (dst->saving) { - /* we came from mailbox_save_using_mail() */ - mail_log_append_mail_message(ctx, dst, - MAIL_LOG_EVENT_SAVE, "save"); - } else { - desc = t_strdup_printf("copy from %s", - str_sanitize(mailbox_get_name(src->box), - MAILBOX_NAME_LOG_LEN)); - mail_log_append_mail_message(ctx, dst, - MAIL_LOG_EVENT_COPY, desc); - } + desc = t_strdup_printf("copy from %s", + str_sanitize(mailbox_get_name(src->box), + MAILBOX_NAME_LOG_LEN)); + mail_log_append_mail_message(ctx, dst, + MAIL_LOG_EVENT_COPY, desc); } static void mail_log_mail_expunge(void *txn, struct mail *mail)
--- a/src/plugins/notify/notify-storage.c Sat May 19 22:40:08 2012 +0300 +++ b/src/plugins/notify/notify-storage.c Sun May 20 03:25:04 2012 +0300 @@ -102,8 +102,15 @@ ctx->dest_mail = lt->tmp_mail; } - if ((ret = lbox->super.copy(ctx, mail)) == 0) + if ((ret = lbox->super.copy(ctx, mail)) < 0) + return -1; + + if (ctx->saving) { + /* we came from mailbox_save_using_mail() */ + notify_contexts_mail_save(ctx->dest_mail); + } else { notify_contexts_mail_copy(mail, ctx->dest_mail); + } return ret; }
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/plugins/pop3-migration/Makefile.am Sun May 20 03:25:04 2012 +0300 @@ -0,0 +1,17 @@ +AM_CPPFLAGS = \ + -I$(top_srcdir)/src/lib \ + -I$(top_srcdir)/src/lib-mail \ + -I$(top_srcdir)/src/lib-index \ + -I$(top_srcdir)/src/lib-storage + +NOPLUGIN_LDFLAGS = +lib05_pop3_migration_plugin_la_LDFLAGS = -module -avoid-version + +module_LTLIBRARIES = \ + lib05_pop3_migration_plugin.la + +lib05_pop3_migration_plugin_la_SOURCES = \ + pop3-migration-plugin.c + +noinst_HEADERS = \ + pop3-migration-plugin.h
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/plugins/pop3-migration/pop3-migration-plugin.c Sun May 20 03:25:04 2012 +0300 @@ -0,0 +1,632 @@ +/* Copyright (c) 2007-2012 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "istream.h" +#include "istream-header-filter.h" +#include "sha1.h" +#include "message-size.h" +#include "mail-namespace.h" +#include "mail-search-build.h" +#include "mail-storage-private.h" +#include "pop3-migration-plugin.h" + +#define POP3_MIGRATION_CONTEXT(obj) \ + MODULE_CONTEXT(obj, pop3_migration_storage_module) +#define POP3_MIGRATION_MAIL_CONTEXT(obj) \ + MODULE_CONTEXT(obj, pop3_migration_mail_module) + +struct pop3_uidl_map { + uint32_t pop3_seq; + uint32_t imap_uid; + + /* UIDL */ + const char *pop3_uidl; + /* LIST size */ + uoff_t size; + /* sha1(TOP 0) - set only when needed */ + unsigned char hdr_sha1[SHA1_RESULTLEN]; + unsigned int hdr_sha1_set:1; +}; + +struct imap_msg_map { + uint32_t uid, pop3_seq; + uoff_t psize; + const char *pop3_uidl; + + /* sha1(header) - set only when needed */ + unsigned char hdr_sha1[SHA1_RESULTLEN]; + unsigned int hdr_sha1_set:1; +}; + +struct pop3_migration_mail_storage { + union mail_storage_module_context module_ctx; + + const char *pop3_box_vname; + struct mailbox *pop3_box; + ARRAY_DEFINE(pop3_uidl_map, struct pop3_uidl_map); + + unsigned int all_mailboxes:1; + unsigned int pop3_all_hdr_sha1_set:1; +}; + +struct pop3_migration_mailbox { + union mailbox_module_context module_ctx; + + ARRAY_DEFINE(imap_msg_map, struct imap_msg_map); + unsigned int first_unfound_idx; + + unsigned int uidl_synced:1; + unsigned int uidl_sync_failed:1; + unsigned int uidl_ordered:1; +}; + +static const char *hdr_hash_skip_headers[] = { + "Content-Length", + "Status", + "X-IMAP", + "X-IMAPbase", + "X-Keywords", + "X-Message-Flag", + "X-Status", + "X-UID", + "X-UIDL" +}; +const char *pop3_migration_plugin_version = DOVECOT_VERSION; + +static MODULE_CONTEXT_DEFINE_INIT(pop3_migration_storage_module, + &mail_storage_module_register); +static MODULE_CONTEXT_DEFINE_INIT(pop3_migration_mail_module, + &mail_module_register); + +static int imap_msg_map_uid_cmp(const struct imap_msg_map *map1, + const struct imap_msg_map *map2) +{ + if (map1->uid < map2->uid) + return -1; + if (map1->uid > map2->uid) + return 1; + return 0; +} + +static int pop3_uidl_map_pop3_seq_cmp(const struct pop3_uidl_map *map1, + const struct pop3_uidl_map *map2) +{ + if (map1->pop3_seq < map2->pop3_seq) + return -1; + if (map1->pop3_seq > map2->pop3_seq) + return 1; + return 0; +} + +static int pop3_uidl_map_hdr_cmp(const struct pop3_uidl_map *map1, + const struct pop3_uidl_map *map2) +{ + return memcmp(map1->hdr_sha1, map2->hdr_sha1, sizeof(map1->hdr_sha1)); +} + +static int imap_msg_map_hdr_cmp(const struct imap_msg_map *map1, + const struct imap_msg_map *map2) +{ + return memcmp(map1->hdr_sha1, map2->hdr_sha1, sizeof(map1->hdr_sha1)); +} + +static int get_hdr_sha1(struct mail *mail, unsigned char sha1[SHA1_RESULTLEN]) +{ + struct message_size hdr_size; + struct istream *input, *input2; + const unsigned char *data; + size_t size; + struct sha1_ctxt sha1_ctx; + + if (mail_get_hdr_stream(mail, &hdr_size, &input) < 0) { + i_error("pop3_migration: Failed to get header for msg %u: %s", + mail->seq, mailbox_get_last_error(mail->box, NULL)); + return -1; + } + input2 = i_stream_create_limit(input, hdr_size.physical_size); + /* hide headers that might change or be different in IMAP vs. POP3 */ + input = i_stream_create_header_filter(input2, + HEADER_FILTER_EXCLUDE | HEADER_FILTER_NO_CR, + hdr_hash_skip_headers, + N_ELEMENTS(hdr_hash_skip_headers), + null_header_filter_callback, NULL); + i_stream_unref(&input2); + + sha1_init(&sha1_ctx); + while (i_stream_read_data(input, &data, &size, 0) > 0) { + sha1_loop(&sha1_ctx, data, size); + i_stream_skip(input, size); + } + if (input->stream_errno != 0) { + i_error("pop3_migration: Failed to read header for msg %u: %m", + mail->seq); + i_stream_unref(&input); + return -1; + } + sha1_result(&sha1_ctx, sha1); + i_stream_unref(&input); + return 0; +} + +static int pop3_mailbox_open(struct mail_storage *storage) +{ + struct pop3_migration_mail_storage *mstorage = + POP3_MIGRATION_CONTEXT(storage); + struct mail_namespace *ns; + + if (mstorage->pop3_box != NULL) + return 0; + + ns = mail_namespace_find(storage->user->namespaces, + mstorage->pop3_box_vname); + if (ns == NULL) { + i_error("pop3_migration: Namespace not found for mailbox %s", + mstorage->pop3_box_vname); + return -1; + } + mstorage->pop3_box = mailbox_alloc(ns->list, mstorage->pop3_box_vname, + MAILBOX_FLAG_READONLY | + MAILBOX_FLAG_POP3_SESSION); + mstorage->all_mailboxes = + mail_user_plugin_getenv(storage->user, + "pop3_migration_all_mailboxes") != NULL; + return 0; +} + +static int pop3_map_read(struct mail_storage *storage) +{ + struct pop3_migration_mail_storage *mstorage = + POP3_MIGRATION_CONTEXT(storage); + struct mailbox *pop3_box = mstorage->pop3_box; + struct mailbox_transaction_context *t; + struct mail_search_args *search_args; + struct mail_search_context *ctx; + struct mail *mail; + struct pop3_uidl_map *map; + const char *uidl; + uoff_t size; + int ret = 0; + + if (array_is_created(&mstorage->pop3_uidl_map)) { + /* already read these, just reset the imap_uids */ + array_foreach_modifiable(&mstorage->pop3_uidl_map, map) + map->imap_uid = 0; + return 0; + } + i_array_init(&mstorage->pop3_uidl_map, 128); + + if (mailbox_sync(pop3_box, 0) < 0) { + i_error("pop3_migration: Couldn't sync mailbox %s: %s", + pop3_box->vname, mailbox_get_last_error(pop3_box, NULL)); + return -1; + } + + t = mailbox_transaction_begin(pop3_box, 0); + search_args = mail_search_build_init(); + mail_search_build_add_all(search_args); + ctx = mailbox_search_init(t, search_args, NULL, + MAIL_FETCH_VIRTUAL_SIZE, NULL); + mail_search_args_unref(&search_args); + + while (mailbox_search_next(ctx, &mail)) { + if (mail_get_virtual_size(mail, &size) < 0) { + i_error("pop3_migration: Failed to get size for msg %u: %s", + mail->seq, + mailbox_get_last_error(pop3_box, NULL)); + ret = -1; + break; + } + if (mail_get_special(mail, MAIL_FETCH_UIDL_BACKEND, &uidl) < 0) { + i_error("pop3_migration: Failed to get UIDL for msg %u: %s", + mail->seq, + mailbox_get_last_error(pop3_box, NULL)); + ret = -1; + break; + } + if (*uidl == '\0') { + i_warning("pop3_migration: UIDL for msg %u is empty", + mail->seq); + continue; + } + + map = array_append_space(&mstorage->pop3_uidl_map); + map->pop3_seq = mail->seq; + map->pop3_uidl = p_strdup(storage->pool, uidl); + map->size = size; + } + + if (mailbox_search_deinit(&ctx) < 0) + ret = -1; + (void)mailbox_transaction_commit(&t); + return ret; +} + +static int pop3_map_read_hdr_hashes(struct mail_storage *storage, + unsigned first_seq) +{ + struct pop3_migration_mail_storage *mstorage = + POP3_MIGRATION_CONTEXT(storage); + struct mailbox_transaction_context *t; + struct mail_search_args *search_args; + struct mail_search_context *ctx; + struct mail *mail; + struct pop3_uidl_map *map; + int ret = 0; + + if (mstorage->pop3_all_hdr_sha1_set) + return 0; + if (mstorage->all_mailboxes) { + /* we may be matching against multiple mailboxes. + read all the hashes only once. */ + first_seq = 1; + } + + t = mailbox_transaction_begin(mstorage->pop3_box, 0); + search_args = mail_search_build_init(); + mail_search_build_add_seqset(search_args, first_seq, + array_count(&mstorage->pop3_uidl_map)+1); + ctx = mailbox_search_init(t, search_args, NULL, + MAIL_FETCH_STREAM_HEADER, NULL); + mail_search_args_unref(&search_args); + + while (mailbox_search_next(ctx, &mail)) { + map = array_idx_modifiable(&mstorage->pop3_uidl_map, + mail->seq-1); + + if (get_hdr_sha1(mail, map->hdr_sha1) < 0) + ret = -1; + else + map->hdr_sha1_set = TRUE; + } + + if (mailbox_search_deinit(&ctx) < 0) + ret = -1; + (void)mailbox_transaction_commit(&t); + if (ret == 0 && first_seq == 1) + mstorage->pop3_all_hdr_sha1_set = TRUE; + return ret; +} + +static int imap_map_read(struct mailbox *box) +{ + struct pop3_migration_mailbox *mbox = POP3_MIGRATION_CONTEXT(box); + struct mailbox_status status; + struct mailbox_transaction_context *t; + struct mail_search_args *search_args; + struct mail_search_context *ctx; + struct mail *mail; + struct imap_msg_map *map; + uoff_t psize; + int ret = 0; + + mailbox_get_open_status(box, STATUS_MESSAGES, &status); + + i_assert(!array_is_created(&mbox->imap_msg_map)); + p_array_init(&mbox->imap_msg_map, box->pool, status.messages); + + t = mailbox_transaction_begin(box, 0); + search_args = mail_search_build_init(); + mail_search_build_add_all(search_args); + ctx = mailbox_search_init(t, search_args, NULL, + MAIL_FETCH_PHYSICAL_SIZE, NULL); + mail_search_args_unref(&search_args); + + while (mailbox_search_next(ctx, &mail)) { + if (mail_get_physical_size(mail, &psize) < 0) { + i_error("pop3_migration: Failed to get psize for imap uid %u: %s", + mail->uid, + mailbox_get_last_error(box, NULL)); + ret = -1; + break; + } + + map = array_append_space(&mbox->imap_msg_map); + map->uid = mail->uid; + map->psize = psize; + } + + if (mailbox_search_deinit(&ctx) < 0) + ret = -1; + (void)mailbox_transaction_commit(&t); + return ret; +} + +static int imap_map_read_hdr_hashes(struct mailbox *box) +{ + struct pop3_migration_mailbox *mbox = POP3_MIGRATION_CONTEXT(box); + struct mailbox_transaction_context *t; + struct mail_search_args *search_args; + struct mail_search_context *ctx; + struct mail *mail; + struct imap_msg_map *map; + int ret = 0; + + t = mailbox_transaction_begin(box, 0); + search_args = mail_search_build_init(); + mail_search_build_add_seqset(search_args, mbox->first_unfound_idx+1, + array_count(&mbox->imap_msg_map)+1); + ctx = mailbox_search_init(t, search_args, NULL, + MAIL_FETCH_STREAM_HEADER, NULL); + mail_search_args_unref(&search_args); + + while (mailbox_search_next(ctx, &mail)) { + map = array_idx_modifiable(&mbox->imap_msg_map, mail->seq-1); + + if (get_hdr_sha1(mail, map->hdr_sha1) < 0) + ret = -1; + else + map->hdr_sha1_set = TRUE; + } + + if (mailbox_search_deinit(&ctx) < 0) + ret = -1; + (void)mailbox_transaction_commit(&t); + return ret; +} + +static bool pop3_uidl_assign_by_size(struct mailbox *box) +{ + struct pop3_migration_mailbox *mbox = POP3_MIGRATION_CONTEXT(box); + struct pop3_migration_mail_storage *mstorage = + POP3_MIGRATION_CONTEXT(box->storage); + struct pop3_uidl_map *pop3_map; + struct imap_msg_map *imap_map; + unsigned int i, pop3_count, imap_count, count; + + pop3_map = array_get_modifiable(&mstorage->pop3_uidl_map, &pop3_count); + imap_map = array_get_modifiable(&mbox->imap_msg_map, &imap_count); + count = I_MIN(pop3_count, imap_count); + + /* see if we can match the messages using sizes */ + for (i = 0; i < count; i++) { + if (pop3_map[i].size != imap_map[i].psize) + break; + if (i+1 < count && pop3_map[i].size == pop3_map[i+1].size) { + /* two messages with same size, don't trust them */ + break; + } + + pop3_map[i].imap_uid = imap_map[i].uid; + imap_map[i].pop3_uidl = pop3_map[i].pop3_uidl; + imap_map[i].pop3_seq = pop3_map[i].pop3_seq; + } + mbox->first_unfound_idx = i; + return i == count; +} + +static int pop3_uidl_assign_by_hdr_hash(struct mailbox *box) +{ + struct pop3_migration_mail_storage *mstorage = + POP3_MIGRATION_CONTEXT(box->storage); + struct pop3_migration_mailbox *mbox = POP3_MIGRATION_CONTEXT(box); + struct pop3_uidl_map *pop3_map; + struct imap_msg_map *imap_map; + unsigned int pop3_idx, imap_idx, pop3_count, imap_count; + unsigned int first_seq, missing_uids_count; + int ret; + + first_seq = mbox->first_unfound_idx+1; + if (pop3_map_read_hdr_hashes(box->storage, first_seq) < 0 || + imap_map_read_hdr_hashes(box) < 0) + return -1; + + array_sort(&mstorage->pop3_uidl_map, pop3_uidl_map_hdr_cmp); + array_sort(&mbox->imap_msg_map, imap_msg_map_hdr_cmp); + + pop3_map = array_get_modifiable(&mstorage->pop3_uidl_map, &pop3_count); + imap_map = array_get_modifiable(&mbox->imap_msg_map, &imap_count); + + pop3_idx = imap_idx = 0; + while (pop3_idx < pop3_count && imap_idx < imap_count) { + if (!pop3_map[pop3_idx].hdr_sha1_set || + pop3_map[pop3_idx].imap_uid != 0) { + pop3_idx++; + continue; + } + if (!imap_map[imap_idx].hdr_sha1_set || + imap_map[imap_idx].pop3_uidl != NULL) { + imap_idx++; + continue; + } + ret = memcmp(pop3_map[pop3_idx].hdr_sha1, + imap_map[imap_idx].hdr_sha1, + sizeof(pop3_map[pop3_idx].hdr_sha1)); + if (ret < 0) + pop3_idx++; + else if (ret > 0) + imap_idx++; + else { + pop3_map[pop3_idx].imap_uid = imap_map[imap_idx].uid; + imap_map[imap_idx].pop3_uidl = + pop3_map[pop3_idx].pop3_uidl; + imap_map[imap_idx].pop3_seq = + pop3_map[pop3_idx].pop3_seq; + } + } + missing_uids_count = 0; + for (pop3_idx = 0; pop3_idx < pop3_count; pop3_idx++) { + if (pop3_map[pop3_idx].imap_uid == 0) + missing_uids_count++; + } + if (missing_uids_count > 0 && !mstorage->all_mailboxes) { + i_warning("pop3_migration: %u POP3 messages have no " + "matching IMAP messages", missing_uids_count); + } + array_sort(&mstorage->pop3_uidl_map, pop3_uidl_map_pop3_seq_cmp); + array_sort(&mbox->imap_msg_map, imap_msg_map_uid_cmp); + return 0; +} + +static int pop3_migration_uidl_sync(struct mailbox *box) +{ + struct pop3_migration_mailbox *mbox = POP3_MIGRATION_CONTEXT(box); + struct pop3_migration_mail_storage *mstorage = + POP3_MIGRATION_CONTEXT(box->storage); + const struct pop3_uidl_map *pop3_map; + unsigned int i, count; + uint32_t prev_uid; + + if (mbox->uidl_synced) + return 0; + + if (pop3_mailbox_open(box->storage) < 0) + return -1; + + if (pop3_map_read(box->storage) < 0 || imap_map_read(box) < 0) + return -1; + + if (!pop3_uidl_assign_by_size(box)) { + /* everything wasn't assigned, figure out the rest with + header hashes */ + if (pop3_uidl_assign_by_hdr_hash(box) < 0) + return -1; + } + + /* see if the POP3 UIDL order is the same as IMAP UID order */ + mbox->uidl_ordered = TRUE; + pop3_map = array_get(&mstorage->pop3_uidl_map, &count); + prev_uid = 0; + for (i = 0; i < count; i++) { + if (pop3_map[i].imap_uid == 0) + continue; + + if (prev_uid > pop3_map[i].imap_uid) { + mbox->uidl_ordered = FALSE; + break; + } + prev_uid = pop3_map[i].imap_uid; + } + + mbox->uidl_synced = TRUE; + return 0; +} + +static int +pop3_migration_get_special(struct mail *_mail, enum mail_fetch_field field, + const char **value_r) +{ + struct mail_private *mail = (struct mail_private *)_mail; + union mail_module_context *mmail = POP3_MIGRATION_MAIL_CONTEXT(mail); + struct pop3_migration_mailbox *mbox = POP3_MIGRATION_CONTEXT(_mail->box); + struct imap_msg_map map_key, *map; + + if (field == MAIL_FETCH_UIDL_BACKEND || + field == MAIL_FETCH_POP3_ORDER) { + if (mbox->uidl_sync_failed || + pop3_migration_uidl_sync(_mail->box) < 0) { + mbox->uidl_sync_failed = TRUE; + mail_storage_set_error(_mail->box->storage, + MAIL_ERROR_TEMP, + "POP3 UIDLs couldn't be synced"); + return -1; + } + + memset(&map_key, 0, sizeof(map_key)); + map_key.uid = _mail->uid; + map = array_bsearch(&mbox->imap_msg_map, &map_key, + imap_msg_map_uid_cmp); + if (map != NULL && map->pop3_uidl != NULL) { + if (field == MAIL_FETCH_UIDL_BACKEND) + *value_r = map->pop3_uidl; + else + *value_r = t_strdup_printf("%u", map->pop3_seq); + return 0; + } + /* not found from POP3 server, fallback to default */ + } + return mmail->super.get_special(_mail, field, value_r); +} + +static void pop3_migration_mail_allocated(struct mail *_mail) +{ + struct pop3_migration_mail_storage *mstorage = + POP3_MIGRATION_CONTEXT(_mail->box->storage); + struct mail_private *mail = (struct mail_private *)_mail; + struct mail_vfuncs *v = mail->vlast; + union mail_module_context *mmail; + struct mail_namespace *ns; + + if (mstorage == NULL || + (!mstorage->all_mailboxes && !_mail->box->inbox_user)) { + /* assigns UIDLs only for INBOX */ + return; + } + + ns = mail_namespace_find(_mail->box->storage->user->namespaces, + mstorage->pop3_box_vname); + if (ns == mailbox_get_namespace(_mail->box)) { + /* we're accessing the pop3-migration namespace itself */ + return; + } + + mmail = p_new(mail->pool, union mail_module_context, 1); + mmail->super = *v; + mail->vlast = &mmail->super; + + v->get_special = pop3_migration_get_special; + MODULE_CONTEXT_SET_SELF(mail, pop3_migration_mail_module, mmail); +} + +static void pop3_migration_mailbox_allocated(struct mailbox *box) +{ + struct mailbox_vfuncs *v = box->vlast; + struct pop3_migration_mailbox *mbox; + + mbox = p_new(box->pool, struct pop3_migration_mailbox, 1); + mbox->module_ctx.super = *v; + box->vlast = &mbox->module_ctx.super; + + MODULE_CONTEXT_SET(box, pop3_migration_storage_module, mbox); +} + +static void pop3_migration_mail_storage_destroy(struct mail_storage *storage) +{ + struct pop3_migration_mail_storage *mstorage = + POP3_MIGRATION_CONTEXT(storage); + + if (mstorage->pop3_box != NULL) + mailbox_free(&mstorage->pop3_box); + if (array_is_created(&mstorage->pop3_uidl_map)) + array_free(&mstorage->pop3_uidl_map); + + if (mstorage->module_ctx.super.destroy != NULL) + mstorage->module_ctx.super.destroy(storage); +} + +static void pop3_migration_mail_storage_created(struct mail_storage *storage) +{ + struct pop3_migration_mail_storage *mstorage; + struct mail_storage_vfuncs *v = storage->vlast; + const char *pop3_box_vname; + + pop3_box_vname = mail_user_plugin_getenv(storage->user, + "pop3_migration_mailbox"); + if (pop3_box_vname == NULL) + return; + + mstorage = p_new(storage->pool, struct pop3_migration_mail_storage, 1); + mstorage->module_ctx.super = *v; + storage->vlast = &mstorage->module_ctx.super; + v->destroy = pop3_migration_mail_storage_destroy; + + mstorage->pop3_box_vname = p_strdup(storage->pool, pop3_box_vname); + + MODULE_CONTEXT_SET(storage, pop3_migration_storage_module, mstorage); +} + +static struct mail_storage_hooks pop3_migration_mail_storage_hooks = { + .mail_allocated = pop3_migration_mail_allocated, + .mailbox_allocated = pop3_migration_mailbox_allocated, + .mail_storage_created = pop3_migration_mail_storage_created +}; + +void pop3_migration_plugin_init(struct module *module) +{ + mail_storage_hooks_add(module, &pop3_migration_mail_storage_hooks); +} + +void pop3_migration_plugin_deinit(void) +{ + mail_storage_hooks_remove(&pop3_migration_mail_storage_hooks); +}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/plugins/pop3-migration/pop3-migration-plugin.h Sun May 20 03:25:04 2012 +0300 @@ -0,0 +1,7 @@ +#ifndef POP3_MIGRATION_PLUGIN_H +#define POP3_MIGRATION_PLUGIN_H + +void pop3_migration_plugin_init(struct module *module); +void pop3_migration_plugin_deinit(void); + +#endif
--- a/src/plugins/quota/quota-count.c Sat May 19 22:40:08 2012 +0300 +++ b/src/plugins/quota/quota-count.c Sun May 20 03:25:04 2012 +0300 @@ -83,7 +83,12 @@ } if (mailbox_list_iter_deinit(&ctx) < 0) ret = -1; - + if (ns->prefix_len > 0 && ret == 0 && + (ns->prefix_len != 6 || strncasecmp(ns->prefix, "INBOX", 5) != 0)) { + /* if the namespace prefix itself exists, count it also */ + const char *name = t_strndup(ns->prefix, ns->prefix_len-1); + ret = quota_count_mailbox(root, ns, name, bytes, count); + } return ret; }
--- a/src/plugins/quota/quota-private.h Sat May 19 22:40:08 2012 +0300 +++ b/src/plugins/quota/quota-private.h Sun May 20 03:25:04 2012 +0300 @@ -133,6 +133,9 @@ /* how many bytes/mails can be saved until limit is reached. (set once, not updated by bytes_used/count_used) */ uint64_t bytes_ceil, count_ceil; + /* how many bytes/mails we are over quota (either *_ceil or *_over + is always zero) */ + uint64_t bytes_over, count_over; struct mail *tmp_mail;
--- a/src/plugins/quota/quota-storage.c Sat May 19 22:40:08 2012 +0300 +++ b/src/plugins/quota/quota-storage.c Sun May 20 03:25:04 2012 +0300 @@ -298,7 +298,15 @@ /* try to look up the size. this works only if it's cached. */ if (qbox->expunge_qt->tmp_mail == NULL) { + /* FIXME: ugly kludge to open the transaction for sync_view. + box->view may not have all the new messages that + sync_notify() notifies about, and those messages would + cause a quota recalculation. */ + struct mail_index_view *box_view = box->view; + if (box->tmp_sync_view != NULL) + box->view = box->tmp_sync_view; qbox->expunge_trans = mailbox_transaction_begin(box, 0); + box->view = box_view; qbox->expunge_qt->tmp_mail = mail_alloc(qbox->expunge_trans, MAIL_FETCH_PHYSICAL_SIZE, NULL);
--- a/src/plugins/quota/quota.c Sat May 19 22:40:08 2012 +0300 +++ b/src/plugins/quota/quota.c Sun May 20 03:25:04 2012 +0300 @@ -723,7 +723,7 @@ return -1; warning = array_append_space(&root_set->warning_rules); - warning->command = i_strdup(p+1); + warning->command = p_strdup(root_set->set->pool, p+1); warning->rule = rule; warning->reverse = reverse; @@ -940,7 +940,7 @@ struct quota_root *const *roots; const char *mailbox_name; unsigned int i, count; - uint64_t bytes_limit, count_limit, current, limit, ceil; + uint64_t bytes_limit, count_limit, current, limit, diff; int ret; ctx->limits_set = TRUE; @@ -964,9 +964,16 @@ QUOTA_NAME_STORAGE_BYTES, ¤t, &limit); if (ret > 0) { - ceil = limit < current ? 0 : limit - current; - if (ctx->bytes_ceil > ceil) - ctx->bytes_ceil = ceil; + if (limit < current) { + ctx->bytes_ceil = 0; + diff = current - limit; + if (ctx->bytes_over < diff) + ctx->bytes_over = diff; + } else { + diff = limit - current; + if (ctx->bytes_ceil > diff) + ctx->bytes_ceil = diff; + } } else if (ret < 0) { ctx->failed = TRUE; return -1; @@ -978,9 +985,16 @@ QUOTA_NAME_MESSAGES, ¤t, &limit); if (ret > 0) { - ceil = limit < current ? 0 : limit - current; - if (ctx->count_ceil > ceil) - ctx->count_ceil = ceil; + if (limit < current) { + ctx->count_ceil = 0; + diff = current - limit; + if (ctx->count_over < diff) + ctx->count_over = diff; + } else { + diff = limit - current; + if (ctx->count_ceil > diff) + ctx->count_ceil = diff; + } } else if (ret < 0) { ctx->failed = TRUE; return -1;
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/plugins/replication/Makefile.am Sun May 20 03:25:04 2012 +0300 @@ -0,0 +1,25 @@ +AM_CPPFLAGS = \ + -I$(top_srcdir)/src/lib \ + -I$(top_srcdir)/src/lib-mail \ + -I$(top_srcdir)/src/lib-imap \ + -I$(top_srcdir)/src/lib-index \ + -I$(top_srcdir)/src/lib-storage \ + -I$(top_srcdir)/src/replication \ + -I$(top_srcdir)/src/plugins/notify + +NOPLUGIN_LDFLAGS = +lib20_replication_plugin_la_LDFLAGS = -module -avoid-version + +module_LTLIBRARIES = \ + lib20_replication_plugin.la + +if DOVECOT_PLUGIN_DEPS +lib20_replication_plugin_la_LIBADD = \ + ../notify/lib15_notify_plugin.la +endif + +lib20_replication_plugin_la_SOURCES = \ + replication-plugin.c + +noinst_HEADERS = \ + replication-plugin.h
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/plugins/replication/replication-plugin.c Sun May 20 03:25:04 2012 +0300 @@ -0,0 +1,371 @@ +/* Copyright (c) 2012 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "str.h" +#include "strescape.h" +#include "fd-set-nonblock.h" +#include "ioloop.h" +#include "network.h" +#include "write-full.h" +#include "mail-user.h" +#include "mail-namespace.h" +#include "mail-storage-private.h" +#include "notify-plugin.h" +#include "replication-common.h" +#include "replication-plugin.h" + +#include <stdlib.h> + +#define REPLICATION_SOCKET_NAME "replication-notify" +#define REPLICATION_FIFO_NAME "replication-notify-fifo" +#define REPLICATION_NOTIFY_DELAY_MSECS 500 +#define REPLICATION_SYNC_TIMEOUT_SECS 10 + +#define REPLICATION_USER_CONTEXT(obj) \ + MODULE_CONTEXT(obj, replication_user_module) + +struct replication_user { + union mail_user_module_context module_ctx; + + const char *socket_path; + + struct timeout *to; + enum replication_priority priority; + unsigned int sync_secs; +}; + +struct replication_mail_txn_context { + struct mail_namespace *ns; + bool new_messages; +}; + +static MODULE_CONTEXT_DEFINE_INIT(replication_user_module, + &mail_user_module_register); +static int fifo_fd; +static bool fifo_failed; +static char *fifo_path; + +static int +replication_fifo_notify(struct mail_user *user, + enum replication_priority priority) +{ + string_t *str; + ssize_t ret; + + if (fifo_failed) + return -1; + if (fifo_fd == -1) { + fifo_fd = open(fifo_path, O_WRONLY); + if (fifo_fd == -1) { + i_error("open(%s) failed: %m", fifo_path); + fifo_failed = TRUE; + return -1; + } + fd_set_nonblock(fifo_fd, TRUE); + } + /* <username> \t <priority> */ + str = t_str_new(256); + str_tabescape_write(str, user->username); + str_append_c(str, '\t'); + switch (priority) { + case REPLICATION_PRIORITY_NONE: + case REPLICATION_PRIORITY_SYNC: + i_unreached(); + case REPLICATION_PRIORITY_LOW: + str_append(str, "low"); + break; + case REPLICATION_PRIORITY_HIGH: + str_append(str, "high"); + break; + } + str_append_c(str, '\n'); + ret = write(fifo_fd, str_data(str), str_len(str)); + if (ret == 0) { + /* busy, try again later */ + return 0; + } + if (ret != (ssize_t)str_len(str)) { + if (ret < 0) + i_error("write(%s) failed: %m", fifo_path); + else + i_error("write(%s) wrote partial data", fifo_path); + if (close(fifo_fd) < 0) + i_error("close(%s) failed: %m", fifo_path); + fifo_fd = -1; + return -1; + } + return 1; +} + +static void replication_notify_now(struct mail_user *user) +{ + struct replication_user *ruser = REPLICATION_USER_CONTEXT(user); + int ret; + + i_assert(ruser->priority != REPLICATION_PRIORITY_NONE); + i_assert(ruser->priority != REPLICATION_PRIORITY_SYNC); + + if ((ret = replication_fifo_notify(user, ruser->priority)) < 0 && + !fifo_failed) { + /* retry once, in case replication server was restarted */ + ret = replication_fifo_notify(user, ruser->priority); + } + if (ret != 0) { + timeout_remove(&ruser->to); + ruser->priority = REPLICATION_PRIORITY_NONE; + } +} + +static int replication_notify_sync(struct mail_user *user) +{ + struct replication_user *ruser = REPLICATION_USER_CONTEXT(user); + string_t *str; + char buf[1024]; + int fd; + ssize_t ret; + + fd = net_connect_unix(ruser->socket_path); + if (fd == -1) { + i_error("net_connect_unix(%s) failed: %m", ruser->socket_path); + return -1; + } + net_set_nonblock(fd, FALSE); + + /* <username> \t "sync" */ + str = t_str_new(256); + str_tabescape_write(str, user->username); + str_append(str, "\tsync\n"); + alarm(ruser->sync_secs); + if (write_full(fd, str_data(str), str_len(str)) < 0) { + i_error("write(%s) failed: %m", ruser->socket_path); + ret = -1; + } else { + /* + | - */ + ret = read(fd, buf, sizeof(buf)); + if (ret < 0) { + if (ret != EINTR) { + i_error("read(%s) failed: %m", + ruser->socket_path); + } else { + i_warning("replication(%s): Sync failure: " + "Timeout in %u secs", + user->username, ruser->sync_secs); + } + } else if (ret == 0) { + i_error("read(%s) failed: EOF", ruser->socket_path); + ret = -1; + } else if (buf[0] == '+') { + /* success */ + ret = 0; + } else if (buf[0] == '-') { + /* failure */ + if (buf[ret-1] == '\n') ret--; + i_warning("replication(%s): Sync failure: %s", + user->username, t_strndup(buf+1, ret-1)); + ret = -1; + } else { + i_warning("replication(%s): " + "Remote sent invalid input: %s", + user->username, t_strndup(buf, ret)); + } + } + alarm(0); + if (close(fd) < 0) + i_error("close(%s) failed: %m", ruser->socket_path); + return ret; +} + +static void replication_notify(struct mail_namespace *ns, + enum replication_priority priority) +{ + struct replication_user *ruser; + + if (ns->user->dsyncing) { + /* we're running dsync, which means that the remote is telling + us about a change. don't trigger a replication back to it */ + return; + } + + if (ns->owner == NULL) { + /* public namespace. we can't handle this for now. */ + return; + } + ruser = REPLICATION_USER_CONTEXT(ns->owner); + + if (priority == REPLICATION_PRIORITY_SYNC) { + if (replication_notify_sync(ns->owner) == 0) { + timeout_remove(&ruser->to); + ruser->priority = REPLICATION_PRIORITY_NONE; + return; + } + /* sync replication failed, try as "high" via fifo */ + priority = REPLICATION_PRIORITY_HIGH; + } + + if (ruser->priority < priority) + ruser->priority = priority; + if (ruser->to == NULL) { + ruser->to = timeout_add(REPLICATION_NOTIFY_DELAY_MSECS, + replication_notify_now, ns->owner); + } +} + +static void * +replication_mail_transaction_begin(struct mailbox_transaction_context *t) +{ + struct replication_mail_txn_context *ctx; + + ctx = i_new(struct replication_mail_txn_context, 1); + ctx->ns = mailbox_get_namespace(t->box); + return ctx; +} + +static void replication_mail_save(void *txn, struct mail *mail ATTR_UNUSED) +{ + struct replication_mail_txn_context *ctx = + (struct replication_mail_txn_context *)txn; + + ctx->new_messages = TRUE; +} + +static void replication_mail_copy(void *txn, struct mail *src ATTR_UNUSED, + struct mail *dst ATTR_UNUSED) +{ + struct replication_mail_txn_context *ctx = + (struct replication_mail_txn_context *)txn; + + ctx->new_messages = TRUE; +} + +static void +replication_mail_transaction_commit(void *txn, + struct mail_transaction_commit_changes *changes) +{ + struct replication_mail_txn_context *ctx = + (struct replication_mail_txn_context *)txn; + struct replication_user *ruser = + REPLICATION_USER_CONTEXT(ctx->ns->user); + enum replication_priority priority; + + if (ctx->new_messages || changes->changed) { + priority = !ctx->new_messages ? REPLICATION_PRIORITY_LOW : + ruser->sync_secs == 0 ? REPLICATION_PRIORITY_HIGH : + REPLICATION_PRIORITY_SYNC; + replication_notify(ctx->ns, priority); + } + i_free(ctx); +} + +static void replication_mailbox_create(struct mailbox *box) +{ + replication_notify(mailbox_get_namespace(box), + REPLICATION_PRIORITY_LOW); +} + +static void +replication_mailbox_delete_commit(void *txn ATTR_UNUSED, + struct mailbox *box) +{ + replication_notify(mailbox_get_namespace(box), + REPLICATION_PRIORITY_LOW); +} + +static void +replication_mailbox_rename(struct mailbox *src ATTR_UNUSED, + struct mailbox *dest, + bool rename_children ATTR_UNUSED) +{ + replication_notify(mailbox_get_namespace(dest), + REPLICATION_PRIORITY_LOW); +} + +static void replication_mailbox_set_subscribed(struct mailbox *box, + bool subscribed ATTR_UNUSED) +{ + replication_notify(mailbox_get_namespace(box), + REPLICATION_PRIORITY_LOW); +} + +static void replication_user_deinit(struct mail_user *user) +{ + struct replication_user *ruser = REPLICATION_USER_CONTEXT(user); + + if (ruser->to != NULL) { + replication_notify_now(user); + if (ruser->to != NULL) { + i_warning("%s: Couldn't send final notification " + "due to fifo being busy", fifo_path); + timeout_remove(&ruser->to); + } + } + + ruser->module_ctx.super.deinit(user); +} + +static void replication_user_created(struct mail_user *user) +{ + struct mail_user_vfuncs *v = user->vlast; + struct replication_user *ruser; + const char *value; + + ruser = p_new(user->pool, struct replication_user, 1); + ruser->module_ctx.super = *v; + user->vlast = &ruser->module_ctx.super; + v->deinit = replication_user_deinit; + MODULE_CONTEXT_SET(user, replication_user_module, ruser); + + if (fifo_path == NULL) { + /* we'll assume that all users have the same base_dir. + they really should. */ + fifo_path = i_strconcat(user->set->base_dir, + "/"REPLICATION_FIFO_NAME, NULL); + } + ruser->socket_path = p_strconcat(user->pool, user->set->base_dir, + "/"REPLICATION_SOCKET_NAME, NULL); + value = mail_user_plugin_getenv(user, "replication_sync_timeout"); + if (value != NULL && str_to_uint(value, &ruser->sync_secs) < 0) { + i_error("replication(%s): " + "Invalid replication_sync_timeout value: %s", + user->username, value); + } +} + +static const struct notify_vfuncs replication_vfuncs = { + .mail_transaction_begin = replication_mail_transaction_begin, + .mail_save = replication_mail_save, + .mail_copy = replication_mail_copy, + .mail_transaction_commit = replication_mail_transaction_commit, + .mailbox_create = replication_mailbox_create, + .mailbox_delete_commit = replication_mailbox_delete_commit, + .mailbox_rename = replication_mailbox_rename, + .mailbox_set_subscribed = replication_mailbox_set_subscribed +}; + +static struct notify_context *replication_ctx; + +static struct mail_storage_hooks replication_mail_storage_hooks = { + .mail_user_created = replication_user_created +}; + +void replication_plugin_init(struct module *module) +{ + fifo_fd = -1; + replication_ctx = notify_register(&replication_vfuncs); + mail_storage_hooks_add(module, &replication_mail_storage_hooks); +} + +void replication_plugin_deinit(void) +{ + if (fifo_fd != -1) { + if (close(fifo_fd) < 0) + i_error("close(%s) failed: %m", fifo_path); + fifo_fd = -1; + } + i_free_and_null(fifo_path); + + mail_storage_hooks_remove(&replication_mail_storage_hooks); + notify_unregister(replication_ctx); +} + +const char *replication_plugin_dependencies[] = { "notify", NULL };
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/plugins/replication/replication-plugin.h Sun May 20 03:25:04 2012 +0300 @@ -0,0 +1,9 @@ +#ifndef REPLICATION_PLUGIN_H +#define REPLICATION_PLUGIN_H + +extern const char *replication_plugin_dependencies[]; + +void replication_plugin_init(struct module *module); +void replication_plugin_deinit(void); + +#endif
--- a/src/plugins/stats/stats-connection.c Sat May 19 22:40:08 2012 +0300 +++ b/src/plugins/stats/stats-connection.c Sun May 20 03:25:04 2012 +0300 @@ -14,8 +14,24 @@ int fd; char *path; + + bool open_failed; }; +static bool stats_connection_open(struct stats_connection *conn) +{ + if (conn->open_failed) + return FALSE; + + conn->fd = open(conn->path, O_WRONLY); + if (conn->fd == -1) { + i_error("stats: open(%s) failed: %m", conn->path); + conn->open_failed = TRUE; + return FALSE; + } + return TRUE; +} + struct stats_connection * stats_connection_create(const char *path) { @@ -24,9 +40,7 @@ conn = i_new(struct stats_connection, 1); conn->refcount = 1; conn->path = i_strdup(path); - conn->fd = open(path, O_WRONLY); - if (conn->fd == -1) - i_error("stats: open(%s) failed: %m", path); + stats_connection_open(conn); return conn; } @@ -57,8 +71,10 @@ static bool pipe_warned = FALSE; ssize_t ret; - if (conn->fd == -1) - return; + if (conn->fd == -1) { + if (!stats_connection_open(conn)) + return; + } if (str_len(str) > PIPE_BUF && !pipe_warned) { i_warning("stats update sent more bytes that PIPE_BUF " @@ -69,11 +85,13 @@ ret = write(conn->fd, str_data(str), str_len(str)); if (ret != (ssize_t)str_len(str)) { - if (ret < 0) - i_error("write(%s) failed: %m", conn->path); - else if ((size_t)ret != str_len(str)) + if (ret < 0) { + /* don't log EPIPE errors. they can happen when + Dovecot is stopped. */ + if (errno != EPIPE) + i_error("write(%s) failed: %m", conn->path); + } else if ((size_t)ret != str_len(str)) i_error("write(%s) wrote partial update", conn->path); - /* this shouldn't happen, just stop sending updates */ if (close(conn->fd) < 0) i_error("close(%s) failed: %m", conn->path); conn->fd = -1;
--- a/src/plugins/stats/stats-plugin.c Sat May 19 22:40:08 2012 +0300 +++ b/src/plugins/stats/stats-plugin.c Sun May 20 03:25:04 2012 +0300 @@ -18,7 +18,9 @@ /* If session isn't refreshed every 15 minutes, it's dropped. Must be smaller than MAIL_SESSION_IDLE_TIMEOUT_MSECS in stats server */ #define SESSION_STATS_FORCE_REFRESH_SECS (5*60) +#define REFRESH_CHECK_INTERVAL 100 #define MAIL_STATS_SOCKET_NAME "stats-mail" +#define PROC_IO_PATH "/proc/self/io" #define USECS_PER_SEC 1000000 @@ -42,6 +44,9 @@ static MODULE_CONTEXT_DEFINE_INIT(stats_storage_module, &mail_storage_module_register); +static bool proc_io_disabled = FALSE; +static int proc_io_fd = -1; + static struct stats_connection *global_stats_conn = NULL; static struct mail_user *stats_global_user = NULL; static unsigned int stats_user_count = 0; @@ -105,44 +110,50 @@ return 0; } +static int process_io_open(void) +{ + if (proc_io_fd == -1) { + if (proc_io_disabled) + return -1; + proc_io_fd = open(PROC_IO_PATH, O_RDONLY); + if (proc_io_fd == -1) { + if (errno != ENOENT) + i_error("open(%s) failed: %m", PROC_IO_PATH); + proc_io_disabled = TRUE; + return -1; + } + } + return proc_io_fd; +} + static void process_read_io_stats(struct mail_stats *stats) { - const char *path = "/proc/self/io"; - static bool io_disabled = FALSE; char buf[1024]; int fd, ret; - if (io_disabled) + if ((fd = process_io_open()) == -1) return; - fd = open(path, O_RDONLY); - if (fd == -1) { - if (errno != ENOENT) - i_error("open(%s) failed: %m", path); - io_disabled = TRUE; - return; - } - ret = read(fd, buf, sizeof(buf)); + ret = pread(fd, buf, sizeof(buf), 0); if (ret <= 0) { if (ret == -1) - i_error("read(%s) failed: %m", path); + i_error("read(%s) failed: %m", PROC_IO_PATH); else - i_error("read(%s) returned EOF", path); + i_error("read(%s) returned EOF", PROC_IO_PATH); } else if (ret == sizeof(buf)) { /* just shouldn't happen.. */ - i_error("%s is larger than expected", path); - io_disabled = TRUE; + i_error("%s is larger than expected", PROC_IO_PATH); + proc_io_disabled = TRUE; } else { buf[ret] = '\0'; T_BEGIN { if (process_io_buffer_parse(buf, stats) < 0) { - i_error("Invalid input in file %s", path); - io_disabled = TRUE; + i_error("Invalid input in file %s", + PROC_IO_PATH); + proc_io_disabled = TRUE; } } T_END; } - if (close(fd) < 0) - i_error("close(%s) failed: %m", path); } void mail_stats_get(struct stats_user *suser, struct mail_stats *stats_r) @@ -165,77 +176,6 @@ user_trans_stats_get(suser, &stats_r->trans_stats); } -static struct mailbox_transaction_context * -stats_transaction_begin(struct mailbox *box, - enum mailbox_transaction_flags flags) -{ - struct stats_user *suser = STATS_USER_CONTEXT(box->storage->user); - struct stats_mailbox *sbox = STATS_CONTEXT(box); - struct mailbox_transaction_context *trans; - struct stats_transaction_context *strans; - - trans = sbox->module_ctx.super.transaction_begin(box, flags); - trans->stats_track = TRUE; - - strans = i_new(struct stats_transaction_context, 1); - strans->trans = trans; - DLLIST_PREPEND(&suser->transactions, strans); - - MODULE_CONTEXT_SET(trans, stats_storage_module, strans); - return trans; -} - -static void stats_transaction_free(struct stats_user *suser, - struct stats_transaction_context *strans) -{ - DLLIST_REMOVE(&suser->transactions, strans); - - trans_stats_add(&suser->session_stats.trans_stats, - &strans->trans->stats); -} - -static int -stats_transaction_commit(struct mailbox_transaction_context *ctx, - struct mail_transaction_commit_changes *changes_r) -{ - struct stats_transaction_context *strans = STATS_CONTEXT(ctx); - struct stats_mailbox *sbox = STATS_CONTEXT(ctx->box); - struct stats_user *suser = STATS_USER_CONTEXT(ctx->box->storage->user); - - stats_transaction_free(suser, strans); - return sbox->module_ctx.super.transaction_commit(ctx, changes_r); -} - -static void -stats_transaction_rollback(struct mailbox_transaction_context *ctx) -{ - struct stats_transaction_context *strans = STATS_CONTEXT(ctx); - struct stats_mailbox *sbox = STATS_CONTEXT(ctx->box); - struct stats_user *suser = STATS_USER_CONTEXT(ctx->box->storage->user); - - stats_transaction_free(suser, strans); - sbox->module_ctx.super.transaction_rollback(ctx); -} - -static void stats_mailbox_allocated(struct mailbox *box) -{ - struct mailbox_vfuncs *v = box->vlast; - struct stats_mailbox *sbox; - struct stats_user *suser = STATS_USER_CONTEXT(box->storage->user); - - if (suser == NULL) - return; - - sbox = p_new(box->pool, struct stats_mailbox, 1); - sbox->module_ctx.super = *v; - box->vlast = &sbox->module_ctx.super; - - v->transaction_begin = stats_transaction_begin; - v->transaction_commit = stats_transaction_commit; - v->transaction_rollback = stats_transaction_rollback; - MODULE_CONTEXT_SET(box, stats_storage_module, sbox); -} - static void stats_io_activate(void *context) { struct mail_user *user = context; @@ -358,8 +298,9 @@ return FALSE; } -static bool session_stats_need_send(struct stats_user *suser, bool *changed_r, - unsigned int *to_next_secs_r) +static bool +session_stats_need_send(struct stats_user *suser, time_t now, + bool *changed_r, unsigned int *to_next_secs_r) { unsigned int diff; @@ -374,7 +315,7 @@ *changed_r = FALSE; if (!suser->session_sent_duplicate) { - if (suser->last_session_update != ioloop_time) { + if (suser->last_session_update != now) { /* send one duplicate notification so stats reader knows that this session is idle now */ return TRUE; @@ -383,7 +324,7 @@ return FALSE; } - diff = ioloop_time - suser->last_session_update; + diff = now - suser->last_session_update; if (diff < SESSION_STATS_FORCE_REFRESH_SECS) { *to_next_secs_r = SESSION_STATS_FORCE_REFRESH_SECS - diff; return FALSE; @@ -395,11 +336,12 @@ { struct stats_user *suser = STATS_USER_CONTEXT(user); unsigned int to_next_secs; + time_t now = time(NULL); bool changed; - if (session_stats_need_send(suser, &changed, &to_next_secs)) { + if (session_stats_need_send(suser, now, &changed, &to_next_secs)) { suser->session_sent_duplicate = !changed; - suser->last_session_update = ioloop_time; + suser->last_session_update = now; suser->last_sent_session_stats = suser->session_stats; stats_connection_send_session(suser->stats_conn, user, &suser->session_stats); @@ -412,6 +354,103 @@ session_stats_refresh_timeout, user); } +static struct mailbox_transaction_context * +stats_transaction_begin(struct mailbox *box, + enum mailbox_transaction_flags flags) +{ + struct stats_user *suser = STATS_USER_CONTEXT(box->storage->user); + struct stats_mailbox *sbox = STATS_CONTEXT(box); + struct mailbox_transaction_context *trans; + struct stats_transaction_context *strans; + + trans = sbox->module_ctx.super.transaction_begin(box, flags); + trans->stats_track = TRUE; + + strans = i_new(struct stats_transaction_context, 1); + strans->trans = trans; + DLLIST_PREPEND(&suser->transactions, strans); + + MODULE_CONTEXT_SET(trans, stats_storage_module, strans); + return trans; +} + +static void stats_transaction_free(struct stats_user *suser, + struct stats_transaction_context *strans) +{ + DLLIST_REMOVE(&suser->transactions, strans); + + trans_stats_add(&suser->session_stats.trans_stats, + &strans->trans->stats); +} + +static int +stats_transaction_commit(struct mailbox_transaction_context *ctx, + struct mail_transaction_commit_changes *changes_r) +{ + struct stats_transaction_context *strans = STATS_CONTEXT(ctx); + struct stats_mailbox *sbox = STATS_CONTEXT(ctx->box); + struct stats_user *suser = STATS_USER_CONTEXT(ctx->box->storage->user); + + stats_transaction_free(suser, strans); + return sbox->module_ctx.super.transaction_commit(ctx, changes_r); +} + +static void +stats_transaction_rollback(struct mailbox_transaction_context *ctx) +{ + struct stats_transaction_context *strans = STATS_CONTEXT(ctx); + struct stats_mailbox *sbox = STATS_CONTEXT(ctx->box); + struct stats_user *suser = STATS_USER_CONTEXT(ctx->box->storage->user); + + stats_transaction_free(suser, strans); + sbox->module_ctx.super.transaction_rollback(ctx); +} + +static bool stats_search_next_nonblock(struct mail_search_context *ctx, + struct mail **mail_r, bool *tryagain_r) +{ + struct stats_mailbox *sbox = STATS_CONTEXT(ctx->transaction->box); + struct mail_user *user = ctx->transaction->box->storage->user; + struct stats_user *suser = STATS_USER_CONTEXT(user); + bool ret; + + ret = sbox->module_ctx.super. + search_next_nonblock(ctx, mail_r, tryagain_r); + if (!ret && !*tryagain_r) { + /* end of search */ + return FALSE; + } + + if (*tryagain_r || + ++suser->refresh_check_counter % REFRESH_CHECK_INTERVAL == 0) { + /* a) retrying, so this is a long running search. + b) we've returned enough matches */ + if (time(NULL) != suser->last_session_update) + session_stats_refresh(user); + } + return ret; +} + +static void stats_mailbox_allocated(struct mailbox *box) +{ + struct mailbox_vfuncs *v = box->vlast; + struct stats_mailbox *sbox; + struct stats_user *suser = STATS_USER_CONTEXT(box->storage->user); + + if (suser == NULL) + return; + + sbox = p_new(box->pool, struct stats_mailbox, 1); + sbox->module_ctx.super = *v; + box->vlast = &sbox->module_ctx.super; + + v->transaction_begin = stats_transaction_begin; + v->transaction_commit = stats_transaction_commit; + v->transaction_rollback = stats_transaction_rollback; + v->search_next_nonblock = stats_search_next_nonblock; + MODULE_CONTEXT_SET(box, stats_storage_module, sbox); +} + static void session_stats_refresh_timeout(struct mail_user *user) { if (stats_global_user != NULL) @@ -428,7 +467,7 @@ if (stats_global_user == NULL) stats_add_session(user); - last_update_secs = ioloop_time - suser->last_session_update; + last_update_secs = time(NULL) - suser->last_session_update; if (last_update_secs >= suser->refresh_secs) { if (stats_global_user != NULL) stats_add_session(user); @@ -530,7 +569,7 @@ suser->stats_conn = global_stats_conn; guid_128_generate(suser->session_guid); - suser->last_session_update = ioloop_time; + suser->last_session_update = time(NULL); suser->ioloop_ctx = ioloop_ctx; io_loop_context_add_callbacks(ioloop_ctx,
--- a/src/plugins/stats/stats-plugin.h Sat May 19 22:40:08 2012 +0300 +++ b/src/plugins/stats/stats-plugin.h Sun May 20 03:25:04 2012 +0300 @@ -6,6 +6,8 @@ #include "mail-user.h" #include "mail-storage-private.h" +#include <sys/time.h> + #define STATS_USER_CONTEXT(obj) \ MODULE_CONTEXT(obj, stats_user_module) @@ -33,6 +35,7 @@ unsigned int refresh_secs; bool track_commands; + unsigned int refresh_check_counter; /* current session statistics */ struct mail_stats session_stats;
--- a/src/plugins/trash/trash-plugin.c Sat May 19 22:40:08 2012 +0300 +++ b/src/plugins/trash/trash-plugin.c Sun May 20 03:25:04 2012 +0300 @@ -97,13 +97,15 @@ } static int trash_try_clean_mails(struct quota_transaction_context *ctx, - uint64_t size_needed) + uint64_t size_needed, + unsigned int count_needed) { struct trash_user *tuser = TRASH_USER_CONTEXT(ctx->quota->user); struct trash_mailbox *trashes; unsigned int i, j, count, oldest_idx; time_t oldest, received = 0; - uint64_t size, size_expunged = 0, expunged_count = 0; + uint64_t size, size_expunged = 0; + unsigned int expunged_count = 0; int ret = 0; trashes = array_get_modifiable(&tuser->trash_boxes, &count); @@ -139,7 +141,8 @@ mail_expunge(trashes[oldest_idx].mail); expunged_count++; size_expunged += size; - if (size_expunged >= size_needed) + if (size_expunged >= size_needed && + expunged_count >= count_needed) break; trashes[oldest_idx].mail = NULL; } else { @@ -158,9 +161,11 @@ trash->mail = NULL; (void)mailbox_search_deinit(&trash->search_ctx); - if (size_expunged >= size_needed) + if (size_expunged >= size_needed && + expunged_count >= count_needed) { (void)mailbox_transaction_commit(&trash->trans); - else { + (void)mailbox_sync(trash->box, 0); + } else { /* couldn't get enough space, don't expunge anything */ mailbox_transaction_rollback(&trash->trans); } @@ -175,14 +180,33 @@ (unsigned long long)size_needed, (unsigned long long)size_expunged); } - return FALSE; + return 0; + } + if (expunged_count < count_needed) { + if (ctx->quota->user->mail_debug) { + i_debug("trash plugin: Failed to remove enough messages " + "(needed %u messages, expunged only %u messages)", + count_needed, expunged_count); + } + return 0; } - ctx->bytes_used = ctx->bytes_used > (int64_t)size_expunged ? - ctx->bytes_used - size_expunged : 0; - ctx->count_used = ctx->count_used > (int64_t)expunged_count ? - ctx->count_used - expunged_count : 0; - return TRUE; + if (ctx->bytes_over > 0) { + /* user is over quota. drop the over-bytes first. */ + i_assert(ctx->bytes_over <= size_expunged); + size_expunged -= ctx->bytes_over; + ctx->bytes_over = 0; + } + if (ctx->count_over > 0) { + /* user is over quota. drop the over-count first. */ + i_assert(ctx->count_over <= expunged_count); + expunged_count -= ctx->count_over; + ctx->count_over = 0; + } + + ctx->bytes_ceil += size_expunged; + ctx->count_ceil += expunged_count; + return 1; } static int @@ -210,11 +234,11 @@ } /* not enough space. try deleting some from mailbox. */ - ret = trash_try_clean_mails(ctx, size); + ret = trash_try_clean_mails(ctx, size + ctx->bytes_over, + 1 + ctx->count_over); if (ret <= 0) return 0; } - return 0; }
--- a/src/plugins/virtual/virtual-mail.c Sat May 19 22:40:08 2012 +0300 +++ b/src/plugins/virtual/virtual-mail.c Sun May 20 03:25:04 2012 +0300 @@ -117,8 +117,8 @@ i_assert(!saving); - mail_index_lookup_ext(mail->box->view, seq, mbox->virtual_ext_id, - &data, &expunged); + mail_index_lookup_ext(mail->transaction->view, seq, + mbox->virtual_ext_id, &data, &expunged); vrec = data; bbox = virtual_backend_box_lookup(mbox, vrec->mailbox_id); @@ -131,7 +131,7 @@ vmail->imail.data.seq = seq; mail->seq = seq; - mail_index_lookup_uid(mail->box->view, seq, &mail->uid); + mail_index_lookup_uid(mail->transaction->view, seq, &mail->uid); if (!vmail->lost) { mail->expunged = vmail->backend_mail->expunged; @@ -148,7 +148,7 @@ { uint32_t seq; - if (!mail_index_lookup_seq(mail->box->view, uid, &seq)) + if (!mail_index_lookup_seq(mail->transaction->view, uid, &seq)) return FALSE; virtual_mail_set_seq(mail, seq, FALSE);
--- a/src/pop3-login/client.c Sat May 19 22:40:08 2012 +0300 +++ b/src/pop3-login/client.c Sun May 20 03:25:04 2012 +0300 @@ -56,6 +56,9 @@ args_ok = FALSE; else client->common.remote_port = remote_port; + } else if (strncasecmp(*tmp, "SESSION=", 8) == 0) { + client->common.session_id = + p_strdup(client->common.pool, *tmp + 8); } else if (strncasecmp(*tmp, "TTL=", 4) == 0) { if (str_to_uint(*tmp + 4, &client->common.proxy_ttl) < 0) args_ok = FALSE;
--- a/src/pop3-login/pop3-proxy.c Sat May 19 22:40:08 2012 +0300 +++ b/src/pop3-login/pop3-proxy.c Sun May 20 03:25:04 2012 +0300 @@ -41,9 +41,10 @@ if (client->proxy_xclient) { /* remote supports XCLIENT, send it */ (void)o_stream_send_str(output, t_strdup_printf( - "XCLIENT ADDR=%s PORT=%u TTL=%u\r\n", + "XCLIENT ADDR=%s PORT=%u SESSION=%s TTL=%u\r\n", net_ip2addr(&client->common.ip), client->common.remote_port, + client_get_session_id(&client->common), client->common.proxy_ttl - 1)); client->common.proxy_state = POP3_PROXY_XCLIENT; } else {
--- a/src/pop3/Makefile.am Sat May 19 22:40:08 2012 +0300 +++ b/src/pop3/Makefile.am Sun May 20 03:25:04 2012 +0300 @@ -19,12 +19,15 @@ ../lib-storage/mail-search-parser-imap.o endif -libs = \ +pop3_LDADD = \ + $(unused_objects) \ $(LIBDOVECOT_STORAGE) \ - $(unused_objects) - -pop3_LDADD = $(libs) $(LIBDOVECOT) $(MODULE_LIBS) -pop3_DEPENDENCIES = $(libs) $(LIBDOVECOT_DEPS) + $(LIBDOVECOT) \ + $(MODULE_LIBS) +pop3_DEPENDENCIES = \ + $(unused_objects) \ + $(LIBDOVECOT_STORAGE_DEPS) \ + $(LIBDOVECOT_DEPS) pop3_SOURCES = \ main.c \
--- a/src/pop3/main.c Sat May 19 22:40:08 2012 +0300 +++ b/src/pop3/main.c Sun May 20 03:25:04 2012 +0300 @@ -116,7 +116,8 @@ if (set->verbose_proctitle) verbose_proctitle = TRUE; - client = client_create(fd_in, fd_out, mail_user, user, set); + client = client_create(fd_in, fd_out, input->session_id, + mail_user, user, set); if (client != NULL) T_BEGIN { client_add_input(client, input_buf); } T_END; @@ -164,6 +165,7 @@ input.remote_ip = client->auth_req.remote_ip; input.username = username; input.userdb_fields = extra_fields; + input.session_id = client->session_id; buffer_create_const_data(&input_buf, client->data, client->auth_req.data_size);
--- a/src/pop3/pop3-client.c Sat May 19 22:40:08 2012 +0300 +++ b/src/pop3/pop3-client.c Sun May 20 03:25:04 2012 +0300 @@ -6,6 +6,7 @@ #include "network.h" #include "istream.h" #include "ostream.h" +#include "crc32.h" #include "str.h" #include "llist.h" #include "hostpid.h" @@ -192,6 +193,7 @@ array_free(&msgnum_to_seq_map); return ret; } + i_assert(msgnum == client->messages_count); client->trans = t; client->message_sizes = @@ -274,7 +276,8 @@ return mask; } -struct client *client_create(int fd_in, int fd_out, struct mail_user *user, +struct client *client_create(int fd_in, int fd_out, const char *session_id, + struct mail_user *user, struct mail_storage_service_user *service_user, const struct pop3_settings *set) { @@ -297,6 +300,7 @@ client->service_user = service_user; client->v = pop3_client_vfuncs; client->set = set; + client->session_id = p_strdup(pool, session_id); client->fd_in = fd_in; client->fd_out = fd_out; client->input = i_stream_create_fd(fd_in, MAX_INBUF_SIZE, FALSE); @@ -350,15 +354,20 @@ return NULL; } - if (var_has_key(set->pop3_logout_format, 'u', "uidl_change") && - client->messages_count > 0) - client->message_uidl_hashes_save = TRUE; - client->uidl_keymask = parse_uidl_keymask(client->mail_set->pop3_uidl_format); if (client->uidl_keymask == 0) i_fatal("Invalid pop3_uidl_format"); + if (var_has_key(set->pop3_logout_format, 'u', "uidl_change")) { + /* logging uidl_change. we need hashes of the UIDLs */ + client->message_uidls_save = TRUE; + } else if (strcmp(set->pop3_uidl_duplicates, "allow") != 0) { + /* UIDL duplicates aren't allowed, so we'll need to + keep track of them */ + client->message_uidls_save = TRUE; + } + if (!set->pop3_no_flag_updates && client->messages_count > 0) client->seen_bitmask = i_malloc(MSGS_BITMASK_SIZE(client)); @@ -381,12 +390,8 @@ uint32_t i, old_hash, new_hash; unsigned int old_msg_count, new_msg_count; - if (client->message_uidl_hashes == NULL) { - /* UIDL command not given or %u not actually used in format */ - return ""; - } - if (client->message_uidl_hashes_save) { - /* UIDL command not finished */ + if (client->message_uidls == NULL) { + /* UIDL command not given */ return ""; } @@ -394,18 +399,18 @@ old_msg_count = client->lowest_retr_pop3_msn > 0 ? client->lowest_retr_pop3_msn - 1 : client->messages_count; for (i = 0, old_hash = 0; i < old_msg_count; i++) - old_hash ^= client->message_uidl_hashes[i]; + old_hash ^= crc32_str(client->message_uidls[i]); /* assume all except deleted messages were sent to POP3 client */ if (!client->deleted) { for (i = 0, new_hash = 0; i < client->messages_count; i++) - new_hash ^= client->message_uidl_hashes[i]; + new_hash ^= crc32_str(client->message_uidls[i]); } else { for (i = 0, new_hash = 0; i < client->messages_count; i++) { if (client->deleted_bitmask[i / CHAR_BIT] & (1 << (i % CHAR_BIT))) continue; - new_hash ^= client->message_uidl_hashes[i]; + new_hash ^= crc32_str(client->message_uidls[i]); } } @@ -432,6 +437,7 @@ { 'i', NULL, "input" }, { 'o', NULL, "output" }, { 'u', NULL, "uidl_change" }, + { '\0', NULL, "session" }, { '\0', NULL, NULL } }; struct var_expand_table *tab; @@ -449,7 +455,12 @@ tab[6].value = dec2str(client->total_size); tab[7].value = dec2str(client->input->v_offset); tab[8].value = dec2str(client->output->offset); - tab[9].value = client_build_uidl_change_string(client); + if (var_has_key(client->set->pop3_logout_format, + tab[9].key, tab[9].long_key)) + tab[9].value = client_build_uidl_change_string(client); + else + tab[9].value = ""; + tab[10].value = client->session_id; str = t_str_new(128); var_expand(str, client->set->pop3_logout_format, tab); @@ -507,8 +518,9 @@ } mail_user_unref(&client->user); + if (client->uidl_pool != NULL) + pool_unref(&client->uidl_pool); i_free(client->message_sizes); - i_free(client->message_uidl_hashes); i_free(client->deleted_bitmask); i_free(client->seen_bitmask); i_free(client->msgnum_to_seq_map);
--- a/src/pop3/pop3-client.h Sat May 19 22:40:08 2012 +0300 +++ b/src/pop3/pop3-client.h Sun May 20 03:25:04 2012 +0300 @@ -24,6 +24,7 @@ struct client *prev, *next; struct pop3_client_vfuncs v; + const char *session_id; int fd_in, fd_out; struct io *io; @@ -62,7 +63,7 @@ unsigned int retr_count; /* [msgnum] */ - uint32_t *message_uidl_hashes; + const char **message_uidls; uoff_t *message_sizes; /* [msgnum/8] & msgnum%8 */ unsigned char *deleted_bitmask; @@ -71,6 +72,7 @@ /* settings: */ const struct pop3_settings *set; const struct mail_storage_settings *mail_set; + pool_t uidl_pool; enum uidl_keys uidl_keymask; /* Module-specific contexts. */ @@ -80,7 +82,7 @@ unsigned int deleted:1; unsigned int waiting_input:1; unsigned int anvil_sent:1; - unsigned int message_uidl_hashes_save:1; + unsigned int message_uidls_save:1; }; struct pop3_module_register { @@ -98,7 +100,8 @@ /* Create new client with specified input/output handles. socket specifies if the handle is a socket. */ -struct client *client_create(int fd_in, int fd_out, struct mail_user *user, +struct client *client_create(int fd_in, int fd_out, const char *session_id, + struct mail_user *user, struct mail_storage_service_user *service_user, const struct pop3_settings *set); void client_destroy(struct client *client, const char *reason);
--- a/src/pop3/pop3-commands.c Sat May 19 22:40:08 2012 +0300 +++ b/src/pop3/pop3-commands.c Sun May 20 03:25:04 2012 +0300 @@ -4,8 +4,8 @@ #include "array.h" #include "istream.h" #include "ostream.h" +#include "hash.h" #include "str.h" -#include "crc32.h" #include "var-expand.h" #include "message-size.h" #include "mail-storage.h" @@ -550,62 +550,9 @@ bool list_all; }; -static bool pop3_get_uid(struct client *client, struct cmd_uidl_context *ctx, - struct var_expand_table *tab, string_t *str) -{ - char uid_str[MAX_INT_STRLEN]; - const char *uidl; - - if (mail_get_special(ctx->mail, MAIL_FETCH_UIDL_BACKEND, &uidl) == 0 && - *uidl != '\0') { - str_append(str, uidl); - return TRUE; - } - - if (client->set->pop3_reuse_xuidl && - mail_get_first_header(ctx->mail, "X-UIDL", &uidl) > 0) { - str_append(str, uidl); - return FALSE; - } - - if ((client->uidl_keymask & UIDL_UID) != 0) { - i_snprintf(uid_str, sizeof(uid_str), "%u", - ctx->mail->uid); - tab[1].value = uid_str; - } - if ((client->uidl_keymask & UIDL_MD5) != 0) { - if (mail_get_special(ctx->mail, MAIL_FETCH_HEADER_MD5, - &tab[2].value) < 0 || - *tab[2].value == '\0') { - /* broken */ - i_fatal("UIDL: Header MD5 not found " - "(pop3_uidl_format=%%m not supported by storage?)"); - } - } - if ((client->uidl_keymask & UIDL_FILE_NAME) != 0) { - if (mail_get_special(ctx->mail, - MAIL_FETCH_UIDL_FILE_NAME, - &tab[3].value) < 0 || - *tab[3].value == '\0') { - /* broken */ - i_fatal("UIDL: File name not found " - "(pop3_uidl_format=%%f not supported by storage?)"); - } - } - if ((client->uidl_keymask & UIDL_GUID) != 0) { - if (mail_get_special(ctx->mail, MAIL_FETCH_GUID, - &tab[4].value) < 0 || - *tab[4].value == '\0') { - /* broken */ - i_fatal("UIDL: Message GUID not found " - "(pop3_uidl_format=%%g not supported by storage?)"); - } - } - var_expand(str, client->mail_set->pop3_uidl_format, tab); - return FALSE; -} - -static bool list_uids_iter(struct client *client, struct cmd_uidl_context *ctx) +static void +pop3_get_uid(struct client *client, struct mail *mail, string_t *str, + bool *permanent_uidl_r) { static struct var_expand_table static_tab[] = { { 'v', NULL, "uidvalidity" }, @@ -616,20 +563,108 @@ { '\0', NULL, NULL } }; struct var_expand_table *tab; - string_t *str; - int ret; - unsigned int uidl_pos; - bool save_hashes, found = FALSE; + char uid_str[MAX_INT_STRLEN]; + const char *uidl; + + if (mail_get_special(mail, MAIL_FETCH_UIDL_BACKEND, &uidl) == 0 && + *uidl != '\0') { + str_append(str, uidl); + /* UIDL is already permanent */ + *permanent_uidl_r = TRUE; + return; + } + + *permanent_uidl_r = FALSE; + + if (client->set->pop3_reuse_xuidl && + mail_get_first_header(mail, "X-UIDL", &uidl) > 0) { + str_append(str, uidl); + return; + } tab = t_malloc(sizeof(static_tab)); memcpy(tab, static_tab, sizeof(static_tab)); tab[0].value = t_strdup_printf("%u", client->uid_validity); - save_hashes = client->message_uidl_hashes_save && ctx->list_all; - if (save_hashes && client->message_uidl_hashes == NULL) { - client->message_uidl_hashes = - i_new(uint32_t, client->messages_count); + if ((client->uidl_keymask & UIDL_UID) != 0) { + i_snprintf(uid_str, sizeof(uid_str), "%u", + mail->uid); + tab[1].value = uid_str; + } + if ((client->uidl_keymask & UIDL_MD5) != 0) { + if (mail_get_special(mail, MAIL_FETCH_HEADER_MD5, + &tab[2].value) < 0 || + *tab[2].value == '\0') { + /* broken */ + i_fatal("UIDL: Header MD5 not found " + "(pop3_uidl_format=%%m not supported by storage?)"); + } + } + if ((client->uidl_keymask & UIDL_FILE_NAME) != 0) { + if (mail_get_special(mail, MAIL_FETCH_UIDL_FILE_NAME, + &tab[3].value) < 0 || + *tab[3].value == '\0') { + /* broken */ + i_fatal("UIDL: File name not found " + "(pop3_uidl_format=%%f not supported by storage?)"); + } + } + if ((client->uidl_keymask & UIDL_GUID) != 0) { + if (mail_get_special(mail, MAIL_FETCH_GUID, + &tab[4].value) < 0 || + *tab[4].value == '\0') { + /* broken */ + i_fatal("UIDL: Message GUID not found " + "(pop3_uidl_format=%%g not supported by storage?)"); + } } + var_expand(str, client->mail_set->pop3_uidl_format, tab); +} + +static bool +list_uidls_saved_iter(struct client *client, struct cmd_uidl_context *ctx) +{ + bool found = FALSE; + int ret; + + while (ctx->msgnum < client->messages_count) { + uint32_t msgnum = ctx->msgnum++; + + if (client->deleted) { + if (client->deleted_bitmask[msgnum / CHAR_BIT] & + (1 << (msgnum % CHAR_BIT))) + continue; + } + found = TRUE; + + ret = client_send_line(client, + ctx->list_all ? "%u %s" : "+OK %u %s", + msgnum+1, client->message_uidls[msgnum]); + if (ret < 0 || !ctx->list_all) + break; + if (ret == 0) { + /* output is being buffered, continue when there's + more space */ + return FALSE; + } + } + /* finished */ + client->cmd = NULL; + + if (ctx->list_all) + client_send_line(client, "."); + i_free(ctx); + return found; +} + +static bool list_uids_iter(struct client *client, struct cmd_uidl_context *ctx) +{ + string_t *str; + int ret; + bool permanent_uidl, found = FALSE; + + if (client->message_uidls != NULL) + return list_uidls_saved_iter(client, ctx); str = t_str_new(128); while (mailbox_search_next(ctx->search_ctx, &ctx->mail)) { @@ -645,18 +680,13 @@ found = TRUE; str_truncate(str, 0); - str_printfa(str, ctx->list_all ? "%u " : "+OK %u ", msgnum+1); - uidl_pos = str_len(str); - if (!pop3_get_uid(client, ctx, tab, str) && - client->set->pop3_save_uidl) - mail_update_pop3_uidl(ctx->mail, str_c(str) + uidl_pos); + pop3_get_uid(client, ctx->mail, str, &permanent_uidl); + if (client->set->pop3_save_uidl && !permanent_uidl) + mail_update_pop3_uidl(ctx->mail, str_c(str)); - if (save_hashes) { - client->message_uidl_hashes[msgnum] = - crc32_str(str_c(str) + uidl_pos); - } - - ret = client_send_line(client, "%s", str_c(str)); + ret = client_send_line(client, + ctx->list_all ? "%u %s" : "+OK %u %s", + msgnum+1, str_c(str)); if (ret < 0) break; if (ret == 0 && ctx->list_all) { @@ -671,8 +701,6 @@ client->cmd = NULL; - if (save_hashes) - client->message_uidl_hashes_save = FALSE; if (ctx->list_all) client_send_line(client, "."); i_free(ctx); @@ -686,6 +714,74 @@ (void)list_uids_iter(client, ctx); } +static void uidl_rename_duplicate(string_t *uidl, struct hash_table *prev_uidls) +{ + void *key, *value; + unsigned int counter; + + while (hash_table_lookup_full(prev_uidls, str_c(uidl), &key, &value)) { + /* duplicate. the value contains the number of duplicates. */ + counter = POINTER_CAST_TO(value, unsigned int) + 1; + hash_table_update(prev_uidls, key, POINTER_CAST(counter)); + str_printfa(uidl, "-%u", counter); + /* the second lookup really should return NULL, but just in + case of some weird UIDLs do this as many times as needed */ + } +} + +static void client_uidls_save(struct client *client) +{ + struct mail_search_context *search_ctx; + struct mail_search_args *search_args; + struct mail *mail; + struct hash_table *prev_uidls; + string_t *str; + char *uidl; + enum mail_fetch_field wanted_fields; + uint32_t msgnum; + bool permanent_uidl, uidl_duplicates_rename; + + i_assert(client->message_uidls == NULL); + + search_args = pop3_search_build(client, 0); + wanted_fields = 0; + if ((client->uidl_keymask & UIDL_MD5) != 0) + wanted_fields |= MAIL_FETCH_HEADER_MD5; + + search_ctx = mailbox_search_init(client->trans, search_args, + pop3_sort_program, + wanted_fields, NULL); + mail_search_args_unref(&search_args); + + uidl_duplicates_rename = + strcmp(client->set->pop3_uidl_duplicates, "rename") == 0; + prev_uidls = hash_table_create(default_pool, default_pool, 0, + str_hash, (hash_cmp_callback_t *)strcmp); + client->uidl_pool = pool_alloconly_create("message uidls", 1024); + client->message_uidls = p_new(client->uidl_pool, const char *, + client->messages_count); + + str = t_str_new(128); msgnum = 0; + while (mailbox_search_next(search_ctx, &mail)) { + if (client_verify_ordering(client, mail, msgnum) < 0) + i_fatal("Can't finish POP3 UIDL command"); + + str_truncate(str, 0); + pop3_get_uid(client, mail, str, &permanent_uidl); + if (client->set->pop3_save_uidl && !permanent_uidl) + mail_update_pop3_uidl(mail, str_c(str)); + + if (uidl_duplicates_rename) + uidl_rename_duplicate(str, prev_uidls); + uidl = p_strdup(client->uidl_pool, str_c(str)); + client->message_uidls[msgnum] = uidl; + hash_table_insert(prev_uidls, uidl, POINTER_CAST(1)); + msgnum++; + } + (void)mailbox_search_deinit(&search_ctx); + hash_table_destroy(&prev_uidls); +} + static struct cmd_uidl_context * cmd_uidl_init(struct client *client, uint32_t seq) { @@ -693,19 +789,23 @@ struct mail_search_args *search_args; enum mail_fetch_field wanted_fields; - search_args = pop3_search_build(client, seq); + if (client->message_uidls_save && client->message_uidls == NULL) + client_uidls_save(client); ctx = i_new(struct cmd_uidl_context, 1); ctx->list_all = seq == 0; - wanted_fields = 0; - if ((client->uidl_keymask & UIDL_MD5) != 0) - wanted_fields |= MAIL_FETCH_HEADER_MD5; + if (client->message_uidls == NULL) { + wanted_fields = 0; + if ((client->uidl_keymask & UIDL_MD5) != 0) + wanted_fields |= MAIL_FETCH_HEADER_MD5; - ctx->search_ctx = mailbox_search_init(client->trans, search_args, - pop3_sort_program, - wanted_fields, NULL); - mail_search_args_unref(&search_args); + search_args = pop3_search_build(client, seq); + ctx->search_ctx = mailbox_search_init(client->trans, search_args, + pop3_sort_program, + wanted_fields, NULL); + mail_search_args_unref(&search_args); + } if (seq == 0) { client->cmd = cmd_uidl_callback;
--- a/src/pop3/pop3-settings.c Sat May 19 22:40:08 2012 +0300 +++ b/src/pop3/pop3-settings.c Sun May 20 03:25:04 2012 +0300 @@ -70,6 +70,7 @@ DEF(SET_BOOL, pop3_fast_size_lookups), DEF(SET_STR, pop3_client_workarounds), DEF(SET_STR, pop3_logout_format), + DEF(SET_ENUM, pop3_uidl_duplicates), SETTING_DEFINE_LIST_END }; @@ -84,7 +85,8 @@ .pop3_lock_session = FALSE, .pop3_fast_size_lookups = FALSE, .pop3_client_workarounds = "", - .pop3_logout_format = "top=%t/%p, retr=%r/%b, del=%d/%m, size=%s" + .pop3_logout_format = "top=%t/%p, retr=%r/%b, del=%d/%m, size=%s", + .pop3_uidl_duplicates = "allow:rename" }; static const struct setting_parser_info *pop3_setting_dependencies[] = {
--- a/src/pop3/pop3-settings.h Sat May 19 22:40:08 2012 +0300 +++ b/src/pop3/pop3-settings.h Sun May 20 03:25:04 2012 +0300 @@ -22,6 +22,7 @@ bool pop3_fast_size_lookups; const char *pop3_client_workarounds; const char *pop3_logout_format; + const char *pop3_uidl_duplicates; enum pop3_client_workarounds parsed_workarounds; };
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/replication/Makefile.am Sun May 20 03:25:04 2012 +0300 @@ -0,0 +1,4 @@ +SUBDIRS = aggregator replicator + +noinst_HEADERS = \ + replication-common.h
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/replication/aggregator/Makefile.am Sun May 20 03:25:04 2012 +0300 @@ -0,0 +1,26 @@ +pkglibexecdir = $(libexecdir)/dovecot + +pkglibexec_PROGRAMS = aggregator + +AM_CPPFLAGS = \ + -I$(top_srcdir)/src/lib \ + -I$(top_srcdir)/src/lib-settings \ + -I$(top_srcdir)/src/lib-auth \ + -I$(top_srcdir)/src/lib-master \ + -I$(top_srcdir)/src/replication \ + -DPKG_STATEDIR=\""$(statedir)"\" + +aggregator_LDFLAGS = -export-dynamic +aggregator_LDADD = $(LIBDOVECOT) $(MODULE_LIBS) +aggregator_DEPENDENCIES = $(LIBDOVECOT_DEPS) + +aggregator_SOURCES = \ + aggregator.c \ + aggregator-settings.c \ + notify-connection.c \ + replicator-connection.c + +noinst_HEADERS = \ + aggregator-settings.h \ + notify-connection.h \ + replicator-connection.h
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/replication/aggregator/aggregator-settings.c Sun May 20 03:25:04 2012 +0300 @@ -0,0 +1,85 @@ +/* Copyright (c) 2012 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "buffer.h" +#include "settings-parser.h" +#include "service-settings.h" +#include "aggregator-settings.h" + +/* <settings checks> */ +static struct file_listener_settings aggregator_unix_listeners_array[] = { + { "replication-notify", 0600, "", "" } +}; +static struct file_listener_settings *aggregator_unix_listeners[] = { + &aggregator_unix_listeners_array[0] +}; +static buffer_t aggregator_unix_listeners_buf = { + aggregator_unix_listeners, sizeof(aggregator_unix_listeners), { 0, } +}; + +static struct file_listener_settings aggregator_fifo_listeners_array[] = { + { "replication-notify-fifo", 0600, "", "" } +}; +static struct file_listener_settings *aggregator_fifo_listeners[] = { + &aggregator_fifo_listeners_array[0] +}; +static buffer_t aggregator_fifo_listeners_buf = { + aggregator_fifo_listeners, sizeof(aggregator_fifo_listeners), { 0, } +}; +/* </settings checks> */ + +struct service_settings aggregator_service_settings = { + .name = "aggregator", + .protocol = "", + .type = "", + .executable = "aggregator", + .user = "$default_internal_user", + .group = "", + .privileged_group = "", + .extra_groups = "", + .chroot = ".", + + .drop_priv_before_exec = FALSE, + + .process_min_avail = 0, + .process_limit = 0, + .client_limit = 0, + .service_count = 0, + .idle_kill = 0, + .vsz_limit = (uoff_t)-1, + + .unix_listeners = { { &aggregator_unix_listeners_buf, + sizeof(aggregator_unix_listeners[0]) } }, + .fifo_listeners = { { &aggregator_fifo_listeners_buf, + sizeof(aggregator_fifo_listeners[0]) } }, + .inet_listeners = ARRAY_INIT +}; + +#undef DEF +#define DEF(type, name) \ + { type, #name, offsetof(struct aggregator_settings, name), NULL } + +static const struct setting_define aggregator_setting_defines[] = { + DEF(SET_STR, replicator_host), + DEF(SET_UINT, replicator_port), + + SETTING_DEFINE_LIST_END +}; + +const struct aggregator_settings aggregator_default_settings = { + .replicator_host = "replicator", + .replicator_port = 0 +}; + +const struct setting_parser_info aggregator_setting_parser_info = { + .module_name = "aggregator", + .defines = aggregator_setting_defines, + .defaults = &aggregator_default_settings, + + .type_offset = (size_t)-1, + .struct_size = sizeof(struct aggregator_settings), + + .parent_offset = (size_t)-1 +}; + +const struct aggregator_settings *aggregator_settings;
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/replication/aggregator/aggregator-settings.h Sun May 20 03:25:04 2012 +0300 @@ -0,0 +1,12 @@ +#ifndef AGGREGATOR_SETTINGS_H +#define AGGREGATOR_SETTINGS_H + +struct aggregator_settings { + const char *replicator_host; + unsigned int replicator_port; +}; + +extern const struct setting_parser_info aggregator_setting_parser_info; +extern const struct aggregator_settings *aggregator_settings; + +#endif
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/replication/aggregator/aggregator.c Sun May 20 03:25:04 2012 +0300 @@ -0,0 +1,75 @@ +/* Copyright (c) 2012 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "restrict-access.h" +#include "master-service.h" +#include "master-service-settings.h" +#include "aggregator-settings.h" +#include "notify-connection.h" +#include "replicator-connection.h" + +struct replicator_connection *replicator; + +static void client_connected(struct master_service_connection *conn) +{ + master_service_client_connection_accept(conn); + notify_connection_create(conn->fd, conn->fifo); +} + +static void main_preinit(void) +{ + struct ip_addr *ips; + unsigned int ips_count; + const struct aggregator_settings *set; + void **sets; + int ret; + + sets = master_service_settings_get_others(master_service); + set = sets[0]; + + if (set->replicator_port != 0) { + ret = net_gethostbyname(set->replicator_host, &ips, &ips_count); + if (ret != 0) { + i_fatal("replicator_host: gethostbyname(%s) failed: %s", + set->replicator_host, net_gethosterror(ret)); + } + replicator = replicator_connection_create_inet(ips, ips_count, + set->replicator_port, + notify_connection_sync_callback); + } else { + replicator = replicator_connection_create_unix(set->replicator_host, + notify_connection_sync_callback); + } +} + +int main(int argc, char *argv[]) +{ + const struct setting_parser_info *set_roots[] = { + &aggregator_setting_parser_info, + NULL + }; + const char *error; + + master_service = master_service_init("aggregator", 0, + &argc, &argv, NULL); + if (master_getopt(master_service) > 0) + return FATAL_DEFAULT; + + if (master_service_settings_read_simple(master_service, set_roots, + &error) < 0) + i_fatal("Error reading configuration: %s", error); + master_service_init_log(master_service, "aggregator: "); + + main_preinit(); + + restrict_access_by_env(NULL, FALSE); + restrict_access_allow_coredumps(TRUE); + master_service_init_finish(master_service); + + master_service_run(master_service, client_connected); + + notify_connections_destroy_all(); + replicator_connection_destroy(&replicator); + master_service_deinit(&master_service); + return 0; +}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/replication/aggregator/notify-connection.c Sun May 20 03:25:04 2012 +0300 @@ -0,0 +1,154 @@ +/* Copyright (c) 2012 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "ioloop.h" +#include "network.h" +#include "istream.h" +#include "ostream.h" +#include "llist.h" +#include "strescape.h" +#include "master-service.h" +#include "replication-common.h" +#include "replicator-connection.h" +#include "notify-connection.h" + +#define MAX_INBUF_SIZE 8192 + +#define CONNECTION_IS_FIFO(conn) \ + ((conn)->output == NULL) + +struct notify_connection { + struct notify_connection *prev, *next; + int refcount; + + int fd; + struct io *io; + struct istream *input; + struct ostream *output; +}; + +static struct notify_connection *conns = NULL; + +static void notify_connection_unref(struct notify_connection *conn); +static void notify_connection_destroy(struct notify_connection *conn); + +static bool notify_input_error(struct notify_connection *conn) +{ + if (CONNECTION_IS_FIFO(conn)) + return TRUE; + notify_connection_destroy(conn); + return FALSE; +} + +void notify_connection_sync_callback(bool success, void *context) +{ + struct notify_connection *conn = context; + + o_stream_send_str(conn->output, success ? "+\n" : "-\n"); + notify_connection_unref(conn); +} + +static int +notify_input_line(struct notify_connection *conn, const char *line) +{ + const char *const *args; + enum replication_priority priority; + + /* <username> \t <priority> */ + args = t_strsplit_tabescaped(line); + if (str_array_length(args) < 2) { + i_error("Client sent invalid input"); + return -1; + } + if (replication_priority_parse(args[1], &priority) < 0) { + i_error("Client sent invalid priority: %s", args[1]); + return -1; + } + if (priority != REPLICATION_PRIORITY_SYNC) + replicator_connection_notify(replicator, args[0], priority); + else { + conn->refcount++; + replicator_connection_notify_sync(replicator, args[0], conn); + } + return 0; +} + +static void notify_input(struct notify_connection *conn) +{ + const char *line; + int ret; + + switch (i_stream_read(conn->input)) { + case -2: + /* buffer full */ + i_error("Client sent too long line"); + notify_input_error(conn); + return; + case -1: + /* disconnected */ + notify_connection_destroy(conn); + return; + } + + while ((line = i_stream_next_line(conn->input)) != NULL) { + T_BEGIN { + ret = notify_input_line(conn, line); + } T_END; + if (ret < 0) { + if (!notify_input_error(conn)) + return; + } + } +} + +void notify_connection_create(int fd, bool fifo) +{ + struct notify_connection *conn; + + conn = i_new(struct notify_connection, 1); + conn->refcount = 1; + conn->fd = fd; + conn->io = io_add(fd, IO_READ, notify_input, conn); + conn->input = i_stream_create_fd(fd, MAX_INBUF_SIZE, FALSE); + if (!fifo) + conn->output = o_stream_create_fd(fd, (size_t)-1, FALSE); + + DLLIST_PREPEND(&conns, conn); +} + +static void notify_connection_unref(struct notify_connection *conn) +{ + i_assert(conn->refcount > 0); + if (--conn->refcount > 0) + return; + + i_stream_destroy(&conn->input); + if (conn->output != NULL) + o_stream_destroy(&conn->output); + i_free(conn); +} + +static void notify_connection_destroy(struct notify_connection *conn) +{ + i_assert(conn->fd != -1); + + if (!CONNECTION_IS_FIFO(conn)) + master_service_client_connection_destroyed(master_service); + + DLLIST_REMOVE(&conns, conn); + + io_remove(&conn->io); + i_stream_close(conn->input); + if (conn->output != NULL) + o_stream_close(conn->output); + net_disconnect(conn->fd); + conn->fd = -1; + + notify_connection_unref(conn); +} + +void notify_connections_destroy_all(void) +{ + while (conns != NULL) + notify_connection_destroy(conns); +}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/replication/aggregator/notify-connection.h Sun May 20 03:25:04 2012 +0300 @@ -0,0 +1,9 @@ +#ifndef NOTIFY_CONNECTION_H +#define NOTIFY_CONNECTION_H + +void notify_connection_create(int fd, bool fifo); +void notify_connections_destroy_all(void); + +void notify_connection_sync_callback(bool success, void *context); + +#endif
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/replication/aggregator/replicator-connection.c Sun May 20 03:25:04 2012 +0300 @@ -0,0 +1,325 @@ +/* Copyright (c) 2012 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "ioloop.h" +#include "network.h" +#include "istream.h" +#include "ostream.h" +#include "buffer.h" +#include "hash.h" +#include "llist.h" +#include "strescape.h" +#include "replicator-connection.h" + +#define MAX_INBUF_SIZE 1024 +#define REPLICATOR_RECONNECT_MSECS 5000 +#define REPLICATOR_MEMBUF_MAX_SIZE 1024*1024 +#define REPLICATOR_HANDSHAKE "VERSION\treplicator-notify\t1\t0\n" + +struct replicator_connection { + char *path; + struct ip_addr *ips; + unsigned int ips_count, ip_idx, port; + + int fd; + struct io *io; + struct istream *input; + struct ostream *output; + struct timeout *to; + + buffer_t *queue[REPLICATION_PRIORITY_SYNC + 1]; + + struct hash_table *requests; + unsigned int request_id_counter; + replicator_sync_callback_t *callback; +}; + +static void replicator_connection_disconnect(struct replicator_connection *conn); + +static int +replicator_input_line(struct replicator_connection *conn, const char *line) +{ + void *context; + unsigned int id; + + /* <+|-> \t <id> */ + if ((line[0] != '+' && line[0] != '-') || line[1] != '\t' || + str_to_uint(line+2, &id) < 0 || id == 0) { + i_error("Replicator sent invalid input: %s", line); + return -1; + } + + context = hash_table_lookup(conn->requests, POINTER_CAST(id)); + if (context == NULL) { + i_error("Replicator sent invalid ID: %u", id); + return -1; + } + hash_table_remove(conn->requests, context); + conn->callback(line[0] == '+', context); + return 0; +} + +static void replicator_input(struct replicator_connection *conn) +{ + const char *line; + + switch (i_stream_read(conn->input)) { + case -2: + /* buffer full */ + i_error("Replicator sent too long line"); + replicator_connection_disconnect(conn); + return; + case -1: + /* disconnected */ + replicator_connection_disconnect(conn); + return; + } + + while ((line = i_stream_next_line(conn->input)) != NULL) + (void)replicator_input_line(conn, line); +} + +static bool +replicator_send_buf(struct replicator_connection *conn, buffer_t *buf) +{ + const unsigned char *data = buf->data; + unsigned int len = IO_BLOCK_SIZE; + + /* try to send about IO_BLOCK_SIZE amount of data, + but only full lines */ + if (len > buf->used) + len = buf->used; + for (;; len++) { + i_assert(len < buf->used); /* there is always LF */ + if (data[len] == '\n') { + len++; + break; + } + } + + if (o_stream_send(conn->output, data, len) < 0) { + replicator_connection_disconnect(conn); + return FALSE; + } + buffer_delete(buf, 0, len); + return TRUE; +} + +static int replicator_output(struct replicator_connection *conn) +{ + enum replication_priority p; + + if (o_stream_flush(conn->output) < 0) { + replicator_connection_disconnect(conn); + return 1; + } + + for (p = REPLICATION_PRIORITY_SYNC;;) { + if (o_stream_get_buffer_used_size(conn->output) > 0) { + o_stream_set_flush_pending(conn->output, TRUE); + break; + } + /* output buffer is empty, send more data */ + if (conn->queue[p]->used > 0) { + if (!replicator_send_buf(conn, conn->queue[p])) + break; + } else { + if (p == REPLICATION_PRIORITY_LOW) + break; + p--; + } + } + return 1; +} + +static void replicator_connection_connect(struct replicator_connection *conn) +{ + unsigned int n; + int fd = -1; + + if (conn->fd != -1) + return; + + if (conn->port == 0) { + fd = net_connect_unix(conn->path); + if (fd == -1) + i_error("net_connect_unix(%s) failed: %m", conn->path); + } else { + for (n = 0; n < conn->ips_count; n++) { + unsigned int idx = conn->ip_idx; + + conn->ip_idx = (conn->ip_idx + 1) % conn->ips_count; + fd = net_connect_ip(&conn->ips[idx], conn->port, NULL); + if (fd != -1) + break; + i_error("connect(%s, %u) failed: %m", + net_ip2addr(&conn->ips[idx]), conn->port); + } + } + + if (fd == -1) { + if (conn->to == NULL) { + conn->to = timeout_add(REPLICATOR_RECONNECT_MSECS, + replicator_connection_connect, + conn); + } + return; + } + + if (conn->to != NULL) + timeout_remove(&conn->to); + conn->fd = fd; + conn->io = io_add(fd, IO_READ, replicator_input, conn); + conn->input = i_stream_create_fd(fd, MAX_INBUF_SIZE, FALSE); + conn->output = o_stream_create_fd(fd, (size_t)-1, FALSE); + (void)o_stream_send_str(conn->output, REPLICATOR_HANDSHAKE); + o_stream_set_flush_callback(conn->output, replicator_output, conn); +} + +static void replicator_abort_all_requests(struct replicator_connection *conn) +{ + struct hash_iterate_context *iter; + void *key, *value; + + iter = hash_table_iterate_init(conn->requests); + while (hash_table_iterate(iter, &key, &value)) + conn->callback(FALSE, value); + hash_table_iterate_deinit(&iter); + hash_table_clear(conn->requests, TRUE); +} + +static void replicator_connection_disconnect(struct replicator_connection *conn) +{ + if (conn->fd == -1) + return; + + replicator_abort_all_requests(conn); + io_remove(&conn->io); + i_stream_destroy(&conn->input); + o_stream_destroy(&conn->output); + net_disconnect(conn->fd); + conn->fd = -1; +} + +static struct replicator_connection *replicator_connection_create(void) +{ + struct replicator_connection *conn; + unsigned int i; + + conn = i_new(struct replicator_connection, 1); + conn->fd = -1; + conn->requests = hash_table_create(default_pool, default_pool, + 0, NULL, NULL); + for (i = REPLICATION_PRIORITY_LOW; i <= REPLICATION_PRIORITY_SYNC; i++) + conn->queue[i] = buffer_create_dynamic(default_pool, 1024); + return conn; +} + +struct replicator_connection * +replicator_connection_create_unix(const char *path, + replicator_sync_callback_t *callback) +{ + struct replicator_connection *conn; + + conn = replicator_connection_create(); + conn->callback = callback; + conn->path = i_strdup(path); + return conn; +} + +struct replicator_connection * +replicator_connection_create_inet(const struct ip_addr *ips, + unsigned int ips_count, unsigned int port, + replicator_sync_callback_t *callback) +{ + struct replicator_connection *conn; + + conn = replicator_connection_create(); + conn->callback = callback; + conn->ips = i_new(struct ip_addr, ips_count); + memcpy(conn->ips, ips, sizeof(*ips) * ips_count); + conn->ips_count = ips_count; + conn->port = port; + return conn; +} + +void replicator_connection_destroy(struct replicator_connection **_conn) +{ + struct replicator_connection *conn = *_conn; + unsigned int i; + + *_conn = NULL; + replicator_connection_disconnect(conn); + + for (i = REPLICATION_PRIORITY_LOW; i <= REPLICATION_PRIORITY_SYNC; i++) + buffer_free(&conn->queue[i]); + + if (conn->to != NULL) + timeout_remove(&conn->to); + hash_table_destroy(&conn->requests); + i_free(conn); +} + +static void +replicator_send(struct replicator_connection *conn, + enum replication_priority priority, const char *data) +{ + unsigned int data_len = strlen(data); + + if (conn->fd != -1 && + o_stream_get_buffer_used_size(conn->output) == 0) { + /* we can send data immediately */ + o_stream_send(conn->output, data, data_len); + } else if (conn->queue[priority]->used + data_len >= + REPLICATOR_MEMBUF_MAX_SIZE) { + /* FIXME: compress duplicates, start writing to file */ + } else { + /* queue internally to separate queues */ + buffer_append(conn->queue[priority], data, data_len); + if (conn->output != NULL) + o_stream_set_flush_pending(conn->output, TRUE); + } +} + +void replicator_connection_notify(struct replicator_connection *conn, + const char *username, + enum replication_priority priority) +{ + const char *priority_str = ""; + + replicator_connection_connect(conn); + + switch (priority) { + case REPLICATION_PRIORITY_NONE: + case REPLICATION_PRIORITY_SYNC: + i_unreached(); + case REPLICATION_PRIORITY_LOW: + priority_str = "low"; + break; + case REPLICATION_PRIORITY_HIGH: + priority_str = "high"; + break; + } + + T_BEGIN { + replicator_send(conn, priority, t_strdup_printf( + "U\t%s\t%s\n", str_tabescape(username), priority_str)); + } T_END; +} + +void replicator_connection_notify_sync(struct replicator_connection *conn, + const char *username, void *context) +{ + unsigned int id; + + replicator_connection_connect(conn); + + id = ++conn->request_id_counter; + if (id == 0) id++; + hash_table_insert(conn->requests, POINTER_CAST(id), context); + + T_BEGIN { + replicator_send(conn, REPLICATION_PRIORITY_SYNC, t_strdup_printf( + "U\t%s\tsync\t%u\n", str_tabescape(username), id)); + } T_END; +}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/replication/aggregator/replicator-connection.h Sun May 20 03:25:04 2012 +0300 @@ -0,0 +1,25 @@ +#ifndef REPLICATOR_CONNECTION_H +#define REPLICATOR_CONNECTION_H + +#include "replication-common.h" + +typedef void replicator_sync_callback_t(bool success, void *context); + +struct replicator_connection * +replicator_connection_create_unix(const char *path, + replicator_sync_callback_t *callback); +struct replicator_connection * +replicator_connection_create_inet(const struct ip_addr *ips, + unsigned int ips_count, unsigned int port, + replicator_sync_callback_t *callback); +void replicator_connection_destroy(struct replicator_connection **conn); + +void replicator_connection_notify(struct replicator_connection *conn, + const char *username, + enum replication_priority priority); +void replicator_connection_notify_sync(struct replicator_connection *conn, + const char *username, void *context); + +extern struct replicator_connection *replicator; + +#endif
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/replication/replication-common.h Sun May 20 03:25:04 2012 +0300 @@ -0,0 +1,30 @@ +#ifndef REPLICATION_COMMON_H +#define REPLICATION_COMMON_H + +enum replication_priority { + /* user is fully replicated, as far as we know */ + REPLICATION_PRIORITY_NONE = 0, + /* flag changes, expunges, etc. */ + REPLICATION_PRIORITY_LOW, + /* new emails */ + REPLICATION_PRIORITY_HIGH, + /* synchronously wait for new emails to be replicated */ + REPLICATION_PRIORITY_SYNC +}; + +static inline int +replication_priority_parse(const char *str, + enum replication_priority *priority_r) +{ + if (strcmp(str, "low") == 0) + *priority_r = REPLICATION_PRIORITY_LOW; + else if (strcmp(str, "high") == 0) + *priority_r = REPLICATION_PRIORITY_HIGH; + else if (strcmp(str, "sync") == 0) + *priority_r = REPLICATION_PRIORITY_SYNC; + else + return -1; + return 0; +} + +#endif
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/replication/replicator/Makefile.am Sun May 20 03:25:04 2012 +0300 @@ -0,0 +1,30 @@ +pkglibexecdir = $(libexecdir)/dovecot + +pkglibexec_PROGRAMS = replicator + +AM_CPPFLAGS = \ + -I$(top_srcdir)/src/lib \ + -I$(top_srcdir)/src/lib-settings \ + -I$(top_srcdir)/src/lib-auth \ + -I$(top_srcdir)/src/lib-master \ + -I$(top_srcdir)/src/replication \ + -DPKG_STATEDIR=\""$(statedir)"\" + +replicator_LDFLAGS = -export-dynamic +replicator_LDADD = $(LIBDOVECOT) $(MODULE_LIBS) +replicator_DEPENDENCIES = $(LIBDOVECOT_DEPS) + +replicator_SOURCES = \ + doveadm-connection.c \ + replicator.c \ + replicator-brain.c \ + replicator-queue.c \ + replicator-settings.c \ + notify-connection.c + +noinst_HEADERS = \ + doveadm-connection.h \ + replicator-brain.h \ + replicator-queue.h \ + replicator-settings.h \ + notify-connection.h
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/replication/replicator/doveadm-connection.c Sun May 20 03:25:04 2012 +0300 @@ -0,0 +1,195 @@ +/* Copyright (c) 2012 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "ioloop.h" +#include "network.h" +#include "istream.h" +#include "ostream.h" +#include "str.h" +#include "strescape.h" +#include "doveadm-connection.h" + +#include <unistd.h> + +#define DOVEADM_FAIL_TIMEOUT_MSECS (1000*5) +#define DOVEADM_HANDSHAKE "VERSION\tdoveadm-server\t1\t0\n" +#define MAX_INBUF_SIZE 1024 + +struct doveadm_connection { + char *path; + int fd; + struct io *io; + struct istream *input; + struct ostream *output; + struct timeout *to; + + doveadm_callback_t *callback; + void *context; + + time_t last_connect_failure; + unsigned int handshaked:1; + unsigned int end_of_print:1; +}; + +struct doveadm_connection *doveadm_connection_init(const char *path) +{ + struct doveadm_connection *conn; + + conn = i_new(struct doveadm_connection, 1); + conn->path = i_strdup(path); + conn->fd = -1; + return conn; +} + +static void doveadm_callback(struct doveadm_connection *conn, + enum doveadm_reply reply) +{ + doveadm_callback_t *callback = conn->callback; + void *context = conn->context; + + if (conn->to != NULL) + timeout_remove(&conn->to); + + conn->callback = NULL; + conn->context = NULL; + callback(reply, context); +} + +static void doveadm_disconnect(struct doveadm_connection *conn) +{ + if (conn->fd == -1) + return; + + io_remove(&conn->io); + o_stream_destroy(&conn->output); + i_stream_destroy(&conn->input); + if (close(conn->fd) < 0) + i_error("close(doveadm) failed: %m"); + conn->fd = -1; + + if (conn->callback != NULL) + doveadm_callback(conn, DOVEADM_REPLY_FAIL); +} + +void doveadm_connection_deinit(struct doveadm_connection **_conn) +{ + struct doveadm_connection *conn = *_conn; + + *_conn = NULL; + + doveadm_disconnect(conn); + i_free(conn->path); + i_free(conn); +} + +static int doveadm_input_line(struct doveadm_connection *conn, const char *line) +{ + if (!conn->handshaked) { + if (strcmp(line, "+") != 0) { + i_error("%s: Unexpected handshake: %s", + conn->path, line); + return -1; + } + conn->handshaked = TRUE; + return 0; + } + if (conn->callback == NULL) { + i_error("%s: Unexpected input: %s", conn->path, line); + return -1; + } + if (!conn->end_of_print) { + if (line[0] == '\0') + conn->end_of_print = TRUE; + return 0; + } + if (line[0] == '+') + doveadm_callback(conn, DOVEADM_REPLY_OK); + else if (line[0] == '-') { + if (strcmp(line+1, "NOUSER") == 0) + doveadm_callback(conn, DOVEADM_REPLY_NOUSER); + else + doveadm_callback(conn, DOVEADM_REPLY_FAIL); + } else { + i_error("%s: Invalid input: %s", conn->path, line); + return -1; + } + conn->end_of_print = FALSE; + /* FIXME: disconnect after each request for now. + doveadm server's getopt() handling seems to break otherwise */ + return -1; +} + +static void doveadm_input(struct doveadm_connection *conn) +{ + const char *line; + + while ((line = i_stream_read_next_line(conn->input)) != NULL) { + if (doveadm_input_line(conn, line) < 0) { + doveadm_disconnect(conn); + return; + } + } + if (conn->input->eof) + doveadm_disconnect(conn); +} + +static int doveadm_connect(struct doveadm_connection *conn) +{ + if (conn->fd != -1) + return 0; + + if (conn->last_connect_failure == ioloop_time) + return -1; + + conn->fd = net_connect_unix(conn->path); + if (conn->fd == -1) { + i_error("net_connect_unix(%s) failed: %m", conn->path); + conn->last_connect_failure = ioloop_time; + return -1; + } + conn->last_connect_failure = 0; + conn->io = io_add(conn->fd, IO_READ, doveadm_input, conn); + conn->input = i_stream_create_fd(conn->fd, MAX_INBUF_SIZE, FALSE); + conn->output = o_stream_create_fd(conn->fd, (size_t)-1, FALSE); + o_stream_send_str(conn->output, DOVEADM_HANDSHAKE); + return 0; +} + +static void doveadm_fail_timeout(struct doveadm_connection *conn) +{ + doveadm_callback(conn, DOVEADM_REPLY_FAIL); +} + +void doveadm_connection_sync(struct doveadm_connection *conn, + const char *username, bool full, + doveadm_callback_t *callback, void *context) +{ + string_t *cmd; + + i_assert(callback != NULL); + i_assert(!doveadm_connection_is_busy(conn)); + + conn->callback = callback; + conn->context = context; + + if (doveadm_connect(conn) < 0) { + i_assert(conn->to == NULL); + conn->to = timeout_add(DOVEADM_FAIL_TIMEOUT_MSECS, + doveadm_fail_timeout, conn); + } else { + /* <flags> <username> <command> [<args>] */ + cmd = t_str_new(256); + str_append_c(cmd, '\t'); + str_tabescape_write(cmd, username); + str_append(cmd, "\tsync\t-d"); + if (full) + str_append(cmd, "\t-f"); + str_append_c(cmd, '\n'); + o_stream_send(conn->output, str_data(cmd), str_len(cmd)); + } +} + +bool doveadm_connection_is_busy(struct doveadm_connection *conn) +{ + return conn->callback != NULL; +}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/replication/replicator/doveadm-connection.h Sun May 20 03:25:04 2012 +0300 @@ -0,0 +1,20 @@ +#ifndef DOVEADM_CONNECTION_H +#define DOVEADM_CONNECTION_H + +enum doveadm_reply { + DOVEADM_REPLY_OK, + DOVEADM_REPLY_FAIL, + DOVEADM_REPLY_NOUSER +}; + +typedef void doveadm_callback_t(enum doveadm_reply reply, void *context); + +struct doveadm_connection *doveadm_connection_init(const char *path); +void doveadm_connection_deinit(struct doveadm_connection **conn); + +void doveadm_connection_sync(struct doveadm_connection *conn, + const char *username, bool full, + doveadm_callback_t *callback, void *context); +bool doveadm_connection_is_busy(struct doveadm_connection *conn); + +#endif
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/replication/replicator/notify-connection.c Sun May 20 03:25:04 2012 +0300 @@ -0,0 +1,197 @@ +/* Copyright (c) 2012 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "llist.h" +#include "istream.h" +#include "ostream.h" +#include "strescape.h" +#include "master-service.h" +#include "replicator-queue.h" +#include "notify-connection.h" + +#include <unistd.h> + +#define MAX_INBUF_SIZE (1024*64) +#define NOTIFY_CLIENT_PROTOCOL_MAJOR_VERSION 1 +#define NOTIFY_CLIENT_PROTOCOL_MINOR_VERSION 0 + +struct notify_connection { + struct notify_connection *prev, *next; + int refcount; + + int fd; + struct io *io; + struct istream *input; + struct ostream *output; + + struct replicator_queue *queue; + + unsigned int version_received:1; + unsigned int destroyed:1; +}; + +struct notify_sync_request { + struct notify_connection *conn; + unsigned int id; +}; + +static struct notify_connection *connections; + +static void notify_connection_destroy(struct notify_connection *conn); + +static void notify_sync_callback(bool success, void *context) +{ + struct notify_sync_request *request = context; + + o_stream_send_str(request->conn->output, t_strdup_printf( + "%c\t%u\n", success ? '+' : '-', request->id)); + + notify_connection_unref(&request->conn); + i_free(request); +} + +static int +notify_connection_input_line(struct notify_connection *conn, const char *line) +{ + struct notify_sync_request *request; + const char *const *args; + enum replication_priority priority; + unsigned int id; + + /* U \t <username> \t <priority> [\t <sync id>] */ + args = t_strsplit_tabescaped(line); + if (str_array_length(args) < 2) { + i_error("notify client sent invalid input: %s", line); + return -1; + } + if (strcmp(args[0], "U") != 0) { + i_error("notify client sent unknown command: %s", args[0]); + return -1; + } + if (replication_priority_parse(args[2], &priority) < 0) { + i_error("notify client sent invalid priority: %s", args[2]); + return -1; + } + if (priority != REPLICATION_PRIORITY_SYNC) + replicator_queue_add(conn->queue, args[1], priority); + else if (args[3] == NULL || str_to_uint(args[3], &id) < 0) { + i_error("notify client sent invalid sync id: %s", line); + return -1; + } else { + request = i_new(struct notify_sync_request, 1); + request->conn = conn; + request->id = id; + notify_connection_ref(conn); + replicator_queue_add_sync(conn->queue, args[1], + notify_sync_callback, request); + } + return 0; +} + +static void notify_connection_input(struct notify_connection *conn) +{ + const char *line; + int ret; + + switch (i_stream_read(conn->input)) { + case -2: + i_error("BUG: Client connection sent too much data"); + notify_connection_destroy(conn); + return; + case -1: + notify_connection_destroy(conn); + return; + } + + if (!conn->version_received) { + if ((line = i_stream_next_line(conn->input)) == NULL) + return; + + if (!version_string_verify(line, "replicator-notify", + NOTIFY_CLIENT_PROTOCOL_MAJOR_VERSION)) { + i_error("Notify client not compatible with this server " + "(mixed old and new binaries?)"); + notify_connection_destroy(conn); + return; + } + conn->version_received = TRUE; + } + + while ((line = i_stream_next_line(conn->input)) != NULL) { + T_BEGIN { + ret = notify_connection_input_line(conn, line); + } T_END; + if (ret < 0) { + notify_connection_destroy(conn); + break; + } + } +} + +struct notify_connection * +notify_connection_create(int fd, struct replicator_queue *queue) +{ + struct notify_connection *conn; + + i_assert(fd >= 0); + + conn = i_new(struct notify_connection, 1); + conn->refcount = 1; + conn->queue = queue; + conn->fd = fd; + conn->input = i_stream_create_fd(fd, MAX_INBUF_SIZE, FALSE); + conn->output = o_stream_create_fd(fd, (size_t)-1, FALSE); + conn->io = io_add(fd, IO_READ, notify_connection_input, conn); + conn->queue = queue; + + DLLIST_PREPEND(&connections, conn); + return conn; +} + +static void notify_connection_destroy(struct notify_connection *conn) +{ + if (conn->destroyed) + return; + conn->destroyed = TRUE; + + DLLIST_REMOVE(&connections, conn); + + io_remove(&conn->io); + i_stream_close(conn->input); + o_stream_close(conn->output); + if (close(conn->fd) < 0) + i_error("close(notify connection) failed: %m"); + conn->fd = -1; + + notify_connection_unref(&conn); + master_service_client_connection_destroyed(master_service); +} + +void notify_connection_ref(struct notify_connection *conn) +{ + i_assert(conn->refcount > 0); + + conn->refcount++; +} + +void notify_connection_unref(struct notify_connection **_conn) +{ + struct notify_connection *conn = *_conn; + + i_assert(conn->refcount > 0); + + *_conn = NULL; + if (--conn->refcount > 0) + return; + + notify_connection_destroy(conn); + i_stream_unref(&conn->input); + o_stream_unref(&conn->output); + i_free(conn); +} + +void notify_connections_destroy_all(void) +{ + while (connections != NULL) + notify_connection_destroy(connections); +}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/replication/replicator/notify-connection.h Sun May 20 03:25:04 2012 +0300 @@ -0,0 +1,13 @@ +#ifndef NOTIFY_CONNECTION_H +#define NOTIFY_CONNECTION_H + +struct replicator_queue; + +struct notify_connection * +notify_connection_create(int fd, struct replicator_queue *queue); +void notify_connection_ref(struct notify_connection *conn); +void notify_connection_unref(struct notify_connection **conn); + +void notify_connections_destroy_all(void); + +#endif
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/replication/replicator/replicator-brain.c Sun May 20 03:25:04 2012 +0300 @@ -0,0 +1,166 @@ +/* Copyright (c) 2012 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "ioloop.h" +#include "doveadm-connection.h" +#include "replicator-settings.h" +#include "replicator-queue.h" +#include "replicator-brain.h" + +struct replicator_sync_context { + struct replicator_brain *brain; + struct replicator_user *user; +}; + +struct replicator_brain { + pool_t pool; + struct replicator_queue *queue; + const struct replicator_settings *set; + struct timeout *to; + + ARRAY_DEFINE(doveadm_conns, struct doveadm_connection *); +}; + +static void replicator_brain_fill(struct replicator_brain *brain); + +static void replicator_brain_queue_changed(void *context) +{ + struct replicator_brain *brain = context; + + replicator_brain_fill(brain); +} + +struct replicator_brain * +replicator_brain_init(struct replicator_queue *queue, + const struct replicator_settings *set) +{ + struct replicator_brain *brain; + pool_t pool; + + pool = pool_alloconly_create("replication brain", 1024); + brain = p_new(pool, struct replicator_brain, 1); + brain->pool = pool; + brain->queue = queue; + brain->set = set; + p_array_init(&brain->doveadm_conns, pool, 16); + replicator_queue_set_change_callback(queue, + replicator_brain_queue_changed, brain); + replicator_brain_fill(brain); + return brain; +} + +void replicator_brain_deinit(struct replicator_brain **_brain) +{ + struct replicator_brain *brain = *_brain; + struct doveadm_connection **connp; + + *_brain = NULL; + + array_foreach_modifiable(&brain->doveadm_conns, connp) + doveadm_connection_deinit(connp); + if (brain->to != NULL) + timeout_remove(&brain->to); + pool_unref(&brain->pool); +} + +static struct doveadm_connection * +get_doveadm_connection(struct replicator_brain *brain) +{ + struct doveadm_connection *const *connp, *conn = NULL; + + array_foreach(&brain->doveadm_conns, connp) { + if (!doveadm_connection_is_busy(*connp)) + return *connp; + } + if (array_count(&brain->doveadm_conns) == + brain->set->replication_max_conns) + return NULL; + + conn = doveadm_connection_init(brain->set->doveadm_socket_path); + array_append(&brain->doveadm_conns, &conn, 1); + return conn; +} + +static void doveadm_sync_callback(enum doveadm_reply reply, void *context) +{ + struct replicator_sync_context *ctx = context; + + if (reply == DOVEADM_REPLY_NOUSER) { + /* user no longer exists, remove from replication */ + replicator_queue_remove(ctx->brain->queue, &ctx->user); + } else { + ctx->user->last_sync_failed = + reply != DOVEADM_REPLY_OK; + replicator_queue_push(ctx->brain->queue, ctx->user); + } + replicator_brain_fill(ctx->brain); + i_free(ctx); +} + +static bool +doveadm_replicate(struct replicator_brain *brain, struct replicator_user *user) +{ + struct replicator_sync_context *ctx; + struct doveadm_connection *conn; + time_t next_full_sync; + bool full; + + conn = get_doveadm_connection(brain); + if (conn == NULL) + return FALSE; + + next_full_sync = user->last_full_sync + + brain->set->replication_full_sync_interval; + full = next_full_sync <= ioloop_time; + /* update the sync times immediately. if the replication fails we still + wouldn't want it to be retried immediately. */ + user->last_fast_sync = ioloop_time; + if (full) + user->last_full_sync = ioloop_time; + /* reset priority also. if more updates arrive during replication + we'll do another replication to make sure nothing gets lost */ + user->priority = REPLICATION_PRIORITY_NONE; + + ctx = i_new(struct replicator_sync_context, 1); + ctx->brain = brain; + ctx->user = user; + doveadm_connection_sync(conn, user->username, full, + doveadm_sync_callback, ctx); + return TRUE; +} + +static void replicator_brain_timeout(struct replicator_brain *brain) +{ + timeout_remove(&brain->to); + replicator_brain_fill(brain); +} + +static bool replicator_brain_fill_next(struct replicator_brain *brain) +{ + struct replicator_user *user; + unsigned int next_secs; + + user = replicator_queue_pop(brain->queue, &next_secs); + if (user == NULL) { + /* nothing more to do */ + if (brain->to != NULL) + timeout_remove(&brain->to); + brain->to = timeout_add(next_secs * 1000, + replicator_brain_timeout, brain); + return FALSE; + } + + if (!doveadm_replicate(brain, user)) { + /* all connections were full, put the user back to queue */ + replicator_queue_push(brain->queue, user); + return FALSE; + } + /* replication started for the user */ + return TRUE; +} + +static void replicator_brain_fill(struct replicator_brain *brain) +{ + while (replicator_brain_fill_next(brain)) ; +}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/replication/replicator/replicator-brain.h Sun May 20 03:25:04 2012 +0300 @@ -0,0 +1,11 @@ +#ifndef REPLICATOR_BRAIN_H +#define REPLICATOR_BRAIN_H + +struct replicator_settings; + +struct replicator_brain * +replicator_brain_init(struct replicator_queue *queue, + const struct replicator_settings *set); +void replicator_brain_deinit(struct replicator_brain **brain); + +#endif
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/replication/replicator/replicator-queue.c Sun May 20 03:25:04 2012 +0300 @@ -0,0 +1,371 @@ +/* Copyright (c) 2012 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "ioloop.h" +#include "istream.h" +#include "ostream.h" +#include "str.h" +#include "strescape.h" +#include "hash.h" +#include "replicator-queue.h" + +#include <unistd.h> +#include <fcntl.h> + +struct replicator_sync_lookup { + struct replicator_user *user; + + replicator_sync_callback_t *callback; + void *context; + + bool wait_for_next_push; +}; + +struct replicator_queue { + struct priorityq *user_queue; + /* username => struct replicator_user* */ + struct hash_table *user_hash; + + ARRAY_DEFINE(sync_lookups, struct replicator_sync_lookup); + + unsigned int full_sync_interval; + + void (*change_callback)(void *context); + void *change_context; +}; + +static int user_priority_cmp(const void *p1, const void *p2) +{ + const struct replicator_user *user1 = p1, *user2 = p2; + + if (user1->priority > user2->priority) + return -1; + if (user1->priority < user2->priority) + return 1; + + if (user1->priority != REPLICATION_PRIORITY_NONE) { + /* there is something to replicate */ + if (user1->last_fast_sync < user2->last_fast_sync) + return -1; + if (user1->last_fast_sync > user2->last_fast_sync) + return 1; + } else { + /* nothing to replicate, but do still periodic full syncs */ + if (user1->last_full_sync < user2->last_full_sync) + return -1; + if (user1->last_full_sync > user2->last_full_sync) + return 1; + } + return 0; +} + +struct replicator_queue *replicator_queue_init(unsigned int full_sync_interval) +{ + struct replicator_queue *queue; + + queue = i_new(struct replicator_queue, 1); + queue->full_sync_interval = full_sync_interval; + queue->user_queue = priorityq_init(user_priority_cmp, 1024); + queue->user_hash = + hash_table_create(default_pool, default_pool, 1024, + str_hash, (hash_cmp_callback_t *)strcmp); + i_array_init(&queue->sync_lookups, 32); + return queue; +} + +void replicator_queue_deinit(struct replicator_queue **_queue) +{ + struct replicator_queue *queue = *_queue; + struct priorityq_item *item; + + *_queue = NULL; + + queue->change_callback = NULL; + + while ((item = priorityq_pop(queue->user_queue)) != NULL) { + struct replicator_user *user = (struct replicator_user *)item; + + user->popped = TRUE; + replicator_queue_remove(queue, &user); + } + + priorityq_deinit(&queue->user_queue); + hash_table_destroy(&queue->user_hash); + i_assert(array_count(&queue->sync_lookups) == 0); + array_free(&queue->sync_lookups); + i_free(queue); +} + +void replicator_queue_set_change_callback(struct replicator_queue *queue, + void (*callback)(void *context), + void *context) +{ + queue->change_callback = callback; + queue->change_context = context; +} + +static struct replicator_user * +replicator_queue_add_int(struct replicator_queue *queue, const char *username, + enum replication_priority priority) +{ + struct replicator_user *user; + + user = hash_table_lookup(queue->user_hash, username); + if (user == NULL) { + user = i_new(struct replicator_user, 1); + user->username = i_strdup(username); + hash_table_insert(queue->user_hash, user->username, user); + } else { + if (user->priority > priority) { + /* user already has a higher priority than this */ + return user; + } + if (!user->popped) + priorityq_remove(queue->user_queue, &user->item); + } + user->priority = priority; + user->last_update = ioloop_time; + + if (!user->popped) + priorityq_add(queue->user_queue, &user->item); + return user; +} + +struct replicator_user * +replicator_queue_add(struct replicator_queue *queue, const char *username, + enum replication_priority priority) +{ + struct replicator_user *user; + + user = replicator_queue_add_int(queue, username, priority); + if (queue->change_callback != NULL) + queue->change_callback(queue->change_context); + return user; +} + +void replicator_queue_add_sync(struct replicator_queue *queue, + const char *username, + replicator_sync_callback_t *callback, + void *context) +{ + struct replicator_user *user; + struct replicator_sync_lookup *lookup; + + user = replicator_queue_add_int(queue, username, + REPLICATION_PRIORITY_SYNC); + + lookup = array_append_space(&queue->sync_lookups); + lookup->user = user; + lookup->callback = callback; + lookup->context = context; + lookup->wait_for_next_push = user->popped; + + if (queue->change_callback != NULL) + queue->change_callback(queue->change_context); +} + +void replicator_queue_remove(struct replicator_queue *queue, + struct replicator_user **_user) +{ + struct replicator_user *user = *_user; + + *_user = NULL; + if (!user->popped) + priorityq_remove(queue->user_queue, &user->item); + hash_table_remove(queue->user_hash, user->username); + + i_free(user->username); + i_free(user); + + if (queue->change_callback != NULL) + queue->change_callback(queue->change_context); +} + +struct replicator_user * +replicator_queue_pop(struct replicator_queue *queue, + unsigned int *next_secs_r) +{ + struct priorityq_item *item; + struct replicator_user *user; + time_t next_full_sync; + + item = priorityq_peek(queue->user_queue); + if (item == NULL) { + /* no users defined. we shouldn't normally get here */ + *next_secs_r = 3600; + return NULL; + } + user = (struct replicator_user *)item; + + next_full_sync = user->last_full_sync + queue->full_sync_interval; + if (user->priority == REPLICATION_PRIORITY_NONE && + next_full_sync > ioloop_time) { + /* we don't want to do a full sync yet */ + *next_secs_r = next_full_sync - ioloop_time; + return NULL; + } + priorityq_remove(queue->user_queue, &user->item); + user->popped = TRUE; + return user; +} + +static void +replicator_queue_handle_sync_lookups(struct replicator_queue *queue, + struct replicator_user *user) +{ + struct replicator_sync_lookup *lookups; + ARRAY_DEFINE(callbacks, struct replicator_sync_lookup); + unsigned int i, count; + bool success = !user->last_sync_failed; + + t_array_init(&callbacks, 8); + lookups = array_get_modifiable(&queue->sync_lookups, &count); + for (i = 0; i < count; ) { + if (lookups[i].user != user) + i++; + else if (lookups[i].wait_for_next_push) { + /* another sync request came while user was being + replicated */ + i_assert(user->priority == REPLICATION_PRIORITY_SYNC); + lookups[i].wait_for_next_push = FALSE; + i++; + } else { + array_append(&callbacks, &lookups[i], 1); + array_delete(&queue->sync_lookups, i, 1); + } + } + + array_foreach_modifiable(&callbacks, lookups) + lookups->callback(success, lookups->context); +} + +void replicator_queue_push(struct replicator_queue *queue, + struct replicator_user *user) +{ + i_assert(user->popped); + + priorityq_add(queue->user_queue, &user->item); + user->popped = FALSE; + + T_BEGIN { + replicator_queue_handle_sync_lookups(queue, user); + } T_END; +} + +static int +replicator_queue_import_line(struct replicator_queue *queue, const char *line) +{ + const char *const *args, *username; + unsigned int priority; + struct replicator_user *user, tmp_user; + + /* <user> <priority> <last update> <last fast sync> <last full sync> */ + args = t_strsplit_tabescaped(line); + if (str_array_length(args) < 5) + return -1; + + memset(&tmp_user, 0, sizeof(tmp_user)); + username = args[0]; + if (username[0] == '\0' || + str_to_uint(args[1], &priority) < 0 || + str_to_time(args[2], &tmp_user.last_update) < 0 || + str_to_time(args[3], &tmp_user.last_fast_sync) < 0 || + str_to_time(args[3], &tmp_user.last_full_sync) < 0) + return -1; + tmp_user.priority = priority; + + user = hash_table_lookup(queue->user_hash, username); + if (user != NULL) { + if (user->last_update > tmp_user.last_update) { + /* we already have a newer state */ + return 0; + } + if (user->last_update == tmp_user.last_update) { + /* either one of these could be newer. use the one + with higher priority. */ + if (user->priority > tmp_user.priority) + return 0; + } + } + user = replicator_queue_add(queue, tmp_user.username, + tmp_user.priority); + user->last_update = tmp_user.last_update; + user->last_fast_sync = tmp_user.last_fast_sync; + user->last_full_sync = tmp_user.last_full_sync; + return 0; +} + +int replicator_queue_import(struct replicator_queue *queue, const char *path) +{ + struct istream *input; + const char *line; + int fd, ret = 0; + + fd = open(path, O_RDONLY); + if (fd == -1) { + if (errno == ENOENT) + return 0; + i_error("open(%s) failed: %m", path); + return -1; + } + + input = i_stream_create_fd(fd, (size_t)-1, TRUE); + while ((line = i_stream_read_next_line(input)) != NULL) { + T_BEGIN { + ret = replicator_queue_import_line(queue, line); + } T_END; + if (ret < 0) { + i_error("Invalid replicator db record: %s", line); + break; + } + } + if (input->stream_errno != 0) + ret = -1; + i_stream_destroy(&input); + return ret; +} + +static void +replicator_queue_export_user(struct replicator_user *user, string_t *str) +{ + str_tabescape_write(str, user->username); + str_printfa(str, "\t%d\t%lld\t%lld\t%lld", (int)user->priority, + (long long)user->last_update, + (long long)user->last_fast_sync, + (long long)user->last_full_sync); +} + +int replicator_queue_export(struct replicator_queue *queue, const char *path) +{ + struct ostream *output; + struct priorityq_item *const *items; + unsigned int i, count; + string_t *str; + int fd, ret; + + fd = creat(path, 0600); + if (fd == -1) { + i_error("creat(%s) failed: %m", path); + return -1; + } + output = o_stream_create_fd_file(fd, 0, TRUE); + o_stream_cork(output); + + str = t_str_new(128); + items = priorityq_items(queue->user_queue); + count = priorityq_count(queue->user_queue); + for (i = 0; i < count; i++) { + struct replicator_user *user = + (struct replicator_user *)items[i]; + + str_truncate(str, 0); + replicator_queue_export_user(user, str); + if (o_stream_send(output, str_data(str), str_len(str)) < 0) + break; + } + + ret = output->last_failed_errno != 0 ? -1 : 0; + o_stream_destroy(&output); + return ret; +}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/replication/replicator/replicator-queue.h Sun May 20 03:25:04 2012 +0300 @@ -0,0 +1,60 @@ +#ifndef REPLICATOR_QUEUE_H +#define REPLICATOR_QUEUE_H + +#include "priorityq.h" +#include "replication-common.h" + +struct replicator_user { + struct priorityq_item item; + + char *username; + enum replication_priority priority; + /* last time this user's state was updated */ + time_t last_update; + /* last_fast_run is always >= last_full_run. */ + time_t last_fast_sync, last_full_sync; + + /* User isn't currently in replication queue */ + unsigned int popped:1; + /* Last replication sync failed */ + unsigned int last_sync_failed:1; +}; + +typedef void replicator_sync_callback_t(bool success, void *context); + +struct replicator_queue *replicator_queue_init(unsigned int full_sync_interval); +void replicator_queue_deinit(struct replicator_queue **queue); + +/* Call the specified callback when data is added/removed/moved in queue + via _add(), _add_sync() or _remove() functions (not push/pop). */ +void replicator_queue_set_change_callback(struct replicator_queue *queue, + void (*callback)(void *context), + void *context); + +/* Add a user to queue and return it. If the user already exists, it's updated + only if the new priority is higher. */ +struct replicator_user * +replicator_queue_add(struct replicator_queue *queue, const char *username, + enum replication_priority priority); +void replicator_queue_add_sync(struct replicator_queue *queue, + const char *username, + replicator_sync_callback_t *callback, + void *context); +/* Remove user from replication queue and free it. */ +void replicator_queue_remove(struct replicator_queue *queue, + struct replicator_user **user); + +/* Return the next user from replication queue, and remove it from the queue. + If there's nothing to be replicated currently, returns NULL and sets + next_secs_r to when there should be more work to do. */ +struct replicator_user * +replicator_queue_pop(struct replicator_queue *queue, + unsigned int *next_secs_r); +/* Add user back to queue. */ +void replicator_queue_push(struct replicator_queue *queue, + struct replicator_user *user); + +int replicator_queue_import(struct replicator_queue *queue, const char *path); +int replicator_queue_export(struct replicator_queue *queue, const char *path); + +#endif
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/replication/replicator/replicator-settings.c Sun May 20 03:25:04 2012 +0300 @@ -0,0 +1,80 @@ +/* Copyright (c) 2012 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "buffer.h" +#include "settings-parser.h" +#include "service-settings.h" +#include "replicator-settings.h" + +/* <settings checks> */ +static struct file_listener_settings replicator_unix_listeners_array[] = { + { "replicator", 0600, "$default_internal_user", "" } +}; +static struct file_listener_settings *replicator_unix_listeners[] = { + &replicator_unix_listeners_array[0] +}; +static buffer_t replicator_unix_listeners_buf = { + replicator_unix_listeners, sizeof(replicator_unix_listeners), { 0, } +}; +/* </settings checks> */ + +struct service_settings replicator_service_settings = { + .name = "replicator", + .protocol = "", + .type = "", + .executable = "replicator", + .user = "", + .group = "", + .privileged_group = "", + .extra_groups = "", + .chroot = "", + + .drop_priv_before_exec = FALSE, + + .process_min_avail = 0, + .process_limit = 1, + .client_limit = 0, + .service_count = 0, + .idle_kill = -1U, + .vsz_limit = (uoff_t)-1, + + .unix_listeners = { { &replicator_unix_listeners_buf, + sizeof(replicator_unix_listeners[0]) } }, + .fifo_listeners = ARRAY_INIT, + .inet_listeners = ARRAY_INIT +}; + +#undef DEF +#define DEF(type, name) \ + { type, #name, offsetof(struct replicator_settings, name), NULL } + +static const struct setting_define replicator_setting_defines[] = { + DEF(SET_STR, auth_socket_path), + DEF(SET_STR, doveadm_socket_path), + + DEF(SET_TIME, replication_full_sync_interval), + DEF(SET_UINT, replication_max_conns), + + SETTING_DEFINE_LIST_END +}; + +const struct replicator_settings replicator_default_settings = { + .auth_socket_path = "auth-userdb", + .doveadm_socket_path = "doveadm-server", + + .replication_full_sync_interval = 60*60*12, + .replication_max_conns = 10 +}; + +const struct setting_parser_info replicator_setting_parser_info = { + .module_name = "replicator", + .defines = replicator_setting_defines, + .defaults = &replicator_default_settings, + + .type_offset = (size_t)-1, + .struct_size = sizeof(struct replicator_settings), + + .parent_offset = (size_t)-1 +}; + +const struct replicator_settings *replicator_settings;
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/replication/replicator/replicator-settings.h Sun May 20 03:25:04 2012 +0300 @@ -0,0 +1,15 @@ +#ifndef REPLICATOR_SETTINGS_H +#define REPLICATOR_SETTINGS_H + +struct replicator_settings { + const char *auth_socket_path; + const char *doveadm_socket_path; + + unsigned int replication_full_sync_interval; + unsigned int replication_max_conns; +}; + +extern const struct setting_parser_info replicator_setting_parser_info; +extern const struct replicator_settings *replicator_settings; + +#endif
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/replication/replicator/replicator.c Sun May 20 03:25:04 2012 +0300 @@ -0,0 +1,117 @@ +/* Copyright (c) 2012 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "ioloop.h" +#include "restrict-access.h" +#include "auth-master.h" +#include "master-service.h" +#include "master-service-settings.h" +#include "notify-connection.h" +#include "replicator-brain.h" +#include "replicator-queue.h" +#include "replicator-settings.h" + +#define REPLICATOR_AUTH_SERVICE_NAME "replicator" +#define REPLICATOR_DB_DUMP_INTERVAL_MSECS (1000*60*15) +#define REPLICATOR_DB_PATH PKG_STATEDIR"/replicator.db" + +static struct replicator_queue *queue; +static struct replicator_brain *brain; +static const struct replicator_settings *set; +static struct timeout *to_dump; + +static void client_connected(struct master_service_connection *conn) +{ + master_service_client_connection_accept(conn); + (void)notify_connection_create(conn->fd, queue); +} + +static void replication_add_users(struct replicator_queue *queue) +{ + struct auth_master_connection *auth_conn; + struct auth_master_user_list_ctx *ctx; + struct auth_user_info user_info; + struct replicator_user *user; + const char *username; + + auth_conn = auth_master_init(set->auth_socket_path, + AUTH_MASTER_FLAG_NO_IDLE_TIMEOUT); + + memset(&user_info, 0, sizeof(user_info)); + user_info.service = REPLICATOR_AUTH_SERVICE_NAME; + + /* add all users into replication queue, so that we can start doing + full syncs for everyone whose state can't be found */ + ctx = auth_master_user_list_init(auth_conn, NULL, &user_info); + while ((username = auth_master_user_list_next(ctx)) != NULL) { + user = replicator_queue_add(queue, username, + REPLICATION_PRIORITY_NONE); + user->last_update = 0; + } + if (auth_master_user_list_deinit(&ctx) < 0) + i_error("listing users failed, can't replicate existing data"); + auth_master_deinit(&auth_conn); + + /* add updates from replicator db, if it exists */ + (void)replicator_queue_import(queue, REPLICATOR_DB_PATH); +} + +static void replicator_dump_timeout(void *context ATTR_UNUSED) +{ + (void)replicator_queue_export(queue, REPLICATOR_DB_PATH); +} + +static void main_init(void) +{ + void **sets; + + sets = master_service_settings_get_others(master_service); + set = sets[0]; + + queue = replicator_queue_init(set->replication_full_sync_interval); + replication_add_users(queue); + to_dump = timeout_add(REPLICATOR_DB_DUMP_INTERVAL_MSECS, + replicator_dump_timeout, NULL); + brain = replicator_brain_init(queue, set); +} + +static void main_deinit(void) +{ + notify_connections_destroy_all(); + replicator_brain_deinit(&brain); + timeout_remove(&to_dump); + (void)replicator_queue_export(queue, REPLICATOR_DB_PATH); + replicator_queue_deinit(&queue); +} + +int main(int argc, char *argv[]) +{ + const struct setting_parser_info *set_roots[] = { + &replicator_setting_parser_info, + NULL + }; + const enum master_service_flags service_flags = + MASTER_SERVICE_FLAG_NO_IDLE_DIE; + const char *error; + + master_service = master_service_init("replicator", service_flags, + &argc, &argv, NULL); + if (master_getopt(master_service) > 0) + return FATAL_DEFAULT; + + if (master_service_settings_read_simple(master_service, set_roots, + &error) < 0) + i_fatal("Error reading configuration: %s", error); + master_service_init_log(master_service, "replicator: "); + + restrict_access_by_env(NULL, FALSE); + restrict_access_allow_coredumps(TRUE); + master_service_init_finish(master_service); + + main_init(); + master_service_run(master_service, client_connected); + main_deinit(); + + master_service_deinit(&master_service); + return 0; +}
--- a/src/stats/mail-command.c Sat May 19 22:40:08 2012 +0300 +++ b/src/stats/mail-command.c Sun May 20 03:25:04 2012 +0300 @@ -100,11 +100,12 @@ struct mail_command *cmd; struct mail_stats stats, diff_stats; const char *error; - unsigned int cmd_id; - bool done; + unsigned int i, cmd_id; + bool done = FALSE, continued = FALSE; - /* <session guid> <cmd id> <done> <name> <args> [key=value ..] */ - if (str_array_length(args) < 4) { + /* <session guid> <cmd id> [d] <name> <args> [key=value ..] + <session guid> <cmd id> c[d] [key=value ..] */ + if (str_array_length(args) < 3) { *error_r = "UPDATE-CMD: Too few parameters"; return -1; } @@ -115,39 +116,62 @@ *error_r = "UPDATE-CMD: Invalid command id"; return -1; } - if (strcmp(args[2], "0") != 0 && - strcmp(args[2], "1") != 0) { - *error_r = "UPDATE-CMD: Invalid done parameter"; - return -1; - } - done = args[2][0] == '1'; - if (mail_stats_parse(args+5, &stats, error_r) < 0) { - *error_r = t_strconcat("UPDATE-CMD: ", *error_r, NULL); - return -1; + for (i = 0; args[2][i] != '\0'; i++) { + switch (args[2][i]) { + case 'd': + done = TRUE; + break; + case 'c': + continued = TRUE; + break; + default: + *error_r = "UPDATE-CMD: Invalid flags parameter"; + return -1; + } } cmd = mail_command_find(session, cmd_id); - if (cmd == NULL) { + if (!continued) { + /* new command */ + if (cmd != NULL) { + *error_r = "UPDATE-CMD: Duplicate new command id"; + return -1; + } + if (str_array_length(args) < 5) { + *error_r = "UPDATE-CMD: Too few parameters"; + return -1; + } cmd = mail_command_add(session, args[3], args[4]); cmd->id = cmd_id; - cmd->stats = stats; - diff_stats = stats; + session->highest_cmd_id = + I_MAX(session->highest_cmd_id, cmd_id); session->num_cmds++; session->user->num_cmds++; session->user->domain->num_cmds++; if (session->ip != NULL) session->ip->num_cmds++; + args += 5; } else { - if (!mail_stats_diff(&cmd->stats, &stats, &diff_stats, - &error)) { - *error_r = t_strconcat("UPDATE-CMD: stats shrank: ", - error, NULL); - return -1; + if (cmd == NULL) { + /* already expired command, ignore */ + i_warning("UPDATE-CMD: Already expired"); + return 0; } + args += 3; cmd->last_update = ioloop_timeval; - mail_stats_add(&session->stats, &diff_stats); + } + if (mail_stats_parse(args, &stats, error_r) < 0) { + *error_r = t_strconcat("UPDATE-CMD: ", *error_r, NULL); + return -1; } + if (!mail_stats_diff(&cmd->stats, &stats, &diff_stats, &error)) { + *error_r = t_strconcat("UPDATE-CMD: stats shrank: ", + error, NULL); + return -1; + } + mail_stats_add(&cmd->stats, &diff_stats); + if (done) { cmd->id = 0; mail_command_unref(&cmd); @@ -182,7 +206,8 @@ } mail_command_free(stable_mail_commands_head); - if (global_used_memory < stats_settings->memory_limit) + if (global_used_memory < stats_settings->memory_limit || + stable_mail_commands_head == NULL) break; diff = ioloop_time - stable_mail_commands_head->last_update.tv_sec;
--- a/src/stats/mail-domain.c Sat May 19 22:40:08 2012 +0300 +++ b/src/stats/mail-domain.c Sun May 20 03:25:04 2012 +0300 @@ -100,7 +100,8 @@ while (mail_domains_head != NULL && mail_domains_head->refcount == 0) { mail_domain_free(mail_domains_head); - if (global_used_memory < stats_settings->memory_limit) + if (global_used_memory < stats_settings->memory_limit || + mail_domains_head == NULL) break; diff = ioloop_time - mail_domains_head->last_update.tv_sec;
--- a/src/stats/mail-ip.c Sat May 19 22:40:08 2012 +0300 +++ b/src/stats/mail-ip.c Sun May 20 03:25:04 2012 +0300 @@ -96,7 +96,8 @@ while (mail_ips_head != NULL && mail_ips_head->refcount == 0) { mail_ip_free(mail_ips_head); - if (global_used_memory < stats_settings->memory_limit) + if (global_used_memory < stats_settings->memory_limit || + mail_ips_head == NULL) break; diff = ioloop_time - mail_ips_head->last_update.tv_sec;
--- a/src/stats/mail-session.c Sat May 19 22:40:08 2012 +0300 +++ b/src/stats/mail-session.c Sun May 20 03:25:04 2012 +0300 @@ -46,9 +46,10 @@ static void mail_session_idle_timeout(struct mail_session *session) { - i_warning("Session %s (user %s) appears to have crashed, " + i_warning("Session %s (user %s, service %s) appears to have crashed, " "disconnecting it", - guid_128_to_string(session->guid), session->user->name); + guid_128_to_string(session->guid), session->user->name, + session->service); mail_session_disconnect(session); } @@ -276,7 +277,8 @@ i_assert(mail_sessions_head->disconnected); mail_session_free(mail_sessions_head); - if (global_used_memory < stats_settings->memory_limit) + if (global_used_memory < stats_settings->memory_limit || + mail_sessions_head == NULL) break; diff = ioloop_time - mail_sessions_head->last_update.tv_sec;
--- a/src/stats/mail-user.c Sat May 19 22:40:08 2012 +0300 +++ b/src/stats/mail-user.c Sun May 20 03:25:04 2012 +0300 @@ -118,7 +118,8 @@ while (mail_users_head != NULL && mail_users_head->refcount == 0) { mail_user_free(mail_users_head); - if (global_used_memory < stats_settings->memory_limit) + if (global_used_memory < stats_settings->memory_limit || + mail_users_head == NULL) break; diff = ioloop_time - mail_users_head->last_update.tv_sec;
--- a/src/util/Makefile.am Sat May 19 22:40:08 2012 +0300 +++ b/src/util/Makefile.am Sun May 20 03:25:04 2012 +0300 @@ -27,8 +27,13 @@ rawlog_SOURCES = \ rawlog.c -script_login_LDADD = $(LIBDOVECOT_STORAGE) $(LIBDOVECOT) $(MODULE_LIBS) -script_login_DEPENDENCIES = $(LIBDOVECOT_STORAGE) $(LIBDOVECOT_DEPS) +script_login_LDADD = \ + $(LIBDOVECOT_STORAGE) \ + $(LIBDOVECOT) \ + $(MODULE_LIBS) +script_login_DEPENDENCIES = \ + $(LIBDOVECOT_STORAGE_DEPS) \ + $(LIBDOVECOT_DEPS) script_login_SOURCES = \ script-login.c
--- a/src/util/script-login.c Sat May 19 22:40:08 2012 +0300 +++ b/src/util/script-login.c Sun May 20 03:25:04 2012 +0300 @@ -84,7 +84,7 @@ /* put everything to environment */ env_clean(); keys = t_str_new(256); - args = t_strsplit(data_line, "\t"); + args = t_strsplit_tab(data_line); if (str_array_length(args) < 3) i_fatal("Missing input fields");
--- a/update-version.sh Sat May 19 22:40:08 2012 +0300 +++ b/update-version.sh Sun May 20 03:25:04 2012 +0300 @@ -64,4 +64,4 @@ cmp -s "${BUILDDIR}/${VERSION_H}" "${BUILDDIR}/${VERSION_HT}" && \ rm -f "${BUILDDIR}/${VERSION_HT}" || \ - mv "${BUILDDIR}/${VERSION_HT}" "${BUILDDIR}/${VERSION_H}" + mv -f "${BUILDDIR}/${VERSION_HT}" "${BUILDDIR}/${VERSION_H}"